Enable Workload Identity Federation to work with provider

.local/Dockerfile

@@ -0,0 +1,20 @@
+FROM golang:1.22-alpine
+
+ENV CGO_ENABLED=0
+ENV GOROOT=/usr/local/go
+ENV GOPATH=${HOME}/go
+ENV PATH=$PATH:${GOROOT}/bin
+
+RUN apk update && apk add --no-cache \
+    git && \
+    go install github.com/go-delve/delve/cmd/dlv@latest
+
+WORKDIR /secrets-store-csi-driver-provider-gcp-codebase
+
+COPY go.mod go.mod
+RUN go mod download
+
+EXPOSE 30123
+
+# these dlv debug arguments replicate driver args from DaemonSet
+ENTRYPOINT ["/go/bin/dlv", "--listen=:30123", "--accept-multiclient", "--headless=true", "--api-version=2", "debug", "./", "--", "-v", "5"]

.local/README.md

@@ -0,0 +1,29 @@
+# Overview
+This clones the workflow laid out in [secrets store csi driver for local debugging](https://github.com/kubernetes-sigs/secrets-store-csi-driver/tree/main/.local).
+
+Please review the sibling flow with its prerequisites, and then you can pick and choose what is needed on this side.
+
+> NOTE: Steps in this guide are not tested by CI/CD. This is just one of the way to locally debug the code and a good starting point.
+
+The debug driver was used to help flesh out issues with the federation of the workload identity. You must have a pod
+that is attempting to mount the secret driver to have the debug breakpoints hit.
+
+Review [docs/fleet-wif-notes.md](../docs/fleet-wif-notes.md) and the example [mypod.yaml.tmpl](../examples/mypod.yaml.tmpl)
+for more details about setting up a consuming pod.
+
+## Creating a docker image
+- Build docker image from [Dockerfile](Dockerfile):
+
+```sh
+docker build -t debug-driver -f .local/Dockerfile .
+```
+
+## Update the debug-driver.yaml
+Update the following items in .local/debug-driver.yaml:
+
+* In the `workload-id-config` update the `audience` to match the audience of the workload identity pool provider.
+* In the `debug-driver` container, update `driver-volume` to utilize a path on your local machine that is configured to be 
+mounted into the pod.
+* In the `gcp-ksa` volume, update the `audience` to match the audience of the workload identity pool provider.
+
+Deploy the debug-driver and the consuming pod. You can then hook up your IDE to the debug-driver container via delve.

.local/debug-driver.yaml

@@ -0,0 +1,121 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workload-id-config
+  namespace: kube-system
+immutable: false
+data:
+  config: >-
+    {
+      "audience":"//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID",
+      "credential_source":{
+        "file":"/var/run/secrets/tokens/gcp-ksa/token",
+        "format":{
+          "type":"text"
+        }
+      },
+      "subject_token_type":"urn:ietf:params:oauth:token-type:jwt",
+      "token_info_url":"https://sts.googleapis.com/v1/introspect",
+      "token_url":"https://sts.googleapis.com/v1/token",
+      "type":"external_account",
+      "universe_domain":"googleapis.com"
+    }
+
+---
+# Separate deployment to manage actual driver debugging independent of node registrar and livenessprobe
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: debug-driver
+  namespace: kube-system
+spec:
+  replicas: 1 # Keep single replica for debugging
+  selector:
+    matchLabels:
+      app: debug-driver
+  template:
+    metadata:
+      labels:
+        app: debug-driver
+    spec:
+      serviceAccountName: secrets-store-csi-driver-provider-gcp
+      initContainers:
+        - name: chown-provider-mount
+          image: busybox
+          command:
+            - chown
+            - "1000:1000"
+            - /etc/kubernetes/secrets-store-csi-providers
+          volumeMounts:
+            - mountPath: "/etc/kubernetes/secrets-store-csi-providers"
+              name: providervol
+      containers:
+        - name: debug-driver
+          image: debug-driver:latest # dlv debug image built locally
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /secrets-store-csi-driver-provider-gcp-codebase
+              name: driver-volume
+            - name: providervol
+              mountPath: /etc/kubernetes/secrets-store-csi-providers
+              mountPropagation: None
+              readOnly: false
+            - name: "gcp-ksa"
+              mountPath: "/var/run/secrets/tokens/gcp-ksa"
+              readOnly: true
+              mountPropagation: None
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1Gi
+            requests:
+              cpu: 250m
+              memory: 512Mi
+          env:
+            - name: TARGET_DIR
+              value: /etc/kubernetes/secrets-store-csi-providers
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/run/secrets/tokens/gcp-ksa/google-application-credentials.json
+            - name: GAIA_TOKEN_EXCHANGE_ENDPOINT
+              value: https://sts.googleapis.com/v1/token
+      volumes:
+        - name: driver-volume
+          hostPath:
+            path: # /path/to/your/secrets-store-csi-driver-provider-gcp/codebase/on/host
+            type: Directory
+        - name: providervol
+          hostPath:
+            path: /etc/kubernetes/secrets-store-csi-providers
+        - name: "gcp-ksa"
+          projected:
+            defaultMode: 420
+            sources:
+              - serviceAccountToken:
+                  audience: # //iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID
+                  expirationSeconds: 172800
+                  path: token
+              - configMap:
+                  name: workload-id-config
+                  items:
+                      - key: config
+                        path: google-application-credentials.json
+                  optional: false
+
+---
+# Service to connect dlv apis
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-debug
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector:
+    app: debug-driver
+  ports:
+  - name: debug
+    port: 30123
+    targetPort: 30123
+    nodePort: 30123

auth/auth.go

@@ -117,15 +117,20 @@ func (c *Client) TokenSource(ctx context.Context, cfg *config.MountConfig) (oaut
 // its own token. Token creation can be removed once driver implements the requiresRepublish.
 func (c *Client) Token(ctx context.Context, cfg *config.MountConfig) (*oauth2.Token, error) {
 
+	var audience string
 	idPool, idProvider, err := c.gkeWorkloadIdentity(ctx, cfg)
 	if err != nil {
-		idPool, idProvider, err = c.fleetWorkloadIdentity(ctx, cfg)
+		idPool, idProvider, audience, err = c.fleetWorkloadIdentity(ctx, cfg)
 		if err != nil {
 			return nil, err
 		}
 	}
-
-	klog.V(5).InfoS("workload id configured", "pool", idPool, "provider", idProvider)
+	if audience == "" {
+		audience = fmt.Sprintf("identitynamespace:%s:%s", idPool, idProvider)
+		klog.V(5).InfoS("workload id configured", "pool", idPool, "provider", idProvider)
+	} else {
+		klog.V(5).InfoS("workload federation pool audience", audience)
+	}
 
 	// Get iam.gke.io/gcp-service-account annotation to see if the
 	// identitybindingtoken token should be traded for a GCP SA token.
@@ -143,21 +148,21 @@ func (c *Client) Token(ctx context.Context, cfg *config.MountConfig) (*oauth2.To
 	// Obtain a serviceaccount token for the pod.
 	var saTokenVal string
 	if cfg.PodInfo.ServiceAccountTokens != "" {
-		saToken, err := c.extractSAToken(cfg, idPool) // calling function to extract token received from driver.
+		saToken, err := c.extractSAToken(cfg, idPool, audience) // calling function to extract token received from driver.
 		if err != nil {
 			return nil, fmt.Errorf("unable to fetch SA token from driver: %w", err)
 		}
 		saTokenVal = saToken.Token
 	} else {
-		saToken, err := c.generatePodSAToken(ctx, cfg, idPool) // if no token received, provider generates its own token.
+		saToken, err := c.generatePodSAToken(ctx, cfg, idPool, audience) // if no token received, provider generates its own token.
 		if err != nil {
 			return nil, fmt.Errorf("unable to fetch pod token: %w", err)
 		}
 		saTokenVal = saToken.Token
 	}
 
 	// Trade the kubernetes token for an identitybindingtoken token.
-	idBindToken, err := tradeIDBindToken(ctx, c.HTTPClient, saTokenVal, idPool, idProvider)
+	idBindToken, err := tradeIDBindToken(ctx, c.HTTPClient, saTokenVal, audience)
 	if err != nil {
 		return nil, fmt.Errorf("unable to fetch identitybindingtoken: %w", err)
 	}
@@ -179,28 +184,32 @@ func (c *Client) Token(ctx context.Context, cfg *config.MountConfig) (*oauth2.To
 	return &oauth2.Token{AccessToken: gcpSAResp.GetAccessToken()}, nil
 }
 
-func (c *Client) extractSAToken(cfg *config.MountConfig, idPool string) (*authenticationv1.TokenRequestStatus, error) {
+func (c *Client) extractSAToken(cfg *config.MountConfig, idPool, audience string) (*authenticationv1.TokenRequestStatus, error) {
 	audienceTokens := map[string]authenticationv1.TokenRequestStatus{}
 	if err := json.Unmarshal([]byte(cfg.PodInfo.ServiceAccountTokens), &audienceTokens); err != nil {
 		return nil, err
 	}
 	for k, v := range audienceTokens {
-		if k == idPool { // Only returns the token if the audience is the workload identity. Other tokens cannot be used.
+		if k == idPool || k == audience { // Only returns the token if the audience is the workload identity. Other tokens cannot be used.
 			return &v, nil
 		}
 	}
 	return nil, fmt.Errorf("no token has audience value of idPool")
 }
 
-func (c *Client) generatePodSAToken(ctx context.Context, cfg *config.MountConfig, idPool string) (*authenticationv1.TokenRequestStatus, error) {
+func (c *Client) generatePodSAToken(ctx context.Context, cfg *config.MountConfig, idPool, audience string) (*authenticationv1.TokenRequestStatus, error) {
 	ttl := int64((15 * time.Minute).Seconds())
+	_audience := idPool
+	if _audience == "" {
+		_audience = audience
+	}
 	resp, err := c.KubeClient.CoreV1().
 		ServiceAccounts(cfg.PodInfo.Namespace).
 		CreateToken(ctx, cfg.PodInfo.ServiceAccount,
 			&authenticationv1.TokenRequest{
 				Spec: authenticationv1.TokenRequestSpec{
 					ExpirationSeconds: &ttl,
-					Audiences:         []string{idPool},
+					Audiences:         []string{_audience},
 					BoundObjectRef: &authenticationv1.BoundObjectReference{
 						Kind:       "Pod", // Pod and secret are the only valid types
 						APIVersion: "v1",
@@ -243,44 +252,45 @@ func (c *Client) gkeWorkloadIdentity(ctx context.Context, cfg *config.MountConfi
 	return idPool, idProvider, nil
 }
 
-func (c *Client) fleetWorkloadIdentity(ctx context.Context, cfg *config.MountConfig) (string, string, error) {
+func (c *Client) fleetWorkloadIdentity(ctx context.Context, cfg *config.MountConfig) (string, string, string, error) {
 	const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
 	var jsonData []byte
 	var err error
 	if filename := os.Getenv(envVar); filename != "" {
 		jsonData, err = os.ReadFile(filepath.Clean(filename))
 		if err != nil {
-			return "", "", fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
+			return "", "", "", fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
 		}
 	}
 
 	// Parse jsonData as one of the other supported credentials files.
 	var f credentialsFile
 	if err := json.Unmarshal(jsonData, &f); err != nil {
-		return "", "", err
+		return "", "", "", err
 	}
 
 	if f.Type != externalAccountKey {
-		return "", "", fmt.Errorf("google: unexpected credentials type: %v, expected: %v", f.Type, externalAccountKey)
+		return "", "", "", fmt.Errorf("google: unexpected credentials type: %v, expected: %v", f.Type, externalAccountKey)
 	}
 
 	split := strings.SplitN(f.Audience, ":", 3)
 	if split == nil || len(split) < 3 {
-		return "", "", fmt.Errorf("google: unexpected audience value: %v", f.Audience)
+		// If the audience is not in the expected format, return the audience as the audience since this is likely a federated pool.
+		return "", "", f.Audience, nil
 	}
 	idPool := split[1]
 	idProvider := split[2]
 
-	return idPool, idProvider, nil
+	return idPool, idProvider, "", nil
 }
 
-func tradeIDBindToken(ctx context.Context, client *http.Client, k8sToken, idPool, idProvider string) (*oauth2.Token, error) {
+func tradeIDBindToken(ctx context.Context, client *http.Client, k8sToken, audience string) (*oauth2.Token, error) {
 	body, err := json.Marshal(map[string]string{
 		"grant_type":           "urn:ietf:params:oauth:grant-type:token-exchange",
 		"subject_token_type":   "urn:ietf:params:oauth:token-type:jwt",
 		"requested_token_type": "urn:ietf:params:oauth:token-type:access_token",
 		"subject_token":        k8sToken,
-		"audience":             fmt.Sprintf("identitynamespace:%s:%s", idPool, idProvider),
+		"audience":             audience,
 		"scope":                "https://www.googleapis.com/auth/cloud-platform",
 	})
 	if err != nil {