diff --git a/cmd/rbac-manager/main.go b/cmd/rbac-manager/main.go
index 82ae90a0..9257c01c 100644
--- a/cmd/rbac-manager/main.go
+++ b/cmd/rbac-manager/main.go
@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
 
 	"cloud.google.com/go/compute/metadata"
 	"github.com/alecthomas/kingpin"
+	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	directoryv1 "google.golang.org/api/admin/directory/v1"
 	"google.golang.org/api/impersonate"
@@ -112,6 +112,8 @@ func createGoogleDirectory(ctx context.Context, subject string) (*directoryv1.Se
 		return nil, err
 	}
 
+	var ts oauth2.TokenSource
+
 	// If the found credential doesn't contain JSON, try to fallback to workload identity
 	if len(creds.JSON) == 0 {
 		// Get the email address associated with the service account. The account may be empty
@@ -128,21 +130,25 @@ func createGoogleDirectory(ctx context.Context, subject string) (*directoryv1.Se
 			Subject:         subject,
 		}
 
-		ts, err := impersonate.CredentialsTokenSource(ctx, config, option.WithCredentials(creds))
+		// Impersonation (as itself) is required as the federated access token obtained from the GCE
+		// metadata server is not sufficient for acting as the subject via domain-wide delegation.
+		// For delegation to work, we need to sign a JWT with the the "sub" claim set to subject -
+		// this happens implicitly through impersonation.
+		ts, err = impersonate.CredentialsTokenSource(ctx, config)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		conf, err := google.JWTConfigFromJSON(creds.JSON, scopes...)
 		if err != nil {
 			return nil, err
 		}
 
-		return directoryv1.NewService(ctx, option.WithTokenSource(ts))
-	}
+		// Access to the directory API must be signed with a Subject to enable domain selection.
+		conf.Subject = subject
 
-	conf, err := google.JWTConfigFromJSON(creds.JSON, strings.Join(scopes, " "))
-	if err != nil {
-		return nil, err
+		ts = conf.TokenSource(ctx)
 	}
 
-	// Access to the directory API must be signed with a Subject to enable domain selection.
-	conf.Subject = subject
-
-	return directoryv1.NewService(ctx, option.WithHTTPClient(conf.Client(ctx)))
+	return directoryv1.NewService(ctx, option.WithTokenSource(ts))
 }