From b60a7b62e51c50335cbaf235e0fa162db48bdca8 Mon Sep 17 00:00:00 2001 From: Darren Dowker Date: Thu, 18 Apr 2024 18:36:00 -0700 Subject: [PATCH 1/3] workload identity changes --- broker/fragment/store_gcs.go | 62 +++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/broker/fragment/store_gcs.go b/broker/fragment/store_gcs.go index 010f55cc..c88c4852 100644 --- a/broker/fragment/store_gcs.go +++ b/broker/fragment/store_gcs.go @@ -2,7 +2,6 @@ package fragment import ( "context" - "fmt" "io" "net/url" "strings" @@ -153,31 +152,44 @@ func (s *gcsBackend) gcsClient(ep *url.URL) (cfg GSStoreConfig, client *storage. creds, err := google.FindDefaultCredentials(ctx, storage.ScopeFullControl) if err != nil { return - } else if creds.JSON == nil { - err = fmt.Errorf("use of GCS requires that a service-account private key be supplied with application default credentials") - return - } - conf, err := google.JWTConfigFromJSON(creds.JSON, storage.ScopeFullControl) - if err != nil { - return - } - client, err = storage.NewClient(ctx, option.WithTokenSource(conf.TokenSource(ctx))) - if err != nil { - return - } - opts = storage.SignedURLOptions{ - GoogleAccessID: conf.Email, - PrivateKey: conf.PrivateKey, + } else if creds.JSON != nil { + conf, err := google.JWTConfigFromJSON(creds.JSON, storage.ScopeFullControl) + if err != nil { + return + } + client, err = storage.NewClient(ctx, option.WithTokenSource(conf.TokenSource(ctx))) + if err != nil { + return + } + opts = storage.SignedURLOptions{ + GoogleAccessID: conf.Email, + PrivateKey: conf.PrivateKey, + } + s.client, s.signedURLOptions = client, opts + + log.WithFields(log.Fields{ + "ProjectID": creds.ProjectID, + "GoogleAccessID": conf.Email, + "PrivateKeyID": conf.PrivateKeyID, + "Subject": conf.Subject, + "Scopes": conf.Scopes, + }).Info("constructed new GCS client") + } else { + // Possible to use GCS without a service account (e.g. with a GCE instance and workload identity). + client, err = storage.NewClient(ctx, option.WithTokenSource(creds.TokenSource)) + if err != nil { + return + } + + s.client = client + + // TODO(djd): do i need to form a signed URL options or change this file to work without one? + // Looking at SignGet() it seems like I do need to form signed URL options. Just give it a try. + + log.WithFields(log.Fields{ + "ProjectID": creds.ProjectID, + }).Info("constructed new GCS client without JWT") } - s.client, s.signedURLOptions = client, opts - - log.WithFields(log.Fields{ - "ProjectID": creds.ProjectID, - "GoogleAccessID": conf.Email, - "PrivateKeyID": conf.PrivateKeyID, - "Subject": conf.Subject, - "Scopes": conf.Scopes, - }).Info("constructed new GCS client") return } From 86f8ba3e96d78c88d9d197a4d30a889f03de62bd Mon Sep 17 00:00:00 2001 From: Darren Dowker Date: Thu, 18 Apr 2024 19:00:08 -0700 Subject: [PATCH 2/3] get past CI error --- broker/fragment/store_gcs.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/broker/fragment/store_gcs.go b/broker/fragment/store_gcs.go index c88c4852..6b441016 100644 --- a/broker/fragment/store_gcs.go +++ b/broker/fragment/store_gcs.go @@ -12,6 +12,7 @@ import ( log "github.com/sirupsen/logrus" pb "go.gazette.dev/core/broker/protocol" "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" "google.golang.org/api/iterator" "google.golang.org/api/option" ) @@ -132,6 +133,8 @@ func (s *gcsBackend) Remove(ctx context.Context, fragment pb.Fragment) error { } func (s *gcsBackend) gcsClient(ep *url.URL) (cfg GSStoreConfig, client *storage.Client, opts storage.SignedURLOptions, err error) { + var conf *jwt.Config + if err = parseStoreArgs(ep, &cfg); err != nil { return } @@ -153,7 +156,7 @@ func (s *gcsBackend) gcsClient(ep *url.URL) (cfg GSStoreConfig, client *storage. if err != nil { return } else if creds.JSON != nil { - conf, err := google.JWTConfigFromJSON(creds.JSON, storage.ScopeFullControl) + conf, err = google.JWTConfigFromJSON(creds.JSON, storage.ScopeFullControl) if err != nil { return } From 45427bcbb666bbc3c0617cb2321be2179b85745f Mon Sep 17 00:00:00 2001 From: Darren Dowker Date: Wed, 24 Apr 2024 11:36:33 -0700 Subject: [PATCH 3/3] correct comment --- broker/fragment/store_gcs.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/broker/fragment/store_gcs.go b/broker/fragment/store_gcs.go index 6b441016..4fa7e5c3 100644 --- a/broker/fragment/store_gcs.go +++ b/broker/fragment/store_gcs.go @@ -186,8 +186,7 @@ func (s *gcsBackend) gcsClient(ep *url.URL) (cfg GSStoreConfig, client *storage. s.client = client - // TODO(djd): do i need to form a signed URL options or change this file to work without one? - // Looking at SignGet() it seems like I do need to form signed URL options. Just give it a try. + // Note: SignGet() also works with empty signedURLOptions. log.WithFields(log.Fields{ "ProjectID": creds.ProjectID,