From 10f184698a48e3b9fc20cb19493e0c686063c708 Mon Sep 17 00:00:00 2001 From: Darren Dowker Date: Thu, 6 Jun 2024 13:56:19 -0700 Subject: [PATCH] clean up debug logs --- broker/client/reader.go | 14 +++----------- broker/fragment/store_gcs.go | 12 ------------ mainboilerplate/runconsumer/run_consumer.go | 10 +++++----- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/broker/client/reader.go b/broker/client/reader.go index 235b0b2b..74cd267f 100644 --- a/broker/client/reader.go +++ b/broker/client/reader.go @@ -173,18 +173,10 @@ func (r *Reader) Read(p []byte) (n int, err error) { // If the frame preceding EOF provided a fragment URL, open it directly. if !r.Request.MetadataOnly && r.Response.Status == pb.Status_OK && r.Response.FragmentUrl != "" { - log.WithFields(log.Fields{ - "FragmentUrl": r.Response.FragmentUrl, - }).Warn("reader handle FragmentUrl") - if TransformSignedURLs { + if SkipSignedURLs { fragURL := r.Response.Fragment.BackingStore.URL() - log.WithFields(log.Fields{ - "url": fmt.Sprintf("%+v", fragURL), - "fragment": fmt.Sprintf("%+v", *r.Response.Fragment), - "offset": r.Request.Offset, - }).Warn("reader handle url") if fragURL.Scheme != "gs" { - return 0, fmt.Errorf("TransformSignedURL unsupported scheme: %s", fragURL.Scheme) + return 0, fmt.Errorf("SkipSignedURL unsupported scheme: %s", fragURL.Scheme) } if r.direct, err = OpenUnsignedFragmentURL(r.ctx, *r.Response.Fragment, r.Request.Offset, fragURL); err == nil { @@ -466,7 +458,7 @@ var ( // stores_test.go, which is in broker/fragment, imports broker/client so we cannot import broker/fragment here // to avoid a cycle. Instead we will repeat a subset of store_gcs.go. -var TransformSignedURLs = false +var SkipSignedURLs = false var gcs = &gcsBackend{} type gcsBackend struct { diff --git a/broker/fragment/store_gcs.go b/broker/fragment/store_gcs.go index 6d7b8542..159c5b8b 100644 --- a/broker/fragment/store_gcs.go +++ b/broker/fragment/store_gcs.go @@ -44,10 +44,6 @@ func (s *gcsBackend) SignGet(ep *url.URL, fragment pb.Fragment, d time.Duration) } if DisableSignedUrls { - log.WithFields(log.Fields{ - "ep": fmt.Sprintf("%+v", *ep), - "fragment": fmt.Sprintf("%+v", fragment), - }).Info("signGet disable signed urls") u := &url.URL{ Path: fmt.Sprintf("/%s/%s", cfg.bucket, cfg.rewritePath(cfg.prefix, fragment.ContentPath())), } @@ -68,10 +64,6 @@ func (s *gcsBackend) Exists(ctx context.Context, ep *url.URL, fragment pb.Fragme if err != nil { return false, err } - log.WithFields(log.Fields{ - "ep": fmt.Sprintf("%+v", *ep), - "fragment": fmt.Sprintf("%+v", fragment), - }).Info("Exists fragment") _, err = client.Bucket(cfg.bucket).Object(cfg.rewritePath(cfg.prefix, fragment.ContentPath())).Attrs(ctx) if err == nil { exists = true @@ -86,10 +78,6 @@ func (s *gcsBackend) Open(ctx context.Context, ep *url.URL, fragment pb.Fragment if err != nil { return nil, err } - log.WithFields(log.Fields{ - "ep": fmt.Sprintf("%+v", *ep), - "fragment": fmt.Sprintf("%+v", fragment), - }).Info("Open fragment") return client.Bucket(cfg.bucket).Object(cfg.rewritePath(cfg.prefix, fragment.ContentPath())).NewReader(ctx) } diff --git a/mainboilerplate/runconsumer/run_consumer.go b/mainboilerplate/runconsumer/run_consumer.go index 34b61f06..10da412c 100644 --- a/mainboilerplate/runconsumer/run_consumer.go +++ b/mainboilerplate/runconsumer/run_consumer.go @@ -76,10 +76,10 @@ type Config interface { type BaseConfig struct { Consumer struct { mbp.ServiceConfig - Limit uint32 `long:"limit" env:"LIMIT" default:"32" description:"Maximum number of Shards this consumer process will allocate"` - MaxHotStandbys uint32 `long:"max-hot-standbys" env:"MAX_HOT_STANDBYS" default:"3" description:"Maximum effective hot standbys of any one shard, which upper-bounds its stated hot-standbys."` - WatchDelay time.Duration `long:"watch-delay" env:"WATCH_DELAY" default:"30ms" description:"Delay applied to the application of watched Etcd events. Larger values amortize the processing of fast-changing Etcd keys."` - TransformSignedURLs bool `long:"transform-signed-urls" env:"TRANSFORM_SIGNED_URLS" description:"When a signed URL is received, transform it into an unsigned URL. This is useful when clients do not require the signing."` + Limit uint32 `long:"limit" env:"LIMIT" default:"32" description:"Maximum number of Shards this consumer process will allocate"` + MaxHotStandbys uint32 `long:"max-hot-standbys" env:"MAX_HOT_STANDBYS" default:"3" description:"Maximum effective hot standbys of any one shard, which upper-bounds its stated hot-standbys."` + WatchDelay time.Duration `long:"watch-delay" env:"WATCH_DELAY" default:"30ms" description:"Delay applied to the application of watched Etcd events. Larger values amortize the processing of fast-changing Etcd keys."` + SkipSignedURLs bool `long:"skip-signed-urls" env:"SKIP_SIGNED_URLS" description:"When a signed URL is received, use fragment info instead to retrieve data with auth header. This is useful when clients do not wish/require the signing."` } `group:"Consumer" namespace:"consumer" env-namespace:"CONSUMER"` Broker struct { @@ -126,7 +126,7 @@ func (sc Cmd) Execute(args []string) error { mbp.Must(err, "building Server instance") // Arize avoidance of using signed URLs. - client.TransformSignedURLs = bc.Consumer.TransformSignedURLs + client.SkipSignedURLs = bc.Consumer.SkipSignedURLs if bc.Broker.Cache.Size <= 0 { log.Warn("--broker.cache.size is disabled; consider setting > 0")