From 38ab54229c53a4c173a6a8bf348199352fee4d70 Mon Sep 17 00:00:00 2001 From: Max Holland Date: Mon, 12 Aug 2024 10:42:10 +0100 Subject: [PATCH] tweak retries and add time taken log --- catalyst-uploader.go | 3 ++- core/uploader.go | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst-uploader.go b/catalyst-uploader.go index ca95efd..cf843ac 100644 --- a/catalyst-uploader.go +++ b/catalyst-uploader.go @@ -99,6 +99,7 @@ func run() int { return 1 } + start := time.Now() out, err := core.Upload(os.Stdin, uri, WaitBetweenWrites, *timeout, *storageFallbackURLs) if err != nil { glog.Errorf("Uploader failed for %s: %s", uri.Redacted(), err) @@ -109,7 +110,7 @@ func run() int { if out != nil { respHeaders = out.UploaderResponseHeaders } - glog.Infof("Uploader succeeded for %s. storageRequestID=%s Etag=%s", uri.Redacted(), respHeaders.Get("X-Amz-Request-Id"), respHeaders.Get("Etag")) + glog.Infof("Uploader succeeded for %s. storageRequestID=%s Etag=%s timeTaken=%vms", uri.Redacted(), respHeaders.Get("X-Amz-Request-Id"), respHeaders.Get("Etag"), time.Since(start).Milliseconds()) // success, write uploaded file details to stdout if glog.V(5) { err = json.NewEncoder(stdout).Encode(map[string]string{"uri": uri.Redacted()}) diff --git a/core/uploader.go b/core/uploader.go index c31e7d1..49ba0ab 100644 --- a/core/uploader.go +++ b/core/uploader.go @@ -13,11 +13,10 @@ import ( "strings" "time" - "golang.org/x/sync/errgroup" - "github.com/cenkalti/backoff/v4" "github.com/golang/glog" "github.com/livepeer/go-tools/drivers" + "golang.org/x/sync/errgroup" ) type ByteCounter struct { @@ -43,7 +42,7 @@ func NoRetries() backoff.BackOff { } func UploadRetryBackoff() backoff.BackOff { - return newExponentialBackOffExecutor(2*time.Minute, 5*time.Minute, 1*time.Hour) + return newExponentialBackOffExecutor(30*time.Second, 4*time.Minute, 15*time.Minute) } func SingleRequestRetryBackoff() backoff.BackOff {