From ce56e5c1bc1bb7d763f96a7e5d33673318ab27cf Mon Sep 17 00:00:00 2001
From: Max Holland <max@livepeer.org>
Date: Wed, 14 Feb 2024 10:25:57 +0000
Subject: [PATCH] Increase retry delays

We can sometimes see a short period of issues from storj, our existing retries were very close together (a few seconds in total) so would only catch extremely short periods of problems.

Also removing the fatal logging since it prints huge goroutine traces into our logging
---
 catalyst-uploader.go | 13 +++++++------
 core/uploader.go     |  6 +++---
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/catalyst-uploader.go b/catalyst-uploader.go
index f8e6601..b1f588c 100644
--- a/catalyst-uploader.go
+++ b/catalyst-uploader.go
@@ -24,7 +24,8 @@ func main() {
 func run() int {
 	err := flag.Set("logtostderr", "true")
 	if err != nil {
-		glog.Fatal(err)
+		glog.Error(err)
+		return 1
 	}
 	// cmd line args
 	describe := flag.Bool("j", false, "Describe supported storage services in JSON format and exit")
@@ -38,7 +39,7 @@ func run() int {
 	}
 
 	if flag.NArg() == 0 {
-		glog.Fatal("Destination URI is not specified. See -h for usage.")
+		glog.Error("Destination URI is not specified. See -h for usage.")
 		return 1
 	}
 
@@ -48,19 +49,19 @@ func run() int {
 
 	output := flag.Arg(0)
 	if output == "" {
-		glog.Fatal("Object store URI was empty")
+		glog.Error("Object store URI was empty")
 		return 1
 	}
 
 	uri, err := url.Parse(output)
 	if err != nil {
-		glog.Fatalf("Failed to parse URI: %s", err)
+		glog.Errorf("Failed to parse URI: %s", err)
 		return 1
 	}
 
 	out, err := core.Upload(os.Stdin, uri, WaitBetweenWrites, *timeout)
 	if err != nil {
-		glog.Fatalf("Uploader failed for %s: %s", uri.Redacted(), err)
+		glog.Errorf("Uploader failed for %s: %s", uri.Redacted(), err)
 		return 1
 	}
 
@@ -73,7 +74,7 @@ func run() int {
 	if glog.V(5) {
 		err = json.NewEncoder(stdout).Encode(map[string]string{"uri": uri.Redacted()})
 		if err != nil {
-			glog.Fatal(err)
+			glog.Error(err)
 			return 1
 		}
 	}
diff --git a/core/uploader.go b/core/uploader.go
index b1ca96f..df95b1a 100644
--- a/core/uploader.go
+++ b/core/uploader.go
@@ -29,15 +29,15 @@ func (bc *ByteCounter) Write(p []byte) (n int, err error) {
 
 func newExponentialBackOffExecutor() *backoff.ExponentialBackOff {
 	backOff := backoff.NewExponentialBackOff()
-	backOff.InitialInterval = 200 * time.Millisecond
-	backOff.MaxInterval = 5 * time.Second
+	backOff.InitialInterval = 10 * time.Second
+	backOff.MaxInterval = 1 * time.Minute
 	backOff.MaxElapsedTime = 0 // don't impose a timeout as part of the retries
 
 	return backOff
 }
 
 func UploadRetryBackoff() backoff.BackOff {
-	return backoff.WithMaxRetries(newExponentialBackOffExecutor(), 3)
+	return backoff.WithMaxRetries(newExponentialBackOffExecutor(), 7)
 }
 
 const segmentWriteTimeout = 5 * time.Minute