backoff: Fix usage of backoff library (#1306)

* thumbnails: Avoid reusing same backoff instance

These are not meant to be reused. After 10 failures we
would basically turnoff retries for that code.

* backoff: Call Reset after configuring exp backoffs

So many quirks in this lib. This is also required and if
we don't call Reset() we're basically ignoring any InitialInterval
that is configured.

* Remove trailing =

clients/object_store_client.go

@@ -185,7 +185,7 @@ func newExponentialBackOffExecutor() *backoff.ExponentialBackOff {
 	backOff.InitialInterval = 200 * time.Millisecond
 	backOff.MaxInterval = maxRetryInterval
 	backOff.MaxElapsedTime = 0 // don't impose a timeout as part of the retries
-
+	backOff.Reset()
 	return backOff
 }
 

pipeline/ffmpeg.go

@@ -407,5 +407,6 @@ func retries(retries uint64) backoff.BackOff {
 	backOff.InitialInterval = 1 * time.Second
 	backOff.MaxInterval = 30 * time.Second
 	backOff.MaxElapsedTime = 0 // don't impose a timeout as part of the retries
+	backOff.Reset()
 	return backoff.WithMaxRetries(backOff, retries)
 }

thumbnails/thumbnails.go

@@ -25,8 +25,10 @@ const resolution = "854:480"
 const vttFilename = "thumbnails.vtt"
 const outputDir = "thumbnails"
 
-// Wait a maximum of 5 mins for thumbnails to finish
-var thumbWaitBackoff = backoff.WithMaxRetries(backoff.NewConstantBackOff(30*time.Second), 10)
+func thumbWaitBackoff() backoff.BackOff {
+	// Wait a maximum of 5 mins for thumbnails to finish
+	return backoff.WithMaxRetries(backoff.NewConstantBackOff(30*time.Second), 10)
+}
 
 func getMediaManifest(requestID string, input string) (*m3u8.MediaPlaylist, error) {
 	var (
@@ -102,7 +104,7 @@ func GenerateThumbsVTT(requestID string, input string, output *url.URL) error {
 				rc.Close()
 			}
 			return err
-		}, thumbWaitBackoff)
+		}, thumbWaitBackoff())
 		if err != nil {
 			return fmt.Errorf("failed to find thumb %s: %w", filename, err)
 		}

video/probe.go

@@ -59,6 +59,7 @@ func (p Probe) runProbe(url string, ffProbeOptions ...string) (iv InputVideo, er
 	backOff.InitialInterval = 500 * time.Millisecond
 	backOff.MaxInterval = 2 * time.Second
 	backOff.MaxElapsedTime = 0 // don't impose a timeout as part of the retries
+	backOff.Reset()
 	err = backoff.Retry(operation, backoff.WithMaxRetries(backOff, 3))
 	if err != nil {
 		return InputVideo{}, fmt.Errorf("error probing: %w", err)