clients: Download segments from backup URL

clients/input_copy.go

@@ -287,6 +287,30 @@ func GetFile(ctx context.Context, requestID, url string, dStorage *DStorageDownl
 	}
 }
 
+func GetFileWithBackup(ctx context.Context, requestID, url string, dStorage *DStorageDownload) (io.ReadCloser, error) {
+	rc, err := GetFile(ctx, requestID, url, dStorage)
+	if err == nil {
+		return rc, nil
+	}
+
+	backupURL := config.GetStorageBackupURL(url)
+	if backupURL == "" {
+		return nil, err
+	}
+	rc, backupErr := GetFile(ctx, requestID, backupURL, dStorage)
+	if backupErr == nil {
+		return rc, nil
+	}
+
+	// prioritize retriable errors in the response so we don't skip retries
+	if !xerrors.IsUnretriable(err) {
+		return nil, err
+	} else if !xerrors.IsUnretriable(backupErr) {
+		return nil, backupErr
+	}
+	return nil, err
+}
+
 var retryableHttpClient = newRetryableHttpClient()
 
 func newRetryableHttpClient() *http.Client {

clients/manifest.go

@@ -332,7 +332,7 @@ func ClipInputManifest(requestID, sourceURL, clipTargetUrl string, startTimeUnix
 		segmentURL := sourceSegmentURLs[v.SeqId].URL
 		dStorage := NewDStorageDownload()
 		err = backoff.Retry(func() error {
-			rc, err := GetFile(context.Background(), requestID, segmentURL.String(), dStorage)
+			rc, err := GetFileWithBackup(context.Background(), requestID, segmentURL.String(), dStorage)
 			if err != nil {
 				return fmt.Errorf("error clipping: failed to download segment %d: %w", v.SeqId, err)
 			}

thumbnails/thumbnails.go

@@ -169,7 +169,7 @@ func GenerateThumbsFromManifest(requestID, input string, output *url.URL) error
 			)
 			// save the segment to memory
 			err = backoff.Retry(func() error {
-				rc, err = clients.GetFile(context.Background(), requestID, segURL.String(), nil)
+				rc, err = clients.GetFileWithBackup(context.Background(), requestID, segURL.String(), nil)
 				return err
 			}, clients.DownloadRetryBackoff())
 			if err != nil {

transcode/transcode.go

@@ -536,7 +536,7 @@ func transcodeSegment(
 	err := backoff.Retry(func() error {
 		ctx, cancel := context.WithTimeout(context.Background(), clients.MaxCopyFileDuration)
 		defer cancel()
-		rc, err := clients.GetFile(ctx, transcodeRequest.RequestID, segment.Input.URL.String(), nil)
+		rc, err := clients.GetFileWithBackup(ctx, transcodeRequest.RequestID, segment.Input.URL.String(), nil)
 		if err != nil {
 			return fmt.Errorf("failed to download source segment %q: %w", segment.Input, err)
 		}