Merge pull request #1869 from ASFHyP3/improve-retry

Use exponential backoff retry strategy

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.1.2]
+### Changed
+- AWS Batch jobs are now retried twice, once after 10 minutes and once after 60 minutes, to reduce the number of jobs that fail due to transient errors, such as Earthdata Login and Sentinel-1 distribution outages.
+
 ## [4.1.1]
 ### Added
 - New DEM coverage map that allows COP90 tiles to fill the COP30 gaps over Azerbaijan and Armenia.

apps/step-function.json.j2

@@ -215,10 +215,7 @@
         "ShareIdentifier": "default",
         "SchedulingPriorityOverride.$": "$.priority",
         "Parameters.$": "$.job_parameters",
-        "ContainerOverrides.$": "$.container_overrides",
-        "RetryStrategy": {
-          "Attempts": 3
-        }
+        "ContainerOverrides.$": "$.container_overrides"
       },
       "ResultPath": "$.results.processing_results.step_{{ loop.index0 }}",
       "Next": "{% if not loop.last %}{{ loop.nextitem['name'] }}{% else %}GET_FILES{% endif %}",
@@ -230,6 +227,14 @@
           ],
           "MaxAttempts": 2
         },
+        {
+          "ErrorEquals": [
+            "States.TaskFailed"
+          ],
+          "IntervalSeconds": 600,
+          "MaxAttempts": 2,
+          "BackoffRate": 6.0
+        },
         {
           "ErrorEquals": [
             "States.ALL"