split into two burst job specs

ASFHyP3 · Oct 1, 2024 · a3c337e · a3c337e
1 parent 4ad3805
commit a3c337e
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 33 deletions.
diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml
@@ -25,6 +25,7 @@ jobs:
             deploy_ref: refs/heads/multi-burst-sandbox
             job_files: >-
               job_spec/INSAR_ISCE_BURST.yml
+              job_spec/INSAR_ISCE_MULTI_BURST.yml
               job_spec/AUTORIFT.yml
               job_spec/RTC_GAMMA.yml
               job_spec/WATER_MAP.yml

diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py
@@ -1,6 +1,7 @@
 import json
 import os
 import sys
+from copy import deepcopy
 from pathlib import Path
 
 import requests
@@ -135,7 +136,20 @@ def get_multipolygon_from_geojson(input_file):
     return MultiPolygon(polygons)
 
 
-def validate_jobs(jobs):
+def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]:
+    jobs = deepcopy(jobs)
+    for job in jobs:
+        if job['job_type'] == 'INSAR_ISCE_BURST':
+            job_parameters = job['job_parameters']
+            job_parameters['reference'], job_parameters['secondary'] = (
+                [granule] for granule in job_parameters.pop('granules')
+            )
+    return jobs
+
+
+def validate_jobs(jobs: list[dict]) -> None:
+    jobs = convert_single_burst_jobs(jobs)
+
     granules = get_granules(jobs)
     granule_metadata = get_cmr_metadata(granules)
 

diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py
@@ -65,7 +65,8 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str:
 def lambda_handler(event: dict, _) -> dict:
     job_type, job_parameters = event['job_type'], event['job_parameters']
 
-    if job_type == 'INSAR_ISCE_BURST':
+    # TODO rename to INSAR_ISCE_BURST after the two burst types are merged
+    if job_type == 'INSAR_ISCE_MULTI_BURST':
         memory = get_insar_isce_burst_memory(job_parameters)
         omp_num_threads = INSAR_ISCE_BURST_OMP_NUM_THREADS[memory]
         return get_container_overrides(memory, omp_num_threads)

diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml
@@ -1,38 +1,23 @@
 INSAR_ISCE_BURST:
   required_parameters:
-    - reference
-    - secondary
+    - granules
   parameters:
-    reference:
+    granules:
       default:  '""'
       api_schema:
         type: array
-        minItems: 1
-        maxItems: 30
+        minItems: 2
+        maxItems: 2
         example:
           - S1_136231_IW2_20200604T022312_VV_7C85-BURST
-        items:
-          description: Name of the reference Sentinel-1 SLC IW burst granule to process
-          type: string
-          pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$'
-          minLength: 43
-          maxLength: 43
-          example: S1_136231_IW2_20200604T022312_VV_7C85-BURST
-    secondary:
-      default:  '""'
-      api_schema:
-        type: array
-        minItems: 1
-        maxItems: 30
-        example:
           - S1_136231_IW2_20200616T022313_VV_5D11-BURST
         items:
-          description: Name of the secondary Sentinel-1 SLC IW burst granule to process
+          description: Name of the Sentinel-1 SLC IW burst granule to process
           type: string
-          pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$'
+          pattern: "^S1_\\d{6}_IW.*-BURST"
           minLength: 43
           maxLength: 43
-          example: S1_136231_IW2_20200616T022313_VV_5D11-BURST
+          example: S1_136231_IW2_20200604T022312_VV_7C85-BURST
     bucket_prefix:
       default:  '""'
     apply_water_mask:
@@ -67,6 +52,8 @@ INSAR_ISCE_BURST:
       command:
         - ++process
         - insar_tops_burst
+        - ++omp-num-threads
+        - '1'
         - --bucket
         - '!Ref Bucket'
         - --bucket-prefix
@@ -75,13 +62,10 @@ INSAR_ISCE_BURST:
         - Ref::apply_water_mask
         - --looks
         - Ref::looks
-        - --reference
-        - Ref::reference
-        - --secondary
-        - Ref::secondary
-      timeout: 126000 # 35 hours
+        - Ref::granules
+      timeout: 5400
       vcpu: 1
-      memory: 4 # Memory is always overridden by the step function
+      memory: 7600
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
diff --git a/job_spec/INSAR_ISCE_MULTI_BURST.yml b/job_spec/INSAR_ISCE_MULTI_BURST.yml
@@ -0,0 +1,87 @@
+INSAR_ISCE_MULTI_BURST:
+  required_parameters:
+    - reference
+    - secondary
+  parameters:
+    reference:
+      default:  '""'
+      api_schema:
+        type: array
+        minItems: 1
+        maxItems: 30
+        example:
+          - S1_136231_IW2_20200604T022312_VV_7C85-BURST
+        items:
+          description: Name of the reference Sentinel-1 SLC IW burst granule to process
+          type: string
+          pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$'
+          minLength: 43
+          maxLength: 43
+          example: S1_136231_IW2_20200604T022312_VV_7C85-BURST
+    secondary:
+      default:  '""'
+      api_schema:
+        type: array
+        minItems: 1
+        maxItems: 30
+        example:
+          - S1_136231_IW2_20200616T022313_VV_5D11-BURST
+        items:
+          description: Name of the secondary Sentinel-1 SLC IW burst granule to process
+          type: string
+          pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$'
+          minLength: 43
+          maxLength: 43
+          example: S1_136231_IW2_20200616T022313_VV_5D11-BURST
+    bucket_prefix:
+      default:  '""'
+    apply_water_mask:
+      api_schema:
+        description: Sets pixels over coastal and large inland waterbodies as invalid for phase unwrapping.
+        default: false
+        type: boolean
+    looks:
+      api_schema:
+        description: Number of looks to take in range and azimuth
+        type: string
+        default: 20x4
+        enum:
+          - 20x4
+          - 10x2
+          - 5x1
+  cost_profiles:
+    EDC:
+      cost: 1.0
+    DEFAULT:
+      cost: 1.0
+  validators:
+    - check_dem_coverage
+    - check_valid_polarizations
+    - check_same_burst_ids
+    - check_not_antimeridian
+  compute_environment:
+    name: 'Default'
+  tasks:
+    - name: ''
+      image: ghcr.io/asfhyp3/hyp3-isce2
+      command:
+        - ++process
+        - insar_tops_burst
+        - --bucket
+        - '!Ref Bucket'
+        - --bucket-prefix
+        - Ref::bucket_prefix
+        - --apply-water-mask
+        - Ref::apply_water_mask
+        - --looks
+        - Ref::looks
+        - --reference
+        - Ref::reference
+        - --secondary
+        - Ref::secondary
+      timeout: 126000 # 35 hours
+      vcpu: 1
+      memory: 4 # Memory is always overridden by the step function
+      secrets:
+        - EARTHDATA_USERNAME
+        - EARTHDATA_PASSWORD
diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py
@@ -424,11 +424,19 @@ def test_validate_jobs():
             'job_parameters': {}
         },
         {
-            'job_type': 'INSAR_ISCE_BURST',
+            # TODO rename to INSAR_ISCE_BURST after the two burst types are merged
+            'job_type': 'INSAR_ISCE_MULTI_BURST',
             'job_parameters': {
                 'reference': [valid_burst_pair[0]],
                 'secondary': [valid_burst_pair[1]]
             }
+        },
+        {
+            # TODO remove this test case after the two burst types are merged
+            'job_type': 'INSAR_ISCE_BURST',
+            'job_parameters': {
+                'granules': [valid_burst_pair[0], valid_burst_pair[1]]
+            }
         }
     ]
     validation.validate_jobs(jobs)
@@ -457,7 +465,8 @@ def test_validate_jobs():
 
     jobs = [
         {
-            'job_type': 'INSAR_ISCE_BURST',
+            # TODO rename to INSAR_ISCE_BURST after the two burst types are merged
+            'job_type': 'INSAR_ISCE_MULTI_BURST',
             'job_parameters': {
                 'reference': [invalid_burst_pair[0]],
                 'secondary': [invalid_burst_pair[1]]
@@ -466,3 +475,15 @@ def test_validate_jobs():
     ]
     with raises(validation.GranuleValidationError):
         validation.validate_jobs(jobs)
+
+    # TODO remove this test case after the two burst types are merged
+    jobs = [
+        {
+            'job_type': 'INSAR_ISCE_BURST',
+            'job_parameters': {
+                'granules': [invalid_burst_pair[0], invalid_burst_pair[1]]
+            }
+        }
+    ]
+    with raises(validation.GranuleValidationError):
+        validation.validate_jobs(jobs)
diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py
@@ -16,7 +16,8 @@
 
 def mock_insar_isce_burst_job(looks: str, bursts: int) -> dict:
     return {
-        'job_type': 'INSAR_ISCE_BURST',
+        # TODO rename to INSAR_ISCE_BURST after the two burst types are merged
+        'job_type': 'INSAR_ISCE_MULTI_BURST',
         'job_parameters': {
             'looks': looks,
             'reference': ' '.join('foo' for _ in range(bursts)),