From fd5745f595b82526bb70ce26df894db285a1561e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 12:54:51 -0500 Subject: [PATCH 001/102] support for multi burst insar --- .github/workflows/deploy-daac.yml | 1 + apps/api/src/hyp3_api/util.py | 5 +- job_spec/INSAR_MULTI_BURST.yml | 85 +++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 job_spec/INSAR_MULTI_BURST.yml diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index c6af17100..036b1ea73 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -49,6 +49,7 @@ jobs: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml + job_spec/INSAR_MULTI_BURST.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 1500 expanded_max_vcpus: 3000 diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index f5ecd6a88..934d00608 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -9,7 +9,10 @@ class TokenDeserializeError(Exception): def get_granules(jobs: list[dict]) -> set[str]: - return {granule for job in jobs for granule in job['job_parameters'].get('granules', [])} + granules = {granule for job in jobs for granule in job['job_parameters'].get('granules', [])} + granules.union({granule for job in jobs for granule in job['job_parameters'].get('reference', [])}) + granules.union({granule for job in jobs for granule in job['job_parameters'].get('secondary', [])}) + return granules def serialize(payload: dict): diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml new file mode 100644 index 000000000..2acaa855c --- /dev/null +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -0,0 +1,85 @@ +INSAR_MULTI_BURST: + required_parameters: + - reference + - secondary + parameters: + reference: + default: '""' + api_schema: + type: array + minItems: 1 + example: + - S1_136231_IW2_20200604T022312_VV_7C85-BURST + items: + description: List of the reference Sentinel-1 SLC IW burst granules to process + type: string + pattern: "^S1_\\d{6}_IW.*-BURST" + minLength: 43 + maxLength: 43 + example: S1_136231_IW2_20200604T022312_VV_7C85-BURST + secondary: + default: '""' + api_schema: + type: array + minItems: 1 + example: + - S1_136231_IW2_20200616T022313_VV_5D11-BURST + items: + description: List of the secondary Sentinel-1 SLC IW burst granules to process + type: string + pattern: "^S1_\\d{6}_IW.*-BURST" + minLength: 43 + maxLength: 43 + example: S1_136231_IW2_20200616T022313_VV_5D11-BURST + bucket_prefix: + default: '""' + apply_water_mask: + api_schema: + description: Sets pixels over coastal and large inland waterbodies as invalid for phase unwrapping. + default: false + type: boolean + looks: + api_schema: + description: Number of looks to take in range and azimuth + type: string + default: 20x4 + enum: + - 20x4 + - 10x2 + - 5x1 + cost_profiles: + EDC: + cost: 1.0 + DEFAULT: + cost: 1.0 + validators: + - check_dem_coverage + - check_valid_polarizations + - check_same_burst_ids + - check_not_antimeridian + tasks: + - name: '' + image: ghcr.io/asfhyp3/hyp3-isce2 + command: + - ++process + - insar_tops_burst + - ++omp-num-threads + - '1' + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::bucket_prefix + - --apply-water-mask + - Ref::apply_water_mask + - --looks + - Ref::looks + - --reference + - Ref::reference + - --secondary + - Ref::secondary + timeout: 5400 + vcpu: 1 + memory: 7600 + secrets: + - EARTHDATA_USERNAME + - EARTHDATA_PASSWORD From e7c8c9173d28946c6953b9734a2606975379eae4 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 15:37:37 -0500 Subject: [PATCH 002/102] change back to granules --- apps/api/src/hyp3_api/util.py | 5 +---- job_spec/INSAR_MULTI_BURST.yml | 32 ++++++++------------------------ 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index 934d00608..f5ecd6a88 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -9,10 +9,7 @@ class TokenDeserializeError(Exception): def get_granules(jobs: list[dict]) -> set[str]: - granules = {granule for job in jobs for granule in job['job_parameters'].get('granules', [])} - granules.union({granule for job in jobs for granule in job['job_parameters'].get('reference', [])}) - granules.union({granule for job in jobs for granule in job['job_parameters'].get('secondary', [])}) - return granules + return {granule for job in jobs for granule in job['job_parameters'].get('granules', [])} def serialize(payload: dict): diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 2acaa855c..893aa3f3a 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -1,36 +1,23 @@ INSAR_MULTI_BURST: required_parameters: - - reference - - secondary + - granules parameters: - reference: + granules: default: '""' api_schema: type: array - minItems: 1 + minItems: 2 + maxItems: 2 example: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST + - S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST + - S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST items: - description: List of the reference Sentinel-1 SLC IW burst granules to process + description: Name of the Sentinel-1 SLC IW burst granule to process type: string pattern: "^S1_\\d{6}_IW.*-BURST" minLength: 43 maxLength: 43 example: S1_136231_IW2_20200604T022312_VV_7C85-BURST - secondary: - default: '""' - api_schema: - type: array - minItems: 1 - example: - - S1_136231_IW2_20200616T022313_VV_5D11-BURST - items: - description: List of the secondary Sentinel-1 SLC IW burst granules to process - type: string - pattern: "^S1_\\d{6}_IW.*-BURST" - minLength: 43 - maxLength: 43 - example: S1_136231_IW2_20200616T022313_VV_5D11-BURST bucket_prefix: default: '""' apply_water_mask: @@ -73,10 +60,7 @@ INSAR_MULTI_BURST: - Ref::apply_water_mask - --looks - Ref::looks - - --reference - - Ref::reference - - --secondary - - Ref::secondary + - Ref::granules timeout: 5400 vcpu: 1 memory: 7600 From fe0f185ab91d1158d0f540396e6bdbdfb0e24ddd Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 15:41:06 -0500 Subject: [PATCH 003/102] updated changelog for multi_burst --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0555562ec..93df39d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [7.8.0] + +### Added +- Added `INSAR_MULTI_BURST` job type that can merge adjacent bursts into an SLC and then perform InSAR. + + ## [7.7.2] ### Change From eefe263681102e3a38fbfa33dce302d327fb9498 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 16:12:57 -0500 Subject: [PATCH 004/102] regex to match 1 or more bursts --- job_spec/INSAR_MULTI_BURST.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 893aa3f3a..fa623b76b 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -14,10 +14,10 @@ INSAR_MULTI_BURST: items: description: Name of the Sentinel-1 SLC IW burst granule to process type: string - pattern: "^S1_\\d{6}_IW.*-BURST" + pattern: "^((S1_\\d{6}_IW\\d_\\d{8}T\\d{6}_[VH][VH]_([\\d]|[A-F]){4}-BURST)($|[ \\t]{1,}))+" minLength: 43 maxLength: 43 - example: S1_136231_IW2_20200604T022312_VV_7C85-BURST + example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST bucket_prefix: default: '""' apply_water_mask: From 4f19f38fa4d9f3c75de4523afbde81e4d0c255bd Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 16:15:19 -0500 Subject: [PATCH 005/102] cleaner regex --- job_spec/INSAR_MULTI_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index fa623b76b..4b63693c6 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -14,7 +14,7 @@ INSAR_MULTI_BURST: items: description: Name of the Sentinel-1 SLC IW burst granule to process type: string - pattern: "^((S1_\\d{6}_IW\\d_\\d{8}T\\d{6}_[VH][VH]_([\\d]|[A-F]){4}-BURST)($|[ \\t]{1,}))+" + pattern: "^((S1_\\d{6}_IW\\d_\\d{8}T\\d{6}_[VH]{2}_([\\dA-F]){4}-BURST)($|[ \\t]{1,}))+" minLength: 43 maxLength: 43 example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST From 5ad1a2cec5dcb36c5ecdad16fbd1a9d83bceeb69 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 16:15:50 -0500 Subject: [PATCH 006/102] removed max length --- job_spec/INSAR_MULTI_BURST.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 4b63693c6..4beb8eb75 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -16,7 +16,6 @@ INSAR_MULTI_BURST: type: string pattern: "^((S1_\\d{6}_IW\\d_\\d{8}T\\d{6}_[VH]{2}_([\\dA-F]){4}-BURST)($|[ \\t]{1,}))+" minLength: 43 - maxLength: 43 example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST bucket_prefix: default: '""' From 73f26bb9be9a6ca815b663410f91e93092a2001d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 12:52:32 -0500 Subject: [PATCH 007/102] updated regex --- job_spec/INSAR_MULTI_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 4beb8eb75..b816c649b 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -14,7 +14,7 @@ INSAR_MULTI_BURST: items: description: Name of the Sentinel-1 SLC IW burst granule to process type: string - pattern: "^((S1_\\d{6}_IW\\d_\\d{8}T\\d{6}_[VH]{2}_([\\dA-F]){4}-BURST)($|[ \\t]{1,}))+" + pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST bucket_prefix: From 7408de1614f1799d9a04c26dcddf1cbae3146ff0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:20:07 +0000 Subject: [PATCH 008/102] Bump setuptools from 72.1.0 to 72.2.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 72.1.0 to 72.2.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v72.1.0...v72.2.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 11f6b8327..3a1b5f238 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -15,6 +15,6 @@ flake8==7.1.1 flake8-import-order==0.18.2 flake8-blind-except==0.2.1 flake8-builtins==2.5.0 -setuptools==72.1.0 +setuptools==72.2.0 openapi-spec-validator==0.7.1 cfn-lint==1.9.7 From 80879596a9834a231eee6867a17439c65b01ecec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:20:12 +0000 Subject: [PATCH 009/102] Bump cfn-lint from 1.9.7 to 1.10.3 Bumps [cfn-lint](https://github.com/aws-cloudformation/cfn-lint) from 1.9.7 to 1.10.3. - [Release notes](https://github.com/aws-cloudformation/cfn-lint/releases) - [Changelog](https://github.com/aws-cloudformation/cfn-lint/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-cloudformation/cfn-lint/compare/v1.9.7...v1.10.3) --- updated-dependencies: - dependency-name: cfn-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 11f6b8327..c29d8b516 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -17,4 +17,4 @@ flake8-blind-except==0.2.1 flake8-builtins==2.5.0 setuptools==72.1.0 openapi-spec-validator==0.7.1 -cfn-lint==1.9.7 +cfn-lint==1.10.3 From 5594006868bf9172479d19ea54fce5fb6207bfc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:20:21 +0000 Subject: [PATCH 010/102] Bump openapi-core from 0.19.2 to 0.19.3 Bumps [openapi-core](https://github.com/python-openapi/openapi-core) from 0.19.2 to 0.19.3. - [Release notes](https://github.com/python-openapi/openapi-core/releases) - [Commits](https://github.com/python-openapi/openapi-core/compare/0.19.2...0.19.3) --- updated-dependencies: - dependency-name: openapi-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-apps-api.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt index 3a4d5aec4..7d4b7ff50 100644 --- a/requirements-apps-api.txt +++ b/requirements-apps-api.txt @@ -1,7 +1,7 @@ flask==2.2.5 Flask-Cors==4.0.1 jsonschema==4.23.0 -openapi-core==0.19.2 +openapi-core==0.19.3 prance==23.6.21.0 PyJWT==2.9.0 requests==2.32.3 From f4d69ab2de9d5226ff8f56493711d1d4e460ae68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:20:29 +0000 Subject: [PATCH 011/102] Bump boto3 from 1.34.159 to 1.35.0 Bumps [boto3](https://github.com/boto/boto3) from 1.34.159 to 1.35.0. - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.34.159...1.35.0) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- requirements-apps-disable-private-dns.txt | 2 +- requirements-apps-start-execution-manager.txt | 2 +- requirements-apps-start-execution-worker.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-all.txt b/requirements-all.txt index 11f6b8327..3df5726d2 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.34.159 +boto3==1.35.0 jinja2==3.1.4 moto[dynamodb]==5.0.12 pytest==8.3.2 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index 16c7ac7aa..4518d8791 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.34.159 +boto3==1.35.0 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index 00ad69582..19299088c 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.34.159 +boto3==1.35.0 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index aca0bc7ac..355394f48 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.34.159 +boto3==1.35.0 ./lib/lambda_logging/ From bbf5aa3e88bfc7665a82abf835bdf85ebad66338 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:47:44 -0500 Subject: [PATCH 012/102] changed insar_multi_burst to use reference and secondary --- apps/api/src/hyp3_api/util.py | 5 ++++- job_spec/INSAR_MULTI_BURST.yml | 31 ++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index f5ecd6a88..01a70f2f0 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -9,7 +9,10 @@ class TokenDeserializeError(Exception): def get_granules(jobs: list[dict]) -> set[str]: - return {granule for job in jobs for granule in job['job_parameters'].get('granules', [])} + granules = set() + for key in ['granules', 'reference', 'secondary']: + granules.union({granule for job in jobs for granule in job['job_parameters'].get(key, [])}) + return granules def serialize(payload: dict): diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index b816c649b..382e26c25 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -1,18 +1,32 @@ INSAR_MULTI_BURST: required_parameters: - - granules + - reference + - secondary parameters: - granules: + reference: default: '""' api_schema: type: array - minItems: 2 - maxItems: 2 + minItems: 1 + maxItems: 50 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST - - S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST items: - description: Name of the Sentinel-1 SLC IW burst granule to process + description: Name of the reference Sentinel-1 SLC IW burst granule to process + type: string + pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' + minLength: 43 + example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST + secondary: + default: '""' + api_schema: + type: array + minItems: 1 + maxItems: 50 + example: + - S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST + items: + description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 @@ -59,7 +73,10 @@ INSAR_MULTI_BURST: - Ref::apply_water_mask - --looks - Ref::looks - - Ref::granules + - --reference + - Ref::reference + - --secondary + - Ref::secondary timeout: 5400 vcpu: 1 memory: 7600 From d5143e3453acbc7fd1e7dae150c2cacef87ec089 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:28:47 -0500 Subject: [PATCH 013/102] better examples --- job_spec/INSAR_MULTI_BURST.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 382e26c25..da42e1192 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -10,13 +10,15 @@ INSAR_MULTI_BURST: minItems: 1 maxItems: 50 example: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST + - S1_136231_IW2_20200604T022312_VV_7C85-BURST + - S1_136232_IW2_20200604T022312_VV_7C85-BURST items: description: Name of the reference Sentinel-1 SLC IW burst granule to process type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 - example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST + example: + - S1_136231_IW2_20200604T022312_VV_7C85-BURST secondary: default: '""' api_schema: @@ -24,13 +26,14 @@ INSAR_MULTI_BURST: minItems: 1 maxItems: 50 example: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST + - S1_136231_IW2_20200616T022313_VV_5D11-BURST + - S1_136232_IW2_20200616T022313_VV_5D11-BURST items: description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 - example: S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022312_VV_7C85-BURST + example: S1_136231_IW2_20200616T022313_VV_5D11-BURST bucket_prefix: default: '""' apply_water_mask: From 8e00156516ea97402b32072d66dc1575a618e9b2 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:33:27 -0500 Subject: [PATCH 014/102] changed from {} to set() --- tests/test_api/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api/test_util.py b/tests/test_api/test_util.py index 033911d8d..918d7f46b 100644 --- a/tests/test_api/test_util.py +++ b/tests/test_api/test_util.py @@ -17,7 +17,7 @@ def test_get_granules(): {'job_parameters': {'granules': ['F', 'F']}}, {'job_parameters': {}}, ] - ) == {'A', 'B', 'C', 'D', 'E', 'F'} + ) == set('A', 'B', 'C', 'D', 'E', 'F') def test_serialize_token(): From 11c6a0458b9f1e462addba4c493fc32c577e8d90 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:34:36 -0500 Subject: [PATCH 015/102] Update job_spec/INSAR_MULTI_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_MULTI_BURST.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index da42e1192..077196f9c 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -17,6 +17,7 @@ INSAR_MULTI_BURST: type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 + maxLength: 43 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST secondary: From 1607914773b4f46db3bdf73acfee9c65711c2c69 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:34:41 -0500 Subject: [PATCH 016/102] Update job_spec/INSAR_MULTI_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_MULTI_BURST.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 077196f9c..74244ba75 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -34,6 +34,7 @@ INSAR_MULTI_BURST: type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 + maxLength: 43 example: S1_136231_IW2_20200616T022313_VV_5D11-BURST bucket_prefix: default: '""' From c8aeac03e374c42a18c61eb7887a05c207930170 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:36:06 -0500 Subject: [PATCH 017/102] Update job_spec/INSAR_MULTI_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_MULTI_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 74244ba75..cf5dd037e 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -67,7 +67,7 @@ INSAR_MULTI_BURST: image: ghcr.io/asfhyp3/hyp3-isce2 command: - ++process - - insar_tops_burst + - insar_tops_multi_burst - ++omp-num-threads - '1' - --bucket From 92fa14990487f0b48adfee7500f102b681275d3f Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:37:09 -0500 Subject: [PATCH 018/102] cast to set --- tests/test_api/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api/test_util.py b/tests/test_api/test_util.py index 918d7f46b..9fab25588 100644 --- a/tests/test_api/test_util.py +++ b/tests/test_api/test_util.py @@ -17,7 +17,7 @@ def test_get_granules(): {'job_parameters': {'granules': ['F', 'F']}}, {'job_parameters': {}}, ] - ) == set('A', 'B', 'C', 'D', 'E', 'F') + ) == set({'A', 'B', 'C', 'D', 'E', 'F'}) def test_serialize_token(): From c56db78792d2b900fb82b90db09ef9aa51fa7231 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:38:21 -0500 Subject: [PATCH 019/102] Update CHANGELOG.md Co-authored-by: Jake Herrmann --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93df39d4a..f40064598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [7.8.0] ### Added -- Added `INSAR_MULTI_BURST` job type that can merge adjacent bursts into an SLC and then perform InSAR. +- Added `INSAR_MULTI_BURST` job type to `hyp3-test` that can merge adjacent bursts into an SLC and then perform InSAR. ## [7.7.2] From 446ad753835ecad427f35408256f7cec5787b018 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:45:21 -0500 Subject: [PATCH 020/102] granules = --- apps/api/src/hyp3_api/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index 01a70f2f0..33d0b7cb1 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -11,7 +11,7 @@ class TokenDeserializeError(Exception): def get_granules(jobs: list[dict]) -> set[str]: granules = set() for key in ['granules', 'reference', 'secondary']: - granules.union({granule for job in jobs for granule in job['job_parameters'].get(key, [])}) + granules = granules.union({granule for job in jobs for granule in job['job_parameters'].get(key, [])}) return granules From dafae415daec3fc5353cfa3fe36d6cccf88e6d05 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:50:54 -0500 Subject: [PATCH 021/102] added reference and secondary cases to test_get_granules --- tests/test_api/test_util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_api/test_util.py b/tests/test_api/test_util.py index 9fab25588..3a8945500 100644 --- a/tests/test_api/test_util.py +++ b/tests/test_api/test_util.py @@ -13,11 +13,14 @@ def test_get_granules(): {'job_parameters': {'granules': ['A']}}, {'job_parameters': {'granules': ['B']}}, {'job_parameters': {'granules': ['C', 'D']}}, + {'job_parameters': {'secondary': ['J']}}, {'job_parameters': {'granules': ['C', 'D', 'E']}}, {'job_parameters': {'granules': ['F', 'F']}}, + {'job_parameters': {'reference': ['G', 'H', 'H', 'I']}}, + {'job_parameters': {'secondary': []}}, {'job_parameters': {}}, ] - ) == set({'A', 'B', 'C', 'D', 'E', 'F'}) + ) == set({'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'}) def test_serialize_token(): From 566279d458d3d23cb3da7985a0fad36a3658975f Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 16:31:39 -0500 Subject: [PATCH 022/102] remove cast to set --- tests/test_api/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api/test_util.py b/tests/test_api/test_util.py index 3a8945500..ba5b37bbb 100644 --- a/tests/test_api/test_util.py +++ b/tests/test_api/test_util.py @@ -20,7 +20,7 @@ def test_get_granules(): {'job_parameters': {'secondary': []}}, {'job_parameters': {}}, ] - ) == set({'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'}) + ) == {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'} def test_serialize_token(): From 19e3e8a95dee09be4e8dcbb79e38402425c3193b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 16:42:00 -0500 Subject: [PATCH 023/102] Update apps/api/src/hyp3_api/util.py Co-authored-by: Jake Herrmann --- apps/api/src/hyp3_api/util.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/api/src/hyp3_api/util.py b/apps/api/src/hyp3_api/util.py index 33d0b7cb1..78d2dd02b 100644 --- a/apps/api/src/hyp3_api/util.py +++ b/apps/api/src/hyp3_api/util.py @@ -9,10 +9,12 @@ class TokenDeserializeError(Exception): def get_granules(jobs: list[dict]) -> set[str]: - granules = set() - for key in ['granules', 'reference', 'secondary']: - granules = granules.union({granule for job in jobs for granule in job['job_parameters'].get(key, [])}) - return granules + return { + granule + for key in ['granules', 'reference', 'secondary'] + for job in jobs + for granule in job['job_parameters'].get(key, []) + } def serialize(payload: dict): From 214f5a0d0fd012f7ae40166a006712f0c2ed51d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 21:49:24 +0000 Subject: [PATCH 024/102] Bump moto[dynamodb] from 5.0.12 to 5.0.13 Bumps [moto[dynamodb]](https://github.com/getmoto/moto) from 5.0.12 to 5.0.13. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.0.12...5.0.13) --- updated-dependencies: - dependency-name: moto[dynamodb] dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 3df5726d2..64163d86d 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -7,7 +7,7 @@ -r requirements-apps-update-db.txt boto3==1.35.0 jinja2==3.1.4 -moto[dynamodb]==5.0.12 +moto[dynamodb]==5.0.13 pytest==8.3.2 PyYAML==6.0.2 responses==0.25.3 From 89b15cccf902227cd457234e070d0d3fb398eace Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 19 Aug 2024 15:08:08 -0800 Subject: [PATCH 025/102] fix INSAR_MULTI_BURST example scene names --- job_spec/INSAR_MULTI_BURST.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index cf5dd037e..93652e4f9 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -11,7 +11,7 @@ INSAR_MULTI_BURST: maxItems: 50 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST - - S1_136232_IW2_20200604T022312_VV_7C85-BURST + - S1_136232_IW2_20200604T022315_VV_7C85-BURST items: description: Name of the reference Sentinel-1 SLC IW burst granule to process type: string @@ -28,7 +28,7 @@ INSAR_MULTI_BURST: maxItems: 50 example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST - - S1_136232_IW2_20200616T022313_VV_5D11-BURST + - S1_136232_IW2_20200616T022316_VV_5D11-BURST items: description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string From c228650aea7a4681ffe7401b16f21fb4b93ff135 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 20 Aug 2024 09:22:05 -0800 Subject: [PATCH 026/102] 30 max bursts --- job_spec/INSAR_MULTI_BURST.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 93652e4f9..5c4f0df57 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -8,7 +8,7 @@ INSAR_MULTI_BURST: api_schema: type: array minItems: 1 - maxItems: 50 + maxItems: 30 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST - S1_136232_IW2_20200604T022315_VV_7C85-BURST @@ -25,7 +25,7 @@ INSAR_MULTI_BURST: api_schema: type: array minItems: 1 - maxItems: 50 + maxItems: 30 example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST - S1_136232_IW2_20200616T022316_VV_5D11-BURST From 5039ca09807a6a3872397f8579c0a7eba6e83a82 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 20 Aug 2024 09:22:54 -0800 Subject: [PATCH 027/102] Update job_spec/INSAR_MULTI_BURST.yml Co-authored-by: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> --- job_spec/INSAR_MULTI_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 5c4f0df57..1d0a3a8ea 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -13,7 +13,7 @@ INSAR_MULTI_BURST: - S1_136231_IW2_20200604T022312_VV_7C85-BURST - S1_136232_IW2_20200604T022315_VV_7C85-BURST items: - description: Name of the reference Sentinel-1 SLC IW burst granule to process + description: Name of the reference Sentinel-1 SLC IW burst granule(s) to process type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 From e4dd8b40e3c235b148f996ee91e306acba41bbb9 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 20 Aug 2024 09:23:01 -0800 Subject: [PATCH 028/102] Update job_spec/INSAR_MULTI_BURST.yml Co-authored-by: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> --- job_spec/INSAR_MULTI_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml index 1d0a3a8ea..e09dea02f 100644 --- a/job_spec/INSAR_MULTI_BURST.yml +++ b/job_spec/INSAR_MULTI_BURST.yml @@ -30,7 +30,7 @@ INSAR_MULTI_BURST: - S1_136231_IW2_20200616T022313_VV_5D11-BURST - S1_136232_IW2_20200616T022316_VV_5D11-BURST items: - description: Name of the secondary Sentinel-1 SLC IW burst granule to process + description: Name of the secondary Sentinel-1 SLC IW burst granule(s) to process type: string pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 From 9511e333ecdd1e8c5ff7267e6d518383bcf5dca0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 13:53:05 -0500 Subject: [PATCH 029/102] support more than 2 granules for burst validation --- apps/api/src/hyp3_api/validation.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index f935219d5..283eaabd2 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -77,21 +77,22 @@ def check_dem_coverage(granule_metadata): def check_same_burst_ids(granule_metadata): - ref_burst_id, sec_burst_id = [granule['name'].split('_')[1] for granule in granule_metadata] - if ref_burst_id != sec_burst_id: + burst_ids = [granule['name'].split('_')[1] for granule in granule_metadata] + not_matching = [burst_id for burst_id in set(burst_ids) if burst_ids.count(burst_id) == 1] + if not_matching: raise GranuleValidationError( - f'The requested scenes do not have the same burst ID: {ref_burst_id} and {sec_burst_id}' + f'The requests scenes have burst IDs with no matching pairs: {not_matching}.' ) def check_valid_polarizations(granule_metadata): - ref_polarization, sec_polarization = [granule['name'].split('_')[4] for granule in granule_metadata] - if ref_polarization != sec_polarization: + polarizations = set(granule['name'].split('_')[4] for granule in granule_metadata) + if len(polarizations) > 1: raise GranuleValidationError( - f'The requested scenes do not have the same polarization: {ref_polarization} and {sec_polarization}' + f'The requested scenes need to have the same polarization, got: {", ".join(polarizations)}' ) - if ref_polarization != 'VV' and ref_polarization != 'HH': - raise GranuleValidationError(f'Only VV and HH polarizations are currently supported, got: {ref_polarization}') + if not polarizations.issubset({'VV', 'HH'}): + raise GranuleValidationError(f'Only VV and HH polarizations are currently supported, got: {polarizations.pop()}') def check_not_antimeridian(granule_metadata): From ae04d2a6898e2e17be855941e6005e06b4e88ca4 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 13:53:22 -0500 Subject: [PATCH 030/102] updated validation tests for multi burst --- tests/test_api/test_validation.py | 217 +++++++++++++++++++++++------- 1 file changed, 172 insertions(+), 45 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 3ab648bae..c67812383 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -100,59 +100,186 @@ def test_check_dem_coverage(): def test_check_same_burst_ids(): - valid_case = [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - } + valid_cases = [ + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + } + ], + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + } + ], + [ + { + 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' + }, + ] ] - invalid_case = [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136232_IW2_20200616T022313_HH_5D11-BURST' - } + invalid_cases = [ + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136232_IW2_20200616T022313_HH_5D11-BURST' + } + ], + [ + { + 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136233_IW2_20200604T022316_VV_7C85-BURST' + }, + ], + [ + { + 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' + }, + { + 'name': 'S1_136233_IW2_20200604T022316_VV_7C85-BURST' + }, + ] ] - - validation.check_same_burst_ids(valid_case) - with raises(validation.GranuleValidationError, match=r'.*do not have the same burst ID.*'): - validation.check_same_burst_ids(invalid_case) + for valid_case in valid_cases: + validation.check_same_burst_ids(valid_case) + for invalid_case in invalid_cases: + with raises(validation.GranuleValidationError, match=r'.*burst IDs with no matching pairs*'): + validation.check_same_burst_ids(invalid_case) def test_check_valid_polarizations(): - valid_case = [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - } + valid_cases = [ + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + } + ], + [ + { + 'name': 'S1_136231_IW2_20200604T022312_HH_7C85-BURST' + }, + { + 'name': 'S1_136232_IW2_20200604T022316_HH_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_HH_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200616T022315_HH_5D11-BURST' + } + ], + [ + { + 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' + }, + ] ] - different_polarizations = [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_HH_5D11-BURST' - } + invalid_cases_no_match = [ + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136232_IW2_20200616T022313_HH_5D11-BURST' + } + ], + [ + { + 'name': 'S1_136232_IW2_20200616T022315_HH_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200604T022316_HH_7C85-BURST' + }, + ] ] - unsupported_polarizations = [ - { - 'name': 'S1_136231_IW2_20200604T022312_VH_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VH_5D11-BURST' - } + invalid_cases_unsupported = [ + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136232_IW2_20200616T022315_VH_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_VH_7C85-BURST' + }, + ], + [ + { + 'name': 'S1_136232_IW2_20200616T022315_HV_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200604T022312_HV_7C85-BURST' + }, + ] ] - - validation.check_valid_polarizations(valid_case) - with raises(validation.GranuleValidationError, match=r'.*do not have the same polarization.*'): - validation.check_valid_polarizations(different_polarizations) - with raises(validation.GranuleValidationError, match=r'.*Only VV and HH.*'): - validation.check_valid_polarizations(unsupported_polarizations) + for valid_case in valid_cases: + validation.check_valid_polarizations(valid_case) + for (invalid_case_no_match, invalid_case_unsupported) in zip(invalid_cases_no_match, invalid_cases_unsupported): + with raises(validation.GranuleValidationError, match=r'.*need to have the same polarization*'): + validation.check_valid_polarizations(invalid_case_no_match) + with raises(validation.GranuleValidationError, match=r'.*currently supported*'): + validation.check_valid_polarizations(invalid_case_unsupported) def test_check_granules_exist(): From 8d60963da512ffc69b1e0b9aa1c7a484ddb7c0d3 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 13:55:22 -0500 Subject: [PATCH 031/102] INSAR_MULTI_BURST to INSAR_ISCE_BURST --- .github/workflows/deploy-daac.yml | 1 - job_spec/INSAR_ISCE_BURST.yml | 37 ++++++++++--- job_spec/INSAR_MULTI_BURST.yml | 90 ------------------------------- 3 files changed, 29 insertions(+), 99 deletions(-) delete mode 100644 job_spec/INSAR_MULTI_BURST.yml diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index 036b1ea73..c6af17100 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -49,7 +49,6 @@ jobs: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml - job_spec/INSAR_MULTI_BURST.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 1500 expanded_max_vcpus: 3000 diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 512f7d276..2cfaed263 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -1,23 +1,41 @@ INSAR_ISCE_BURST: required_parameters: - - granules + - reference + - secondary parameters: - granules: + reference: default: '""' api_schema: type: array - minItems: 2 - maxItems: 2 + minItems: 1 + maxItems: 30 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST + - S1_136232_IW2_20200604T022312_VV_7C85-BURST + items: + description: Name of the reference Sentinel-1 SLC IW burst granule to process + type: string + pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' + minLength: 43 + maxLength: 43 + example: + - S1_136231_IW2_20200604T022312_VV_7C85-BURST + secondary: + default: '""' + api_schema: + type: array + minItems: 1 + maxItems: 50 + example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST + - S1_136232_IW2_20200616T022313_VV_5D11-BURST items: - description: Name of the Sentinel-1 SLC IW burst granule to process + description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string - pattern: "^S1_\\d{6}_IW.*-BURST" + pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 maxLength: 43 - example: S1_136231_IW2_20200604T022312_VV_7C85-BURST + example: S1_136231_IW2_20200616T022313_VV_5D11-BURST bucket_prefix: default: '""' apply_water_mask: @@ -60,7 +78,10 @@ INSAR_ISCE_BURST: - Ref::apply_water_mask - --looks - Ref::looks - - Ref::granules + - --reference + - Ref::reference + - --secondary + - Ref::secondary timeout: 5400 vcpu: 1 memory: 7600 diff --git a/job_spec/INSAR_MULTI_BURST.yml b/job_spec/INSAR_MULTI_BURST.yml deleted file mode 100644 index cf5dd037e..000000000 --- a/job_spec/INSAR_MULTI_BURST.yml +++ /dev/null @@ -1,90 +0,0 @@ -INSAR_MULTI_BURST: - required_parameters: - - reference - - secondary - parameters: - reference: - default: '""' - api_schema: - type: array - minItems: 1 - maxItems: 50 - example: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST - - S1_136232_IW2_20200604T022312_VV_7C85-BURST - items: - description: Name of the reference Sentinel-1 SLC IW burst granule to process - type: string - pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' - minLength: 43 - maxLength: 43 - example: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST - secondary: - default: '""' - api_schema: - type: array - minItems: 1 - maxItems: 50 - example: - - S1_136231_IW2_20200616T022313_VV_5D11-BURST - - S1_136232_IW2_20200616T022313_VV_5D11-BURST - items: - description: Name of the secondary Sentinel-1 SLC IW burst granule to process - type: string - pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' - minLength: 43 - maxLength: 43 - example: S1_136231_IW2_20200616T022313_VV_5D11-BURST - bucket_prefix: - default: '""' - apply_water_mask: - api_schema: - description: Sets pixels over coastal and large inland waterbodies as invalid for phase unwrapping. - default: false - type: boolean - looks: - api_schema: - description: Number of looks to take in range and azimuth - type: string - default: 20x4 - enum: - - 20x4 - - 10x2 - - 5x1 - cost_profiles: - EDC: - cost: 1.0 - DEFAULT: - cost: 1.0 - validators: - - check_dem_coverage - - check_valid_polarizations - - check_same_burst_ids - - check_not_antimeridian - tasks: - - name: '' - image: ghcr.io/asfhyp3/hyp3-isce2 - command: - - ++process - - insar_tops_multi_burst - - ++omp-num-threads - - '1' - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::bucket_prefix - - --apply-water-mask - - Ref::apply_water_mask - - --looks - - Ref::looks - - --reference - - Ref::reference - - --secondary - - Ref::secondary - timeout: 5400 - vcpu: 1 - memory: 7600 - secrets: - - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD From 518484c9c484749b2ec6a2c56a5c098131470628 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 14:02:30 -0500 Subject: [PATCH 032/102] updated examples --- job_spec/INSAR_ISCE_BURST.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 2cfaed263..77d1f7f79 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -9,9 +9,11 @@ INSAR_ISCE_BURST: type: array minItems: 1 maxItems: 30 - example: + examples: + - S1_136231_IW2_20200604T022312_VV_7C85-BURST + - S1_136232_IW2_20200604T022315_VV_7C85-BURST + examples: - S1_136231_IW2_20200604T022312_VV_7C85-BURST - - S1_136232_IW2_20200604T022312_VV_7C85-BURST items: description: Name of the reference Sentinel-1 SLC IW burst granule to process type: string @@ -26,9 +28,11 @@ INSAR_ISCE_BURST: type: array minItems: 1 maxItems: 50 - example: + examples: - S1_136231_IW2_20200616T022313_VV_5D11-BURST - - S1_136232_IW2_20200616T022313_VV_5D11-BURST + - S1_136232_IW2_20200616T022316_VV_5D11-BURST + examples: + - S1_136232_IW2_20200616T022316_VV_5D11-BURST items: description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string From 0734ba9d3e8849c845e548b5d059adb98289cb1f Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 14:09:27 -0500 Subject: [PATCH 033/102] updated changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f40064598..6486d4c0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [7.8.0] +## [8.0.0] ### Added -- Added `INSAR_MULTI_BURST` job type to `hyp3-test` that can merge adjacent bursts into an SLC and then perform InSAR. +- `INSAR_ISCE_BURST` now has the ability to process multiple adjacent bursts into one interferogram. + +### Changed +- `INSAR_ISCE_BURST` now takes in `reference` and `secondary` parameters rather than `granules`. ## [7.7.2] From f521d68e4aca56895dffd70913f70919707ae05d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 14:16:36 -0500 Subject: [PATCH 034/102] single burst example and 30 max lim --- job_spec/INSAR_ISCE_BURST.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 77d1f7f79..6661fc533 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -9,10 +9,7 @@ INSAR_ISCE_BURST: type: array minItems: 1 maxItems: 30 - examples: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST - - S1_136232_IW2_20200604T022315_VV_7C85-BURST - examples: + example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST items: description: Name of the reference Sentinel-1 SLC IW burst granule to process @@ -27,12 +24,9 @@ INSAR_ISCE_BURST: api_schema: type: array minItems: 1 - maxItems: 50 - examples: + maxItems: 30 + example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST - - S1_136232_IW2_20200616T022316_VV_5D11-BURST - examples: - - S1_136232_IW2_20200616T022316_VV_5D11-BURST items: description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string From 0c183a3b6f60ff0fd008241f724e742e3ebfa8ab Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 14:18:25 -0500 Subject: [PATCH 035/102] line too long --- apps/api/src/hyp3_api/validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 283eaabd2..d5dfe98be 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -92,7 +92,9 @@ def check_valid_polarizations(granule_metadata): f'The requested scenes need to have the same polarization, got: {", ".join(polarizations)}' ) if not polarizations.issubset({'VV', 'HH'}): - raise GranuleValidationError(f'Only VV and HH polarizations are currently supported, got: {polarizations.pop()}') + raise GranuleValidationError( + f'Only VV and HH polarizations are currently supported, got: {polarizations.pop()}' + ) def check_not_antimeridian(granule_metadata): From 988417ba213c2cfd5d2700e1c5b8020c4789cbea Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 14:21:00 -0500 Subject: [PATCH 036/102] unsupported polarization match --- tests/test_api/test_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index c67812383..248b14d4e 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -252,10 +252,10 @@ def test_check_valid_polarizations(): invalid_cases_unsupported = [ [ { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + 'name': 'S1_136231_IW2_20200604T022312_VH_7C85-BURST' }, { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + 'name': 'S1_136231_IW2_20200616T022313_VH_5D11-BURST' }, { 'name': 'S1_136232_IW2_20200616T022315_VH_5D11-BURST' From b8aecaa980321a4a670804d09b970a121f2c9cf1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 20 Aug 2024 14:24:19 -0500 Subject: [PATCH 037/102] better changelog --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6486d4c0f..92e360b86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.0.0] -### Added +### Changed - `INSAR_ISCE_BURST` now has the ability to process multiple adjacent bursts into one interferogram. +- `INSAR_ISCE_BURST` now uses `reference` and `secondary` parameters which each accept a list of 1 or more scene names. -### Changed -- `INSAR_ISCE_BURST` now takes in `reference` and `secondary` parameters rather than `granules`. +# Removed +- `INSAR_ISCE_BURST` no longer accepts a `granules` parameter. ## [7.7.2] From 261f7366f1e9d91aed2dce4e497c578488bacd50 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 20 Aug 2024 13:36:39 -0800 Subject: [PATCH 038/102] add hyp3-multi-burst-sandbox deployment --- .../workflows/deploy-multi-burst-sandbox.yml | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/deploy-multi-burst-sandbox.yml diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml new file mode 100644 index 000000000..4e9d26b44 --- /dev/null +++ b/.github/workflows/deploy-multi-burst-sandbox.yml @@ -0,0 +1,80 @@ +name: Deploy Multi-Burst Sandbox Stack to AWS + +on: + push: + branches: + - multi-burst-sandbox + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - environment: hyp3-multi-burst-sandbox + domain: hyp3-multi-burst-sandbox.asf.alaska.edu + template_bucket: cf-templates-1hz9ldhhl4ahu-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: DEFAULT + deploy_ref: refs/heads/multi-burst-sandbox + job_files: >- + job_spec/INSAR_ISCE_BURST.yml + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 640 + expanded_max_vcpus: 640 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + distribution_url: '' + + environment: + name: ${{ matrix.environment }} + url: https://${{ matrix.domain }} + + steps: + - uses: actions/checkout@v4.1.7 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} + aws-region: ${{ secrets.AWS_REGION }} + + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - uses: ./.github/actions/deploy-hyp3 + with: + TEMPLATE_BUCKET: ${{ matrix.template_bucket }} + STACK_NAME: ${{ matrix.environment }} + DOMAIN_NAME: ${{ matrix.domain }} + API_NAME: ${{ matrix.environment }} + CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }} + IMAGE_TAG: ${{ matrix.image_tag }} + PRODUCT_LIFETIME: ${{ matrix.product_lifetime_in_days }} + VPC_ID: ${{ secrets.VPC_ID }} + SUBNET_IDS: ${{ secrets.SUBNET_IDS }} + SECRET_ARN: ${{ secrets.SECRET_ARN }} + CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} + DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }} + DEFAULT_APPLICATION_STATUS: ${{ matrix.default_application_status }} + COST_PROFILE: ${{ matrix.cost_profile }} + JOB_FILES: ${{ matrix.job_files }} + DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} + EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} + MONTHLY_BUDGET: ${{ secrets.MONTHLY_BUDGET }} + REQUIRED_SURPLUS: ${{ matrix.required_surplus }} + ORIGIN_ACCESS_IDENTITY_ID: ${{ secrets.ORIGIN_ACCESS_IDENTITY_ID }} + SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} + AMI_ID: ${{ matrix.ami_id }} + INSTANCE_TYPES: ${{ matrix.instance_types }} + DISTRIBUTION_URL: ${{ matrix.distribution_url }} + AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }} From 42d4b5317225d0294f9d0b8ba004ff602df7d709 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:28:08 -0500 Subject: [PATCH 039/102] Update CHANGELOG.md Co-authored-by: Jake Herrmann --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92e360b86..62b2d4396 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.0.0] ### Changed -- `INSAR_ISCE_BURST` now has the ability to process multiple adjacent bursts into one interferogram. -- `INSAR_ISCE_BURST` now uses `reference` and `secondary` parameters which each accept a list of 1 or more scene names. +- `INSAR_ISCE_BURST` now accepts `reference` and `secondary` parameters, each of which is a list containing a single granule name. In a future release, `INSAR_ISCE_BURST` will support processing multiple adjacent bursts into one interferogram, and each parameter will be a list of one or more granule names. # Removed - `INSAR_ISCE_BURST` no longer accepts a `granules` parameter. From 5049bd9124ad91dff998d289af28ce240146a50d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:28:20 -0500 Subject: [PATCH 040/102] Update CHANGELOG.md Co-authored-by: Jake Herrmann --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b2d4396..174d11d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `INSAR_ISCE_BURST` now accepts `reference` and `secondary` parameters, each of which is a list containing a single granule name. In a future release, `INSAR_ISCE_BURST` will support processing multiple adjacent bursts into one interferogram, and each parameter will be a list of one or more granule names. -# Removed +### Removed - `INSAR_ISCE_BURST` no longer accepts a `granules` parameter. From ee88d1f319f08a46e1ec1daca21f7202bfaf7c02 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:28:40 -0500 Subject: [PATCH 041/102] Update job_spec/INSAR_ISCE_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 6661fc533..1904d21bf 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -8,7 +8,7 @@ INSAR_ISCE_BURST: api_schema: type: array minItems: 1 - maxItems: 30 + maxItems: 1 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST items: From 42c9ac4735c0557d0a25638040607abbdd5e9218 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:29:01 -0500 Subject: [PATCH 042/102] Update job_spec/INSAR_ISCE_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 1904d21bf..55cef4683 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -24,7 +24,7 @@ INSAR_ISCE_BURST: api_schema: type: array minItems: 1 - maxItems: 30 + maxItems: 1 example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST items: From d467d5e47ad3309485b2c490e7264e408a511b87 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:29:24 -0500 Subject: [PATCH 043/102] Update job_spec/INSAR_ISCE_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_ISCE_BURST.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 55cef4683..31f39c169 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -17,8 +17,7 @@ INSAR_ISCE_BURST: pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' minLength: 43 maxLength: 43 - example: - - S1_136231_IW2_20200604T022312_VV_7C85-BURST + example: S1_136231_IW2_20200604T022312_VV_7C85-BURST secondary: default: '""' api_schema: From 586765ac57a5c459a9202830200f230bcb94214b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:29:56 -0500 Subject: [PATCH 044/102] Update apps/api/src/hyp3_api/validation.py Co-authored-by: Jake Herrmann --- apps/api/src/hyp3_api/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index d5dfe98be..4f8921bf8 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -81,7 +81,7 @@ def check_same_burst_ids(granule_metadata): not_matching = [burst_id for burst_id in set(burst_ids) if burst_ids.count(burst_id) == 1] if not_matching: raise GranuleValidationError( - f'The requests scenes have burst IDs with no matching pairs: {not_matching}.' + f'The requested scenes have burst IDs with no matching pairs: {not_matching}.' ) From 5836b33d3967c795db4515c7e2ad23ef17e0e74e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 11:46:32 -0500 Subject: [PATCH 045/102] only 2 burst ids --- apps/api/src/hyp3_api/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 4f8921bf8..bff7b7a37 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -78,7 +78,7 @@ def check_dem_coverage(granule_metadata): def check_same_burst_ids(granule_metadata): burst_ids = [granule['name'].split('_')[1] for granule in granule_metadata] - not_matching = [burst_id for burst_id in set(burst_ids) if burst_ids.count(burst_id) == 1] + not_matching = [burst_id for burst_id in set(burst_ids) if burst_ids.count(burst_id) != 2] if not_matching: raise GranuleValidationError( f'The requested scenes have burst IDs with no matching pairs: {not_matching}.' From f96cb5a0d8e085358283a9fe87c5c04d4ab28b0d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 12:22:53 -0500 Subject: [PATCH 046/102] add 3 burst_id invalid case --- tests/test_api/test_validation.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 248b14d4e..8432ec458 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -147,6 +147,17 @@ def test_check_same_burst_ids(): 'name': 'S1_136232_IW2_20200616T022313_HH_5D11-BURST' } ], + [ + { + 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + } + ], [ { 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' From 26566f6a732e1f9bdd4a77c6777ffec7636f7fdf Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 14:21:21 -0500 Subject: [PATCH 047/102] modify validation to pass job and granule_metadata for flexibility --- apps/api/src/hyp3_api/validation.py | 22 +-- tests/test_api/test_validation.py | 204 ++++++++++++++-------------- 2 files changed, 117 insertions(+), 109 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index bff7b7a37..599f99c66 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -70,22 +70,26 @@ def check_granules_exist(granules, granule_metadata): raise GranuleValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}') -def check_dem_coverage(granule_metadata): +def check_dem_coverage(_, granule_metadata): bad_granules = [g['name'] for g in granule_metadata if not has_sufficient_coverage(g['polygon'])] if bad_granules: raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}') -def check_same_burst_ids(granule_metadata): - burst_ids = [granule['name'].split('_')[1] for granule in granule_metadata] - not_matching = [burst_id for burst_id in set(burst_ids) if burst_ids.count(burst_id) != 2] - if not_matching: +def check_same_burst_ids(job, _): + ref_ids = [ref.split('_')[1] for ref in job['job_parameters'].get('reference', [])] + sec_ids = [sec.split('_')[1] for sec in job['job_parameters'].get('secondary', [])] + if ref_ids != sec_ids: raise GranuleValidationError( - f'The requested scenes have burst IDs with no matching pairs: {not_matching}.' + f'The requested scenes have burst IDs with no matching pairs.' + ) + if len(set(ref_ids)) != len(ref_ids): + raise GranuleValidationError( + f'The requested scenes have more than 1 pair with the same burst ID.' ) -def check_valid_polarizations(granule_metadata): +def check_valid_polarizations(_, granule_metadata): polarizations = set(granule['name'].split('_')[4] for granule in granule_metadata) if len(polarizations) > 1: raise GranuleValidationError( @@ -97,7 +101,7 @@ def check_valid_polarizations(granule_metadata): ) -def check_not_antimeridian(granule_metadata): +def check_not_antimeridian(_, granule_metadata): for granule in granule_metadata: bbox = granule['polygon'].bounds if abs(bbox[0] - bbox[2]) > 180.0 and bbox[0] * bbox[2] < 0.0: @@ -132,4 +136,4 @@ def validate_jobs(jobs): job_granule_metadata = [granule for granule in granule_metadata if granule['name'] in get_granules([job])] module = sys.modules[__name__] validator = getattr(module, validator_name) - validator(job_granule_metadata) + validator(job, job_granule_metadata) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 8432ec458..6ec0e3501 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -18,11 +18,11 @@ def test_not_antimeridian(): bad_rect = rectangle(51.7, 51.3, 179.7, -179.3) good_granules = [{'polygon': good_rect, 'name': 'good'}, {'polygon': good_rect, 'name': 'good'}] - validation.check_not_antimeridian(good_granules) + validation.check_not_antimeridian({}, good_granules) bad_granules = [{'polygon': good_rect, 'name': 'good'}, {'polygon': bad_rect, 'name': 'bad'}] with raises(validation.GranuleValidationError, match=r'.*crosses the antimeridian.*'): - validation.check_not_antimeridian(bad_granules) + validation.check_not_antimeridian({}, bad_granules) def test_has_sufficient_coverage(): @@ -84,118 +84,119 @@ def test_check_dem_coverage(): covered2 = {'name': 'covered2', 'polygon': rectangle(-62, -90, 180, -180)} not_covered = {'name': 'not_covered', 'polygon': rectangle(-20, -30, 70, 100)} - validation.check_dem_coverage([]) - validation.check_dem_coverage([covered1]) - validation.check_dem_coverage([covered2]) - validation.check_dem_coverage([covered1, covered2]) + validation.check_dem_coverage({}, []) + validation.check_dem_coverage({}, [covered1]) + validation.check_dem_coverage({}, [covered2]) + validation.check_dem_coverage({}, [covered1, covered2]) with raises(validation.GranuleValidationError) as e: - validation.check_dem_coverage([not_covered]) + validation.check_dem_coverage({}, [not_covered]) assert 'not_covered' in str(e) with raises(validation.GranuleValidationError) as e: - validation.check_dem_coverage([covered1, not_covered]) + validation.check_dem_coverage({}, [covered1, not_covered]) assert 'not_covered' in str(e) assert 'covered1' not in str(e) def test_check_same_burst_ids(): - valid_cases = [ - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + valid_jobs = [ + { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], + 'secondary': ['S1_136231_IW2_20200616T022313_VV_5D11-BURST'] } - ], - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200616T022313_VV_5411-BURST', + 'S1_136232_IW2_20200616T022345_VV_5D13-BURST' + ] } - ], - [ - { - 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' - }, - ] + } ] - invalid_cases = [ - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136232_IW2_20200616T022313_HH_5D11-BURST' + invalid_jobs_not_matching = [ + { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], + 'secondary': ['S1_136232_IW2_20200616T022313_VV_5D11-BURST'] } - ], - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + }, + { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], + 'secondary': [ + 'S1_136232_IW2_20200616T022313_VV_5D11-BURST', + 'S1_136233_IW2_20200616T022313_VV_5D11-BURST' + ] } - ], - [ - { - 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136233_IW2_20200604T022316_VV_7C85-BURST' - }, - ], - [ - { - 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' - }, - { - 'name': 'S1_136233_IW2_20200604T022316_VV_7C85-BURST' - }, - ] + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136232_IW2_20200604T123455_VV_ABC5-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_VV_5D11-BURST', + 'S1_136233_IW2_20200617T123213_VV_5E13-BURST' + ] + } + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136232_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136231_IW2_20200604T123455_VV_ABC5-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_VV_5D11-BURST', + 'S1_136233_IW2_20200617T123213_VV_5E13-BURST' + ] + } + } ] - for valid_case in valid_cases: - validation.check_same_burst_ids(valid_case) - for invalid_case in invalid_cases: + invalid_jobs_duplicate = [ + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136231_IW2_20200604T123455_VV_ABC5-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_VV_5D11-BURST', + 'S1_136231_IW2_20200617T123213_VV_5E13-BURST' + ] + } + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136231_IW2_20200604T123455_VV_ABC5-BURST', + 'S1_136232_IW2_20200604T125455_VV_ABC6-BURST', + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_VV_5D11-BURST', + 'S1_136231_IW2_20200617T123213_VV_5E13-BURST', + 'S1_136232_IW2_20200604T123475_VV_ABC7-BURST', + ] + } + } + ] + for valid_job in valid_jobs: + validation.check_same_burst_ids(valid_job, {}) + for invalid_job in invalid_jobs_not_matching: with raises(validation.GranuleValidationError, match=r'.*burst IDs with no matching pairs*'): - validation.check_same_burst_ids(invalid_case) - + validation.check_same_burst_ids(invalid_job, {}) + for invalid_job in invalid_jobs_duplicate: + with raises(validation.GranuleValidationError, match=r'.*more than 1 pair with the same burst ID*'): + validation.check_same_burst_ids(invalid_job, {}) def test_check_valid_polarizations(): valid_cases = [ @@ -285,12 +286,12 @@ def test_check_valid_polarizations(): ] ] for valid_case in valid_cases: - validation.check_valid_polarizations(valid_case) + validation.check_valid_polarizations({}, valid_case) for (invalid_case_no_match, invalid_case_unsupported) in zip(invalid_cases_no_match, invalid_cases_unsupported): with raises(validation.GranuleValidationError, match=r'.*need to have the same polarization*'): - validation.check_valid_polarizations(invalid_case_no_match) + validation.check_valid_polarizations({}, invalid_case_no_match) with raises(validation.GranuleValidationError, match=r'.*currently supported*'): - validation.check_valid_polarizations(invalid_case_unsupported) + validation.check_valid_polarizations({}, invalid_case_unsupported) def test_check_granules_exist(): @@ -368,6 +369,9 @@ def test_validate_jobs(): granule_with_dem_coverage = 'S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8' granule_without_dem_coverage = 'S1A_IW_GRDH_1SDV_20201219T222530_20201219T222555_035761_042F72_8378' + valid_ref_burst = 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + valid_sec_burst = 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + granule_polygon_pairs = [ (granule_with_dem_coverage, [['13.705972 -91.927132 14.452647 -91.773392 14.888498 -94.065727 ' From 49f0ab3ac19063a4e698ddb498d58982aa452c4b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 14:29:37 -0500 Subject: [PATCH 048/102] INSAR_ISCE_BURST cases in test_validate_jobs --- tests/test_api/test_validation.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 6ec0e3501..701622a5f 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -161,7 +161,7 @@ def test_check_same_burst_ids(): } } ] - invalid_jobs_duplicate = [ + invalid_jobs_duplicate = [ { 'job_parameters': { 'reference': [ @@ -196,7 +196,8 @@ def test_check_same_burst_ids(): validation.check_same_burst_ids(invalid_job, {}) for invalid_job in invalid_jobs_duplicate: with raises(validation.GranuleValidationError, match=r'.*more than 1 pair with the same burst ID*'): - validation.check_same_burst_ids(invalid_job, {}) + validation.check_same_burst_ids(invalid_job, {}) + def test_check_valid_polarizations(): valid_cases = [ @@ -371,6 +372,7 @@ def test_validate_jobs(): valid_ref_burst = 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' valid_sec_burst = 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + invalid_sec_burst = 'S1_136232_IW2_20200616T022313_VV_5D11-BURST' granule_polygon_pairs = [ (granule_with_dem_coverage, @@ -412,6 +414,13 @@ def test_validate_jobs(): 'job_type': 'ARIA_RAIDER', 'job_parameters': {} }, + { + 'job_type': 'INSAR_ISCE_BURST', + 'job_parameters': { + 'reference': [valid_ref_burst], + 'secondary': [valid_sec_burst] + } + } ] validation.validate_jobs(jobs) @@ -436,3 +445,15 @@ def test_validate_jobs(): ] with raises(validation.GranuleValidationError): validation.validate_jobs(jobs) + + jobs = [ + { + 'job_type': 'INSAR_ISCE_BURST', + 'job_parameters': { + 'reference': [valid_ref_burst], + 'secondary': [invalid_sec_burst] + } + } + ] + with raises(validation.GranuleValidationError): + validation.validate_jobs(jobs) From 8cc4e6f2f5dbca678e5a2f006cab4df907878e70 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 21 Aug 2024 14:31:08 -0500 Subject: [PATCH 049/102] flake8 --- apps/api/src/hyp3_api/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 599f99c66..aef49270e 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -81,11 +81,11 @@ def check_same_burst_ids(job, _): sec_ids = [sec.split('_')[1] for sec in job['job_parameters'].get('secondary', [])] if ref_ids != sec_ids: raise GranuleValidationError( - f'The requested scenes have burst IDs with no matching pairs.' + 'The requested scenes have burst IDs with no matching pairs.' ) if len(set(ref_ids)) != len(ref_ids): raise GranuleValidationError( - f'The requested scenes have more than 1 pair with the same burst ID.' + 'The requested scenes have more than 1 pair with the same burst ID.' ) From fc5eeb7883918f14f56aab0729e946ce5d94df76 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 12:24:29 -0500 Subject: [PATCH 050/102] refactors --- apps/api/src/hyp3_api/validation.py | 22 ++- tests/test_api/test_validation.py | 203 ++++++++++++++-------------- 2 files changed, 116 insertions(+), 109 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index aef49270e..52cead800 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -77,20 +77,30 @@ def check_dem_coverage(_, granule_metadata): def check_same_burst_ids(job, _): - ref_ids = [ref.split('_')[1] for ref in job['job_parameters'].get('reference', [])] - sec_ids = [sec.split('_')[1] for sec in job['job_parameters'].get('secondary', [])] + refs = job['job_parameters']['reference'] + secs = job['job_parameters']['secondary'] + ref_ids = [ref.split('_')[1] for ref in refs] + sec_ids = [sec.split('_')[1] for sec in secs] + if len(ref_ids) != len(sec_ids): + error_message = ( + f'Number of reference and secondary scenes must match, got: ' + f'{len(ref_ids)} references and {len(sec_ids)} secondaries' + ) + raise GranuleValidationError(error_message) if ref_ids != sec_ids: + bad_id_index = [ref_ids.index(ref) for (ref, sec) in zip(ref_ids, sec_ids) if ref != sec][0] raise GranuleValidationError( - 'The requested scenes have burst IDs with no matching pairs.' + f'Burst IDs do not match for {refs[bad_id_index]} and {secs[bad_id_index]}.' ) if len(set(ref_ids)) != len(ref_ids): + duplicate_pair_id = [ref_id for ref_id in ref_ids if ref_ids.count(ref_id) > 1][0] raise GranuleValidationError( - 'The requested scenes have more than 1 pair with the same burst ID.' + f'The requested scenes have more than 1 pair with the following burst ID: {duplicate_pair_id}.' ) -def check_valid_polarizations(_, granule_metadata): - polarizations = set(granule['name'].split('_')[4] for granule in granule_metadata) +def check_valid_polarizations(job, _): + polarizations = set(granule.split('_')[4] for granule in get_granules([job])) if len(polarizations) > 1: raise GranuleValidationError( f'The requested scenes need to have the same polarization, got: {", ".join(polarizations)}' diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 701622a5f..2e51b615f 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -120,6 +120,15 @@ def test_check_same_burst_ids(): } } ] + invalid_job_different_lengths = { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], + 'secondary': [ + 'S1_136232_IW2_20200616T022313_VV_5D11-BURST', + 'S1_136233_IW2_20200616T022313_VV_5D11-BURST' + ] + } + } invalid_jobs_not_matching = [ { 'job_parameters': { @@ -127,15 +136,6 @@ def test_check_same_burst_ids(): 'secondary': ['S1_136232_IW2_20200616T022313_VV_5D11-BURST'] } }, - { - 'job_parameters': { - 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], - 'secondary': [ - 'S1_136232_IW2_20200616T022313_VV_5D11-BURST', - 'S1_136233_IW2_20200616T022313_VV_5D11-BURST' - ] - } - }, { 'job_parameters': { 'reference': [ @@ -191,108 +191,99 @@ def test_check_same_burst_ids(): ] for valid_job in valid_jobs: validation.check_same_burst_ids(valid_job, {}) + with raises(validation.GranuleValidationError, match=r'.*Number of reference and secondary scenes must match*'): + validation.check_same_burst_ids(invalid_job_different_lengths, {}) for invalid_job in invalid_jobs_not_matching: - with raises(validation.GranuleValidationError, match=r'.*burst IDs with no matching pairs*'): + with raises(validation.GranuleValidationError, match=r'.*Burst IDs do not match*'): validation.check_same_burst_ids(invalid_job, {}) for invalid_job in invalid_jobs_duplicate: - with raises(validation.GranuleValidationError, match=r'.*more than 1 pair with the same burst ID*'): + with raises(validation.GranuleValidationError, match=r'.*The requested scenes have more than 1 pair*'): validation.check_same_burst_ids(invalid_job, {}) def test_check_valid_polarizations(): - valid_cases = [ - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + valid_jobs = [ + { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], + 'secondary': ['S1_136231_IW2_20200616T022313_VV_5D11-BURST'] } - ], - [ - { - 'name': 'S1_136231_IW2_20200604T022312_HH_7C85-BURST' - }, - { - 'name': 'S1_136232_IW2_20200604T022316_HH_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_HH_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200616T022315_HH_5D11-BURST' + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_HH_7C85-BURST', + 'S1_136232_IW2_20200616T022315_HH_5D11-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200616T022313_HH_5411-BURST', + 'S1_136232_IW2_20200616T022345_HH_5D13-BURST' + ] } - ], - [ - { - 'name': 'S1_136232_IW2_20200616T022315_VV_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200604T022316_VV_7C85-BURST' - }, - ] + } ] - invalid_cases_no_match = [ - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136232_IW2_20200616T022313_HH_5D11-BURST' + invalid_jobs_not_matching = [ + { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VV_7C85-BURST'], + 'secondary': ['S1_136232_IW2_20200616T022313_HH_5D11-BURST'] } - ], - [ - { - 'name': 'S1_136232_IW2_20200616T022315_HH_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200604T022316_HH_7C85-BURST' - }, - ] + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136232_IW2_20200604T123455_VV_ABC5-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_VV_5D11-BURST', + 'S1_136233_IW2_20200617T123213_HH_5E13-BURST' + ] + } + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136232_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136231_IW2_20200604T123455_HH_ABC5-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_VV_5D11-BURST', + 'S1_136233_IW2_20200617T123213_HH_5E13-BURST' + ] + } + } ] - invalid_cases_unsupported = [ - [ - { - 'name': 'S1_136231_IW2_20200604T022312_VH_7C85-BURST' - }, - { - 'name': 'S1_136231_IW2_20200616T022313_VH_5D11-BURST' - }, - { - 'name': 'S1_136232_IW2_20200616T022315_VH_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_VH_7C85-BURST' - }, - ], - [ - { - 'name': 'S1_136232_IW2_20200616T022315_HV_5D11-BURST' - }, - { - 'name': 'S1_136231_IW2_20200604T022312_HV_7C85-BURST' - }, - ] + invalid_jobs_unsupported = [ + { + 'job_parameters': { + 'reference': ['S1_136231_IW2_20200604T022312_VH_7C85-BURST'], + 'secondary': ['S1_136231_IW2_20200617T022313_VH_5D11-BURST'] + } + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_HV_7C85-BURST', + 'S1_136231_IW2_20200604T123455_HV_ABC5-BURST', + 'S1_136232_IW2_20200604T125455_HV_ABC6-BURST', + ], + 'secondary': [ + 'S1_136231_IW2_20200617T022313_HV_5D11-BURST', + 'S1_136231_IW2_20200617T123213_HV_5E13-BURST', + 'S1_136232_IW2_20200604T123475_HV_ABC7-BURST', + ] + } + } ] - for valid_case in valid_cases: - validation.check_valid_polarizations({}, valid_case) - for (invalid_case_no_match, invalid_case_unsupported) in zip(invalid_cases_no_match, invalid_cases_unsupported): + for valid_job in valid_jobs: + validation.check_valid_polarizations(valid_job, {}) + for invalid_job in invalid_jobs_not_matching: with raises(validation.GranuleValidationError, match=r'.*need to have the same polarization*'): - validation.check_valid_polarizations({}, invalid_case_no_match) + validation.check_valid_polarizations(invalid_job, {}) + for invalid_job in invalid_jobs_unsupported: with raises(validation.GranuleValidationError, match=r'.*currently supported*'): - validation.check_valid_polarizations({}, invalid_case_unsupported) + validation.check_valid_polarizations(invalid_job, {}) def test_check_granules_exist(): @@ -370,9 +361,15 @@ def test_validate_jobs(): granule_with_dem_coverage = 'S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8' granule_without_dem_coverage = 'S1A_IW_GRDH_1SDV_20201219T222530_20201219T222555_035761_042F72_8378' - valid_ref_burst = 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - valid_sec_burst = 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - invalid_sec_burst = 'S1_136232_IW2_20200616T022313_VV_5D11-BURST' + valid_burst_pair = ( + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + ) + + invalid_burst_pair = ( + 'S1_136231_IW2_20200616T022313_VV_5D11-BURST', + 'S1_136232_IW2_20200604T022315_VV_7C85-BURST' + ) granule_polygon_pairs = [ (granule_with_dem_coverage, @@ -417,8 +414,8 @@ def test_validate_jobs(): { 'job_type': 'INSAR_ISCE_BURST', 'job_parameters': { - 'reference': [valid_ref_burst], - 'secondary': [valid_sec_burst] + 'reference': [valid_burst_pair[0]], + 'secondary': [valid_burst_pair[1]] } } ] @@ -450,8 +447,8 @@ def test_validate_jobs(): { 'job_type': 'INSAR_ISCE_BURST', 'job_parameters': { - 'reference': [valid_ref_burst], - 'secondary': [invalid_sec_burst] + 'reference': [invalid_burst_pair[0]], + 'secondary': [invalid_burst_pair[1]] } } ] From ae32b23d073848592431bb98502ab54a62e3ef77 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 12:27:28 -0500 Subject: [PATCH 051/102] Update job_spec/INSAR_ISCE_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 31f39c169..4f2b0693e 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -14,7 +14,7 @@ INSAR_ISCE_BURST: items: description: Name of the reference Sentinel-1 SLC IW burst granule to process type: string - pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' + pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$' minLength: 43 maxLength: 43 example: S1_136231_IW2_20200604T022312_VV_7C85-BURST From cbd66f39a9e1a1dfe54834fbe0257d2d590b49a4 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 12:30:46 -0500 Subject: [PATCH 052/102] Update job_spec/INSAR_ISCE_BURST.yml Co-authored-by: Jake Herrmann --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 4f2b0693e..ae0ba6eda 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -29,7 +29,7 @@ INSAR_ISCE_BURST: items: description: Name of the secondary Sentinel-1 SLC IW burst granule to process type: string - pattern: '^((S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST)($|[ \t]+))+$' + pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$' minLength: 43 maxLength: 43 example: S1_136231_IW2_20200616T022313_VV_5D11-BURST From 3ddc4338f6004450afa4afe3089570485fdb44fa Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 12:39:35 -0500 Subject: [PATCH 053/102] flake8 --- apps/api/src/hyp3_api/validation.py | 4 ++-- tests/test_api/test_validation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 52cead800..bb4497e70 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -83,8 +83,8 @@ def check_same_burst_ids(job, _): sec_ids = [sec.split('_')[1] for sec in secs] if len(ref_ids) != len(sec_ids): error_message = ( - f'Number of reference and secondary scenes must match, got: ' - f'{len(ref_ids)} references and {len(sec_ids)} secondaries' + f'Number of reference and secondary scenes must match, got: ' + f'{len(ref_ids)} references and {len(sec_ids)} secondaries' ) raise GranuleValidationError(error_message) if ref_ids != sec_ids: diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 2e51b615f..b7e079ef1 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -281,7 +281,7 @@ def test_check_valid_polarizations(): for invalid_job in invalid_jobs_not_matching: with raises(validation.GranuleValidationError, match=r'.*need to have the same polarization*'): validation.check_valid_polarizations(invalid_job, {}) - for invalid_job in invalid_jobs_unsupported: + for invalid_job in invalid_jobs_unsupported: with raises(validation.GranuleValidationError, match=r'.*currently supported*'): validation.check_valid_polarizations(invalid_job, {}) From f150ff53cbc310a7e4044e088c9951d2ea22aac0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 12:54:28 -0500 Subject: [PATCH 054/102] Update apps/api/src/hyp3_api/validation.py Co-authored-by: Jake Herrmann --- apps/api/src/hyp3_api/validation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index bb4497e70..672fa29cb 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -82,11 +82,10 @@ def check_same_burst_ids(job, _): ref_ids = [ref.split('_')[1] for ref in refs] sec_ids = [sec.split('_')[1] for sec in secs] if len(ref_ids) != len(sec_ids): - error_message = ( + raise GranuleValidationError( f'Number of reference and secondary scenes must match, got: ' f'{len(ref_ids)} references and {len(sec_ids)} secondaries' ) - raise GranuleValidationError(error_message) if ref_ids != sec_ids: bad_id_index = [ref_ids.index(ref) for (ref, sec) in zip(ref_ids, sec_ids) if ref != sec][0] raise GranuleValidationError( From 045f250d6a61d2f5ab46214ea813ba821265fe2e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 12:58:23 -0500 Subject: [PATCH 055/102] better loop --- apps/api/src/hyp3_api/validation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index bb4497e70..d0f3d5742 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -87,11 +87,11 @@ def check_same_burst_ids(job, _): f'{len(ref_ids)} references and {len(sec_ids)} secondaries' ) raise GranuleValidationError(error_message) - if ref_ids != sec_ids: - bad_id_index = [ref_ids.index(ref) for (ref, sec) in zip(ref_ids, sec_ids) if ref != sec][0] - raise GranuleValidationError( - f'Burst IDs do not match for {refs[bad_id_index]} and {secs[bad_id_index]}.' - ) + for i in range(len(ref_ids)): + if ref_ids[i] != sec_ids[i]: + raise GranuleValidationError( + f'Burst IDs do not match for {refs[i]} and {secs[i]}.' + ) if len(set(ref_ids)) != len(ref_ids): duplicate_pair_id = [ref_id for ref_id in ref_ids if ref_ids.count(ref_id) > 1][0] raise GranuleValidationError( From 211004eb126e52e8fa8e87146b5f639afa9f51c7 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 22 Aug 2024 13:24:16 -0500 Subject: [PATCH 056/102] Update apps/api/src/hyp3_api/validation.py Co-authored-by: Jake Herrmann --- apps/api/src/hyp3_api/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 4a7bdba4f..7918b3cb3 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -92,7 +92,7 @@ def check_same_burst_ids(job, _): f'Burst IDs do not match for {refs[i]} and {secs[i]}.' ) if len(set(ref_ids)) != len(ref_ids): - duplicate_pair_id = [ref_id for ref_id in ref_ids if ref_ids.count(ref_id) > 1][0] + duplicate_pair_id = next(ref_id for ref_id in ref_ids if ref_ids.count(ref_id) > 1) raise GranuleValidationError( f'The requested scenes have more than 1 pair with the following burst ID: {duplicate_pair_id}.' ) From b5a4156fda0bb626405b8ee67db0d0144c47a853 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 22 Aug 2024 10:35:55 -0800 Subject: [PATCH 057/102] Add burst job reformatting to Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 174d11d33..416907c73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `INSAR_ISCE_BURST` now accepts `reference` and `secondary` parameters, each of which is a list containing a single granule name. In a future release, `INSAR_ISCE_BURST` will support processing multiple adjacent bursts into one interferogram, and each parameter will be a list of one or more granule names. +- All previous `INSAR_ISCE_BURST` jobs have been reformatted to replace the `granules` parameter with the `reference` and `secondary` parameters. ### Removed - `INSAR_ISCE_BURST` no longer accepts a `granules` parameter. From 97372673872851a03ae50fdd57c45c4c742aaabc Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 30 Aug 2024 11:18:26 -0800 Subject: [PATCH 058/102] allow 30 burst pairs --- job_spec/INSAR_ISCE_BURST.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index ae0ba6eda..92ded6f83 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -8,7 +8,7 @@ INSAR_ISCE_BURST: api_schema: type: array minItems: 1 - maxItems: 1 + maxItems: 30 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST items: @@ -23,7 +23,7 @@ INSAR_ISCE_BURST: api_schema: type: array minItems: 1 - maxItems: 1 + maxItems: 30 example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST items: From 0a892ec489ff8d441f20625fe6b737d817d7f0e7 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 3 Sep 2024 12:37:30 -0500 Subject: [PATCH 059/102] better burst id validation --- apps/api/src/hyp3_api/validation.py | 4 ++-- tests/test_api/test_validation.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 7918b3cb3..8334b9a49 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -79,8 +79,8 @@ def check_dem_coverage(_, granule_metadata): def check_same_burst_ids(job, _): refs = job['job_parameters']['reference'] secs = job['job_parameters']['secondary'] - ref_ids = [ref.split('_')[1] for ref in refs] - sec_ids = [sec.split('_')[1] for sec in secs] + ref_ids = ['_'.join([ref.split('_')[1], ref.split('_')[2]]) for ref in refs] + sec_ids = ['_'.join([sec.split('_')[1], sec.split('_')[2]]) for sec in secs] if len(ref_ids) != len(sec_ids): raise GranuleValidationError( f'Number of reference and secondary scenes must match, got: ' diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index b7e079ef1..10faea966 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -118,6 +118,18 @@ def test_check_same_burst_ids(): 'S1_136232_IW2_20200616T022345_VV_5D13-BURST' ] } + }, + { + 'job_parameters': { + 'reference': [ + 'S1_136231_IW2_20200604T022312_VV_7C85-BURST', + 'S1_136231_IW3_20200616T022315_VV_5D11-BURST' + ], + 'secondary': [ + 'S1_136231_IW2_20200616T022313_VV_5411-BURST', + 'S1_136231_IW3_20200616T022345_VV_5D13-BURST' + ] + } } ] invalid_job_different_lengths = { From dc5195cdbb1da1226bb4b94878f45b440d364313 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 3 Sep 2024 12:59:19 -0500 Subject: [PATCH 060/102] Update apps/api/src/hyp3_api/validation.py Co-authored-by: Jake Herrmann --- apps/api/src/hyp3_api/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 8334b9a49..41478896b 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -79,8 +79,8 @@ def check_dem_coverage(_, granule_metadata): def check_same_burst_ids(job, _): refs = job['job_parameters']['reference'] secs = job['job_parameters']['secondary'] - ref_ids = ['_'.join([ref.split('_')[1], ref.split('_')[2]]) for ref in refs] - sec_ids = ['_'.join([sec.split('_')[1], sec.split('_')[2]]) for sec in secs] + ref_ids = ['_'.join(ref.split('_')[1:3]) for ref in refs] + sec_ids = ['_'.join(sec.split('_')[1:3]) for sec in secs] if len(ref_ids) != len(sec_ids): raise GranuleValidationError( f'Number of reference and secondary scenes must match, got: ' From 580b60b33b4cbb07550d22ad4b3f8de9c1c16a57 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 4 Sep 2024 12:43:06 -0800 Subject: [PATCH 061/102] increase insar burst memory --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index aa699fae7..af18d54fe 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -83,7 +83,7 @@ INSAR_ISCE_BURST: - Ref::secondary timeout: 5400 vcpu: 1 - memory: 7600 + memory: 15500 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD From 80ab87c2eb3c09cb3fb7993b966f39ccbd82a9ff Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 14:11:34 -0800 Subject: [PATCH 062/102] add override_memory lambda --- Makefile | 5 +- .../override-memory/override-memory-cf.yml.j2 | 76 +++++++++ apps/override-memory/src/override_memory.py | 27 +++ apps/step-function.json.j2 | 154 ++---------------- apps/workflow-cf.yml.j2 | 12 ++ tests/test_override_memory.py | 2 + 6 files changed, 132 insertions(+), 144 deletions(-) create mode 100644 apps/override-memory/override-memory-cf.yml.j2 create mode 100644 apps/override-memory/src/override_memory.py create mode 100644 tests/test_override_memory.py diff --git a/Makefile b/Makefile index c97f2c118..ed74663ae 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ API = ${PWD}/apps/api/src CHECK_PROCESSING_TIME = ${PWD}/apps/check-processing-time/src GET_FILES = ${PWD}/apps/get-files/src HANDLE_BATCH_EVENT = ${PWD}/apps/handle-batch-event/src +OVERRIDE_MEMORY = ${PWD}/apps/override-memory/src SCALE_CLUSTER = ${PWD}/apps/scale-cluster/src START_EXECUTION_MANAGER = ${PWD}/apps/start-execution-manager/src START_EXECUTION_WORKER = ${PWD}/apps/start-execution-worker/src @@ -9,7 +10,7 @@ DISABLE_PRIVATE_DNS = ${PWD}/apps/disable-private-dns/src UPDATE_DB = ${PWD}/apps/update-db/src UPLOAD_LOG = ${PWD}/apps/upload-log/src DYNAMO = ${PWD}/lib/dynamo -export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} +export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${OVERRIDE_MEMORY}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} build: render @@ -44,7 +45,7 @@ render: static: flake8 openapi-validate cfn-lint flake8: - flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib + flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,override_memory,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib openapi-validate: render openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml diff --git a/apps/override-memory/override-memory-cf.yml.j2 b/apps/override-memory/override-memory-cf.yml.j2 new file mode 100644 index 000000000..001d5fee3 --- /dev/null +++ b/apps/override-memory/override-memory-cf.yml.j2 @@ -0,0 +1,76 @@ +AWSTemplateFormatVersion: 2010-09-09 + +{% if security_environment == 'EDC' %} +Parameters: + + SecurityGroupId: + Type: String + + SubnetIds: + Type: CommaDelimitedList +{% endif %} + +Outputs: + + LambdaArn: + Value: !GetAtt Lambda.Arn + +Resources: + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/aws/lambda/${Lambda}" + RetentionInDays: 90 + + Role: + Type: {{ 'Custom::JplRole' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::Role' }} + Properties: + {% if security_environment in ('JPL', 'JPL-public') %} + ServiceToken: !ImportValue Custom::JplRole::ServiceToken + Path: /account-managed/hyp3/ + {% endif %} + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Action: sts:AssumeRole + Principal: + Service: lambda.amazonaws.com + Effect: Allow + ManagedPolicyArns: + - !Ref Policy + {% if security_environment == 'EDC' %} + - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole + {% endif %} + + Policy: + Type: {{ 'Custom::JplPolicy' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::ManagedPolicy' }} + Properties: + {% if security_environment in ('JPL', 'JPL-public') %} + ServiceToken: !ImportValue Custom::JplPolicy::ServiceToken + Path: /account-managed/hyp3/ + {% endif %} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" + + Lambda: + Type: AWS::Lambda::Function + Properties: + Code: src/ + Handler: override_memory.lambda_handler + MemorySize: 128 + Role: !GetAtt Role.Arn + Runtime: python3.9 + Timeout: 30 + {% if security_environment == 'EDC' %} + VpcConfig: + SecurityGroupIds: + - !Ref SecurityGroupId + SubnetIds: !Ref SubnetIds + {% endif %} diff --git a/apps/override-memory/src/override_memory.py b/apps/override-memory/src/override_memory.py new file mode 100644 index 000000000..1a27c8aee --- /dev/null +++ b/apps/override-memory/src/override_memory.py @@ -0,0 +1,27 @@ +def get_resource_requirements(memory: int) -> dict: + return { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': memory, + } + ] + } + + +def lambda_handler(event: dict, _) -> dict: + job_type, job_parameters = event['job_type'], event['job_parameters'] + + if job_type == 'AUTORIFT' and job_parameters['granules'][0].startswith('S2'): + return get_resource_requirements(7875) + + if job_type == 'AUTORIFT' and job_parameters['granules'][0].startswith('L'): + return get_resource_requirements(15750) + + if job_type == 'RTC_GAMMA' and job_parameters['resolution'] in ['10', '20']: + return get_resource_requirements(63200) + + if job_type in ['WATER_MAP', 'WATER_MAP_EQ'] and job_parameters['resolution'] in ['10', '20']: + return get_resource_requirements(126000) + + return {} diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 68506f211..9bd081c67 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -36,153 +36,23 @@ "Type": "Pass", "InputPath": "$.job_id", "ResultPath": "$.job_parameters.bucket_prefix", - "Next": "INSPECT_MEMORY_REQUIREMENTS" + "Next": "OVERRIDE_MEMORY" }, - "INSPECT_MEMORY_REQUIREMENTS": { - "Type": "Choice", - "Choices": [ - { - "And": [ - { - "Variable": "$.job_type", - "StringEquals": "AUTORIFT" - }, - { - "Variable": "$.job_parameters.granules", - "StringMatches": "S2*" - } - ], - "Next": "USE_SENTINEL2_MEMORY" - }, - { - "And": [ - { - "Variable": "$.job_type", - "StringEquals": "AUTORIFT" - }, - { - "Variable": "$.job_parameters.granules", - "StringMatches": "L*" - } - ], - "Next": "USE_LANDSAT_MEMORY" - }, - { - "And": [ - { - "Variable": "$.job_type", - "StringEquals": "RTC_GAMMA" - }, - { - "Variable": "$.job_parameters.resolution", - "IsPresent": true - }, - { - "Or": [ - { - "Variable": "$.job_parameters.resolution", - "StringEquals": "10" - }, - { - "Variable": "$.job_parameters.resolution", - "StringEquals": "20" - } - ] - } - ], - "Next": "USE_10M_RTC_MEMORY" - }, + "OVERRIDE_MEMORY": { + "Type": "Task", + "Resource": "${OverrideMemoryLambdaArn}", + "Parameters": { + "job_type.$": "$.job_type", + "job_parameters.$": "$.job_parameters" + }, + "Retry": [ { - "And": [ - { - "Or": [ - { - "Variable": "$.job_type", - "StringEquals": "WATER_MAP" - }, - { - "Variable": "$.job_type", - "StringEquals": "WATER_MAP_EQ" - } - ] - }, - { - "Variable": "$.job_parameters.resolution", - "IsPresent": true - }, - { - "Or": [ - { - "Variable": "$.job_parameters.resolution", - "StringEquals": "10" - }, - { - "Variable": "$.job_parameters.resolution", - "StringEquals": "20" - } - ] - } + "ErrorEquals": [ + "States.ALL" ], - "Next": "USE_10M_WATER_MAP_MEMORY" + "MaxAttempts": 2 } ], - "Default": "USE_DEFAULT_MEMORY" - }, - "USE_SENTINEL2_MEMORY": { - "Type": "Pass", - "Result": { - "ResourceRequirements": [ - { - "Type": "MEMORY", - "Value": "7875" - } - ] - }, - "ResultPath": "$.container_overrides", - "Next": "INSPECT_JOB_TYPE" - }, - "USE_LANDSAT_MEMORY": { - "Type": "Pass", - "Result": { - "ResourceRequirements": [ - { - "Type": "MEMORY", - "Value": "15750" - } - ] - }, - "ResultPath": "$.container_overrides", - "Next": "INSPECT_JOB_TYPE" - }, - "USE_10M_RTC_MEMORY": { - "Type": "Pass", - "Result": { - "ResourceRequirements": [ - { - "Type": "MEMORY", - "Value": "63200" - } - ] - }, - "ResultPath": "$.container_overrides", - "Next": "INSPECT_JOB_TYPE" - }, - "USE_10M_WATER_MAP_MEMORY": { - "Type": "Pass", - "Result": { - "ResourceRequirements": [ - { - "Type": "MEMORY", - "Value": "126000" - } - ] - }, - "ResultPath": "$.container_overrides", - "Next": "INSPECT_JOB_TYPE" - }, - "USE_DEFAULT_MEMORY": { - "Type": "Pass", - "Result": {}, "ResultPath": "$.container_overrides", "Next": "INSPECT_JOB_TYPE" }, diff --git a/apps/workflow-cf.yml.j2 b/apps/workflow-cf.yml.j2 index ec740395b..cab2b359a 100644 --- a/apps/workflow-cf.yml.j2 +++ b/apps/workflow-cf.yml.j2 @@ -103,6 +103,7 @@ Resources: {% endfor %} {% endfor %} UpdateDBLambdaArn: !GetAtt UpdateDB.Outputs.LambdaArn + OverrideMemoryLambdaArn: !GetAtt OverrideMemory.Outputs.LambdaArn GetFilesLambdaArn: !GetAtt GetFiles.Outputs.LambdaArn CheckProcessingTimeLambdaArn: !GetAtt CheckProcessingTime.Outputs.LambdaArn UploadLogLambdaArn: !GetAtt UploadLog.Outputs.LambdaArn @@ -162,6 +163,7 @@ Resources: Action: lambda:InvokeFunction Resource: - !GetAtt UpdateDB.Outputs.LambdaArn + - !GetAtt OverrideMemory.Outputs.LambdaArn - !GetAtt GetFiles.Outputs.LambdaArn - !GetAtt CheckProcessingTime.Outputs.LambdaArn - !GetAtt UploadLog.Outputs.LambdaArn @@ -177,6 +179,16 @@ Resources: {% endif %} TemplateURL: update-db/update-db-cf.yml + OverrideMemory: + Type: AWS::CloudFormation::Stack + Properties: + {% if security_environment == 'EDC' %} + Parameters: + SecurityGroupId: !Ref SecurityGroupId + SubnetIds: !Join [",", !Ref SubnetIds] + {% endif %} + TemplateURL: override-memory/override-memory-cf.yml + GetFiles: Type: AWS::CloudFormation::Stack Properties: diff --git a/tests/test_override_memory.py b/tests/test_override_memory.py new file mode 100644 index 000000000..6e5725874 --- /dev/null +++ b/tests/test_override_memory.py @@ -0,0 +1,2 @@ +def test_override_memory(): + raise NotImplementedError() From b44a607a741badc72489ce6ffc203c1de5d92f94 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 14:46:35 -0800 Subject: [PATCH 063/102] test override_memory --- apps/override-memory/src/override_memory.py | 14 +- tests/test_override_memory.py | 140 +++++++++++++++++++- 2 files changed, 148 insertions(+), 6 deletions(-) diff --git a/apps/override-memory/src/override_memory.py b/apps/override-memory/src/override_memory.py index 1a27c8aee..f1f37580b 100644 --- a/apps/override-memory/src/override_memory.py +++ b/apps/override-memory/src/override_memory.py @@ -1,3 +1,9 @@ +AUTORIFT_S2_MEMORY = 7875 +AUTORIFT_LANDSAT_MEMORY = 15750 +RTC_GAMMA_10M_MEMORY = 63200 +WATER_MAP_10M_MEMORY = 126000 + + def get_resource_requirements(memory: int) -> dict: return { 'ResourceRequirements': [ @@ -13,15 +19,15 @@ def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] if job_type == 'AUTORIFT' and job_parameters['granules'][0].startswith('S2'): - return get_resource_requirements(7875) + return get_resource_requirements(AUTORIFT_S2_MEMORY) if job_type == 'AUTORIFT' and job_parameters['granules'][0].startswith('L'): - return get_resource_requirements(15750) + return get_resource_requirements(AUTORIFT_LANDSAT_MEMORY) if job_type == 'RTC_GAMMA' and job_parameters['resolution'] in ['10', '20']: - return get_resource_requirements(63200) + return get_resource_requirements(RTC_GAMMA_10M_MEMORY) if job_type in ['WATER_MAP', 'WATER_MAP_EQ'] and job_parameters['resolution'] in ['10', '20']: - return get_resource_requirements(126000) + return get_resource_requirements(WATER_MAP_10M_MEMORY) return {} diff --git a/tests/test_override_memory.py b/tests/test_override_memory.py index 6e5725874..e965aa55e 100644 --- a/tests/test_override_memory.py +++ b/tests/test_override_memory.py @@ -1,2 +1,138 @@ -def test_override_memory(): - raise NotImplementedError() +from override_memory import ( + AUTORIFT_S2_MEMORY, + AUTORIFT_LANDSAT_MEMORY, + RTC_GAMMA_10M_MEMORY, + WATER_MAP_10M_MEMORY, + lambda_handler, +) + + +def test_override_memory_default(): + assert lambda_handler( + { + 'job_type': 'foo', + 'job_parameters': {}, + }, + None, + ) == {} + + +def test_override_memory_autorift(): + assert lambda_handler( + { + 'job_type': 'AUTORIFT', + 'job_parameters': {'granules': ['S2B_MSIL1C_20200105T152259_N0208_R039_T13CES_20200105T181230']}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': AUTORIFT_S2_MEMORY, + } + ] + } + assert lambda_handler( + { + 'job_type': 'AUTORIFT', + 'job_parameters': {'granules': ['LC08_L1GT_118112_20210107_20210107_02_T2']}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': AUTORIFT_LANDSAT_MEMORY, + } + ] + } + + +def test_override_memory_rtc_gamma(): + assert lambda_handler( + { + 'job_type': 'RTC_GAMMA', + 'job_parameters': {'resolution': '10'}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': RTC_GAMMA_10M_MEMORY, + } + ] + } + assert lambda_handler( + { + 'job_type': 'RTC_GAMMA', + 'job_parameters': {'resolution': '20'}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': RTC_GAMMA_10M_MEMORY, + } + ] + } + + +def test_override_memory_water_map(): + assert lambda_handler( + { + 'job_type': 'WATER_MAP', + 'job_parameters': {'resolution': '10'}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': WATER_MAP_10M_MEMORY, + } + ] + } + assert lambda_handler( + { + 'job_type': 'WATER_MAP', + 'job_parameters': {'resolution': '20'}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': WATER_MAP_10M_MEMORY, + } + ] + } + assert lambda_handler( + { + 'job_type': 'WATER_MAP_EQ', + 'job_parameters': {'resolution': '10'}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': WATER_MAP_10M_MEMORY, + } + ] + } + assert lambda_handler( + { + 'job_type': 'WATER_MAP_EQ', + 'job_parameters': {'resolution': '20'}, + }, + None, + ) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': WATER_MAP_10M_MEMORY, + } + ] + } From 9ae92350c5ee6675e71d10ecf6104b5e5289c4fd Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 14:48:42 -0800 Subject: [PATCH 064/102] import order --- tests/test_override_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_override_memory.py b/tests/test_override_memory.py index e965aa55e..722cab8da 100644 --- a/tests/test_override_memory.py +++ b/tests/test_override_memory.py @@ -1,6 +1,6 @@ from override_memory import ( - AUTORIFT_S2_MEMORY, AUTORIFT_LANDSAT_MEMORY, + AUTORIFT_S2_MEMORY, RTC_GAMMA_10M_MEMORY, WATER_MAP_10M_MEMORY, lambda_handler, From 70773715a92c9f4616eb7a39b1ff26d715939151 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 14:59:11 -0800 Subject: [PATCH 065/102] add job types to multi-burst-sandbox --- .github/workflows/deploy-multi-burst-sandbox.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml index 4e9d26b44..f80a7cf4b 100644 --- a/.github/workflows/deploy-multi-burst-sandbox.yml +++ b/.github/workflows/deploy-multi-burst-sandbox.yml @@ -25,6 +25,10 @@ jobs: deploy_ref: refs/heads/multi-burst-sandbox job_files: >- job_spec/INSAR_ISCE_BURST.yml + job_spec/AUTORIFT.yml + job_spec/RTC_GAMMA.yml + job_spec/WATER_MAP.yml + job_spec/WATER_MAP_EQ.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge default_max_vcpus: 640 expanded_max_vcpus: 640 From 0c12d18561473233ce2b6db760ded7ccd6eda587 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 15:11:30 -0800 Subject: [PATCH 066/102] memory as str --- apps/override-memory/src/override_memory.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/override-memory/src/override_memory.py b/apps/override-memory/src/override_memory.py index f1f37580b..8690da138 100644 --- a/apps/override-memory/src/override_memory.py +++ b/apps/override-memory/src/override_memory.py @@ -1,10 +1,10 @@ -AUTORIFT_S2_MEMORY = 7875 -AUTORIFT_LANDSAT_MEMORY = 15750 -RTC_GAMMA_10M_MEMORY = 63200 -WATER_MAP_10M_MEMORY = 126000 +AUTORIFT_S2_MEMORY = '7875' +AUTORIFT_LANDSAT_MEMORY = '15750' +RTC_GAMMA_10M_MEMORY = '63200' +WATER_MAP_10M_MEMORY = '126000' -def get_resource_requirements(memory: int) -> dict: +def get_resource_requirements(memory: str) -> dict: return { 'ResourceRequirements': [ { From 79e76fccfbecdf12a63344a2e502eec83d1f4cc4 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 15:34:20 -0800 Subject: [PATCH 067/102] rename tests --- tests/test_override_memory.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_override_memory.py b/tests/test_override_memory.py index 722cab8da..af6acc8ee 100644 --- a/tests/test_override_memory.py +++ b/tests/test_override_memory.py @@ -17,7 +17,7 @@ def test_override_memory_default(): ) == {} -def test_override_memory_autorift(): +def test_override_memory_autorift_s2(): assert lambda_handler( { 'job_type': 'AUTORIFT', @@ -32,6 +32,9 @@ def test_override_memory_autorift(): } ] } + + +def test_override_memory_autorift_landsat(): assert lambda_handler( { 'job_type': 'AUTORIFT', @@ -48,7 +51,7 @@ def test_override_memory_autorift(): } -def test_override_memory_rtc_gamma(): +def test_override_memory_rtc_gamma_10m(): assert lambda_handler( { 'job_type': 'RTC_GAMMA', @@ -79,7 +82,7 @@ def test_override_memory_rtc_gamma(): } -def test_override_memory_water_map(): +def test_override_memory_water_map_10m(): assert lambda_handler( { 'job_type': 'WATER_MAP', From b003f56a2435f4fbaa77edc5f0780d10db9f048c Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 15:38:02 -0800 Subject: [PATCH 068/102] fix autorift memory checks --- apps/override-memory/src/override_memory.py | 4 ++-- tests/test_override_memory.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/override-memory/src/override_memory.py b/apps/override-memory/src/override_memory.py index 8690da138..9e00598fe 100644 --- a/apps/override-memory/src/override_memory.py +++ b/apps/override-memory/src/override_memory.py @@ -18,10 +18,10 @@ def get_resource_requirements(memory: str) -> dict: def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] - if job_type == 'AUTORIFT' and job_parameters['granules'][0].startswith('S2'): + if job_type == 'AUTORIFT' and job_parameters['granules'].startswith('S2'): return get_resource_requirements(AUTORIFT_S2_MEMORY) - if job_type == 'AUTORIFT' and job_parameters['granules'][0].startswith('L'): + if job_type == 'AUTORIFT' and job_parameters['granules'].startswith('L'): return get_resource_requirements(AUTORIFT_LANDSAT_MEMORY) if job_type == 'RTC_GAMMA' and job_parameters['resolution'] in ['10', '20']: diff --git a/tests/test_override_memory.py b/tests/test_override_memory.py index af6acc8ee..c28f5893c 100644 --- a/tests/test_override_memory.py +++ b/tests/test_override_memory.py @@ -21,7 +21,7 @@ def test_override_memory_autorift_s2(): assert lambda_handler( { 'job_type': 'AUTORIFT', - 'job_parameters': {'granules': ['S2B_MSIL1C_20200105T152259_N0208_R039_T13CES_20200105T181230']}, + 'job_parameters': {'granules': 'S2B_'}, }, None, ) == { @@ -38,7 +38,7 @@ def test_override_memory_autorift_landsat(): assert lambda_handler( { 'job_type': 'AUTORIFT', - 'job_parameters': {'granules': ['LC08_L1GT_118112_20210107_20210107_02_T2']}, + 'job_parameters': {'granules': 'LC08_'}, }, None, ) == { From ed6b67003ef6cf75899642d6cb5886f8ed002564 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 5 Sep 2024 15:54:01 -0800 Subject: [PATCH 069/102] revert burst memory increase --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index af18d54fe..aa699fae7 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -83,7 +83,7 @@ INSAR_ISCE_BURST: - Ref::secondary timeout: 5400 vcpu: 1 - memory: 15500 + memory: 7600 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD From 727b158118995da110fbe36ebcc9a7049aaa2f4a Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 6 Sep 2024 10:09:04 -0800 Subject: [PATCH 070/102] rename lambda to set_container_overrides --- Makefile | 6 +++--- .../set-container-overrides-cf.yml.j2} | 2 +- .../src/set_container_overrides.py} | 0 apps/step-function.json.j2 | 6 +++--- apps/workflow-cf.yml.j2 | 8 ++++---- ...ide_memory.py => test_set_container_overrides.py} | 12 ++++++------ 6 files changed, 17 insertions(+), 17 deletions(-) rename apps/{override-memory/override-memory-cf.yml.j2 => set-container-overrides/set-container-overrides-cf.yml.j2} (97%) rename apps/{override-memory/src/override_memory.py => set-container-overrides/src/set_container_overrides.py} (100%) rename tests/{test_override_memory.py => test_set_container_overrides.py} (90%) diff --git a/Makefile b/Makefile index ed74663ae..9f3005d8f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ API = ${PWD}/apps/api/src CHECK_PROCESSING_TIME = ${PWD}/apps/check-processing-time/src GET_FILES = ${PWD}/apps/get-files/src HANDLE_BATCH_EVENT = ${PWD}/apps/handle-batch-event/src -OVERRIDE_MEMORY = ${PWD}/apps/override-memory/src +SET_CONTAINER_OVERRIDES = ${PWD}/apps/set-container-overrides/src SCALE_CLUSTER = ${PWD}/apps/scale-cluster/src START_EXECUTION_MANAGER = ${PWD}/apps/start-execution-manager/src START_EXECUTION_WORKER = ${PWD}/apps/start-execution-worker/src @@ -10,7 +10,7 @@ DISABLE_PRIVATE_DNS = ${PWD}/apps/disable-private-dns/src UPDATE_DB = ${PWD}/apps/update-db/src UPLOAD_LOG = ${PWD}/apps/upload-log/src DYNAMO = ${PWD}/lib/dynamo -export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${OVERRIDE_MEMORY}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} +export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SET_CONTAINER_OVERRIDES}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} build: render @@ -45,7 +45,7 @@ render: static: flake8 openapi-validate cfn-lint flake8: - flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,override_memory,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib + flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,set_container_overrides,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib openapi-validate: render openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml diff --git a/apps/override-memory/override-memory-cf.yml.j2 b/apps/set-container-overrides/set-container-overrides-cf.yml.j2 similarity index 97% rename from apps/override-memory/override-memory-cf.yml.j2 rename to apps/set-container-overrides/set-container-overrides-cf.yml.j2 index 001d5fee3..ea91675e4 100644 --- a/apps/override-memory/override-memory-cf.yml.j2 +++ b/apps/set-container-overrides/set-container-overrides-cf.yml.j2 @@ -63,7 +63,7 @@ Resources: Type: AWS::Lambda::Function Properties: Code: src/ - Handler: override_memory.lambda_handler + Handler: set_container_overrides.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn Runtime: python3.9 diff --git a/apps/override-memory/src/override_memory.py b/apps/set-container-overrides/src/set_container_overrides.py similarity index 100% rename from apps/override-memory/src/override_memory.py rename to apps/set-container-overrides/src/set_container_overrides.py diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 9bd081c67..f7885e52a 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -36,11 +36,11 @@ "Type": "Pass", "InputPath": "$.job_id", "ResultPath": "$.job_parameters.bucket_prefix", - "Next": "OVERRIDE_MEMORY" + "Next": "SET_CONTAINER_OVERRIDES" }, - "OVERRIDE_MEMORY": { + "SET_CONTAINER_OVERRIDES": { "Type": "Task", - "Resource": "${OverrideMemoryLambdaArn}", + "Resource": "${SetContainerOverridesLambdaArn}", "Parameters": { "job_type.$": "$.job_type", "job_parameters.$": "$.job_parameters" diff --git a/apps/workflow-cf.yml.j2 b/apps/workflow-cf.yml.j2 index cab2b359a..60f451674 100644 --- a/apps/workflow-cf.yml.j2 +++ b/apps/workflow-cf.yml.j2 @@ -103,7 +103,7 @@ Resources: {% endfor %} {% endfor %} UpdateDBLambdaArn: !GetAtt UpdateDB.Outputs.LambdaArn - OverrideMemoryLambdaArn: !GetAtt OverrideMemory.Outputs.LambdaArn + SetContainerOverridesLambdaArn: !GetAtt SetContainerOverrides.Outputs.LambdaArn GetFilesLambdaArn: !GetAtt GetFiles.Outputs.LambdaArn CheckProcessingTimeLambdaArn: !GetAtt CheckProcessingTime.Outputs.LambdaArn UploadLogLambdaArn: !GetAtt UploadLog.Outputs.LambdaArn @@ -163,7 +163,7 @@ Resources: Action: lambda:InvokeFunction Resource: - !GetAtt UpdateDB.Outputs.LambdaArn - - !GetAtt OverrideMemory.Outputs.LambdaArn + - !GetAtt SetContainerOverrides.Outputs.LambdaArn - !GetAtt GetFiles.Outputs.LambdaArn - !GetAtt CheckProcessingTime.Outputs.LambdaArn - !GetAtt UploadLog.Outputs.LambdaArn @@ -179,7 +179,7 @@ Resources: {% endif %} TemplateURL: update-db/update-db-cf.yml - OverrideMemory: + SetContainerOverrides: Type: AWS::CloudFormation::Stack Properties: {% if security_environment == 'EDC' %} @@ -187,7 +187,7 @@ Resources: SecurityGroupId: !Ref SecurityGroupId SubnetIds: !Join [",", !Ref SubnetIds] {% endif %} - TemplateURL: override-memory/override-memory-cf.yml + TemplateURL: set-container-overrides/set-container-overrides-cf.yml GetFiles: Type: AWS::CloudFormation::Stack diff --git a/tests/test_override_memory.py b/tests/test_set_container_overrides.py similarity index 90% rename from tests/test_override_memory.py rename to tests/test_set_container_overrides.py index c28f5893c..70e9cd351 100644 --- a/tests/test_override_memory.py +++ b/tests/test_set_container_overrides.py @@ -1,4 +1,4 @@ -from override_memory import ( +from set_container_overrides import ( AUTORIFT_LANDSAT_MEMORY, AUTORIFT_S2_MEMORY, RTC_GAMMA_10M_MEMORY, @@ -7,7 +7,7 @@ ) -def test_override_memory_default(): +def test_set_container_overrides_default(): assert lambda_handler( { 'job_type': 'foo', @@ -17,7 +17,7 @@ def test_override_memory_default(): ) == {} -def test_override_memory_autorift_s2(): +def test_set_container_overrides_autorift_s2(): assert lambda_handler( { 'job_type': 'AUTORIFT', @@ -34,7 +34,7 @@ def test_override_memory_autorift_s2(): } -def test_override_memory_autorift_landsat(): +def test_set_container_overrides_autorift_landsat(): assert lambda_handler( { 'job_type': 'AUTORIFT', @@ -51,7 +51,7 @@ def test_override_memory_autorift_landsat(): } -def test_override_memory_rtc_gamma_10m(): +def test_set_container_overrides_rtc_gamma_10m(): assert lambda_handler( { 'job_type': 'RTC_GAMMA', @@ -82,7 +82,7 @@ def test_override_memory_rtc_gamma_10m(): } -def test_override_memory_water_map_10m(): +def test_set_container_overrides_water_map_10m(): assert lambda_handler( { 'job_type': 'WATER_MAP', From d5039b706a072863151fbd2b215db69cf4dd579d Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 6 Sep 2024 11:32:29 -0800 Subject: [PATCH 071/102] burst memory intervals --- .../src/set_container_overrides.py | 31 +++++++++++++++++++ tests/test_set_container_overrides.py | 4 +++ 2 files changed, 35 insertions(+) diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-container-overrides/src/set_container_overrides.py index 9e00598fe..8da36e694 100644 --- a/apps/set-container-overrides/src/set_container_overrides.py +++ b/apps/set-container-overrides/src/set_container_overrides.py @@ -15,9 +15,40 @@ def get_resource_requirements(memory: str) -> dict: } +def get_insar_isce_burst_memory(job_parameters: dict) -> str: + # TODO: + # - update max bursts in job spec and/or add validator to allow diff # of bursts depending on looks? + # - refine based on more data + # - refactor lookup structure and/or pull out constants? + bursts = len(job_parameters['reference']) + looks = job_parameters['looks'] + if looks == '5x1': + if bursts < 3: + return '7500' + if bursts < 5: + return '15500' + if bursts < 13: + return '31500' + if bursts < 26: + return '63500' + if looks == '10x2': + if bursts < 10: + return '7500' + if bursts < 23: + return '15500' + if bursts < 31: + return '31500' + if looks == '20x4': + return '7500' + raise ValueError(f'No memory value for {bursts} bursts and {looks} looks') + + def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] + if job_type == 'INSAR_ISCE_BURST': + return get_resource_requirements(get_insar_isce_burst_memory(job_parameters)) + if job_type == 'AUTORIFT' and job_parameters['granules'].startswith('S2'): return get_resource_requirements(AUTORIFT_S2_MEMORY) diff --git a/tests/test_set_container_overrides.py b/tests/test_set_container_overrides.py index 70e9cd351..1cc8741d4 100644 --- a/tests/test_set_container_overrides.py +++ b/tests/test_set_container_overrides.py @@ -17,6 +17,10 @@ def test_set_container_overrides_default(): ) == {} +def test_set_container_overrides_insar_isce_burst(): + raise NotImplementedError() + + def test_set_container_overrides_autorift_s2(): assert lambda_handler( { From a334f2086230c5444b4e13d76b4b1744b1d57842 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 6 Sep 2024 11:38:37 -0800 Subject: [PATCH 072/102] burst interval --- apps/set-container-overrides/src/set_container_overrides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-container-overrides/src/set_container_overrides.py index 8da36e694..d5b812214 100644 --- a/apps/set-container-overrides/src/set_container_overrides.py +++ b/apps/set-container-overrides/src/set_container_overrides.py @@ -23,7 +23,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: bursts = len(job_parameters['reference']) looks = job_parameters['looks'] if looks == '5x1': - if bursts < 3: + if bursts < 2: return '7500' if bursts < 5: return '15500' From 5bb28a0b89501b1a4ece9dd2b69631b5ad98d94a Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 6 Sep 2024 11:39:48 -0800 Subject: [PATCH 073/102] todo --- apps/set-container-overrides/src/set_container_overrides.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-container-overrides/src/set_container_overrides.py index d5b812214..c2b57f127 100644 --- a/apps/set-container-overrides/src/set_container_overrides.py +++ b/apps/set-container-overrides/src/set_container_overrides.py @@ -20,6 +20,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: # - update max bursts in job spec and/or add validator to allow diff # of bursts depending on looks? # - refine based on more data # - refactor lookup structure and/or pull out constants? + # - can we slightly reduce the 500M padding in order to bump up some of the intervals? bursts = len(job_parameters['reference']) looks = job_parameters['looks'] if looks == '5x1': From 265254d71d1e56fa634248b3a36e91f6a490e3a0 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 6 Sep 2024 15:39:44 -0800 Subject: [PATCH 074/102] split bursts list, catch error --- .../src/set_container_overrides.py | 2 +- apps/step-function.json.j2 | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-container-overrides/src/set_container_overrides.py index c2b57f127..767f27fb8 100644 --- a/apps/set-container-overrides/src/set_container_overrides.py +++ b/apps/set-container-overrides/src/set_container_overrides.py @@ -21,7 +21,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: # - refine based on more data # - refactor lookup structure and/or pull out constants? # - can we slightly reduce the 500M padding in order to bump up some of the intervals? - bursts = len(job_parameters['reference']) + bursts = len(job_parameters['reference'].split(' ')) looks = job_parameters['looks'] if looks == '5x1': if bursts < 2: diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index f7885e52a..19ab738b3 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -53,6 +53,15 @@ "MaxAttempts": 2 } ], + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "JOB_FAILED", + "ResultPath": "$.container_overrides", + } + ], "ResultPath": "$.container_overrides", "Next": "INSPECT_JOB_TYPE" }, From 2493f43f457571762dd1b9455dc921e44336d97b Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 9 Sep 2024 14:57:58 -0800 Subject: [PATCH 075/102] adjust memory intervals --- .../src/set_container_overrides.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-container-overrides/src/set_container_overrides.py index 767f27fb8..57d2fcab8 100644 --- a/apps/set-container-overrides/src/set_container_overrides.py +++ b/apps/set-container-overrides/src/set_container_overrides.py @@ -26,21 +26,24 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: if looks == '5x1': if bursts < 2: return '7500' - if bursts < 5: + if bursts < 4: return '15500' - if bursts < 13: + if bursts < 11: return '31500' if bursts < 26: return '63500' if looks == '10x2': if bursts < 10: return '7500' - if bursts < 23: + if bursts < 21: return '15500' if bursts < 31: return '31500' if looks == '20x4': - return '7500' + if bursts < 23: + return '7500' + if bursts < 31: + return '15500' raise ValueError(f'No memory value for {bursts} bursts and {looks} looks') From 0a227d0c42a6657899ab20f1f67d01d3c2ee055f Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 9 Sep 2024 15:08:28 -0800 Subject: [PATCH 076/102] increase insar burst timeout --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index aa699fae7..28947a247 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -81,7 +81,7 @@ INSAR_ISCE_BURST: - Ref::reference - --secondary - Ref::secondary - timeout: 5400 + timeout: 43200 # 12 hr vcpu: 1 memory: 7600 secrets: From a4943320f2ae4dab968ab3cacb881f80f76d1854 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 10 Sep 2024 09:59:49 -0800 Subject: [PATCH 077/102] disallow 25 bursts at 5x1 looks --- apps/set-container-overrides/src/set_container_overrides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-container-overrides/src/set_container_overrides.py index 57d2fcab8..0315528d6 100644 --- a/apps/set-container-overrides/src/set_container_overrides.py +++ b/apps/set-container-overrides/src/set_container_overrides.py @@ -30,7 +30,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: return '15500' if bursts < 11: return '31500' - if bursts < 26: + if bursts < 25: return '63500' if looks == '10x2': if bursts < 10: From e8796e102815919f09b74e8c140d18d8ca09e2aa Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 10 Sep 2024 14:35:28 -0800 Subject: [PATCH 078/102] rename set_container_overrides to set_batch_overrides --- Makefile | 6 +++--- .../set-batch-overrides-cf.yml.j2} | 2 +- .../src/set_batch_overrides.py} | 0 apps/step-function.json.j2 | 6 +++--- apps/workflow-cf.yml.j2 | 8 ++++---- ...er_overrides.py => test_set_batch_overrides.py} | 14 +++++++------- 6 files changed, 18 insertions(+), 18 deletions(-) rename apps/{set-container-overrides/set-container-overrides-cf.yml.j2 => set-batch-overrides/set-batch-overrides-cf.yml.j2} (97%) rename apps/{set-container-overrides/src/set_container_overrides.py => set-batch-overrides/src/set_batch_overrides.py} (100%) rename tests/{test_set_container_overrides.py => test_set_batch_overrides.py} (89%) diff --git a/Makefile b/Makefile index 9f3005d8f..c76c9a847 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ API = ${PWD}/apps/api/src CHECK_PROCESSING_TIME = ${PWD}/apps/check-processing-time/src GET_FILES = ${PWD}/apps/get-files/src HANDLE_BATCH_EVENT = ${PWD}/apps/handle-batch-event/src -SET_CONTAINER_OVERRIDES = ${PWD}/apps/set-container-overrides/src +SET_BATCH_OVERRIDES = ${PWD}/apps/set-batch-overrides/src SCALE_CLUSTER = ${PWD}/apps/scale-cluster/src START_EXECUTION_MANAGER = ${PWD}/apps/start-execution-manager/src START_EXECUTION_WORKER = ${PWD}/apps/start-execution-worker/src @@ -10,7 +10,7 @@ DISABLE_PRIVATE_DNS = ${PWD}/apps/disable-private-dns/src UPDATE_DB = ${PWD}/apps/update-db/src UPLOAD_LOG = ${PWD}/apps/upload-log/src DYNAMO = ${PWD}/lib/dynamo -export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SET_CONTAINER_OVERRIDES}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} +export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SET_BATCH_OVERRIDES}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} build: render @@ -45,7 +45,7 @@ render: static: flake8 openapi-validate cfn-lint flake8: - flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,set_container_overrides,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib + flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,set_batch_overrides,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib openapi-validate: render openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml diff --git a/apps/set-container-overrides/set-container-overrides-cf.yml.j2 b/apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 similarity index 97% rename from apps/set-container-overrides/set-container-overrides-cf.yml.j2 rename to apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 index ea91675e4..57362c9bf 100644 --- a/apps/set-container-overrides/set-container-overrides-cf.yml.j2 +++ b/apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 @@ -63,7 +63,7 @@ Resources: Type: AWS::Lambda::Function Properties: Code: src/ - Handler: set_container_overrides.lambda_handler + Handler: set_batch_overrides.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn Runtime: python3.9 diff --git a/apps/set-container-overrides/src/set_container_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py similarity index 100% rename from apps/set-container-overrides/src/set_container_overrides.py rename to apps/set-batch-overrides/src/set_batch_overrides.py diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 19ab738b3..2ca9eb2bc 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -36,11 +36,11 @@ "Type": "Pass", "InputPath": "$.job_id", "ResultPath": "$.job_parameters.bucket_prefix", - "Next": "SET_CONTAINER_OVERRIDES" + "Next": "SET_BATCH_OVERRIDES" }, - "SET_CONTAINER_OVERRIDES": { + "SET_BATCH_OVERRIDES": { "Type": "Task", - "Resource": "${SetContainerOverridesLambdaArn}", + "Resource": "${SetBatchOverridesLambdaArn}", "Parameters": { "job_type.$": "$.job_type", "job_parameters.$": "$.job_parameters" diff --git a/apps/workflow-cf.yml.j2 b/apps/workflow-cf.yml.j2 index 60f451674..0859387c6 100644 --- a/apps/workflow-cf.yml.j2 +++ b/apps/workflow-cf.yml.j2 @@ -103,7 +103,7 @@ Resources: {% endfor %} {% endfor %} UpdateDBLambdaArn: !GetAtt UpdateDB.Outputs.LambdaArn - SetContainerOverridesLambdaArn: !GetAtt SetContainerOverrides.Outputs.LambdaArn + SetBatchOverridesLambdaArn: !GetAtt SetBatchOverrides.Outputs.LambdaArn GetFilesLambdaArn: !GetAtt GetFiles.Outputs.LambdaArn CheckProcessingTimeLambdaArn: !GetAtt CheckProcessingTime.Outputs.LambdaArn UploadLogLambdaArn: !GetAtt UploadLog.Outputs.LambdaArn @@ -163,7 +163,7 @@ Resources: Action: lambda:InvokeFunction Resource: - !GetAtt UpdateDB.Outputs.LambdaArn - - !GetAtt SetContainerOverrides.Outputs.LambdaArn + - !GetAtt SetBatchOverrides.Outputs.LambdaArn - !GetAtt GetFiles.Outputs.LambdaArn - !GetAtt CheckProcessingTime.Outputs.LambdaArn - !GetAtt UploadLog.Outputs.LambdaArn @@ -179,7 +179,7 @@ Resources: {% endif %} TemplateURL: update-db/update-db-cf.yml - SetContainerOverrides: + SetBatchOverrides: Type: AWS::CloudFormation::Stack Properties: {% if security_environment == 'EDC' %} @@ -187,7 +187,7 @@ Resources: SecurityGroupId: !Ref SecurityGroupId SubnetIds: !Join [",", !Ref SubnetIds] {% endif %} - TemplateURL: set-container-overrides/set-container-overrides-cf.yml + TemplateURL: set-batch-overrides/set-batch-overrides-cf.yml GetFiles: Type: AWS::CloudFormation::Stack diff --git a/tests/test_set_container_overrides.py b/tests/test_set_batch_overrides.py similarity index 89% rename from tests/test_set_container_overrides.py rename to tests/test_set_batch_overrides.py index 1cc8741d4..3375b97dc 100644 --- a/tests/test_set_container_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -1,4 +1,4 @@ -from set_container_overrides import ( +from set_batch_overrides import ( AUTORIFT_LANDSAT_MEMORY, AUTORIFT_S2_MEMORY, RTC_GAMMA_10M_MEMORY, @@ -7,7 +7,7 @@ ) -def test_set_container_overrides_default(): +def test_set_batch_overrides_default(): assert lambda_handler( { 'job_type': 'foo', @@ -17,11 +17,11 @@ def test_set_container_overrides_default(): ) == {} -def test_set_container_overrides_insar_isce_burst(): +def test_set_batch_overrides_insar_isce_burst(): raise NotImplementedError() -def test_set_container_overrides_autorift_s2(): +def test_set_batch_overrides_autorift_s2(): assert lambda_handler( { 'job_type': 'AUTORIFT', @@ -38,7 +38,7 @@ def test_set_container_overrides_autorift_s2(): } -def test_set_container_overrides_autorift_landsat(): +def test_set_batch_overrides_autorift_landsat(): assert lambda_handler( { 'job_type': 'AUTORIFT', @@ -55,7 +55,7 @@ def test_set_container_overrides_autorift_landsat(): } -def test_set_container_overrides_rtc_gamma_10m(): +def test_set_batch_overrides_rtc_gamma_10m(): assert lambda_handler( { 'job_type': 'RTC_GAMMA', @@ -86,7 +86,7 @@ def test_set_container_overrides_rtc_gamma_10m(): } -def test_set_container_overrides_water_map_10m(): +def test_set_batch_overrides_water_map_10m(): assert lambda_handler( { 'job_type': 'WATER_MAP', From 5fc11928122cae039a8f2ed18383ae80e60d1d11 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 10 Sep 2024 15:49:32 -0800 Subject: [PATCH 079/102] allow 5x1 burst insar up to 30 bursts --- apps/set-batch-overrides/src/set_batch_overrides.py | 3 ++- job_spec/INSAR_ISCE_BURST.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 0315528d6..2cc22825a 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -17,7 +17,6 @@ def get_resource_requirements(memory: str) -> dict: def get_insar_isce_burst_memory(job_parameters: dict) -> str: # TODO: - # - update max bursts in job spec and/or add validator to allow diff # of bursts depending on looks? # - refine based on more data # - refactor lookup structure and/or pull out constants? # - can we slightly reduce the 500M padding in order to bump up some of the intervals? @@ -32,6 +31,8 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: return '31500' if bursts < 25: return '63500' + if bursts < 31: + return '127500' if looks == '10x2': if bursts < 10: return '7500' diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 28947a247..83bc228bb 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -81,7 +81,7 @@ INSAR_ISCE_BURST: - Ref::reference - --secondary - Ref::secondary - timeout: 43200 # 12 hr + timeout: 126000 # 35 hours vcpu: 1 memory: 7600 secrets: From daf6368ec8362607de017361172643c6e28f553f Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 09:57:20 -0800 Subject: [PATCH 080/102] memory constants --- .../src/set_batch_overrides.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 2cc22825a..a43b03bf1 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -2,6 +2,11 @@ AUTORIFT_LANDSAT_MEMORY = '15750' RTC_GAMMA_10M_MEMORY = '63200' WATER_MAP_10M_MEMORY = '126000' +INSAR_ISCE_BURST_MEMORY_8G = '7500' +INSAR_ISCE_BURST_MEMORY_16G = '15500' +INSAR_ISCE_BURST_MEMORY_32G = '31500' +INSAR_ISCE_BURST_MEMORY_64G = '63500' +INSAR_ISCE_BURST_MEMORY_128G = '127500' def get_resource_requirements(memory: str) -> dict: @@ -16,35 +21,31 @@ def get_resource_requirements(memory: str) -> dict: def get_insar_isce_burst_memory(job_parameters: dict) -> str: - # TODO: - # - refine based on more data - # - refactor lookup structure and/or pull out constants? - # - can we slightly reduce the 500M padding in order to bump up some of the intervals? bursts = len(job_parameters['reference'].split(' ')) looks = job_parameters['looks'] if looks == '5x1': if bursts < 2: - return '7500' + return INSAR_ISCE_BURST_MEMORY_8G if bursts < 4: - return '15500' + return INSAR_ISCE_BURST_MEMORY_16G if bursts < 11: - return '31500' + return INSAR_ISCE_BURST_MEMORY_32G if bursts < 25: - return '63500' + return INSAR_ISCE_BURST_MEMORY_64G if bursts < 31: - return '127500' + return INSAR_ISCE_BURST_MEMORY_128G if looks == '10x2': if bursts < 10: - return '7500' + return INSAR_ISCE_BURST_MEMORY_8G if bursts < 21: - return '15500' + return INSAR_ISCE_BURST_MEMORY_16G if bursts < 31: - return '31500' + return INSAR_ISCE_BURST_MEMORY_32G if looks == '20x4': if bursts < 23: - return '7500' + return INSAR_ISCE_BURST_MEMORY_8G if bursts < 31: - return '15500' + return INSAR_ISCE_BURST_MEMORY_16G raise ValueError(f'No memory value for {bursts} bursts and {looks} looks') From 735607019e0848feefec6e7692a75870298f7a8d Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 11:58:29 -0800 Subject: [PATCH 081/102] add tests for insar burst memory --- .../src/set_batch_overrides.py | 3 +- tests/test_set_batch_overrides.py | 123 +++++++++++++++++- 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index a43b03bf1..d28fa4af4 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -2,6 +2,7 @@ AUTORIFT_LANDSAT_MEMORY = '15750' RTC_GAMMA_10M_MEMORY = '63200' WATER_MAP_10M_MEMORY = '126000' + INSAR_ISCE_BURST_MEMORY_8G = '7500' INSAR_ISCE_BURST_MEMORY_16G = '15500' INSAR_ISCE_BURST_MEMORY_32G = '31500' @@ -21,8 +22,8 @@ def get_resource_requirements(memory: str) -> dict: def get_insar_isce_burst_memory(job_parameters: dict) -> str: - bursts = len(job_parameters['reference'].split(' ')) looks = job_parameters['looks'] + bursts = len(job_parameters['reference'].split(' ')) if looks == '5x1': if bursts < 2: return INSAR_ISCE_BURST_MEMORY_8G diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 3375b97dc..ae90f3654 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -1,12 +1,29 @@ +import pytest + from set_batch_overrides import ( AUTORIFT_LANDSAT_MEMORY, AUTORIFT_S2_MEMORY, RTC_GAMMA_10M_MEMORY, WATER_MAP_10M_MEMORY, + INSAR_ISCE_BURST_MEMORY_8G, + INSAR_ISCE_BURST_MEMORY_16G, + INSAR_ISCE_BURST_MEMORY_32G, + INSAR_ISCE_BURST_MEMORY_64G, + INSAR_ISCE_BURST_MEMORY_128G, lambda_handler, ) +def mock_insar_isce_burst_job(looks: str, bursts: int) -> dict: + return { + 'job_type': 'INSAR_ISCE_BURST', + 'job_parameters': { + 'looks': looks, + 'reference': ' '.join('foo' for _ in range(bursts)), + } + } + + def test_set_batch_overrides_default(): assert lambda_handler( { @@ -17,8 +34,110 @@ def test_set_batch_overrides_default(): ) == {} -def test_set_batch_overrides_insar_isce_burst(): - raise NotImplementedError() +def test_set_batch_overrides_insar_isce_burst_5x1(): + assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=1), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_8G, + } + ] + } + for n in range(2, 4): + assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_16G, + } + ] + } + for n in range(4, 11): + assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_32G, + } + ] + } + for n in range(11, 25): + assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_64G, + } + ] + } + for n in range(25, 31): + assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_128G, + } + ] + } + + +def test_set_batch_overrides_insar_isce_burst_10x2(): + for n in range(1, 10): + assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_8G, + } + ] + } + for n in range(10, 21): + assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_16G, + } + ] + } + for n in range(21, 31): + assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_32G, + } + ] + } + + +def test_set_batch_overrides_insar_isce_burst_20x4(): + for n in range(1, 23): + assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_8G, + } + ] + } + for n in range(23, 31): + assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { + 'ResourceRequirements': [ + { + 'Type': 'MEMORY', + 'Value': INSAR_ISCE_BURST_MEMORY_16G, + } + ] + } + + +def test_set_batch_overrides_insar_isce_burst_value_error(): + with pytest.raises(ValueError, match=r'^No memory value for.*'): + lambda_handler(mock_insar_isce_burst_job('20x4', bursts=31), None) + + with pytest.raises(ValueError, match=r'^No memory value for.*'): + lambda_handler(mock_insar_isce_burst_job('foo', bursts=1), None) def test_set_batch_overrides_autorift_s2(): From 0af281f428c567a97d7354f0a934f511059a3f0f Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 12:01:11 -0800 Subject: [PATCH 082/102] set default insar burst memory to 0 --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 83bc228bb..bf7ff7107 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -83,7 +83,7 @@ INSAR_ISCE_BURST: - Ref::secondary timeout: 126000 # 35 hours vcpu: 1 - memory: 7600 + memory: 0 # Memory is always overridden by the step function secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD From 898f62471d9c3bb2675bbf89ccd1e00b8a6ec981 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 14:25:22 -0800 Subject: [PATCH 083/102] set omp_num_threads for multi-burst insar --- .../src/set_batch_overrides.py | 19 ++++++++++-- job_spec/INSAR_ISCE_BURST.yml | 2 -- tests/test_set_batch_overrides.py | 30 ++++++++++++------- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index d28fa4af4..47e002497 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -9,9 +9,17 @@ INSAR_ISCE_BURST_MEMORY_64G = '63500' INSAR_ISCE_BURST_MEMORY_128G = '127500' +INSAR_ISCE_BURST_OMP_NUM_THREADS = { + INSAR_ISCE_BURST_MEMORY_8G: '1', + INSAR_ISCE_BURST_MEMORY_16G: '2', + INSAR_ISCE_BURST_MEMORY_32G: '4', + INSAR_ISCE_BURST_MEMORY_64G: '8', + INSAR_ISCE_BURST_MEMORY_128G: '16', +} -def get_resource_requirements(memory: str) -> dict: - return { + +def get_resource_requirements(memory: str, omp_num_threads: str = None) -> dict: + resource_requirements = { 'ResourceRequirements': [ { 'Type': 'MEMORY', @@ -19,6 +27,9 @@ def get_resource_requirements(memory: str) -> dict: } ] } + if omp_num_threads is not None: + resource_requirements['Environment'] = [{'Name': 'OMP_NUM_THREADS', 'Value': omp_num_threads}] + return resource_requirements def get_insar_isce_burst_memory(job_parameters: dict) -> str: @@ -54,7 +65,9 @@ def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] if job_type == 'INSAR_ISCE_BURST': - return get_resource_requirements(get_insar_isce_burst_memory(job_parameters)) + memory = get_insar_isce_burst_memory(job_parameters) + omp_num_threads = INSAR_ISCE_BURST_OMP_NUM_THREADS[memory] + return get_resource_requirements(memory, omp_num_threads) if job_type == 'AUTORIFT' and job_parameters['granules'].startswith('S2'): return get_resource_requirements(AUTORIFT_S2_MEMORY) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index bf7ff7107..046405223 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -67,8 +67,6 @@ INSAR_ISCE_BURST: command: - ++process - insar_tops_burst - - ++omp-num-threads - - '1' - --bucket - '!Ref Bucket' - --bucket-prefix diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index ae90f3654..86ca6ba26 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -41,7 +41,8 @@ def test_set_batch_overrides_insar_isce_burst_5x1(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_8G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '1'}], } for n in range(2, 4): assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { @@ -50,7 +51,8 @@ def test_set_batch_overrides_insar_isce_burst_5x1(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_16G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '2'}], } for n in range(4, 11): assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { @@ -59,7 +61,8 @@ def test_set_batch_overrides_insar_isce_burst_5x1(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_32G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '4'}], } for n in range(11, 25): assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { @@ -68,7 +71,8 @@ def test_set_batch_overrides_insar_isce_burst_5x1(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_64G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '8'}], } for n in range(25, 31): assert lambda_handler(mock_insar_isce_burst_job('5x1', bursts=n), None) == { @@ -77,7 +81,8 @@ def test_set_batch_overrides_insar_isce_burst_5x1(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_128G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '16'}], } @@ -89,7 +94,8 @@ def test_set_batch_overrides_insar_isce_burst_10x2(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_8G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '1'}], } for n in range(10, 21): assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { @@ -98,7 +104,8 @@ def test_set_batch_overrides_insar_isce_burst_10x2(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_16G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '2'}], } for n in range(21, 31): assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { @@ -107,7 +114,8 @@ def test_set_batch_overrides_insar_isce_burst_10x2(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_32G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '4'}], } @@ -119,7 +127,8 @@ def test_set_batch_overrides_insar_isce_burst_20x4(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_8G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '1'}], } for n in range(23, 31): assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { @@ -128,7 +137,8 @@ def test_set_batch_overrides_insar_isce_burst_20x4(): 'Type': 'MEMORY', 'Value': INSAR_ISCE_BURST_MEMORY_16G, } - ] + ], + 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '2'}], } From 363025293fd33173eb0ac849145584a7113716de Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 14:28:12 -0800 Subject: [PATCH 084/102] comment --- apps/set-batch-overrides/src/set_batch_overrides.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 47e002497..596803973 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -9,6 +9,7 @@ INSAR_ISCE_BURST_MEMORY_64G = '63500' INSAR_ISCE_BURST_MEMORY_128G = '127500' +# vCPU = Memory/8 for r6 instance types INSAR_ISCE_BURST_OMP_NUM_THREADS = { INSAR_ISCE_BURST_MEMORY_8G: '1', INSAR_ISCE_BURST_MEMORY_16G: '2', From b1b3bf63de52c83231ae51689aaf6e4b7c6b5e01 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 14:29:35 -0800 Subject: [PATCH 085/102] flake8 import order --- tests/test_set_batch_overrides.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 86ca6ba26..fd65de10b 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -3,13 +3,13 @@ from set_batch_overrides import ( AUTORIFT_LANDSAT_MEMORY, AUTORIFT_S2_MEMORY, - RTC_GAMMA_10M_MEMORY, - WATER_MAP_10M_MEMORY, - INSAR_ISCE_BURST_MEMORY_8G, + INSAR_ISCE_BURST_MEMORY_128G, INSAR_ISCE_BURST_MEMORY_16G, INSAR_ISCE_BURST_MEMORY_32G, INSAR_ISCE_BURST_MEMORY_64G, - INSAR_ISCE_BURST_MEMORY_128G, + INSAR_ISCE_BURST_MEMORY_8G, + RTC_GAMMA_10M_MEMORY, + WATER_MAP_10M_MEMORY, lambda_handler, ) From fef9152c590e577aff2ddda119176046705f04cd Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 14:40:15 -0800 Subject: [PATCH 086/102] rename function --- .../src/set_batch_overrides.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 596803973..781e6fc30 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -19,8 +19,8 @@ } -def get_resource_requirements(memory: str, omp_num_threads: str = None) -> dict: - resource_requirements = { +def get_container_overrides(memory: str, omp_num_threads: str = None) -> dict: + container_overrides = { 'ResourceRequirements': [ { 'Type': 'MEMORY', @@ -29,8 +29,8 @@ def get_resource_requirements(memory: str, omp_num_threads: str = None) -> dict: ] } if omp_num_threads is not None: - resource_requirements['Environment'] = [{'Name': 'OMP_NUM_THREADS', 'Value': omp_num_threads}] - return resource_requirements + container_overrides['Environment'] = [{'Name': 'OMP_NUM_THREADS', 'Value': omp_num_threads}] + return container_overrides def get_insar_isce_burst_memory(job_parameters: dict) -> str: @@ -68,18 +68,18 @@ def lambda_handler(event: dict, _) -> dict: if job_type == 'INSAR_ISCE_BURST': memory = get_insar_isce_burst_memory(job_parameters) omp_num_threads = INSAR_ISCE_BURST_OMP_NUM_THREADS[memory] - return get_resource_requirements(memory, omp_num_threads) + return get_container_overrides(memory, omp_num_threads) if job_type == 'AUTORIFT' and job_parameters['granules'].startswith('S2'): - return get_resource_requirements(AUTORIFT_S2_MEMORY) + return get_container_overrides(AUTORIFT_S2_MEMORY) if job_type == 'AUTORIFT' and job_parameters['granules'].startswith('L'): - return get_resource_requirements(AUTORIFT_LANDSAT_MEMORY) + return get_container_overrides(AUTORIFT_LANDSAT_MEMORY) if job_type == 'RTC_GAMMA' and job_parameters['resolution'] in ['10', '20']: - return get_resource_requirements(RTC_GAMMA_10M_MEMORY) + return get_container_overrides(RTC_GAMMA_10M_MEMORY) if job_type in ['WATER_MAP', 'WATER_MAP_EQ'] and job_parameters['resolution'] in ['10', '20']: - return get_resource_requirements(WATER_MAP_10M_MEMORY) + return get_container_overrides(WATER_MAP_10M_MEMORY) return {} From b6f15091f9459580b2bdb2a4bf64b818eb3a0a84 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 14:47:31 -0800 Subject: [PATCH 087/102] set burst insar memory default to 4 --- job_spec/INSAR_ISCE_BURST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 046405223..9ebd4445f 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -81,7 +81,7 @@ INSAR_ISCE_BURST: - Ref::secondary timeout: 126000 # 35 hours vcpu: 1 - memory: 0 # Memory is always overridden by the step function + memory: 4 # Memory is always overridden by the step function secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD From e88627162814f1826249ec3ef9fd560511d4b4b9 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 11 Sep 2024 14:56:15 -0800 Subject: [PATCH 088/102] json trailing comma --- apps/step-function.json.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 2ca9eb2bc..bc168208d 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -59,7 +59,7 @@ "States.ALL" ], "Next": "JOB_FAILED", - "ResultPath": "$.container_overrides", + "ResultPath": "$.container_overrides" } ], "ResultPath": "$.container_overrides", From de08391fa2a2fafbccb0f360898a938219fd6678 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 12 Sep 2024 11:01:51 -0800 Subject: [PATCH 089/102] adjust memory interval for 10x2 --- apps/set-batch-overrides/src/set_batch_overrides.py | 2 +- tests/test_set_batch_overrides.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 781e6fc30..26989146d 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -48,7 +48,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: if bursts < 31: return INSAR_ISCE_BURST_MEMORY_128G if looks == '10x2': - if bursts < 10: + if bursts < 8: return INSAR_ISCE_BURST_MEMORY_8G if bursts < 21: return INSAR_ISCE_BURST_MEMORY_16G diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index fd65de10b..089172a60 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -87,7 +87,7 @@ def test_set_batch_overrides_insar_isce_burst_5x1(): def test_set_batch_overrides_insar_isce_burst_10x2(): - for n in range(1, 10): + for n in range(1, 8): assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { 'ResourceRequirements': [ { @@ -97,7 +97,7 @@ def test_set_batch_overrides_insar_isce_burst_10x2(): ], 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '1'}], } - for n in range(10, 21): + for n in range(8, 21): assert lambda_handler(mock_insar_isce_burst_job('10x2', bursts=n), None) == { 'ResourceRequirements': [ { From a3c337ef5c4d7f2a9752d2eba33cc679f4a330e6 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 30 Sep 2024 17:04:19 -0800 Subject: [PATCH 090/102] split into two burst job specs --- .../workflows/deploy-multi-burst-sandbox.yml | 1 + apps/api/src/hyp3_api/validation.py | 16 +++- .../src/set_batch_overrides.py | 3 +- job_spec/INSAR_ISCE_BURST.yml | 40 +++------ job_spec/INSAR_ISCE_MULTI_BURST.yml | 87 +++++++++++++++++++ tests/test_api/test_validation.py | 25 +++++- tests/test_set_batch_overrides.py | 3 +- 7 files changed, 142 insertions(+), 33 deletions(-) create mode 100644 job_spec/INSAR_ISCE_MULTI_BURST.yml diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml index f80a7cf4b..098245c2a 100644 --- a/.github/workflows/deploy-multi-burst-sandbox.yml +++ b/.github/workflows/deploy-multi-burst-sandbox.yml @@ -25,6 +25,7 @@ jobs: deploy_ref: refs/heads/multi-burst-sandbox job_files: >- job_spec/INSAR_ISCE_BURST.yml + job_spec/INSAR_ISCE_MULTI_BURST.yml job_spec/AUTORIFT.yml job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 41478896b..374954694 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -1,6 +1,7 @@ import json import os import sys +from copy import deepcopy from pathlib import Path import requests @@ -135,7 +136,20 @@ def get_multipolygon_from_geojson(input_file): return MultiPolygon(polygons) -def validate_jobs(jobs): +def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: + jobs = deepcopy(jobs) + for job in jobs: + if job['job_type'] == 'INSAR_ISCE_BURST': + job_parameters = job['job_parameters'] + job_parameters['reference'], job_parameters['secondary'] = ( + [granule] for granule in job_parameters.pop('granules') + ) + return jobs + + +def validate_jobs(jobs: list[dict]) -> None: + jobs = convert_single_burst_jobs(jobs) + granules = get_granules(jobs) granule_metadata = get_cmr_metadata(granules) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 26989146d..36ad124c9 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -65,7 +65,8 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] - if job_type == 'INSAR_ISCE_BURST': + # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + if job_type == 'INSAR_ISCE_MULTI_BURST': memory = get_insar_isce_burst_memory(job_parameters) omp_num_threads = INSAR_ISCE_BURST_OMP_NUM_THREADS[memory] return get_container_overrides(memory, omp_num_threads) diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 9ebd4445f..d741b5aa4 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -1,38 +1,23 @@ INSAR_ISCE_BURST: required_parameters: - - reference - - secondary + - granules parameters: - reference: + granules: default: '""' api_schema: type: array - minItems: 1 - maxItems: 30 + minItems: 2 + maxItems: 2 example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST - items: - description: Name of the reference Sentinel-1 SLC IW burst granule to process - type: string - pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$' - minLength: 43 - maxLength: 43 - example: S1_136231_IW2_20200604T022312_VV_7C85-BURST - secondary: - default: '""' - api_schema: - type: array - minItems: 1 - maxItems: 30 - example: - S1_136231_IW2_20200616T022313_VV_5D11-BURST items: - description: Name of the secondary Sentinel-1 SLC IW burst granule to process + description: Name of the Sentinel-1 SLC IW burst granule to process type: string - pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$' + pattern: "^S1_\\d{6}_IW.*-BURST" minLength: 43 maxLength: 43 - example: S1_136231_IW2_20200616T022313_VV_5D11-BURST + example: S1_136231_IW2_20200604T022312_VV_7C85-BURST bucket_prefix: default: '""' apply_water_mask: @@ -67,6 +52,8 @@ INSAR_ISCE_BURST: command: - ++process - insar_tops_burst + - ++omp-num-threads + - '1' - --bucket - '!Ref Bucket' - --bucket-prefix @@ -75,13 +62,10 @@ INSAR_ISCE_BURST: - Ref::apply_water_mask - --looks - Ref::looks - - --reference - - Ref::reference - - --secondary - - Ref::secondary - timeout: 126000 # 35 hours + - Ref::granules + timeout: 5400 vcpu: 1 - memory: 4 # Memory is always overridden by the step function + memory: 7600 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD diff --git a/job_spec/INSAR_ISCE_MULTI_BURST.yml b/job_spec/INSAR_ISCE_MULTI_BURST.yml new file mode 100644 index 000000000..1326793fb --- /dev/null +++ b/job_spec/INSAR_ISCE_MULTI_BURST.yml @@ -0,0 +1,87 @@ +INSAR_ISCE_MULTI_BURST: + required_parameters: + - reference + - secondary + parameters: + reference: + default: '""' + api_schema: + type: array + minItems: 1 + maxItems: 30 + example: + - S1_136231_IW2_20200604T022312_VV_7C85-BURST + items: + description: Name of the reference Sentinel-1 SLC IW burst granule to process + type: string + pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$' + minLength: 43 + maxLength: 43 + example: S1_136231_IW2_20200604T022312_VV_7C85-BURST + secondary: + default: '""' + api_schema: + type: array + minItems: 1 + maxItems: 30 + example: + - S1_136231_IW2_20200616T022313_VV_5D11-BURST + items: + description: Name of the secondary Sentinel-1 SLC IW burst granule to process + type: string + pattern: '^S1_\d{6}_IW\d_\d{8}T\d{6}_[VH]{2}_([\dA-F]){4}-BURST$' + minLength: 43 + maxLength: 43 + example: S1_136231_IW2_20200616T022313_VV_5D11-BURST + bucket_prefix: + default: '""' + apply_water_mask: + api_schema: + description: Sets pixels over coastal and large inland waterbodies as invalid for phase unwrapping. + default: false + type: boolean + looks: + api_schema: + description: Number of looks to take in range and azimuth + type: string + default: 20x4 + enum: + - 20x4 + - 10x2 + - 5x1 + cost_profiles: + EDC: + cost: 1.0 + DEFAULT: + cost: 1.0 + validators: + - check_dem_coverage + - check_valid_polarizations + - check_same_burst_ids + - check_not_antimeridian + compute_environment: + name: 'Default' + tasks: + - name: '' + image: ghcr.io/asfhyp3/hyp3-isce2 + command: + - ++process + - insar_tops_burst + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::bucket_prefix + - --apply-water-mask + - Ref::apply_water_mask + - --looks + - Ref::looks + - --reference + - Ref::reference + - --secondary + - Ref::secondary + timeout: 126000 # 35 hours + vcpu: 1 + memory: 4 # Memory is always overridden by the step function + secrets: + - EARTHDATA_USERNAME + - EARTHDATA_PASSWORD diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 10faea966..121875d9a 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -424,11 +424,19 @@ def test_validate_jobs(): 'job_parameters': {} }, { - 'job_type': 'INSAR_ISCE_BURST', + # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [valid_burst_pair[0]], 'secondary': [valid_burst_pair[1]] } + }, + { + # TODO remove this test case after the two burst types are merged + 'job_type': 'INSAR_ISCE_BURST', + 'job_parameters': { + 'granules': [valid_burst_pair[0], valid_burst_pair[1]] + } } ] validation.validate_jobs(jobs) @@ -457,7 +465,8 @@ def test_validate_jobs(): jobs = [ { - 'job_type': 'INSAR_ISCE_BURST', + # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [invalid_burst_pair[0]], 'secondary': [invalid_burst_pair[1]] @@ -466,3 +475,15 @@ def test_validate_jobs(): ] with raises(validation.GranuleValidationError): validation.validate_jobs(jobs) + + # TODO remove this test case after the two burst types are merged + jobs = [ + { + 'job_type': 'INSAR_ISCE_BURST', + 'job_parameters': { + 'granules': [invalid_burst_pair[0], invalid_burst_pair[1]] + } + } + ] + with raises(validation.GranuleValidationError): + validation.validate_jobs(jobs) diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 089172a60..29b600197 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -16,7 +16,8 @@ def mock_insar_isce_burst_job(looks: str, bursts: int) -> dict: return { - 'job_type': 'INSAR_ISCE_BURST', + # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'looks': looks, 'reference': ' '.join('foo' for _ in range(bursts)), From 5ca97e7f5ac7878a0a679ece8e804f1b96ee4822 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 30 Sep 2024 17:21:16 -0800 Subject: [PATCH 091/102] Add issue link to TODO comments --- apps/set-batch-overrides/src/set_batch_overrides.py | 2 +- tests/test_api/test_validation.py | 8 ++++---- tests/test_set_batch_overrides.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 36ad124c9..7d3dbd67e 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -65,7 +65,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] - # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged if job_type == 'INSAR_ISCE_MULTI_BURST': memory = get_insar_isce_burst_memory(job_parameters) omp_num_threads = INSAR_ISCE_BURST_OMP_NUM_THREADS[memory] diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 121875d9a..76eab7a23 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -424,7 +424,7 @@ def test_validate_jobs(): 'job_parameters': {} }, { - # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [valid_burst_pair[0]], @@ -432,7 +432,7 @@ def test_validate_jobs(): } }, { - # TODO remove this test case after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 remove this test case after the two burst types are merged 'job_type': 'INSAR_ISCE_BURST', 'job_parameters': { 'granules': [valid_burst_pair[0], valid_burst_pair[1]] @@ -465,7 +465,7 @@ def test_validate_jobs(): jobs = [ { - # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [invalid_burst_pair[0]], @@ -476,7 +476,7 @@ def test_validate_jobs(): with raises(validation.GranuleValidationError): validation.validate_jobs(jobs) - # TODO remove this test case after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 remove this test case after the two burst types are merged jobs = [ { 'job_type': 'INSAR_ISCE_BURST', diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 29b600197..a8c88c051 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -16,7 +16,7 @@ def mock_insar_isce_burst_job(looks: str, bursts: int) -> dict: return { - # TODO rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'looks': looks, From 797240c305c07d17923ecde3777135304c5e82c1 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 30 Sep 2024 17:28:17 -0800 Subject: [PATCH 092/102] add a TODO --- apps/api/src/hyp3_api/validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 374954694..96c1b4641 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -136,6 +136,7 @@ def get_multipolygon_from_geojson(input_file): return MultiPolygon(polygons) +# TODO https://github.com/ASFHyP3/hyp3/issues/2442 remove this function after two burst types are merged def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: jobs = deepcopy(jobs) for job in jobs: From dcda039612d51f3cdf57c6b1b3b14976b2cc4beb Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 30 Sep 2024 17:32:53 -0800 Subject: [PATCH 093/102] delete old changelog entry --- CHANGELOG.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb24c873..a7442e101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,15 +47,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Revert ARIA hyp3 deployments back to C-instance family - including the job-spec CLI parameter `omp-num-threads` to ensure multiple jobs fit on single instance. - Deployments with INSAR_ISCE.yml job specs will now use a dedicated compute environment with on-demand instances instead of spot instances for INSAR_ISCE jobs. -## [8.0.0] - -### Changed -- `INSAR_ISCE_BURST` now accepts `reference` and `secondary` parameters, each of which is a list containing a single granule name. In a future release, `INSAR_ISCE_BURST` will support processing multiple adjacent bursts into one interferogram, and each parameter will be a list of one or more granule names. -- All previous `INSAR_ISCE_BURST` jobs have been reformatted to replace the `granules` parameter with the `reference` and `secondary` parameters. - -### Removed -- `INSAR_ISCE_BURST` no longer accepts a `granules` parameter. - ## [7.7.2] From 9caca9b20ed981861a3c907501ca4e4497f8fb41 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 30 Sep 2024 17:33:27 -0800 Subject: [PATCH 094/102] delete blank line --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7442e101..bc4245065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Revert ARIA hyp3 deployments back to C-instance family - including the job-spec CLI parameter `omp-num-threads` to ensure multiple jobs fit on single instance. - Deployments with INSAR_ISCE.yml job specs will now use a dedicated compute environment with on-demand instances instead of spot instances for INSAR_ISCE jobs. - ## [7.7.2] ### Change From 7aba1cde92a14c8cda1bc0fb23490256c78eae33 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 30 Sep 2024 18:02:14 -0800 Subject: [PATCH 095/102] break long lines --- tests/test_api/test_validation.py | 12 ++++++++---- tests/test_set_batch_overrides.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 76eab7a23..d67fbfce3 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -424,7 +424,8 @@ def test_validate_jobs(): 'job_parameters': {} }, { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 + # rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [valid_burst_pair[0]], @@ -432,7 +433,8 @@ def test_validate_jobs(): } }, { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 remove this test case after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 + # remove this test case after the two burst types are merged 'job_type': 'INSAR_ISCE_BURST', 'job_parameters': { 'granules': [valid_burst_pair[0], valid_burst_pair[1]] @@ -465,7 +467,8 @@ def test_validate_jobs(): jobs = [ { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 + # rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [invalid_burst_pair[0]], @@ -476,7 +479,8 @@ def test_validate_jobs(): with raises(validation.GranuleValidationError): validation.validate_jobs(jobs) - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 remove this test case after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 + # remove this test case after the two burst types are merged jobs = [ { 'job_type': 'INSAR_ISCE_BURST', diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index a8c88c051..6df1df591 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -16,7 +16,8 @@ def mock_insar_isce_burst_job(looks: str, bursts: int) -> dict: return { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged + # TODO https://github.com/ASFHyP3/hyp3/issues/2442 + # rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'looks': looks, From 7da5ddaaeee021bae4ead78503c138b6dfe397bf Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 10:33:16 -0800 Subject: [PATCH 096/102] changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc4245065..d8853b4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [7.10.0] + +### Added +- Added a new `INSAR_ISCE_MULTI_BURST` job type for running multi-burst InSAR. Currently, this job type is restricted to a special `hyp3-multi-burst-sandbox` deployment, which is restricted to HyP3 operators. However, this is an important step toward eventually making multi-burst InSAR available for general users. + +### Changed +- Job validator functions now accept two parameters: the job dictionary and the granule metadata. + - New job parameters `reference` and `secondary` are now supported in addition to the existing `granules` parameter. +- Burst InSAR validators now support multi-burst jobs. +- Replaced the step function's `INSPECT_MEMORY_REQUIREMENTS` step with a new `SET_BATCH_OVERRIDES` step, which calls a Lambda function to dynamically calculate [Batch container overrides](https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html) based on job type and parameters. + ## [7.9.3] ### Fixed - Added missing cloudformation:DeleteStack permission to cloudformation deployment role in ASF-deployment-ci-cf.yml . From d2d5f1f66f9945c00e68e99335b1b21ca91c6047 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 10:35:38 -0800 Subject: [PATCH 097/102] reword --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8853b4bc..3b46424e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Job validator functions now accept two parameters: the job dictionary and the granule metadata. - - New job parameters `reference` and `secondary` are now supported in addition to the existing `granules` parameter. + - Validation now supports `reference` and `secondary` job parameters in addition to the existing `granules` parameter. - Burst InSAR validators now support multi-burst jobs. - Replaced the step function's `INSPECT_MEMORY_REQUIREMENTS` step with a new `SET_BATCH_OVERRIDES` step, which calls a Lambda function to dynamically calculate [Batch container overrides](https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html) based on job type and parameters. From 3b969e70198a085c88150c0c2d7e4439c779868a Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 10:36:20 -0800 Subject: [PATCH 098/102] reword again --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b46424e1..013636301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Job validator functions now accept two parameters: the job dictionary and the granule metadata. - - Validation now supports `reference` and `secondary` job parameters in addition to the existing `granules` parameter. + - Granule metadata validation now supports `reference` and `secondary` job parameters in addition to the existing `granules` parameter. - Burst InSAR validators now support multi-burst jobs. - Replaced the step function's `INSPECT_MEMORY_REQUIREMENTS` step with a new `SET_BATCH_OVERRIDES` step, which calls a Lambda function to dynamically calculate [Batch container overrides](https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html) based on job type and parameters. From 55cb9939e1eda3bce81b71144c31419f91cc78d0 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 10:36:55 -0800 Subject: [PATCH 099/102] de-indent --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013636301..5f5a9bfeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Job validator functions now accept two parameters: the job dictionary and the granule metadata. - - Granule metadata validation now supports `reference` and `secondary` job parameters in addition to the existing `granules` parameter. +- Granule metadata validation now supports `reference` and `secondary` job parameters in addition to the existing `granules` parameter. - Burst InSAR validators now support multi-burst jobs. - Replaced the step function's `INSPECT_MEMORY_REQUIREMENTS` step with a new `SET_BATCH_OVERRIDES` step, which calls a Lambda function to dynamically calculate [Batch container overrides](https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html) based on job type and parameters. From faedcbb74bd67e87aca1bca2dec9de982814bc79 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 10:38:59 -0800 Subject: [PATCH 100/102] reword --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f5a9bfeb..7ebf1b3ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [7.10.0] ### Added -- Added a new `INSAR_ISCE_MULTI_BURST` job type for running multi-burst InSAR. Currently, this job type is restricted to a special `hyp3-multi-burst-sandbox` deployment, which is restricted to HyP3 operators. However, this is an important step toward eventually making multi-burst InSAR available for general users. +- Added a new `INSAR_ISCE_MULTI_BURST` job type for running multi-burst InSAR. Currently, this job type is restricted to a special `hyp3-multi-burst-sandbox` deployment for HyP3 operators. However, this is an important step toward eventually making multi-burst InSAR available for general users. ### Changed - Job validator functions now accept two parameters: the job dictionary and the granule metadata. From da8951101111aec019d30a6ab1b1ca6de4cf312f Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 11:01:24 -0800 Subject: [PATCH 101/102] add TODO --- job_spec/INSAR_ISCE_MULTI_BURST.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/job_spec/INSAR_ISCE_MULTI_BURST.yml b/job_spec/INSAR_ISCE_MULTI_BURST.yml index 1326793fb..e6b3a89e5 100644 --- a/job_spec/INSAR_ISCE_MULTI_BURST.yml +++ b/job_spec/INSAR_ISCE_MULTI_BURST.yml @@ -9,6 +9,7 @@ INSAR_ISCE_MULTI_BURST: type: array minItems: 1 maxItems: 30 + # TODO: provide an example with multiple bursts example: - S1_136231_IW2_20200604T022312_VV_7C85-BURST items: From 3e2789a143a26c1ae9bd6140d4e53f5d4b8942e6 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 1 Oct 2024 11:08:25 -0800 Subject: [PATCH 102/102] refactor ref, sec unpacking --- apps/api/src/hyp3_api/validation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 96c1b4641..f5e40aaf2 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -142,9 +142,8 @@ def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: for job in jobs: if job['job_type'] == 'INSAR_ISCE_BURST': job_parameters = job['job_parameters'] - job_parameters['reference'], job_parameters['secondary'] = ( - [granule] for granule in job_parameters.pop('granules') - ) + ref, sec = job_parameters.pop('granules') + job_parameters['reference'], job_parameters['secondary'] = [ref], [sec] return jobs