From b70244456e0e2cfa67e3457a2f52398edac4df60 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 2 Oct 2024 14:03:08 -0800 Subject: [PATCH 1/6] remove multi-burst TODOs --- apps/api/src/hyp3_api/validation.py | 1 - apps/set-batch-overrides/src/set_batch_overrides.py | 1 - tests/test_api/test_validation.py | 8 -------- tests/test_set_batch_overrides.py | 2 -- 4 files changed, 12 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index f5e40aaf2..0f5acbe5e 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -136,7 +136,6 @@ def get_multipolygon_from_geojson(input_file): return MultiPolygon(polygons) -# TODO https://github.com/ASFHyP3/hyp3/issues/2442 remove this function after two burst types are merged def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: jobs = deepcopy(jobs) for job in jobs: diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 7d3dbd67e..0a9dc6627 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -65,7 +65,6 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: def lambda_handler(event: dict, _) -> dict: job_type, job_parameters = event['job_type'], event['job_parameters'] - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 rename to INSAR_ISCE_BURST after the two burst types are merged if job_type == 'INSAR_ISCE_MULTI_BURST': memory = get_insar_isce_burst_memory(job_parameters) omp_num_threads = INSAR_ISCE_BURST_OMP_NUM_THREADS[memory] diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index d67fbfce3..67ccee8d2 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -424,8 +424,6 @@ def test_validate_jobs(): 'job_parameters': {} }, { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 - # rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [valid_burst_pair[0]], @@ -433,8 +431,6 @@ def test_validate_jobs(): } }, { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 - # remove this test case after the two burst types are merged 'job_type': 'INSAR_ISCE_BURST', 'job_parameters': { 'granules': [valid_burst_pair[0], valid_burst_pair[1]] @@ -467,8 +463,6 @@ def test_validate_jobs(): jobs = [ { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 - # rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'reference': [invalid_burst_pair[0]], @@ -479,8 +473,6 @@ def test_validate_jobs(): with raises(validation.GranuleValidationError): validation.validate_jobs(jobs) - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 - # remove this test case after the two burst types are merged jobs = [ { 'job_type': 'INSAR_ISCE_BURST', diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 6df1df591..375413556 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -16,8 +16,6 @@ def mock_insar_isce_burst_job(looks: str, bursts: int) -> dict: return { - # TODO https://github.com/ASFHyP3/hyp3/issues/2442 - # rename to INSAR_ISCE_BURST after the two burst types are merged 'job_type': 'INSAR_ISCE_MULTI_BURST', 'job_parameters': { 'looks': looks, From 3a725e7f8b77e226e4bcb3ae57f10cd909a44554 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:04:07 +0000 Subject: [PATCH 2/6] Bump boto3 from 1.35.31 to 1.35.34 Bumps [boto3](https://github.com/boto/boto3) from 1.35.31 to 1.35.34. - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.35.31...1.35.34) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- requirements-apps-disable-private-dns.txt | 2 +- requirements-apps-start-execution-manager.txt | 2 +- requirements-apps-start-execution-worker.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-all.txt b/requirements-all.txt index 4084ad6c2..d306dbe96 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.35.31 +boto3==1.35.34 jinja2==3.1.4 moto[dynamodb]==5.0.16 pytest==8.3.3 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index f2beaec12..591888472 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.35.31 +boto3==1.35.34 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index 7dc1b04ee..3a73b98e4 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.35.31 +boto3==1.35.34 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index de3b06772..ad5be7290 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.35.31 +boto3==1.35.34 ./lib/lambda_logging/ From 3c663512a0388176b3565c9ca447ea56f5f337b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:04:14 +0000 Subject: [PATCH 3/6] Bump cfn-lint from 1.15.2 to 1.16.0 Bumps [cfn-lint](https://github.com/aws-cloudformation/cfn-lint) from 1.15.2 to 1.16.0. - [Release notes](https://github.com/aws-cloudformation/cfn-lint/releases) - [Changelog](https://github.com/aws-cloudformation/cfn-lint/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-cloudformation/cfn-lint/compare/v1.15.2...v1.16.0) --- updated-dependencies: - dependency-name: cfn-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 4084ad6c2..0478c76a0 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -17,4 +17,4 @@ flake8-blind-except==0.2.1 flake8-builtins==2.5.0 setuptools==75.1.0 openapi-spec-validator==0.7.1 -cfn-lint==1.15.2 +cfn-lint==1.16.0 From 003d5067e93703352656f1e72738a2f3eeae2282 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:15:28 +0000 Subject: [PATCH 4/6] Bump actions/checkout from 4.2.0 to 4.2.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.0...v4.2.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy-daac.yml | 2 +- .github/workflows/deploy-enterprise-test.yml | 2 +- .github/workflows/deploy-enterprise.yml | 2 +- .github/workflows/deploy-multi-burst-sandbox.yml | 2 +- .github/workflows/static-analysis.yml | 10 +++++----- .github/workflows/tests.yml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index f39d629ea..9abfb18b2 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -66,7 +66,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml index 2e3fcbe4e..ab7b964d6 100644 --- a/.github/workflows/deploy-enterprise-test.yml +++ b/.github/workflows/deploy-enterprise-test.yml @@ -101,7 +101,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index 94b4c22dd..4f77da147 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -258,7 +258,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml index c4a2a4518..95eea8fde 100644 --- a/.github/workflows/deploy-multi-burst-sandbox.yml +++ b/.github/workflows/deploy-multi-burst-sandbox.yml @@ -43,7 +43,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 1b66f1fe3..d94186bc6 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -6,7 +6,7 @@ jobs: flake8: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -23,7 +23,7 @@ jobs: matrix: security_environment: [ASF, EDC, JPL, JPL-public] steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -37,7 +37,7 @@ jobs: openapi-spec-validator: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -50,7 +50,7 @@ jobs: statelint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 @@ -70,7 +70,7 @@ jobs: snyk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: snyk/actions/setup@0.4.0 - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 231f4eeac..7e66b82cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.1 - uses: actions/setup-python@v5 with: From f2f8ddd7f7151894941f59efe033ab4dd79a7d25 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 16 Oct 2024 12:57:29 -0800 Subject: [PATCH 5/6] diversify prod its-live-fleet --- .github/workflows/deploy-enterprise.yml | 6 +++--- CHANGELOG.md | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index 4f77da147..ef33c9591 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -25,9 +25,9 @@ jobs: job_files: >- job_spec/AUTORIFT_ITS_LIVE.yml job_spec/S1_CORRECTION_ITS_LIVE.yml - instance_types: r6id.xlarge,r6idn.xlarge,r5dn.xlarge,r5d.xlarge - default_max_vcpus: 2000 # Max: 10,406 - expanded_max_vcpus: 2000 # Max: 10,406 + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 10000 # Max: 10,406 + expanded_max_vcpus: 10000 # Max: 10,406 required_surplus: 0 security_environment: JPL-public ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id diff --git a/CHANGELOG.md b/CHANGELOG.md index d4a85eb17..c546afe62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [7.12.0] + +### Changed +- The `hyp3-its-live` deployment now uses a greater variety of `r6id[n]` instances. + ## [7.11.0] ### Added From c4fdb06decbc7a4c254fc8c2b42cc58841b533ce Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 16 Oct 2024 13:00:54 -0800 Subject: [PATCH 6/6] keep vcpus at 2K for forward processing --- .github/workflows/deploy-enterprise.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index ef33c9591..98bb769a3 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -26,8 +26,8 @@ jobs: job_spec/AUTORIFT_ITS_LIVE.yml job_spec/S1_CORRECTION_ITS_LIVE.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge - default_max_vcpus: 10000 # Max: 10,406 - expanded_max_vcpus: 10000 # Max: 10,406 + default_max_vcpus: 2000 # Max: 10,406 + expanded_max_vcpus: 2000 # Max: 10,406 required_surplus: 0 security_environment: JPL-public ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id