Merge branch 'develop' into remove-subscriptions

.github/actions/deploy-hyp3/action.yml

@@ -68,6 +68,9 @@ inputs:
   DISTRIBUTION_URL:
     description: "CloudFront Distribution URL for Earthdata Cloud environments"
     required: true
+  AUTH_PUBLIC_KEY:
+    description: "Public key for jwt auth provider"
+    required: true
 
 runs:
   using: "composite"
@@ -111,6 +114,7 @@ runs:
                 SecretArn='${{ inputs.SECRET_ARN }}' \
                 ImageTag='${{ inputs.IMAGE_TAG }}' \
                 ProductLifetimeInDays='${{ inputs.PRODUCT_LIFETIME }}' \
+                AuthPublicKey='${{ inputs.AUTH_PUBLIC_KEY }}' \
                 $DOMAIN_NAME \
                 $CERTIFICATE_ARN \
                 $ORIGIN_ACCESS_IDENTITY_ID \

.github/workflows/changelog.yml

@@ -13,6 +13,6 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/create-jira-issue.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   call-create-jira-issue-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}

.github/workflows/deploy-daac.yml

@@ -29,7 +29,7 @@ jobs:
             expanded_max_vcpus: 3000
             required_surplus: 3000
             security_environment: EDC
-            ami_id: image_id_ecs_amz2
+            ami_id: image_id_s1_ecs_amz2
             distribution_url: 'https://d3gm2hf49xd6jj.cloudfront.net'
 
           - environment: hyp3-edc-uat
@@ -52,17 +52,17 @@ jobs:
             expanded_max_vcpus: 3000
             required_surplus: 3000
             security_environment: EDC
-            ami_id: image_id_ecs_amz2
+            ami_id: image_id_s1_ecs_amz2
             distribution_url: 'https://d1riv60tezqha9.cloudfront.net'
 
     environment:
       name: ${{ matrix.environment }}
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
 
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }}
@@ -98,10 +98,11 @@ jobs:
           AMI_ID: ${{ matrix.ami_id }}
           INSTANCE_TYPES: ${{ matrix.instance_types }}
           DISTRIBUTION_URL: ${{ matrix.distribution_url }}
+          AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }}
 
   call-bump-version-workflow:
     if: github.ref == 'refs/heads/main'
     needs: deploy
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/deploy-enterprise-test.yml

@@ -45,9 +45,9 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
 
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }}
@@ -82,3 +82,4 @@ jobs:
           AMI_ID: ${{ matrix.ami_id }}
           INSTANCE_TYPES: ${{ matrix.instance_types }}
           DISTRIBUTION_URL: ${{ matrix.distribution_url }}
+          AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }}

.github/workflows/deploy-enterprise.yml

@@ -39,9 +39,9 @@ jobs:
             product_lifetime_in_days: 180
             quota: 0
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
-            instance_types: c6id.xlarge
-            default_max_vcpus: 1600
-            expanded_max_vcpus: 1600
+            instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
+            default_max_vcpus: 10000
+            expanded_max_vcpus: 10000
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
@@ -51,12 +51,12 @@ jobs:
             domain: hyp3-tibet-jpl.asf.alaska.edu
             template_bucket: cf-templates-1or0efwqffkgd-us-west-2
             image_tag: latest
-            product_lifetime_in_days: 14
+            product_lifetime_in_days: 60
             quota: 0
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 1600
-            expanded_max_vcpus: 1600
+            default_max_vcpus: 0
+            expanded_max_vcpus: 0
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
@@ -69,7 +69,7 @@ jobs:
             product_lifetime_in_days: 14
             quota: 0
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
-            instance_types: c6id.xlarge
+            instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
             default_max_vcpus: 1600
             expanded_max_vcpus: 1600
             required_surplus: 0
@@ -204,9 +204,9 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
 
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: aws-actions/configure-aws-credentials@v3
         with:
           aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }}
@@ -241,3 +241,4 @@ jobs:
           AMI_ID: ${{ matrix.ami_id }}
           INSTANCE_TYPES: ${{ matrix.instance_types }}
           DISTRIBUTION_URL: ${{ matrix.distribution_url }}
+          AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }}

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2

.github/workflows/release-template-comment.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Add a comment with the release template
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
 
       - name: Comment PR
         uses: thollander/actions-comment-pull-request@v2

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     with:
       release_prefix: HyP3
     secrets:

.github/workflows/static-analysis.yml

@@ -6,7 +6,7 @@ jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
@@ -23,7 +23,7 @@ jobs:
       matrix:
         security_environment: [ASF, EDC, JPL, JPL-public]
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
@@ -37,7 +37,7 @@ jobs:
   openapi-spec-validator:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
@@ -50,7 +50,7 @@ jobs:
   statelint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
@@ -70,7 +70,7 @@ jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
       - uses: snyk/actions/[email protected]
       - uses: actions/setup-python@v4
         with:
@@ -87,4 +87,4 @@ jobs:
           snyk iac test --severity-threshold=high
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v3.6.0
 
       - uses: actions/setup-python@v4
         with:

CHANGELOG.md

@@ -4,6 +4,40 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.10.10]
+### Changed
+- Reduced vCPU limits for `hyp3-tibet-jpl` to 0 from 10,000.
+
+## [3.10.9]
+### Changed
+- The public key for the JWT auth provider is now specified as a GitHub Secret. Fixes https://github.com/ASFHyP3/hyp3/issues/1765
+
+## [3.10.8]
+### Changed
+- HyP3 deployments at JPL now use On Demand instances instead of Spot instances to prevent `INSAR_ISCE` jobs from being interrupted.
+  This *should* be a temporary change. 
+
+## [3.10.7]
+### Changed
+- The `INSAR_ISCE_BURST` job type now validates that polarizations and burst ids are the same.
+
+## [3.10.6]
+### Changed
+- Increased vCPU limits for `hyp3-a19-jpl` and `hyp3-tibet-jpl` from 1,600 to 10,000.
+
+## [3.10.5]
+### Changed
+- Updated INSAR_ISCE job specification for [DockerizedTopsApp](https://github.com/ACCESS-Cloud-Based-InSAR/DockerizedTopsApp) v0.2.4
+- Added larger `c6id` instance types to hyp3-a19-jpl and hyp3-nisar-jpl deployments
+
+## [3.10.4]
+### Changed
+- The `hyp3-edc-uat` and `hyp3-edc-prod` deployments now uses the latest Earthdata Cloud AMI with additional software installed.
+
+## [3.10.3]
+### Changed
+- Increased product lifetime for hyp3-tibet-jpl deployment from 14 days to 60 days.
+
 ## [3.10.2]
 ### Deprecated
 - The Subscriptions feature has been deprecated and will be removed as early as `2023-09-05` (September 5, 2023).

apps/api/src/hyp3_api/validation.py

@@ -86,6 +86,24 @@ def check_dem_coverage(job, granule_metadata):
         raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}')
 
 
+def check_same_burst_ids(job, granule_metadata):
+    ref_burst_id, sec_burst_id = [granule['name'].split('_')[1] for granule in granule_metadata]
+    if ref_burst_id != sec_burst_id:
+        raise GranuleValidationError(
+            f'The requested scenes do not have the same burst ID: {ref_burst_id} and {sec_burst_id}'
+        )
+
+
+def check_valid_polarizations(job, granule_metadata):
+    ref_polarization, sec_polarization = [granule['name'].split('_')[4] for granule in granule_metadata]
+    if ref_polarization != sec_polarization:
+        raise GranuleValidationError(
+            f'The requested scenes do not have the same polarization: {ref_polarization} and {sec_polarization}'
+        )
+    if ref_polarization != 'VV' and ref_polarization != 'HH':
+        raise GranuleValidationError(f'Only VV and HH polarizations are currently supported, got: {ref_polarization}')
+
+
 def format_points(point_string):
     converted_to_float = [float(x) for x in point_string.split(' ')]
     points = [list(t) for t in zip(converted_to_float[1::2], converted_to_float[::2])]

apps/compute-cf.yml.j2

@@ -68,8 +68,13 @@ Resources:
       ServiceRole: !GetAtt BatchServiceRole.Arn
       Type: MANAGED
       ComputeResources:
+        {% if security_environment in ('JPL', 'JPL-public') %}
+        Type: EC2
+        AllocationStrategy: BEST_FIT_PROGRESSIVE
+        {% else %}
         Type: SPOT
         AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
+        {% endif %}
         MinvCpus: 0
         MaxvCpus: !Ref MaxvCpus
         InstanceTypes: !Ref InstanceTypes

apps/main-cf.yml.j2

@@ -23,9 +23,8 @@ Parameters:
     Default: 14
 
   AuthPublicKey:
-    Description: Public key for jwt auth provider, if using https://auth.asf.alaska.edu then keep default.
+    Description: Public key for jwt auth provider
     Type: String
-    Default: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCnUEi9E+KNwH2hq0OXR4YyuVPQoNy6CtxQZv3VNXE+TRkmlo7nZbZXQp3JVQAvVaq+ixwNeSvCDI979UGA+SlUpryWy+XFJNAMDlIz9kBVYkTQdT8uFA8IN/fjN1jioOfgpbDJnSaKbb1/NsiTp9cesQQB0qiW5iUsGUfi3Y2x6r446KFt9oJqDzpR2MumKwmHX8tSe1AFUQG+5P4ncoZKS/2EPwfgTuiLW5f/9Duz4vt3peA/wJzBUWd5k9pOz44e2vAbwwLym2APpa7m740Ftyo2lXAZZkejg5MUUza1das5PjN7srt7jJwkBVqd27e1eKHAM4kzAMxcny3SBW2FC6pg6q04NZmPWN6A7y7GzLYJCBuqBwu4OWEzcc4KVkFyTVV+HGUlLX9n2iM4O5RML6+qx+DuK4Ml1kvEBObJ5ce7s4XaNFcg9XmymYiRpZUQmawD1/E5D1l1JNoXNNv8VlJ21c4QYWfmHIhKUF8/dwaRxl6GYd5CrdseEMZtjss88ncmml2cm+7tcpHHJi9Q/wsvAfV7RjFV5JW1PR2UPrATUpJxL+M9dFiGd0KROpqVQ4r6OLAnQQiOrWUquw/38a6JRPW4Y8MH3+JNJMm6MdhB8lFjEKzPic5qBzLJ4yQkOo8K/k0h0K88kT39tRIKVl4Upxvw3iPoVTDGWbkK1w== [email protected]\n
 
   AuthAlgorithm:
     Description: Algorithm for jwt auth provider, if using https://auth.asf.alaska.edu then keep default.