Merge pull request #270 from ASFHyP3/develop

RELEASE: Hyp3 job type templating

.github/workflows/deploy.yml

@@ -9,9 +9,6 @@ on:
 env:
   AWS_REGION: us-west-2
   TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2
-  RTC_GAMMA_REPOSITORY: 626226570674.dkr.ecr.us-east-1.amazonaws.com/hyp3-rtc-gamma
-  INSAR_GAMMA_REPOSITORY: 626226570674.dkr.ecr.us-east-1.amazonaws.com/hyp3-insar-gamma
-  AUTORIFT_REPOSITORY: 626226570674.dkr.ecr.us-east-1.amazonaws.com/hyp3-autorift
 
   VPC_ID: ${{ secrets.VPC_ID }}
   SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
@@ -55,17 +52,18 @@ jobs:
         with:
           python-version: 3.8
 
-      - name: install dependancies
-        working-directory: api
+      - name: install dependencies and render templates
         run: |
           pip install --upgrade pip
-          pip install -r requirements.txt -t src
+          pip install -r requirements-all.txt
+          python apps/render_cf.py --job-types-file job_types.yml
+          pip install -r apps/api/requirements-api.txt -t apps/api/src
           pip install bump2version
 
       - name: Package and deploy
         run: |
           aws cloudformation package \
-            --template-file cloudformation.yml \
+            --template-file apps/main-cf.yml \
             --s3-bucket ${TEMPLATE_BUCKET} \
             --output-template-file packaged.yml
           aws cloudformation deploy \
@@ -78,9 +76,7 @@ jobs:
                 SubnetIds="${SUBNET_IDS}" \
                 EDLUsername="${EDL_USERNAME}" \
                 EDLPassword="${EDL_PASSWORD}" \
-                RtcGammaImage="${RTC_GAMMA_REPOSITORY}:${IMAGE_TAG}" \
-                InsarGammaImage="${INSAR_GAMMA_REPOSITORY}:${IMAGE_TAG}" \
-                AutoriftImage="${AUTORIFT_REPOSITORY}:${IMAGE_TAG}" \
+                ImageTag="${IMAGE_TAG}" \
                 DomainName="${DOMAIN_NAME}" \
                 CertificateArn="${CERTIFICATE_ARN}"
 

.github/workflows/static-analysis.yml

@@ -26,9 +26,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - run: |
+          pip install -r requirements-all.txt
+          python apps/render_cf.py --job-types-file job_types.yml
+
       - uses: scottbrenner/cfn-lint-action@master
         with:
-          args: "--ignore-checks W3002 --template *.yml **/*.yml"
+          args: "--ignore-checks W3002 --template **/*cf.yml"
 
   openapi-spec-validator:
     runs-on: ubuntu-latest
@@ -44,7 +52,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install openapi-spec-validator click prance
 
-      - run: prance validate --backend=openapi-spec-validator api/src/hyp3_api/api-spec/openapi-spec.yml
+      - run: prance validate --backend=openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml
 
 
   statelint:
@@ -55,6 +63,13 @@ jobs:
         with:
           ruby-version: 2.7
       - run: gem install statelint
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - run: |
+          pip install -r requirements-all.txt
+          python apps/render_cf.py --job-types-file job_types.yml
+
       - run: |
-          sed -i 's/"Resource": "${.*}"/"Resource": "foo:bar"/' step-function/step-function.json
-          statelint step-function/step-function.json
+          sed -i 's/"Resource": "${.*}"/"Resource": "foo:bar"/' apps/step-function.json
+          statelint apps/step-function.json

.github/workflows/tests.yml

@@ -3,7 +3,7 @@ name: Run tests
 on: push
 
 jobs:
-  test-api:
+  pytest:
     runs-on: ubuntu-latest
 
     steps:
@@ -15,16 +15,11 @@ jobs:
 
       - run: |
           python -m pip install --upgrade pip
-          pip install -r requirements-test.txt
+          pip install -r requirements-all.txt
 
-      - name: pytest api
+      - name: run pytest
         env:
-          PYTHONPATH: api/src
+          PYTHONPATH: apps/api/src:apps/get-files/src:apps/start-execution/src
         run: |
-          export $(xargs < api/tests/cfg.env)
-          pytest api
-
-      - name: pyest get_files
-        env:
-          PYTHONPATH: step-function/get-files
-        run: pytest step-function/get-files
+          export $(xargs < tests/api/cfg.env)
+          pytest tests

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.1]
+### Changed
+- Hyp3 now uses jinja templates in defining CloudFormation templates and the StepFunction definition, rendered at buildtime.
+- Job types are now defined only in the API spec and the `job_types.yml` file, no job specific information needs to be added to AWS resource definitions.
+- Static Analysis now requires rendering before run.
+
 ## [0.7.0]
 ### Added
 - Added a new `AUTORIFT` job type for processing a pair of Sentinel-1 SLC IW scenes using [autoRIFT](https://github.com/leiyangleon/autoRIFT). For details, refer to the [hyp3-autorift](https://github.com/ASFHyP3/hyp3-autorift) plugin repository.

README.md

@@ -30,19 +30,30 @@ These resources are required for a successful deployment, but managed separately
 - IAM user and roles for automated CloudFormation deployments (if desired)
 
 ### Stack Parameters
-Review the parameters in [cloudformation.yml](cloudformation.yml) for deploy time configuration options.
+Review the parameters in [cloudformation.yml](apps/main-cf.yml) for deploy time configuration options.
 
 ### Deploy with CloudFormation
 
+- Install dependencies for build and run
+```sh
+pip install -r requirements-all.txt
+```
+
+- Render cloudformation templates
+```sh
+python apps/render_cf.py --job-types-file job_types.yml
+```
+
 - Install API dependencies (requires pip for python 3.8)
+
 ```sh
-pip install -r api/requirements.txt -t api/src
+pip install -r apps/api/requirements-api.txt -t apps/api/src
 ```
 
 - Package the CloudFormation template
 ```sh
 aws cloudformation package \
-            --template-file cloudformation.yml \
+            --template-file apps/main-cf.yml \
             --s3-bucket <CloudFormation artifact bucket> \
             --output-template-file packaged.yml
 ```
@@ -69,24 +80,24 @@ aws cloudformation deploy \
 
 
 ## Testing the API
-The HyP3 API source contains test files in `api/tests/`. To run them you need to do a bit of setup first.
+The HyP3 API source contains test files in `tests/api/`. To run them you need to do a bit of setup first.
 
 - Add hyp3-api to python path
 ```sh
-export PYTHONPATH="${PYTHONPATH}:`pwd`/api/src"
+export PYTHONPATH="${PYTHONPATH}:`pwd`/apps/api/src"
 ```
 - Setup environment variables
 ```sh
-export $(cat api/tests/cfg.env | xargs)
+export $(cat tests/api/cfg.env | xargs)
 ```
 - Install test requirements
 ```sh
-pip install -r api/requirements-test.txt
+pip install -r apps/api/requirements-all.txt
 ```
 
 - Run tests
 ```sh
-pytest api/
+pytest tests/api/
 ```
 
 ## Running the API Locally
@@ -100,11 +111,11 @@ The API can be run locally to verify changes, but must be connected to an existi
   - `AUTH_ALGORITHM=HS256`
 - Add hyp3-api to python path
 ```sh
-export PYTHONPATH="${PYTHONPATH}:`pwd`/api/src"
+export PYTHONPATH="${PYTHONPATH}:`pwd`/apps/api/src"
 ```
 - run API
 ```sh
-python3 api/src/hyp3_api/__main__.py
+python3 apps/api/src/hyp3_api/__main__.py
 ```
 - In order to use you will need to include the following cookie
 ```

api/cloudformation.yml → apps/api/api-cf.yml

@@ -129,7 +129,7 @@ Resources:
           MONTHLY_JOB_QUOTA_PER_USER: !Ref MonthlyJobQuotaPerUser
           SYSTEM_AVAILABLE: !Ref SystemAvailable
       Code: src/
-      Handler: lambda_handler.handler
+      Handler: hyp3_api.lambda_handler.handler
       MemorySize: 3008
       Role: !GetAtt LambdaRole.Arn
       Runtime: python3.8

apps/compute-cf.yml

@@ -0,0 +1,134 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+
+  SubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+
+  AmiId:
+    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
+    Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
+
+  ContentBucket:
+    Type: String
+
+Outputs:
+
+  JobQueueArn:
+    Value: !Ref JobQueue
+
+  TaskRoleArn:
+    Value: !GetAtt TaskRole.Arn
+
+Resources:
+
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: !Sub "Security group for ${AWS::StackName} machines"
+      VpcId: !Ref VpcId
+
+  LaunchTemplate:
+    Type: AWS::EC2::LaunchTemplate
+    Properties:
+      LaunchTemplateData:
+        UserData:
+          Fn::Base64: |-
+            Content-Type: multipart/mixed; boundary="==BOUNDARY=="
+            MIME-Version: 1.0
+
+            --==BOUNDARY==
+            Content-Type: text/cloud-boothook; charset="us-ascii"
+
+            cloud-init-per instance mkfs_ssd mkfs.ext4 /dev/nvme1n1
+            cloud-init-per instance mount_ssd mount /dev/nvme1n1 /var/lib/docker
+
+            --==BOUNDARY==--
+
+  ComputeEnvironment:
+    Type: AWS::Batch::ComputeEnvironment
+    Properties:
+      ServiceRole: !GetAtt BatchServiceRole.Arn
+      Type: MANAGED
+      ComputeResources:
+        Type: SPOT
+        AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
+        MinvCpus: 0
+        MaxvCpus: 400
+        InstanceTypes:
+          - r5d.xlarge
+        ImageId: !Ref AmiId
+        Subnets: !Ref SubnetIds
+        InstanceRole: !Ref InstanceProfile
+        SecurityGroupIds:
+          - !Ref SecurityGroup
+        LaunchTemplate:
+          LaunchTemplateId: !Ref LaunchTemplate
+          Version: $Latest
+        Tags:
+          Name: !Ref AWS::StackName
+
+  JobQueue:
+    Type: AWS::Batch::JobQueue
+    Properties:
+      Priority: 1
+      ComputeEnvironmentOrder:
+        - ComputeEnvironment: !Ref ComputeEnvironment
+          Order: 1
+
+  TaskRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: ecs-tasks.amazonaws.com
+          Effect: Allow
+      Policies:
+        - PolicyName: policy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action: s3:PutObject
+                Resource: !Sub "arn:aws:s3:::${ContentBucket}/*"
+              - Effect: Allow
+                Action: s3:PutObjectTagging
+                Resource: !Sub "arn:aws:s3:::${ContentBucket}/*"
+
+  BatchServiceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: batch.amazonaws.com
+          Effect: Allow
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
+
+  InstanceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: ec2.amazonaws.com
+          Effect: Allow
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
+
+  InstanceProfile:
+    Type: AWS::IAM::InstanceProfile
+    Properties:
+      Roles:
+        - !Ref InstanceRole

step-function/get-files/cloudformation.yml → apps/get-files/get-files-cf.yml

@@ -52,8 +52,8 @@ Resources:
         Variables:
           BUCKET: !Ref Bucket
       Code: src/
-      Handler: main.lambda_handler
+      Handler: get_files.lambda_handler
       MemorySize: 128
       Role: !GetAtt Role.Arn
       Runtime: python3.8
-      Timeout: 30
+      Timeout: 30