[DOP-4170]: Create AWS ECS Task definition to process Snooty Parser c…

…ache updates (#951)

* [DOP-4170]: Create lambda and Dockerfile for custom environment

* [DOP-4170]: Create stack to deploy cache updater

* [DOP-4170]: Update lambda dockerfile

* [DOP-4170]: Update workflow

* [DOP-4170]: Fix typo

* [DOP-4170]: Increase timeout

* [DOP-4170]: Use arm 💪

* [DOP-4170]: Use arm 💪

* [DOP-4170]: Use arm64 💪

* [DOP-4170]: Use arm64 💪

* [DOP-4170]: Format names correctly

* [DOP-4170]: Revert arch

* [DOP-4170]: Log event

* [DOP-4170]: Add git

* [DOP-4170]: Clone repo and run cache build

* [DOP-4170]: Refactor pipeline a bit

* [DOP-4170]: Add credentials

* [DOP-4170]: Refactor each step into a separate function for simplicity

* [DOP-4170]: Refactor each step into a separate function for simplicity

* [DOP-4170]: Clone to tmp directory

* [DOP-4170]: Update dependencies

* [DOP-4170]: Add S3 and uploadCache starting code

* [DOP-4170]: Small changes

* [DOP-4170]: Add s3 upload library

* [DOP-4170]: Add cache upload

* [DOP-4170]: Add try catch

* [DOP-4170]: Update snooty parser version

* [DOP-4170]: Reorder no caching flag

* [DOP-4170]: Use latest snooty parser version

* [DOP-4170]: Refactor caching

* [DOP-4170]: Use debugging snooty branch

* [DOP-4170]: Use worker instead of lambda

* [DOP-4170]: Add fargate task def and custer

* [DOP-4170]: Refactor cache updater to be used for ecs task

* [DOP-4170]: Clean up

* [DOP-4170]: Clean up

* [DOP-4170]: Move cache updater to src directory and add logic to build caches for multiple sites

* [DOP-4170]: Add readme and refactor cdk constructs

* [DOP-4170]: Add VPC for cache updater stack

* [DOP-4170]: Üse correct path

* [DOP-4170]: Üse correct path

* [DOP-4170]: Update cpu architecture

* [DOP-4170]: Add QEMU

* [DOP-4170]: Add Snooty parser version

* [DOP-4170]: Don't use arm

* [DOP-4170]: use buildkit action

* [DOP-4170]: Remove buildkit setup

* [DOP-4170]: don't use slim

* [DOP-4170]: Add log group and update workflow conditions

* [DOP-4170]: Add echo statements

* [DOP-4170]: Add echo statements

* [DOP-4170]: Add echo statements

* [DOP-4170]: Update filter to include base

* [DOP-4170]: Update filter to only look at previous commit

* [DOP-4170]: Revert and figure out later

* [DOP-4170]: Grant task permission to write to S3

* [DOP-4170]: Add readmes

* [DOP-4170]: Add throw statements

* [DOP-4170]: PR feedback

* [DOP-4170]: Undo to check if build passes now

* [DOP-4170]: Add import back in

* [DOP-4170]: Remove cross project imports

* [DOP-4170]: Update environment

* [DOP-4170]: Remove api from dockerfile

* [DOP-4170]: Remove unused imports

.github/workflows/README.md

@@ -0,0 +1,15 @@
+# Docs Worker Pool Workflows
+
+This README describes the various workflows defined for the docs-worker-pool repository.
+
+## Feature Branch Deploys
+
+The feature branch deploy process occurs whenever a developer opens a pull request. It consists of three separate workflows:
+
+1. `deploy-feature-branch.yml` - Creates the initial infrastructure when a PR is opened, this includes draft PRs
+2. `update-feature-branch.yml` - Ran whenever a commit is made to the branch for the PR, and conditionally deploys each stack depending on changes made
+3. `clean-feature-branch.yml` - Ran whenever a PR is merged or closed; deletes all of the infrastructure for the feature branch
+
+### Bugs
+
+Right now, there is a small bug with the `update-feature-branch.yml` workflow. This workflow conditionally deploys the various stacks depending on what files have changed from a commit. The issue is that the custom filter action compares the PR branch to master for every workflow run. This means that if you make a change to `src/app.ts` in the first commit, but only make changes to files in the `api/` directory in subsequent commits, it will still run the deploy for the worker.

.github/workflows/update-feature-branch.yml

@@ -10,22 +10,57 @@ concurrency:
   cancel-in-progress: true
 name: Update Feature Branch Infrastructure
 jobs: 
-  deploy:
+  prep-build:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: '18.x'
+          cache: 'npm'
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            dependencies:
+              - 'package-lock.json'
+              - 'cdk-infra/package-lock.json'
+      - name: Install dependencies
+        if: steps.filter.outputs.dependencies == 'true' || (github.event_name == 'pull_request' && github.event.action == 'opened')
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
+      - name: Cache root node_modules
+        id: cache-root
+        uses: actions/cache@v3
+        with:
+          path: |
+            node_modules
+            cdk-infra/node_modules
+          key: ${{ github.head_ref }}
+  build-webhooks:
+    needs: prep-build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-2
-
-      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '18.x'
+      - uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: |
+            node_modules
+            cdk-infra/node_modules
+          key: ${{ github.head_ref }}
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -34,24 +69,82 @@ jobs:
               - 'api/**'
               - 'cdk-infra/lib/constructs/api/**'
               - 'cdk-infra/utils/**'
-            worker:
-              - 'src/**'
-              - 'cdk-infra/lib/constructs/worker/**'
-              - 'Dockerfile.enhanced'
-              - 'modules/**'
       - name: Update Webhook Stack
         if: steps.filter.outputs.webhooks == 'true'
         run: |
-          npm ci
           cd cdk-infra/
-          npm ci
           npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
             auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks
+  build-worker:
+    needs: prep-build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '18.x'
+      - uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: |
+            node_modules
+            cdk-infra/node_modules
+          key: ${{ github.head_ref }}
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            worker:
+              - 'src/!(cache-updater/**)/**'
+              - 'cdk-infra/lib/constructs/worker/**'
+              - 'Dockerfile.enhanced'
+              - 'modules/**'
       - name: Update Worker Stack
         if: steps.filter.outputs.worker == 'true'
         run: |
-          npm ci
           cd cdk-infra/
-          npm ci
           npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
             auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker
+
+
+  build-cache-updater:
+    needs: prep-build
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            cache:
+              - 'src/cache-updater/**'
+              - 'cdk-infra/lib/constructs/cache-updater/**'
+      - uses: actions/checkout@v4
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '18.x'
+      - uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: |
+            node_modules
+            cdk-infra/node_modules
+          key: ${{ github.head_ref }}
+      - name: Update Cache Updater
+        if: steps.filter.outputs.cache == 'true'
+        run: |
+          cd cdk-infra/
+          npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
+            cache-updater

cdk-infra/bin/cdk-infra.ts

@@ -7,6 +7,7 @@ import { AutoBuilderQueueStack } from '../lib/stacks/auto-builder-queue-stack';
 import { WorkerStack } from '../lib/stacks/worker-stack';
 import { WebhookStack } from '../lib/stacks/webhook-stack';
 import { AutoBuilderVpcStack } from '../lib/stacks/auto-builder-vpc-stack';
+import { CacheUpdaterStack } from '../lib/stacks/cache-updater-stack';
 
 async function main() {
   const app = new cdk.App();
@@ -44,6 +45,8 @@ async function main() {
     webhookSecureStrings,
     env,
   });
+
+  new CacheUpdaterStack(app, 'cache-updater', { vpc });
 }
 
 main();

cdk-infra/lib/constructs/cache-updater/cache-updater-worker-construct.ts

@@ -0,0 +1,58 @@
+import { IVpc } from 'aws-cdk-lib/aws-ec2';
+import { Cluster, ContainerImage, FargateTaskDefinition, LogDrivers } from 'aws-cdk-lib/aws-ecs';
+import { Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
+import { LogGroup } from 'aws-cdk-lib/aws-logs';
+import { Bucket } from 'aws-cdk-lib/aws-s3';
+import { Construct } from 'constructs';
+import path from 'path';
+
+const SNOOTY_CACHE_BUCKET_NAME = 'snooty-parse-cache';
+
+interface CacheUpdaterWorkerConstructProps {
+  vpc: IVpc;
+}
+
+export class CacheUpdaterWorkerConstruct extends Construct {
+  clusterName: string;
+
+  constructor(scope: Construct, id: string, { vpc }: CacheUpdaterWorkerConstructProps) {
+    super(scope, id);
+
+    const cluster = new Cluster(this, 'cacheUpdaterCluster', {
+      vpc,
+    });
+
+    const taskRole = new Role(this, 'cacheUpdateWorkerTaskRole', {
+      assumedBy: new ServicePrincipal('ecs-tasks.amazonaws.com'),
+    });
+
+    const snootyParseCacheBucket = Bucket.fromBucketName(this, SNOOTY_CACHE_BUCKET_NAME, SNOOTY_CACHE_BUCKET_NAME);
+
+    snootyParseCacheBucket.grantWrite(taskRole);
+
+    const taskDefinition = new FargateTaskDefinition(this, 'cacheUpdaterWorker', {
+      cpu: 2048,
+      memoryLimitMiB: 4096,
+      taskRole,
+    });
+
+    const taskDefLogGroup = new LogGroup(this, 'cacheUpdaterWorkerLogGroup');
+
+    taskDefinition.addContainer('cacheUpdaterWorkerImage', {
+      image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), {
+        file: 'src/cache-updater/Dockerfile.cacheUpdater',
+        buildArgs: { SNOOTY_PARSER_VERSION: '0.15.2' },
+        exclude: ['tests/', 'node_modules/', 'cdk-infra/'], // adding this just in case it doesn't pick up our dockerignore
+      }),
+      environment: {
+        SNOOTY_CACHE_BUCKET_NAME,
+      },
+      logging: LogDrivers.awsLogs({
+        streamPrefix: 'cacheupdater',
+        logGroup: taskDefLogGroup,
+      }),
+    });
+
+    this.clusterName = cluster.clusterName;
+  }
+}

cdk-infra/lib/stacks/cache-updater-stack.ts

@@ -0,0 +1,15 @@
+import { Stack } from 'aws-cdk-lib';
+import { Construct } from 'constructs';
+import { CacheUpdaterWorkerConstruct } from '../constructs/cache-updater/cache-updater-worker-construct';
+import { IVpc } from 'aws-cdk-lib/aws-ec2';
+
+interface CacheUpdaterStackProps {
+  vpc: IVpc;
+}
+export class CacheUpdaterStack extends Stack {
+  constructor(scope: Construct, id: string, { vpc }: CacheUpdaterStackProps) {
+    super(scope, id);
+
+    new CacheUpdaterWorkerConstruct(this, 'cache-updater-resources', { vpc });
+  }
+}