Dop 4334

.github/workflows/clean-feature-branch.yml

@@ -6,10 +6,10 @@ on:
       - closed
 
 name: Cleanup Feature Branch Infrastructure
-jobs: 
+jobs:
   destroy:
     runs-on: ubuntu-latest
-    
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
@@ -25,11 +25,11 @@ jobs:
         run: |
           cd cdk-infra/
           npm ci
-          npm run cdk destroy -- -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --force \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks
+          npm run cdk destroy -- -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} --force \
+            auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker \
+            auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-queues \
+            auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks
       - name: Delete cache
         run: gh cache delete ${{github.head_ref}}
-        env: 
+        env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/deploy-feature-branch.yml

@@ -21,23 +21,30 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-2
-      - name: Deploy Feature Branch Infrastructure
+      - name: Install node_modules
         run: |
           npm ci
           cd cdk-infra/
           npm ci
-          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues
-          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker
-          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks --outputs-file outputs.json
-      - name: Get Webhook URL
-        uses: mongodb/docs-worker-actions/comment-pr@main
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Cache root node_modules
         id: cache-root
-        uses: actions/cache@v3
+        uses: actions/cache/save@v4
         with:
           path: |
             node_modules
             cdk-infra/node_modules
           key: ${{ github.head_ref }}
+
+      - name: Deploy Feature Branch Infrastructure
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
+          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-queues
+          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker
+          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks --outputs-file outputs.json
+      - name: Get Webhook URL
+        uses: mongodb/docs-worker-actions/comment-pr@DOP-4334
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          STAGE: dotcomstg

.github/workflows/deploy-stg-ecs.yml

@@ -1,8 +1,8 @@
 on:
   push:
     branches:
-      - "main"
-      - "integration"
+      - 'main'
+      - 'integration'
 concurrency:
   group: environment-stg-${{ github.ref }}
   cancel-in-progress: true

.github/workflows/update-feature-branch.yml

@@ -6,7 +6,7 @@ on:
       - synchronize
 
 concurrency:
-  group: environment-stg-feature-${{ github.ref }}
+  group: environment-dotcomstg-feature-${{ github.ref }}
   cancel-in-progress: true
 name: Update Feature Branch Infrastructure
 jobs:
@@ -34,7 +34,7 @@ jobs:
           npm ci
       - name: Cache root node_modules
         id: cache-root
-        uses: actions/cache@v3
+        uses: actions/cache/save@v4
         with:
           path: |
             node_modules
@@ -54,7 +54,7 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: '18.x'
-      - uses: actions/cache/restore@v3
+      - uses: actions/cache/restore@v4
         id: cache-restore
         with:
           path: |
@@ -73,12 +73,16 @@ jobs:
         if: steps.filter.outputs.webhooks == 'true'
         run: |
           cd cdk-infra/
-          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks
+          npm run deploy:feature:stack -- --hotswap -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \
+            auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks
   build-worker:
     needs: prep-build
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
       - uses: actions/checkout@v4
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -105,9 +109,21 @@ jobs:
               - 'cdk-infra/lib/constructs/worker/**'
               - 'Dockerfile'
               - 'modules/**'
+      # - uses: docker/build-push-action@v5
+      #   if: steps.filter.outputs.worker == 'true'
+      #   name: build Docker image
+      #   with:
+      #     context: .
+      #     tags: autobuilder/enhanced:latest
+      #     build-args: |
+      #       NPM_EMAIL=${{ secrets.NPM_EMAIL }}
+      #       NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }}
+      #     cache-from: type=gha,ref=autobuilder/enhanced:latest
+      #     cache-to: type=gha,mode=max
+      #     outputs: type=tar,dest=cdk-infra/image.tar.gz
       - name: Update Worker Stack
         if: steps.filter.outputs.worker == 'true'
         run: |
           cd cdk-infra/
-          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker
+          npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \
+            auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker

.gitignore

@@ -12,4 +12,6 @@ out
 local.json
 globalConfig.json
 .serverless
-dist
+dist
+onDemandApp.js
+*.js.map

.vscode/launch.json

@@ -9,7 +9,6 @@
       "type": "node",
       "request": "attach",
       "port": 9229,
-      "address": "localhost",
       "localRoot": "${workspaceFolder}",
       "remoteRoot": "/home/docsworker-xlarge",
       "autoAttachChildProcesses": false,

Dockerfile

@@ -1,41 +1,23 @@
-# Build the Typescript app
-FROM node:18.16.0-alpine as ts-compiler
-WORKDIR /home/docsworker-xlarge
-COPY  config config/
-COPY package*.json ./
-COPY tsconfig*.json ./
-RUN npm ci --legacy-peer-deps
-COPY . ./
-RUN npm run build
-
-# install persistence module
-RUN cd ./modules/persistence \
-    && npm ci --legacy-peer-deps \
-    && npm run build
-
-# Build modules
-# OAS Page Builder
-RUN cd ./modules/oas-page-builder \
-    && npm ci --legacy-peer-deps \
-    && npm run build
-
 # where repo work will happen
-FROM ubuntu:20.04
-ARG WORK_DIRECTORY=/home/docsworker-xlarge
+FROM ubuntu:20.04 as initial
+
 ARG SNOOTY_PARSER_VERSION=0.16.1
 ARG SNOOTY_FRONTEND_VERSION=0.16.3
 ARG MUT_VERSION=0.11.1
 ARG REDOC_CLI_VERSION=1.2.3
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL
+ARG WORK_DIRECTORY=/home/docsworker-xlarge
+
+ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH}
+ENV NPM_EMAIL=${NPM_EMAIL}
+ENV DOCKER_BUILDKIT=1
 ENV DEBIAN_FRONTEND=noninteractive
 
+WORKDIR ${WORK_DIRECTORY}
 
 # helper libraries for docs builds
-RUN apt-get update && apt-get install -y vim git unzip rsync curl
-
-
-ENV PATH="${PATH}:/opt/snooty:/opt/mut:/home/docsworker-xlarge/.local/bin"
+RUN apt-get update && apt-get install -y vim git unzip zip rsync time
 
 # get node 18
 # https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab
@@ -44,9 +26,6 @@ RUN curl --location https://deb.nodesource.com/setup_18.x | bash -
 RUN apt-get install --yes nodejs
 RUN apt-get install --yes build-essential
 
-# use npm 8.*
-RUN npm install -g npm@8
-
 # install snooty parser
 RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \
     && unzip -d /opt/ snooty-parser.zip
@@ -55,56 +34,81 @@ RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releas
 RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \
     && unzip -d /opt/ mut.zip
 
+
+ENV PATH="${PATH}:/opt/snooty:/opt/mut:/${WORK_DIRECTORY}/.local/bin"
+
 # setup user and root directory
 RUN useradd -ms /bin/bash docsworker-xlarge
 RUN chmod 755 -R ${WORK_DIRECTORY}
 RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY}
 USER docsworker-xlarge
 
-WORKDIR ${WORK_DIRECTORY}
-
-# get shared.mk
-RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
-
 # install snooty frontend and docs-tools
 RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \
     && cd snooty                                                                                   \
-    && npm ci --legacy-peer-deps --omit=dev                                                        \
-    && git clone --depth 1 https://github.com/mongodb/docs-tools.git                               \
-    && mkdir -p ./static/images                                                                    \
-    && mv ./docs-tools/themes/mongodb/static ./static/docs-tools                                   \
-    && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg
+    && npm ci --legacy-peer-deps --omit=dev                                                        
+
+RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
+
 
-# install redoc fork
 RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \
     # Install dependencies for Redoc CLI
     && cd redoc/ \
     && npm ci --prefix cli/ --omit=dev
 
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/package*.json ./
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/config config/
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/build ./
-RUN npm install
+FROM initial as persistence
 
-# Persistence module copy
-# Create directory and add permissions to allow node module installation
 RUN mkdir -p modules/persistence && chmod 755 modules/persistence
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/package*.json ./modules/persistence/
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/dist ./modules/persistence/
-ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js
-RUN cd ./modules/persistence/ && ls && npm ci --legacy-peer-deps
+COPY modules/persistence/package*.json ./modules/persistence/
+RUN cd ./modules/persistence \
+    && npm ci --legacy-peer-deps 
+# Build persistence module
 
-# OAS Page Builder module copy
-# Create directory and add permissions to allow node module installation
-RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/
-COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/
-RUN cd ./modules/oas-page-builder/ && npm ci --legacy-peer-deps
+COPY --chown=docsworker-xlarge modules/persistence/tsconfig*.json ./modules/persistence
+COPY --chown=docsworker-xlarge modules/persistence/src ./modules/persistence/src/
+COPY --chown=docsworker-xlarge modules/persistence/index.ts ./modules/persistence
 
-# Needed for OAS Page Builder module in shared.mk
-ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js
+RUN cd ./modules/persistence \
+    && npm run build:esbuild
+
+FROM initial as oas
+
+RUN  mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder
+COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/
+RUN cd ./modules/oas-page-builder \
+    && npm ci --legacy-peer-deps 
+# Build modules
+# OAS Page Builder
+COPY --chown=docsworker-xlarge modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder
+COPY --chown=docsworker-xlarge modules/oas-page-builder/src ./modules/oas-page-builder/src/
+COPY --chown=docsworker-xlarge modules/oas-page-builder/index.ts ./modules/oas-page-builder
+
+RUN cd ./modules/oas-page-builder \
+    && npm run build:esbuild
+
+FROM initial as root
+
+COPY --from=persistence --chown=docsworker-xlarge ${WORK_DIRECTORY}/modules/persistence/dist/ ./modules/persistence
+COPY --from=oas --chown=docsworker-xlarge ${WORK_DIRECTORY}/modules/oas-page-builder/dist/ ./modules/oas-page-builder
+
+# Root project build
+COPY package*.json ./
+RUN npm ci --legacy-peer-deps
+
+
+COPY tsconfig*.json ./
+COPY config config/
+COPY api api/
+COPY src src/
+
+RUN npm run build:esbuild
+
+ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js
 ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js
+ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js
 
+RUN mkdir -p modules/persistence && chmod 755 modules/persistence
 RUN mkdir repos && chmod 755 repos
+
 EXPOSE 3000
 CMD ["node", "app.js"]

Dockerfile.local

@@ -5,7 +5,6 @@ ARG SNOOTY_PARSER_VERSION=0.15.2
 ARG SNOOTY_FRONTEND_VERSION=0.15.7
 ARG MUT_VERSION=0.10.7
 ARG REDOC_CLI_VERSION=1.2.3
-ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
 
@@ -21,7 +20,7 @@ RUN apt-get install --yes curl
 RUN curl --location https://deb.nodesource.com/setup_18.x | bash -
 RUN apt-get install --yes nodejs
 RUN apt-get install --yes build-essential
-RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config
+RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config time
 
 
 RUN python3 -m pip install poetry
@@ -61,7 +60,7 @@ RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.co
     # Install dependencies for Redoc CLI
     && cd redoc/ \
     && npm ci --prefix cli/ --omit=dev
-    
+
 FROM initial as persistence
 
 RUN mkdir -p modules/persistence && chmod 755 modules/persistence
@@ -118,4 +117,4 @@ RUN mkdir repos && chmod 755 repos
 
 EXPOSE 3000
 
-CMD ["node", "--inspect-brk=0.0.0.0", "--enable-source-maps", "dist/entrypoints/onDemandApp.js"]
+CMD ["node", "--inspect-brk=0.0.0.0", "--enable-source-maps", "onDemandApp.js"]

api/controllers/v2/github.ts

@@ -42,7 +42,7 @@ async function prepGithubPushPayload(
     error: {},
     result: null,
     payload: {
-      jobType: 'githubPush',
+      jobType: 'productionDeploy',
       source: 'github',
       action: 'push',
       repoName: githubEvent.repository.name,