From d563250105872ffba0537d99493657fa76adcd1b Mon Sep 17 00:00:00 2001 From: Brandon Ly Date: Tue, 6 Feb 2024 11:24:32 -0600 Subject: [PATCH 1/6] Hanging build hotfix (#983) * [HOTFIX]: Use isFeatureBranch context variable * [HOTFIX]: Add IS_FEATURE_BRANCH * [HOTFIX]: Don't use gatsby cloud webhook call --- .github/workflows/deploy-feature-branch.yml | 14 +++++++------- .github/workflows/update-feature-branch.yml | 10 +++++----- .../lib/constructs/worker/worker-env-construct.ts | 12 ++++++++++-- cdk-infra/utils/env.ts | 13 +++++++++++++ src/job/jobHandler.ts | 14 ++++++++++++-- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-feature-branch.yml b/.github/workflows/deploy-feature-branch.yml index 2404ac9ae..57e992671 100644 --- a/.github/workflows/deploy-feature-branch.yml +++ b/.github/workflows/deploy-feature-branch.yml @@ -1,12 +1,12 @@ on: pull_request: branches: - - main + - main types: - opened - reopened name: Initial Feature Branch Deploy -jobs: +jobs: deploy: permissions: write-all runs-on: ubuntu-latest @@ -26,12 +26,12 @@ jobs: npm ci cd cdk-infra/ npm ci - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks --outputs-file outputs.json - - name: Get Webhook URL + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks --outputs-file outputs.json + - name: Get Webhook URL uses: mongodb/docs-worker-actions/comment-pr@main - env: + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Cache root node_modules id: cache-root diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 151a8faed..a7a6f534e 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -9,10 +9,10 @@ concurrency: group: environment-stg-feature-${{ github.ref }} cancel-in-progress: true name: Update Feature Branch Infrastructure -jobs: +jobs: prep-build: runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -73,7 +73,7 @@ jobs: if: steps.filter.outputs.webhooks == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks build-worker: needs: prep-build @@ -109,5 +109,5 @@ jobs: if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker \ No newline at end of file + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index 48a7589c8..1b532a16c 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -2,7 +2,14 @@ import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; import { Construct } from 'constructs'; import { getCdnInvalidatorUrl } from '../../../utils/cdn'; -import { getEnv, envShortToFullName, getIsEnhanced, getUseCustomBuckets, getFeatureName } from '../../../utils/env'; +import { + getEnv, + envShortToFullName, + getIsEnhanced, + getUseCustomBuckets, + getFeatureName, + getIsFeatureBranch, +} from '../../../utils/env'; import { getSearchIndexFolder } from '../../../utils/search-index'; import { getSsmPathPrefix } from '../../../utils/ssm'; @@ -90,11 +97,12 @@ export class WorkerEnvConstruct extends Construct { SEARCH_INDEX_FOLDER: getSearchIndexFolder(env), ENHANCED: `${getIsEnhanced()}`, USE_CUSTOM_BUCKETS: `${getUseCustomBuckets()}`, - FEATURE_NAME: `${getFeatureName()}`, + FEATURE_NAME: getFeatureName(), GATSBY_TEST_SEARCH_UI: 'false', GATSBY_SHOW_CHATBOT: gatsbyUseChatbot, GATSBY_HIDE_UNIFIED_FOOTER_LOCALE: gatsbyHideUnifiedFooterLocale, GATSBY_MARIAN_URL: gatsbyMarianUrl, + IS_FEATURE_BRANCH: getIsFeatureBranch(), }; } } diff --git a/cdk-infra/utils/env.ts b/cdk-infra/utils/env.ts index 9979090fb..7b8969f49 100644 --- a/cdk-infra/utils/env.ts +++ b/cdk-infra/utils/env.ts @@ -92,6 +92,19 @@ export function getSnootyParserVersion(): string { return snootyParserVersion; } + +export function getIsFeatureBranch() { + checkContextInit(); + const isFeatureBranch = contextVarsMap.get('isFeatureBranch'); + + if (!isFeatureBranch) { + console.warn('Error! The context variable isFeatureBranch is not defined.'); + return ''; + } + + return isFeatureBranch; +} + export function getFeatureName(): string { checkContextInit(); diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 73a50a1c3..e9cb47d92 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -114,7 +114,12 @@ export abstract class JobHandler { // completed after the Gatsby Cloud build via the SnootyBuildComplete lambda. const { _id: jobId, user } = this.currJob; const gatsbyCloudSiteId = await this._repoEntitlementsRepo.getGatsbySiteIdByGithubUsername(user); - if (this.currJob.payload.isNextGen && gatsbyCloudSiteId && this.currJob.payload.jobType === 'githubPush') { + if ( + this.currJob.payload.isNextGen && + gatsbyCloudSiteId && + this.currJob.payload.jobType === 'githubPush' && + process.env.IS_FEATURE_BRANCH !== 'true' + ) { this.logger.info( jobId, `User ${user} has a Gatsby Cloud site. The Autobuilder will not mark the build as completed right now.` @@ -359,7 +364,12 @@ export abstract class JobHandler { // Call Gatsby Cloud preview webhook after persistence module finishes for staging builds const isFeaturePreviewWebhookEnabled = process.env.GATSBY_CLOUD_PREVIEW_WEBHOOK_ENABLED?.toLowerCase() === 'true'; - if (key === 'persistence-module' && this.name === 'Staging' && isFeaturePreviewWebhookEnabled) { + if ( + key === 'persistence-module' && + this.name === 'Staging' && + isFeaturePreviewWebhookEnabled && + process.env.IS_FEATURE_BRANCH !== 'true' + ) { await this.callGatsbyCloudWebhook(); } } From 8215c5478ae3a0f9af7a283d172d1c34956c1c2c Mon Sep 17 00:00:00 2001 From: Brandon Ly Date: Wed, 7 Feb 2024 09:23:51 -0600 Subject: [PATCH 2/6] Remove Dockerfile.enhanced (#978) * Add command to task def container * Remove dockerfile.enhanced * [DOP-4334]: Use npm ci * Remove references to dockerfile.enhanced * Reference main --- .../workflows/deploy-prd-enhanced-cache.yml | 6 +- .../workflows/deploy-stg-enhanced-worker.yml | 3 +- .github/workflows/update-feature-branch.yml | 2 +- Dockerfile | 10 +- Dockerfile.enhanced | 120 ------------------ cdk-infra/README.md | 2 +- .../lib/constructs/worker/worker-construct.ts | 2 +- 7 files changed, 12 insertions(+), 133 deletions(-) delete mode 100644 Dockerfile.enhanced diff --git a/.github/workflows/deploy-prd-enhanced-cache.yml b/.github/workflows/deploy-prd-enhanced-cache.yml index 2990a4d32..15390b971 100644 --- a/.github/workflows/deploy-prd-enhanced-cache.yml +++ b/.github/workflows/deploy-prd-enhanced-cache.yml @@ -20,7 +20,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - name: Rebuild Cache if New Snooty Parser Version - uses: mongodb/docs-worker-actions/rebuild-parse-cache@DOP-4294 - env: + uses: mongodb/docs-worker-actions/rebuild-parse-cache@main + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WORKSPACE: ${{ github.workspace }} \ No newline at end of file + WORKSPACE: ${{ github.workspace }} diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 4f3fb2d24..c779ff49b 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -1,6 +1,6 @@ on: push: - paths: ["src/**", "cdk-infra/lib/constructs/worker/**", "Dockerfile.enhanced", "modules/**"] + paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] branches: - 'main' - 'integration' @@ -29,4 +29,3 @@ jobs: npm ci npm run deploy:feature:stack -- -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg auto-builder-stack-enhancedApp-dotcomstg-worker npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg auto-builder-stack-enhancedApp-stg-worker - diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index a7a6f534e..b896e751d 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -103,7 +103,7 @@ jobs: worker: - 'src/!(cache-updater/**)/**' - 'cdk-infra/lib/constructs/worker/**' - - 'Dockerfile.enhanced' + - 'Dockerfile' - 'modules/**' - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' diff --git a/Dockerfile b/Dockerfile index f89fd3ebd..01da8ea20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,19 +4,19 @@ WORKDIR /home/docsworker-xlarge COPY config config/ COPY package*.json ./ COPY tsconfig*.json ./ -RUN npm install +RUN npm ci --legacy-peer-deps COPY . ./ RUN npm run build # install persistence module RUN cd ./modules/persistence \ - && npm install \ + && npm ci --legacy-peer-deps \ && npm run build # Build modules # OAS Page Builder RUN cd ./modules/oas-page-builder \ - && npm install \ + && npm ci --legacy-peer-deps \ && npm run build # where repo work will happen @@ -102,14 +102,14 @@ RUN mkdir -p modules/persistence && chmod 755 modules/persistence COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/package*.json ./modules/persistence/ COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/dist ./modules/persistence/ ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js -RUN cd ./modules/persistence/ && ls && npm install +RUN cd ./modules/persistence/ && ls && npm ci --legacy-peer-deps # OAS Page Builder module copy # Create directory and add permissions to allow node module installation RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/ COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/ -RUN cd ./modules/oas-page-builder/ && npm install +RUN cd ./modules/oas-page-builder/ && npm ci --legacy-peer-deps # Needed for OAS Page Builder module in shared.mk ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js diff --git a/Dockerfile.enhanced b/Dockerfile.enhanced deleted file mode 100644 index a01bf9a94..000000000 --- a/Dockerfile.enhanced +++ /dev/null @@ -1,120 +0,0 @@ -# Build the Typescript app -FROM node:18.16.0-alpine as ts-compiler -WORKDIR /home/docsworker-xlarge -COPY config config/ -COPY package*.json ./ -COPY tsconfig*.json ./ -RUN npm ci --legacy-peer-deps -COPY . ./ -RUN npm run build - -# install persistence module -RUN cd ./modules/persistence \ - && npm ci --legacy-peer-deps \ - && npm run build - -# Build modules -# OAS Page Builder -RUN cd ./modules/oas-page-builder \ - && npm ci --legacy-peer-deps \ - && npm run build - -# where repo work will happen -FROM ubuntu:20.04 -ARG WORK_DIRECTORY=/home/docsworker-xlarge -ARG SNOOTY_PARSER_VERSION=0.15.2 -ARG SNOOTY_FRONTEND_VERSION=0.15.7 -ARG MUT_VERSION=0.10.7 -ARG REDOC_CLI_VERSION=1.2.3 -ARG NPM_BASE_64_AUTH -ARG NPM_EMAIL -ENV DEBIAN_FRONTEND=noninteractive - -# install legacy build environment for docs -RUN apt-get -o Acquire::Check-Valid-Until=false update -RUN apt-get -y install libpython2.7-dev python2.7 git rsync unzip curl -RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py -RUN python2.7 get-pip.py -RUN pip install requests virtualenv virtualenvwrapper py-dateutil -RUN python2.7 -m pip install python-dateutil -RUN virtualenv /venv -RUN /venv/bin/pip install --upgrade --force setuptools -RUN /venv/bin/pip install -r https://raw.githubusercontent.com/mongodb/docs-tools/master/giza/requirements.txt - -# helper libraries for docs builds -RUN apt-get update && apt-get install -y vim git - - -ENV PATH="${PATH}:/opt/snooty:/opt/mut:/home/docsworker-xlarge/.local/bin:/usr/local/lib/python2.7/dist-packages/virtualenv/bin" - -# get node 18 -# https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab -RUN apt-get install --yes curl -RUN curl --location https://deb.nodesource.com/setup_18.x | bash - -RUN apt-get install --yes nodejs -RUN apt-get install --yes build-essential - -# use npm 8.* -RUN npm install -g npm@8 - -# install snooty parser -RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ snooty-parser.zip - -# install mut -RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ mut.zip - -# setup user and root directory -RUN useradd -ms /bin/bash docsworker-xlarge -RUN chmod 755 -R ${WORK_DIRECTORY} -RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY} -USER docsworker-xlarge - -WORKDIR ${WORK_DIRECTORY} - -# get shared.mk -RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk - -# install snooty frontend and docs-tools -RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ - && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev \ - && git clone --depth 1 https://github.com/mongodb/docs-tools.git \ - && mkdir -p ./static/images \ - && mv ./docs-tools/themes/mongodb/static ./static/docs-tools \ - && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg - -# install redoc fork -RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \ - # Install dependencies for Redoc CLI - && cd redoc/ \ - && npm ci --prefix cli/ --omit=dev - -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/package*.json ./ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/config config/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/build ./ -RUN npm install - -# Persistence module copy -# Create directory and add permissions to allow node module installation -RUN mkdir -p modules/persistence && chmod 755 modules/persistence -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/package*.json ./modules/persistence/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/dist ./modules/persistence/ -ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js -RUN cd ./modules/persistence/ && ls && npm ci --legacy-peer-deps - -# OAS Page Builder module copy -# Create directory and add permissions to allow node module installation -RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/ -RUN cd ./modules/oas-page-builder/ && npm ci --legacy-peer-deps - -# Needed for OAS Page Builder module in shared.mk -ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js -ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js - -RUN mkdir repos && chmod 755 repos -EXPOSE 3000 -CMD ["node", "--enable-source-maps", "enhanced/enhancedApp.js"] diff --git a/cdk-infra/README.md b/cdk-infra/README.md index 8ab410c1c..30918f438 100644 --- a/cdk-infra/README.md +++ b/cdk-infra/README.md @@ -71,7 +71,7 @@ To verify the CloudFormation is being generated successfully, you can use the `c npm run cdk synth -- -c enhanced=true -c customFeatureName=enhancedApp > cdk.out/template.yaml ``` -Make sure to update your `~/.aws/credentials` file. The `enhanced` context variable, if set to true, will use the `Dockerfile.enhanced` dockerfile instead of the standard one. The `featureName` context variable is used to provide a different name for a custom stack other than the branch name. In the future for feature branches, the context variable `isFeature` will be used to use the Git branch name and append that to the stack name. +Make sure to update your `~/.aws/credentials` file. The `enhanced` context variable, if set to true, will use the `Dockerfile` dockerfile instead of the standard one. The `featureName` context variable is used to provide a different name for a custom stack other than the branch name. In the future for feature branches, the context variable `isFeature` will be used to use the Git branch name and append that to the stack name. ## MongoDB Enhanced Infrastructure Commands diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 53d75c9fa..8c00607dc 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -65,7 +65,6 @@ export class WorkerConstruct extends Construct { executionRole.addToPolicy(executionRoleSsmPolicy); const containerProps: AssetImageProps = { - file: isEnhanced() ? 'Dockerfile.enhanced' : undefined, buildArgs: { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, @@ -96,6 +95,7 @@ export class WorkerConstruct extends Construct { taskDefinition.addContainer('workerImage', { image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), containerProps), environment: dockerEnvironment, + command: ['node', '--enable-source-maps', 'enhanced/enhancedApp.js'], logging: LogDrivers.awsLogs({ streamPrefix: 'autobuilderworker', logGroup: taskDefLogGroup, From 731ad56b3a3cdde04d18ae002bb4af7ef61d2e7d Mon Sep 17 00:00:00 2001 From: Brandon Ly Date: Thu, 8 Feb 2024 10:50:36 -0600 Subject: [PATCH 3/6] Update name of cache update workflow (#991) --- .github/workflows/deploy-prd-enhanced-cache.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-prd-enhanced-cache.yml b/.github/workflows/deploy-prd-enhanced-cache.yml index 15390b971..dea4ac949 100644 --- a/.github/workflows/deploy-prd-enhanced-cache.yml +++ b/.github/workflows/deploy-prd-enhanced-cache.yml @@ -4,7 +4,7 @@ on: concurrency: group: environment-prd-enhanced-cacheUpdate-${{ github.ref }} cancel-in-progress: true -name: Deploy Production ECS Enhanced Webhooks +name: Run Cache Update Check jobs: deploy-prd: runs-on: ubuntu-latest From 88fc4de41eb9dda980f2d5153313142275d8bfa2 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:20:21 -0500 Subject: [PATCH 4/6] DOP-4356 reverse slack modal sort (#990) * DOP-4356 reverse slack modal sort * DOP-4356 sort versions accurately * DOP-4356 sort versions accurately * DOP-4356 sort versions accurately * DOP-4356 checking sorting again * DOP-4356 sort before splice * DOP-4356 just implement reverse and splice * DOP-4356 retrying sort accurately * DOP-4356 retrying sort accurately * DOP-4356 trying other splicing * DOP-4356 update comments, remove from preprd * DOP-4356 cleanup --- src/services/slack.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/services/slack.ts b/src/services/slack.ts index 4cc0d39f2..4370a8500 100644 --- a/src/services/slack.ts +++ b/src/services/slack.ts @@ -191,13 +191,23 @@ export class SlackConnector implements ISlackConnector { }; reposToShow.push(opt); }); - // THis is the limitation enforced by slack as no more 100 items are allowd in the dropdown + + // This is the limitation enforced by slack as no more 100 items are allowd in the dropdown + //Sort the list so that any inactive versions are at the end and will be truncated if any items must be truncated //'[ERROR] no more than 100 items allowed [json-pointer:/view/blocks/0/element/options]' if (reposToShow.length > 100) { - reposToShow = reposToShow.splice(0, 100); + reposToShow = reposToShow.sort().reverse().splice(0, 100); } - reposToShow.sort(); + + //sort versions like so: 4.1, 4.2, 4.11 + reposToShow.sort((a, b) => { + return b.text.text + .toString() + .replace(/\d+/g, (n) => +n + 100000) + .localeCompare(a.text.text.toString().replace(/\d+/g, (n) => +n + 100000)); + }); + return this._getDropDownView(triggerId, reposToShow); } } From 1a41ce3c2d0ae1c28a9b80626e9f71f2e5abcefd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 12 Feb 2024 11:23:01 -0500 Subject: [PATCH 5/6] DOP-4306: Remove legacy build code from autobuilder (#985) * DOP-4306 removed nextGen params * DOP-4306 reverting some changes * DOP-4306 fix typo in test * DOP-4306 repushing stashed changes * DOP-4306 push 4306 to preprd * DOP-4306 remove make and activate * DOP-4306 remove more legacy tooling * DOP-4306 change make commands to next-gen * DOP-4306 change make commands to next-gen --------- Co-authored-by: Brandon Ly --- .github/workflows/deploy-prd-ecs.yml | 14 ----- .github/workflows/deploy-stg-ecs.yml | 17 +----- Dockerfile.legacy | 2 +- cdk-infra/utils/env.ts | 6 +-- cdk-infra/utils/search-index.ts | 2 - cdk-infra/utils/slack/index.ts | 2 - cdk-infra/utils/url.ts | 4 -- infrastructure/ecs-main/serverless.yml | 29 +--------- infrastructure/ecs-repo/serverless.yml | 2 - serverless.yml | 8 --- .../scripts/local-build/utils/get-env-vars.ts | 6 +-- src/entities/job.ts | 2 - src/job/jobHandler.ts | 54 ++++++------------- src/job/manifestJobHandler.ts | 1 - src/job/productionJobHandler.ts | 23 ++++---- src/job/stagingJobHandler.ts | 15 ++---- tests/data/data.ts | 8 +-- tests/data/jobDef.ts | 1 - tests/unit/job/jobValidator.test.ts | 1 - tests/unit/job/manifestJobHandler.test.ts | 7 ++- tests/unit/job/productionJobHandler.test.ts | 22 +------- tests/unit/job/stagingJobHandler.test.ts | 26 ++------- tests/unit/services/githubCommenter.test.ts | 1 - .../services/shellCommandExecutor.test.ts | 11 ---- tests/utils/jobHandlerTestHelper.ts | 5 +- 25 files changed, 51 insertions(+), 218 deletions(-) diff --git a/.github/workflows/deploy-prd-ecs.yml b/.github/workflows/deploy-prd-ecs.yml index 68c751c87..61df05846 100644 --- a/.github/workflows/deploy-prd-ecs.yml +++ b/.github/workflows/deploy-prd-ecs.yml @@ -32,7 +32,6 @@ jobs: npm ci sls deploy --stage prd sls deploy --stage dotcomprd - sls deploy --stage legacydotcomprd - uses: mr-smithers-excellent/docker-build-push@v5 name: build and push to ECR with: @@ -57,19 +56,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} NPM_EMAIL: ${{ secrets.NPM_EMAIL }} -# - uses: mr-smithers-excellent/docker-build-push@v5 -# name: build and push to ECR Legacy Dotcom -# with: -# image: docs-worker-pool-legacydotcomprd -# tags: latest, ${{ github.sha }} -# registry: ${{ steps.login-ecr.outputs.registry }} -# dockerfile: Dockerfile.legacy -# buildArgs: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} -# NPM_EMAIL: ${{ secrets.NPM_EMAIL }} - name: Deploy ECS run: | cd infrastructure/ecs-main diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2967c5812..c60c38296 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4306" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true @@ -34,7 +35,6 @@ jobs: npm ci sls deploy --stage stg sls deploy --stage dotcomstg - sls deploy --stage legacydotcomstg - uses: mr-smithers-excellent/docker-build-push@v5 name: build and push to ECR with: @@ -59,30 +59,15 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} NPM_EMAIL: ${{ secrets.NPM_EMAIL }} - - uses: mr-smithers-excellent/docker-build-push@v5 - name: build and push to ECR Legacy DOTCOM - with: - image: docs-worker-pool-legacydotcomstg - tags: latest - registry: ${{ steps.login-ecr.outputs.registry }} - dockerfile: Dockerfile.legacy - buildArgs: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} - NPM_EMAIL: ${{ secrets.NPM_EMAIL }} - name: Deploy ECS run: | cd infrastructure/ecs-main npm ci sls deploy --stage stg sls deploy --stage dotcomstg - sls deploy --stage legacydotcomstg - name: Deploy Lambdas run: | npm ci sls deploy --stage stg sls deploy --stage dotcomstg - sls deploy --stage legacydotcomstg sls prune -n 5 diff --git a/Dockerfile.legacy b/Dockerfile.legacy index 06a41338f..e8c990d27 100644 --- a/Dockerfile.legacy +++ b/Dockerfile.legacy @@ -3,7 +3,7 @@ FROM node:14-bullseye-slim ARG MUT_VERSION=0.10.6 - + ENV PATH="${PATH}:/opt/mut" RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/cdk-infra/utils/env.ts b/cdk-infra/utils/env.ts index 7b8969f49..51cc399f1 100644 --- a/cdk-infra/utils/env.ts +++ b/cdk-infra/utils/env.ts @@ -1,8 +1,8 @@ import { Construct } from 'constructs'; import { getCurrentBranch } from './git'; -const snootyEnvs = ['staging', 'production', 'dotcomstg', 'dotcomprd', 'legacydotcomprd', 'legacydotcomstg'] as const; -const autoBuilderEnvs = ['stg', 'prd', 'dev', 'dotcomstg', 'dotcomprd', 'legacydotcomstg', 'legacydotcomprd'] as const; +const snootyEnvs = ['staging', 'production', 'dotcomstg', 'dotcomprd'] as const; +const autoBuilderEnvs = ['stg', 'prd', 'dev', 'dotcomstg', 'dotcomprd'] as const; const autoBuilderContextVariables = [ 'enhanced', 'isFeatureBranch', @@ -24,8 +24,6 @@ const autoBuilderToSnootyEnvMap: Record = { prd: 'production', dotcomprd: 'dotcomprd', dotcomstg: 'dotcomstg', - legacydotcomstg: 'legacydotcomstg', - legacydotcomprd: 'legacydotcomprd', }; export function envShortToFullName(env: string): SnootyEnv { diff --git a/cdk-infra/utils/search-index.ts b/cdk-infra/utils/search-index.ts index 7096aa55b..66403f0f0 100644 --- a/cdk-infra/utils/search-index.ts +++ b/cdk-infra/utils/search-index.ts @@ -6,10 +6,8 @@ export function getSearchIndexFolder(env: AutoBuilderEnv): string { return ''; case 'stg': case 'dotcomstg': - case 'legacydotcomstg': return 'preprd'; case 'dotcomprd': - case 'legacydotcomprd': case 'prd': return 'prd'; } diff --git a/cdk-infra/utils/slack/index.ts b/cdk-infra/utils/slack/index.ts index 27bc5a2ac..b31ef2871 100644 --- a/cdk-infra/utils/slack/index.ts +++ b/cdk-infra/utils/slack/index.ts @@ -5,8 +5,6 @@ const dashboardUrls: Record = { prd: `https://workerpool-boxgs.mongodbstitch.com/pages/job.html?collName=`, dotcomstg: `https://workerpoolstaging-qgeyp.mongodbstitch.com/pages/job.html?collName=`, dotcomprd: `https://workerpool-boxgs.mongodbstitch.com/pages/job.html?collName=`, - legacydotcomstg: `https://workerpoolstaging-qgeyp.mongodbstitch.com/pages/job.html?collName=`, - legacydotcomprd: `https://workerpool-boxgs.mongodbstitch.com/pages/job.html?collName=`, dev: `https://workerpoolstaging-qgeyp.mongodbstitch.com/pages/job.html?collName=`, }; diff --git a/cdk-infra/utils/url.ts b/cdk-infra/utils/url.ts index ebfdb7b47..1ed6cad13 100644 --- a/cdk-infra/utils/url.ts +++ b/cdk-infra/utils/url.ts @@ -4,20 +4,16 @@ const hostUrlMap: Record = { dev: 'docs-dev.mongodb.com', stg: 'mongodbcom-cdn.website.staging.corp.mongodb.com', dotcomstg: 'mongodbcom-cdn.website.staging.corp.mongodb.com', - legacydotcomstg: 'mongodbcom-cdn.website.staging.corp.mongodb.com', prd: 'www.mongodb.com', dotcomprd: 'www.mongodb.com', - legacydotcomprd: 'www.mongodb.com', }; const urlPrefixMap: Record = { dev: 'docs', stg: 'docs-qa', dotcomstg: 'docs-qa', - legacydotcomstg: 'docs-qa', prd: 'docs', dotcomprd: 'docs', - legacydotcomprd: 'docs', }; export const getHostUrl = (env: AutoBuilderEnv): string => hostUrlMap[env]; diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index 603febff7..1186632cc 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -38,8 +38,6 @@ custom: prd: worker-pool-deployment dotcomstg: worker-pool-deployment dotcomprd: worker-pool-deployment - legacydotcomstg: worker-pool-deployment - legacydotcomprd: worker-pool-deployment ecs: port: '80' imageUrl: ${self:custom.accountId.${self:provider.stage}}.dkr.ecr.us-east-2.amazonaws.com/${self:service}-${self:provider.stage}:latest @@ -49,40 +47,31 @@ custom: prd: '4096' dotcomstg: '2048' dotcomprd: '4096' - legacydotcomstg: '2048' - legacydotcomprd: '4096' containerMemory: dev: '8192' stg: '8192' prd: '24576' dotcomstg: '8192' dotcomprd: '24576' - legacydotcomstg: '8192' - legacydotcomprd: '24576' + desiredCount: dev: '4' stg: '4' prd: '4' dotcomstg: '4' dotcomprd: '4' - legacydotcomstg: '4' - legacydotcomprd: '4' minimumHealthyPercent: dev: 100 stg: 100 prd: 100 dotcomstg: 100 dotcomprd: 100 - legacydotcomstg: 100 - legacydotcomprd: 100 maximumPercent: dev: 200 stg: 200 prd: 200 dotcomstg: 200 dotcomprd: 200 - legacydotcomstg: 200 - legacydotcomprd: 200 deregistrationDelaySecs: '10' targetGroupName: docs-worker-pool accountId: @@ -91,8 +80,6 @@ custom: prd: 216656347858 dotcomstg: 216656347858 dotcomprd: 216656347858 - legacydotcomstg: 216656347858 - legacydotcomprd: 216656347858 dbUsername: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/username} dbPassword: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/password} @@ -152,16 +139,12 @@ custom: prd: https://cdnvalidator.devops.prod.corp.mongodb.com/api/v1beta1/distributions/${self:custom.distributionName.${self:provider.stage}}/invalidations dotcomstg: https://cdnvalidator.devops.staging.corp.mongodb.com/api/v1beta1/distributions/${self:custom.distributionName.${self:provider.stage}}/invalidations dotcomprd: https://cdnvalidator.devops.prod.corp.mongodb.com/api/v1beta1/distributions/${self:custom.distributionName.${self:provider.stage}}/invalidations - legacydotcomstg: https://cdnvalidator.devops.staging.corp.mongodb.com/api/v1beta1/distributions/${self:custom.distributionName.${self:provider.stage}}/invalidations - legacydotcomprd: https://cdnvalidator.devops.prod.corp.mongodb.com/api/v1beta1/distributions/${self:custom.distributionName.${self:provider.stage}}/invalidations dev: https://cdnvalidator.devops.staging.corp.mongodb.com/api/v1beta1/distributions/${self:custom.distributionName.${self:provider.stage}}/invalidations distributionName: stg: mongodbcom-staging-docs dotcomstg: mongodbcom-staging-docs - legacydotcomstg: mongodbcom-staging-docs dev: mongodbcom-staging-docs dotcomprd: mongodbcom-prod-docs - legacydotcomprd: mongodbcom-prod-docs prd: mongodbcom-prod-docs env: stg: "staging" @@ -169,8 +152,6 @@ custom: dev: "staging" dotcomstg: "dotcomstg" dotcomprd: "dotcomprd" - legacydotcomstg: "dotcomstg" - legacydotcomprd: "dotcomprd" # TODO: When satisfied with manifest generation, remove '-test' searchIndexBucket: docs-search-indexes-test # TODO: In future, use duplicate mapping of dotcomstg -> preprd to have @@ -181,30 +162,22 @@ custom: prd: "prd" dotcomstg: "preprd" dotcomprd: "prd" - legacydotcomstg: "preprd" - legacydotcomprd: "prd" site: host: dev: "docs-dev.mongodb.com" stg: "mongodbcom-cdn.website.staging.corp.mongodb.com" dotcomstg: "mongodbcom-cdn.website.staging.corp.mongodb.com" - legacydotcomstg: "mongodbcom-cdn.website.staging.corp.mongodb.com" prd: "www.mongodb.com" dotcomprd: "www.mongodb.com" - legacydotcomprd: "www.mongodb.com" prefix: dev: "docs" stg: "docs-qa" dotcomstg: "docs-qa" - legacydotcomstg: "docs-qa" prd: "docs" dotcomprd: "docs" - legacydotcomprd: "docs" errorDoc: dotcomstg: "docs-qa/404/index.html" dotcomprd: "docs/404/index.html" - legacydotcomstg: "docs-qa/404/index.html" - legacydotcomprd: "docs/404/index.html" resources: - ${file(./buckets.yml)} diff --git a/infrastructure/ecs-repo/serverless.yml b/infrastructure/ecs-repo/serverless.yml index f2e0f6cb0..4a39bc2ea 100644 --- a/infrastructure/ecs-repo/serverless.yml +++ b/infrastructure/ecs-repo/serverless.yml @@ -17,8 +17,6 @@ custom: dev: worker-pool-deployment dotcomstg: worker-pool-deployment dotcomprd: worker-pool-deployment - legacydotcomstg: worker-pool-deployment - legacydotcomprd: worker-pool-deployment resources: Resources: diff --git a/serverless.yml b/serverless.yml index 72c4365f0..66a398034 100644 --- a/serverless.yml +++ b/serverless.yml @@ -33,15 +33,11 @@ custom: prd: worker-pool-deployment dotcomstg: worker-pool-deployment dotcomprd: worker-pool-deployment - legacydotcomstg: worker-pool-deployment - legacydotcomprd: worker-pool-deployment searchIndexBucket: dev: "" # I'm just here so I don't get fined # TODO: When satisfied with manifest generation, remove the '-test' dotcomstg: docs-search-indexes-test/preprd dotcomprd: docs-search-indexes-test/prd - legacydotcomstg: docs-search-indexes-test/preprd - legacydotcomprd: docs-search-indexes-test/prd fastlyDochubToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/dochub/token} fastlyDochubServiceId: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/dochub/service_id} fastlyDochubMap: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/dochub_map} @@ -51,8 +47,6 @@ custom: dev: aws.partner/mongodb.com/stitch.trigger/5e00b7afec293c6507679c4d dotcomstg: aws.partner/mongodb.com/stitch.trigger/5e00b7afec293c6507679c4d dotcomprd: aws.partner/mongodb.com/stitch.trigger/5e00b7afec293c6507679c4d - legacydotcomstg: aws.partner/mongodb.com/stitch.trigger/5e00b7afec293c6507679c4d - legacydotcomprd: aws.partner/mongodb.com/stitch.trigger/5e00b7afec293c6507679c4d dochubEventBus: arn:aws:events:${aws:region}:${aws:accountId}:event-bus/${self:custom.docuhubEventSource.${self:provider.stage}} dbUsername: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/username} dbPassword: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/password} @@ -79,8 +73,6 @@ custom: prd: https://workerpool-boxgs.mongodbstitch.com/pages/job.html?collName=${self:custom.jobCollection}&jobId= dotcomstg: https://workerpoolstaging-qgeyp.mongodbstitch.com/pages/job.html?collName=${self:custom.jobCollection}&jobId= dotcomprd: https://workerpool-boxgs.mongodbstitch.com/pages/job.html?collName=${self:custom.jobCollection}&jobId= - legacydotcomstg: https://workerpoolstaging-qgeyp.mongodbstitch.com/pages/job.html?collName=${self:custom.jobCollection}&jobId= - legacydotcomprd: https://workerpool-boxgs.mongodbstitch.com/pages/job.html?collName=${self:custom.jobCollection}&jobId= dev: https://workerpoolstaging-qgeyp.mongodbstitch.com/pages/job.html?collName=${self:custom.jobCollection}&jobId= env: stg: "staging" diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index 37d401ec9..dcc3aef46 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -120,8 +120,8 @@ async function getSecureStrings( return paramsMap; } -const snootyEnvs = ['staging', 'production', 'dotcomstg', 'dotcomprd', 'legacydotcomprd', 'legacydotcomstg'] as const; -const autoBuilderEnvs = ['stg', 'prd', 'dev', 'dotcomstg', 'dotcomprd', 'legacydotcomstg', 'legacydotcomprd'] as const; +const snootyEnvs = ['staging', 'production', 'dotcomstg', 'dotcomprd'] as const; +const autoBuilderEnvs = ['stg', 'prd', 'dev', 'dotcomstg', 'dotcomprd'] as const; type SnootyEnv = (typeof snootyEnvs)[number]; type AutoBuilderEnv = (typeof autoBuilderEnvs)[number]; @@ -134,8 +134,6 @@ const autoBuilderToSnootyEnvMap: Record = { prd: 'production', dotcomprd: 'dotcomprd', dotcomstg: 'dotcomstg', - legacydotcomstg: 'legacydotcomstg', - legacydotcomprd: 'legacydotcomprd', }; function envShortToFullName(env: string): SnootyEnv { diff --git a/src/entities/job.ts b/src/entities/job.ts index fc5be5c7b..eaceaba59 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -49,7 +49,6 @@ export type Payload = { primaryAlias: string | null | undefined; repoBranches: any; stable: boolean | null | undefined; - isNextGen: boolean | null | undefined; regression: boolean | null | undefined; urlSlug: string | null | undefined; prefix: string; @@ -78,7 +77,6 @@ export type EnhancedPayload = { aliased?: boolean | null; primaryAlias?: string | null; stable?: boolean | null; - isNextGen?: boolean | null; repoBranches?: any; regression?: boolean | null; urlSlug?: string | null; diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index e9cb47d92..90717368d 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -1,5 +1,4 @@ -import path from 'path'; -import axios, { AxiosResponse } from 'axios'; +import axios from 'axios'; import { Payload, Job, JobStatus } from '../entities/job'; import { JobRepository } from '../repositories/jobRepository'; import { RepoBranchesRepository } from '../repositories/repoBranchesRepository'; @@ -115,7 +114,6 @@ export abstract class JobHandler { const { _id: jobId, user } = this.currJob; const gatsbyCloudSiteId = await this._repoEntitlementsRepo.getGatsbySiteIdByGithubUsername(user); if ( - this.currJob.payload.isNextGen && gatsbyCloudSiteId && this.currJob.payload.jobType === 'githubPush' && process.env.IS_FEATURE_BRANCH !== 'true' @@ -256,25 +254,20 @@ export abstract class JobHandler { @throwIfJobInterupted() private async prepNextGenBuild(): Promise { - if (this.isbuildNextGen()) { - await this._validator.throwIfBranchNotConfigured(this.currJob); - await this.constructPrefix(); - // TODO: Look into moving the generation of manifestPrefix into the manifestJobHandler, - // as well as reducing difficult-to-debug state changes - // if this payload is NOT aliased or if it's the primary alias, we need the index path - if (!this.currJob.payload.aliased || (this.currJob.payload.aliased && this.currJob.payload.primaryAlias)) { - this.currJob.payload.manifestPrefix = this.constructManifestPrefix(); - this._logger.info(this.currJob._id, `Created payload manifestPrefix: ${this.currJob.payload.manifestPrefix}`); - } + await this._validator.throwIfBranchNotConfigured(this.currJob); + await this.constructPrefix(); + // TODO: Look into moving the generation of manifestPrefix into the manifestJobHandler, + // as well as reducing difficult-to-debug state changes + // if this payload is NOT aliased or if it's the primary alias, we need the index path + if (!this.currJob.payload.aliased || (this.currJob.payload.aliased && this.currJob.payload.primaryAlias)) { + this.currJob.payload.manifestPrefix = this.constructManifestPrefix(); + this._logger.info(this.currJob._id, `Created payload manifestPrefix: ${this.currJob.payload.manifestPrefix}`); + } - this.prepStageSpecificNextGenCommands(); - this.constructEnvVars(); - this.currJob.payload.isNextGen = true; - if (this._currJob.payload.jobType === 'productionDeploy') { - this._validator.throwIfNotPublishable(this._currJob); - } - } else { - this.currJob.payload.isNextGen = false; + this.prepStageSpecificNextGenCommands(); + this.constructEnvVars(); + if (this._currJob.payload.jobType === 'productionDeploy') { + this._validator.throwIfNotPublishable(this._currJob); } } @@ -376,22 +369,12 @@ export abstract class JobHandler { await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Finished Build`); } - private async exeBuild(): Promise { - const resp = await this._commandExecutor.execute(this.currJob.buildCommands); - this.logBuildDetails(resp); - await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Finished Build`); - } - @throwIfJobInterupted() private async executeBuild(): Promise { if (this.currJob.buildCommands && this.currJob.buildCommands.length > 0) { await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}Running Build`); await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}running worker.sh`); - if (this.currJob.payload.isNextGen) { - await this.exeBuildModified(); - } else { - await this.exeBuild(); - } + await this.exeBuildModified(); } else { const error = new AutoBuilderError('No commands to execute', 'BuildError'); await this.logError(error); @@ -475,12 +458,7 @@ export abstract class JobHandler { // TODO: Reduce state changes protected prepBuildCommands(): void { - this.currJob.buildCommands = [ - `. /venv/bin/activate`, - `cd repos/${getDirectory(this.currJob)}`, - `rm -f makefile`, - `make html`, - ]; + this.currJob.buildCommands = [`cd repos/${getDirectory(this.currJob)}`, `rm -f makefile`, `make next-gen-html`]; } public async getEnvironmentVariables(): Promise<{ bucket?: string; url?: string; regression?: string }> { diff --git a/src/job/manifestJobHandler.ts b/src/job/manifestJobHandler.ts index d5c07a4f4..3f755cbd1 100644 --- a/src/job/manifestJobHandler.ts +++ b/src/job/manifestJobHandler.ts @@ -99,7 +99,6 @@ export class ManifestJobHandler extends JobHandler { // For mut-index usage info, see: https://github.com/mongodb/mut/blob/main/mut/index/main.py#L2 this.currJob.deployCommands = [ - '. /venv/bin/activate', `cd repos/${getDirectory(this.currJob)}`, 'echo IGNORE: testing manifest generation deploy commands', 'ls -al', diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index b92cb3911..a7ed1ac28 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -48,25 +48,23 @@ export class ProductionJobHandler extends JobHandler { prepDeployCommands(): void { // TODO: Can we simplify the chain of logic here? this.currJob.deployCommands = [ - '. /venv/bin/activate', `cd repos/${getDirectory(this.currJob)}`, - 'make publish && make deploy', + 'make next-gen-publish && make next-gen-deploy', ]; // TODO: Reduce confusion between job.manifestPrefix and job.payload.manifestPrefix - if (this.currJob.payload.isNextGen) { - this.currJob.manifestPrefix = this.currJob.manifestPrefix ?? this.constructManifestPrefix(); + this.currJob.manifestPrefix = this.currJob.manifestPrefix ?? this.constructManifestPrefix(); + this.currJob.deployCommands[ + this.currJob.deployCommands.length - 1 + ] = `make next-gen-deploy MUT_PREFIX=${this.currJob.payload.mutPrefix}`; + // TODO: Remove when satisfied with new manifestJobHandler infrastructure + if (this.currJob.manifestPrefix) { + const searchFlag = this.currJob.payload.stable ? '-g' : ''; this.currJob.deployCommands[ this.currJob.deployCommands.length - 1 - ] = `make next-gen-deploy MUT_PREFIX=${this.currJob.payload.mutPrefix}`; - // TODO: Remove when satisfied with new manifestJobHandler infrastructure - if (this.currJob.manifestPrefix) { - const searchFlag = this.currJob.payload.stable ? '-g' : ''; - this.currJob.deployCommands[ - this.currJob.deployCommands.length - 1 - ] += ` MANIFEST_PREFIX=${this.currJob.manifestPrefix} GLOBAL_SEARCH_FLAG=${searchFlag}`; - } + ] += ` MANIFEST_PREFIX=${this.currJob.manifestPrefix} GLOBAL_SEARCH_FLAG=${searchFlag}`; } + // have to combine search deploy commands // manifestJobHandler.prepDeployCommands @@ -115,7 +113,6 @@ export class ProductionJobHandler extends JobHandler { return; } const searchCommands = [ - '. /venv/bin/activate', `cd repos/${getDirectory(this.currJob)}`, 'echo IGNORE: testing manifest generation deploy commands', 'ls -al', diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index cb9eaae9b..e543da9fb 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -47,17 +47,10 @@ export class StagingJobHandler extends JobHandler { } prepDeployCommands(): void { - // TODO: Can we make this more readable? - this.currJob.deployCommands = ['. /venv/bin/activate', `cd repos/${getDirectory(this.currJob)}`, 'make stage']; - if (this.currJob.payload.isNextGen) { - if (this.currJob.payload.pathPrefix) { - this.currJob.deployCommands[ - this.currJob.deployCommands.length - 1 - ] = `make next-gen-stage MUT_PREFIX=${this.currJob.payload.mutPrefix}`; - } else { - this.currJob.deployCommands[this.currJob.deployCommands.length - 1] = 'make next-gen-stage'; - } - } + this.currJob.deployCommands = [ + `cd repos/${getDirectory(this.currJob)}`, + `make next-gen-stage${this.currJob.payload.pathPrefix ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + ]; } prepStageSpecificNextGenCommands(): void { diff --git a/tests/data/data.ts b/tests/data/data.ts index ace634776..62a36758b 100644 --- a/tests/data/data.ts +++ b/tests/data/data.ts @@ -138,7 +138,7 @@ export class TestDataProvider { } static getCommonBuildCommands(job: Job): Array { - return [`. /venv/bin/activate`, `cd repos/${job.payload.repoName}`, `rm -f makefile`, `make html`]; + return [`cd repos/${job.payload.repoName}`, `rm -f makefile`, `make next-gen-html`]; } static getExpectedProdBuildNextGenCommands(job: Job): Array { @@ -245,11 +245,11 @@ export class TestDataProvider { } static getCommonDeployCommands(job: Job): Array { - return ['. /venv/bin/activate', `cd repos/${job.payload.repoName}`, 'make publish && make deploy']; + return [`cd repos/${job.payload.repoName}`, 'make next-gen-publish && make next-gen-deploy']; } static getCommonDeployCommandsForStaging(job: Job): Array { - return ['. /venv/bin/activate', `cd repos/${job.payload.repoName}`, 'make stage']; + return [`cd repos/${job.payload.repoName}`, `make next-gen-stage`]; } static getExpectedStageDeployNextGenCommands(job: Job): Array { @@ -265,7 +265,7 @@ export class TestDataProvider { static getExpectedProdDeployNextGenCommands(job: Job): Array { const genericCommands = TestDataProvider.getCommonDeployCommands(job); - // TODO: simplify construction of return value + // TODO: simplify constru ction of return value const ret = Array().concat(genericCommands.slice(0, genericCommands.length - 1), [ `make next-gen-deploy MUT_PREFIX=${job.payload.mutPrefix}`, ]); diff --git a/tests/data/jobDef.ts b/tests/data/jobDef.ts index dd8d40c25..2fee82319 100644 --- a/tests/data/jobDef.ts +++ b/tests/data/jobDef.ts @@ -119,7 +119,6 @@ export const getManifestJobDef = (): Omit => ({ action: 'push', branchName: 'DOCSP-666', isFork: false, - isNextGen: true, manifestPrefix: 'testauth-UsingAlias', mutPrefix: 'compass/UsingAlias', newHead: '38b805e9c7c4f1c364476682e93f9d24a87f470a', diff --git a/tests/unit/job/jobValidator.test.ts b/tests/unit/job/jobValidator.test.ts index 2b6614545..050593b97 100644 --- a/tests/unit/job/jobValidator.test.ts +++ b/tests/unit/job/jobValidator.test.ts @@ -106,7 +106,6 @@ describe('JobValidator Tests', () => { repos: [`${job.payload.repoOwner}/${job.payload.repoName}`], github_username: job.user, }); - job.payload.isNextGen = true; await jobValidator.throwIfJobInvalid(job); expect(repoEntitlementRepository.getRepoEntitlementsByGithubUsername).toHaveBeenCalledTimes(1); }); diff --git a/tests/unit/job/manifestJobHandler.test.ts b/tests/unit/job/manifestJobHandler.test.ts index d91342896..d755b834a 100644 --- a/tests/unit/job/manifestJobHandler.test.ts +++ b/tests/unit/job/manifestJobHandler.test.ts @@ -22,7 +22,7 @@ describe('ManifestJobHandler Tests', () => { test('Execute manifestJob runs successfully and does not queue another manifest job', async () => { const queueManifestJobSpy = jest.spyOn(jobHandlerTestHelper.jobHandler, 'queueManifestJob'); jobHandlerTestHelper.jobHandler.currJob.payload.jobType = 'manifestGeneration'; - jobHandlerTestHelper.setStageForDeploySuccess(true, false); + jobHandlerTestHelper.setStageForDeploySuccess(false); await jobHandlerTestHelper.jobHandler.execute(); jobHandlerTestHelper.verifyManifestSuccess(); expect(queueManifestJobSpy).toBeCalledTimes(0); @@ -42,11 +42,10 @@ describe('ManifestJobHandler Tests', () => { dotcomprd: 'example-prd', }; jobHandlerTestHelper.config.get.calledWith('searchIndexFolder').mockReturnValue(mockFolderConfig); - jobHandlerTestHelper.setStageForDeploySuccess(true, false); + jobHandlerTestHelper.setStageForDeploySuccess(false); await jobHandlerTestHelper.jobHandler.execute(); expect(prepSpy).toBeCalledTimes(1); const o = [ - '. /venv/bin/activate', 'cd repos/testauth', 'echo IGNORE: testing manifest generation deploy commands', 'ls -al', @@ -58,7 +57,7 @@ describe('ManifestJobHandler Tests', () => { test('prepDeployCommands throws error without manifestPrefix', async () => { const prepSpy = jest.spyOn(jobHandlerTestHelper.jobHandler, 'prepDeployCommands'); jobHandlerTestHelper.jobHandler.currJob.payload.jobType = 'manifestGeneration'; - jobHandlerTestHelper.setStageForDeploySuccess(true, false); + jobHandlerTestHelper.setStageForDeploySuccess(false); await jobHandlerTestHelper.jobHandler.execute(); expect(prepSpy).toBeCalledTimes(1); expect(prepSpy).toThrowError(); diff --git a/tests/unit/job/productionJobHandler.test.ts b/tests/unit/job/productionJobHandler.test.ts index 828455550..167b95f9f 100644 --- a/tests/unit/job/productionJobHandler.test.ts +++ b/tests/unit/job/productionJobHandler.test.ts @@ -253,7 +253,7 @@ describe('ProductionJobHandler Tests', () => { }); test('Execute Next Gen Build successfully', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(true, false, { + jobHandlerTestHelper.setStageForDeploySuccess(false, { status: 'success', output: 'Great work', error: null, @@ -334,26 +334,8 @@ describe('ProductionJobHandler Tests', () => { expect(jobHandlerTestHelper.jobRepo.updateWithErrorStatus).toBeCalledWith(jobHandlerTestHelper.job._id, 'Not Good'); }); - test('Execute legacy build successfully purges only updated urls', async () => { - const purgedUrls = jobHandlerTestHelper.setStageForDeploySuccess(false); - await jobHandlerTestHelper.jobHandler.execute(); - expect(jobHandlerTestHelper.job.payload.isNextGen).toEqual(false); - expect(jobHandlerTestHelper.job.buildCommands).toEqual( - TestDataProvider.getCommonBuildCommands(jobHandlerTestHelper.job) - ); - expect(jobHandlerTestHelper.job.deployCommands).toEqual( - TestDataProvider.getCommonDeployCommands(jobHandlerTestHelper.job) - ); - expect(jobHandlerTestHelper.cdnConnector.purge).toBeCalledWith( - jobHandlerTestHelper.job._id, - purgedUrls, - jobHandlerTestHelper.job.payload.prefix - ); - expect(jobHandlerTestHelper.jobRepo.insertPurgedUrls).toBeCalledWith(jobHandlerTestHelper.job._id, purgedUrls); - }); - test('Deploy purge process inserts invalidationStatusUrl', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(true); + jobHandlerTestHelper.setStageForDeploySuccess(); await jobHandlerTestHelper.jobHandler.execute(); expect(jobHandlerTestHelper.jobRepo.insertInvalidationRequestStatusUrl).toBeCalledTimes(1); }); diff --git a/tests/unit/job/stagingJobHandler.test.ts b/tests/unit/job/stagingJobHandler.test.ts index 208b41ee2..f0955bae1 100644 --- a/tests/unit/job/stagingJobHandler.test.ts +++ b/tests/unit/job/stagingJobHandler.test.ts @@ -21,24 +21,9 @@ describe('StagingJobHandler Tests', () => { }).toThrow(`${jobHandlerTestHelper.job._id} is stopped`); }); - test('Execute legacy build runs successfully', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(false, false); - await jobHandlerTestHelper.jobHandler.execute(); - expect(jobHandlerTestHelper.job.payload.isNextGen).toEqual(false); - expect(jobHandlerTestHelper.job.buildCommands).toEqual( - TestDataProvider.getCommonBuildCommands(jobHandlerTestHelper.job) - ); - expect(jobHandlerTestHelper.job.deployCommands).toEqual( - TestDataProvider.getCommonDeployCommandsForStaging(jobHandlerTestHelper.job) - ); - expect(jobHandlerTestHelper.cdnConnector.purge).toHaveBeenCalledTimes(0); - expect(jobHandlerTestHelper.jobRepo.insertPurgedUrls).toHaveBeenCalledTimes(0); - }); - test('Execute nextgen build runs successfully without path prefix', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(true, false); + jobHandlerTestHelper.setStageForDeploySuccess(false); await jobHandlerTestHelper.jobHandler.execute(); - expect(jobHandlerTestHelper.job.payload.isNextGen).toEqual(true); expect(jobHandlerTestHelper.job.buildCommands).toEqual( TestDataProvider.getExpectedStagingBuildNextGenCommands(jobHandlerTestHelper.job) ); @@ -50,12 +35,11 @@ describe('StagingJobHandler Tests', () => { }); test('Execute nextgen build runs successfully with pathprefix', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(true, false); + jobHandlerTestHelper.setStageForDeploySuccess(false); jobHandlerTestHelper.job.payload.repoBranches = TestDataProvider.getRepoBranchesData(jobHandlerTestHelper.job); jobHandlerTestHelper.job.payload.pathPrefix = 'Mutprefix'; jobHandlerTestHelper.job.payload.mutPrefix = 'Mutprefix'; await jobHandlerTestHelper.jobHandler.execute(); - expect(jobHandlerTestHelper.job.payload.isNextGen).toEqual(true); expect(jobHandlerTestHelper.job.buildCommands).toEqual( TestDataProvider.getExpectedStagingBuildNextGenCommands(jobHandlerTestHelper.job) ); @@ -65,9 +49,8 @@ describe('StagingJobHandler Tests', () => { }); test('Execute nextgen build runs successfully and results in summary message', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(true, false); + jobHandlerTestHelper.setStageForDeploySuccess(false); await jobHandlerTestHelper.jobHandler.execute(); - expect(jobHandlerTestHelper.job.payload.isNextGen).toEqual(true); expect(jobHandlerTestHelper.job.buildCommands).toEqual( TestDataProvider.getExpectedStagingBuildNextGenCommands(jobHandlerTestHelper.job) ); @@ -81,7 +64,6 @@ describe('StagingJobHandler Tests', () => { test('Execute nextgen build deploy throws error updates the job with correct error message', async () => { jobHandlerTestHelper.setStageForDeployFailure(null, 'ERROR:BAD ONE'); await jobHandlerTestHelper.jobHandler.execute(); - expect(jobHandlerTestHelper.job.payload.isNextGen).toEqual(true); expect(jobHandlerTestHelper.job.buildCommands).toEqual( TestDataProvider.getExpectedStagingBuildNextGenCommands(jobHandlerTestHelper.job) ); @@ -101,7 +83,7 @@ describe('StagingJobHandler Tests', () => { }); test('Staging deploy with Gatsby Cloud site does not result in job completion', async () => { - jobHandlerTestHelper.setStageForDeploySuccess(true, false, undefined, { hasGatsbySiteId: true }); + jobHandlerTestHelper.setStageForDeploySuccess(false, undefined, { hasGatsbySiteId: true }); await jobHandlerTestHelper.jobHandler.execute(); expect(jobHandlerTestHelper.jobRepo.updateWithStatus).toBeCalledTimes(0); }); diff --git a/tests/unit/services/githubCommenter.test.ts b/tests/unit/services/githubCommenter.test.ts index cb616acad..87f9b9059 100644 --- a/tests/unit/services/githubCommenter.test.ts +++ b/tests/unit/services/githubCommenter.test.ts @@ -43,7 +43,6 @@ const mockPayload = { alias: '', aliased: false, primaryAlias: '', - isNextGen: true, stable: true, includeInGlobalSearch: true, regression: false, diff --git a/tests/unit/services/shellCommandExecutor.test.ts b/tests/unit/services/shellCommandExecutor.test.ts index 6a6dd7db4..1d2889a82 100644 --- a/tests/unit/services/shellCommandExecutor.test.ts +++ b/tests/unit/services/shellCommandExecutor.test.ts @@ -18,17 +18,6 @@ describe('ShellCommandExecutor Tests', () => { }); describe('ShellCommandExecutor Tests', () => { - test('ShellCommandExecutor resp contains stderr on success case (needed for legacy builds)', async () => { - cp.exec.mockImplementation((command, options, callback) => { - callback(null, { stderr: 'test warning', stdout: 'test success' }); - }); - const resp = await commandExecutor.execute([]); - expect(resp.error).toBe('test warning'); - expect(resp.output).toBe('test success'); - expect(resp.status).toBe('success'); - expect(cp.exec).toBeCalledTimes(1); - }); - test('ShellCommandExecutor properly throws on system level error', async () => { cp.exec.mockImplementation((command, options, callback) => { callback(Error('Test error'), { stdErr: 'invalid command', stdout: 'test_repo_project_snooty_name' }); diff --git a/tests/utils/jobHandlerTestHelper.ts b/tests/utils/jobHandlerTestHelper.ts index 9f39f27e6..ffcf6af5d 100644 --- a/tests/utils/jobHandlerTestHelper.ts +++ b/tests/utils/jobHandlerTestHelper.ts @@ -80,13 +80,12 @@ export class JobHandlerTestHelper { } setStageForDeploySuccess( - isNextGen = true, prodDeploy = true, returnValue?: MockReturnValueOnce, setupOptions: SetupOptions = {} ): string[] { this.job.payload.repoBranches = TestDataProvider.getPublishBranchesContent(this.job); - this.setupForSuccess(isNextGen); + this.setupForSuccess(); const publishOutput = TestDataProvider.getPublishOutputWithPurgedUrls(prodDeploy); const { hasGatsbySiteId } = setupOptions; @@ -133,7 +132,6 @@ export class JobHandlerTestHelper { expect(this.repoConnector.checkCommits).toBeCalledTimes(1); expect(this.repoConnector.applyPatch).toBeCalledTimes(1); expect(this.job.buildCommands).toEqual(expectedCommandSet); - expect(this.job.payload.isNextGen).toEqual(true); } verifyManifestSuccess(): void { @@ -143,7 +141,6 @@ export class JobHandlerTestHelper { expect(this.repoConnector.checkCommits).toBeCalledTimes(1); expect(this.repoConnector.applyPatch).toBeCalledTimes(1); expect(this.job.buildCommands).toEqual(expectedCommandSet); - expect(this.job.payload.isNextGen).toEqual(true); } setupForSuccess(rootFileExists = true, nextGenEntry: string = TestDataProvider.nextGenEntryInWorkerFile()): void { From 1d124521ee93bbcee46b4d947a10eede56717c5f Mon Sep 17 00:00:00 2001 From: mmeigs Date: Wed, 14 Feb 2024 11:20:44 -0500 Subject: [PATCH 6/6] DOP-4289 readme (#993) * add readme * format --- .github/pull_request_template.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..446e21827 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,10 @@ +### Stories/Links: + +DOP-NNNN + +### Notes + +### README updates + +- [ ] This PR introduces changes that should be reflected in the README, and I have made those updates. +- [ ] This PR does not introduce changes that should be reflected in the README