diff --git a/.github/workflows/clean-feature-branch.yml b/.github/workflows/clean-feature-branch.yml index 2e0f9fec4..86c4503e7 100644 --- a/.github/workflows/clean-feature-branch.yml +++ b/.github/workflows/clean-feature-branch.yml @@ -6,10 +6,10 @@ on: - closed name: Cleanup Feature Branch Infrastructure -jobs: +jobs: destroy: runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 @@ -25,11 +25,11 @@ jobs: run: | cd cdk-infra/ npm ci - npm run cdk destroy -- -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --force \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks + npm run cdk destroy -- -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} --force \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-queues \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks - name: Delete cache run: gh cache delete ${{github.head_ref}} - env: + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-feature-branch.yml b/.github/workflows/deploy-feature-branch.yml index 57e992671..97662846a 100644 --- a/.github/workflows/deploy-feature-branch.yml +++ b/.github/workflows/deploy-feature-branch.yml @@ -21,23 +21,30 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - - name: Deploy Feature Branch Infrastructure + - name: Install node_modules run: | npm ci cd cdk-infra/ npm ci - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks --outputs-file outputs.json - - name: Get Webhook URL - uses: mongodb/docs-worker-actions/comment-pr@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Cache root node_modules id: cache-root - uses: actions/cache@v3 + uses: actions/cache/save@v4 with: path: | node_modules cdk-infra/node_modules key: ${{ github.head_ref }} + + - name: Deploy Feature Branch Infrastructure + run: | + npm ci + cd cdk-infra/ + npm ci + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-queues + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks --outputs-file outputs.json + - name: Get Webhook URL + uses: mongodb/docs-worker-actions/comment-pr@DOP-4334 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + STAGE: dotcomstg diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..d122336ff 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -1,8 +1,8 @@ on: push: branches: - - "main" - - "integration" + - 'main' + - 'integration' concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index b896e751d..cfd7a29d6 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -6,7 +6,7 @@ on: - synchronize concurrency: - group: environment-stg-feature-${{ github.ref }} + group: environment-dotcomstg-feature-${{ github.ref }} cancel-in-progress: true name: Update Feature Branch Infrastructure jobs: @@ -34,7 +34,7 @@ jobs: npm ci - name: Cache root node_modules id: cache-root - uses: actions/cache@v3 + uses: actions/cache/save@v4 with: path: | node_modules @@ -54,7 +54,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '18.x' - - uses: actions/cache/restore@v3 + - uses: actions/cache/restore@v4 id: cache-restore with: path: | @@ -73,12 +73,16 @@ jobs: if: steps.filter.outputs.webhooks == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks + npm run deploy:feature:stack -- --hotswap -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks build-worker: needs: prep-build runs-on: ubuntu-latest steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: latest - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -105,9 +109,21 @@ jobs: - 'cdk-infra/lib/constructs/worker/**' - 'Dockerfile' - 'modules/**' + # - uses: docker/build-push-action@v5 + # if: steps.filter.outputs.worker == 'true' + # name: build Docker image + # with: + # context: . + # tags: autobuilder/enhanced:latest + # build-args: | + # NPM_EMAIL=${{ secrets.NPM_EMAIL }} + # NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} + # cache-from: type=gha,ref=autobuilder/enhanced:latest + # cache-to: type=gha,mode=max + # outputs: type=tar,dest=cdk-infra/image.tar.gz - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker diff --git a/.gitignore b/.gitignore index da3053aa9..bb7e1b9e0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ out local.json globalConfig.json .serverless -dist \ No newline at end of file +dist +onDemandApp.js +*.js.map \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 929db2703..c34da13e5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,6 @@ "type": "node", "request": "attach", "port": 9229, - "address": "localhost", "localRoot": "${workspaceFolder}", "remoteRoot": "/home/docsworker-xlarge", "autoAttachChildProcesses": false, diff --git a/Dockerfile b/Dockerfile index 3524e1a59..324337f38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,23 @@ -# Build the Typescript app -FROM node:18.16.0-alpine as ts-compiler -WORKDIR /home/docsworker-xlarge -COPY config config/ -COPY package*.json ./ -COPY tsconfig*.json ./ -RUN npm ci --legacy-peer-deps -COPY . ./ -RUN npm run build - -# install persistence module -RUN cd ./modules/persistence \ - && npm ci --legacy-peer-deps \ - && npm run build - -# Build modules -# OAS Page Builder -RUN cd ./modules/oas-page-builder \ - && npm ci --legacy-peer-deps \ - && npm run build - # where repo work will happen -FROM ubuntu:20.04 -ARG WORK_DIRECTORY=/home/docsworker-xlarge +FROM ubuntu:20.04 as initial + ARG SNOOTY_PARSER_VERSION=0.16.1 ARG SNOOTY_FRONTEND_VERSION=0.16.3 ARG MUT_VERSION=0.11.1 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL +ARG WORK_DIRECTORY=/home/docsworker-xlarge + +ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} +ENV NPM_EMAIL=${NPM_EMAIL} +ENV DOCKER_BUILDKIT=1 ENV DEBIAN_FRONTEND=noninteractive +WORKDIR ${WORK_DIRECTORY} # helper libraries for docs builds -RUN apt-get update && apt-get install -y vim git unzip rsync curl - - -ENV PATH="${PATH}:/opt/snooty:/opt/mut:/home/docsworker-xlarge/.local/bin" +RUN apt-get update && apt-get install -y vim git unzip zip rsync time # get node 18 # https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab @@ -44,9 +26,6 @@ RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential -# use npm 8.* -RUN npm install -g npm@8 - # install snooty parser RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ snooty-parser.zip @@ -55,56 +34,81 @@ RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releas RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ mut.zip + +ENV PATH="${PATH}:/opt/snooty:/opt/mut:/${WORK_DIRECTORY}/.local/bin" + # setup user and root directory RUN useradd -ms /bin/bash docsworker-xlarge RUN chmod 755 -R ${WORK_DIRECTORY} RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY} USER docsworker-xlarge -WORKDIR ${WORK_DIRECTORY} - -# get shared.mk -RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk - # install snooty frontend and docs-tools RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev \ - && git clone --depth 1 https://github.com/mongodb/docs-tools.git \ - && mkdir -p ./static/images \ - && mv ./docs-tools/themes/mongodb/static ./static/docs-tools \ - && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg + && npm ci --legacy-peer-deps --omit=dev + +RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk + -# install redoc fork RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \ # Install dependencies for Redoc CLI && cd redoc/ \ && npm ci --prefix cli/ --omit=dev -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/package*.json ./ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/config config/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/build ./ -RUN npm install +FROM initial as persistence -# Persistence module copy -# Create directory and add permissions to allow node module installation RUN mkdir -p modules/persistence && chmod 755 modules/persistence -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/package*.json ./modules/persistence/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/dist ./modules/persistence/ -ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js -RUN cd ./modules/persistence/ && ls && npm ci --legacy-peer-deps +COPY modules/persistence/package*.json ./modules/persistence/ +RUN cd ./modules/persistence \ + && npm ci --legacy-peer-deps +# Build persistence module -# OAS Page Builder module copy -# Create directory and add permissions to allow node module installation -RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/ -RUN cd ./modules/oas-page-builder/ && npm ci --legacy-peer-deps +COPY --chown=docsworker-xlarge modules/persistence/tsconfig*.json ./modules/persistence +COPY --chown=docsworker-xlarge modules/persistence/src ./modules/persistence/src/ +COPY --chown=docsworker-xlarge modules/persistence/index.ts ./modules/persistence -# Needed for OAS Page Builder module in shared.mk -ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js +RUN cd ./modules/persistence \ + && npm run build:esbuild + +FROM initial as oas + +RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder +COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/ +RUN cd ./modules/oas-page-builder \ + && npm ci --legacy-peer-deps +# Build modules +# OAS Page Builder +COPY --chown=docsworker-xlarge modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder +COPY --chown=docsworker-xlarge modules/oas-page-builder/src ./modules/oas-page-builder/src/ +COPY --chown=docsworker-xlarge modules/oas-page-builder/index.ts ./modules/oas-page-builder + +RUN cd ./modules/oas-page-builder \ + && npm run build:esbuild + +FROM initial as root + +COPY --from=persistence --chown=docsworker-xlarge ${WORK_DIRECTORY}/modules/persistence/dist/ ./modules/persistence +COPY --from=oas --chown=docsworker-xlarge ${WORK_DIRECTORY}/modules/oas-page-builder/dist/ ./modules/oas-page-builder + +# Root project build +COPY package*.json ./ +RUN npm ci --legacy-peer-deps + + +COPY tsconfig*.json ./ +COPY config config/ +COPY api api/ +COPY src src/ + +RUN npm run build:esbuild + +ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js +ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js +RUN mkdir -p modules/persistence && chmod 755 modules/persistence RUN mkdir repos && chmod 755 repos + EXPOSE 3000 CMD ["node", "app.js"] diff --git a/Dockerfile.local b/Dockerfile.local index 4382caf4d..3a74911b5 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -5,7 +5,6 @@ ARG SNOOTY_PARSER_VERSION=0.15.2 ARG SNOOTY_FRONTEND_VERSION=0.15.7 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.3 -ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge @@ -21,7 +20,7 @@ RUN apt-get install --yes curl RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential -RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config +RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config time RUN python3 -m pip install poetry @@ -61,7 +60,7 @@ RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.co # Install dependencies for Redoc CLI && cd redoc/ \ && npm ci --prefix cli/ --omit=dev - + FROM initial as persistence RUN mkdir -p modules/persistence && chmod 755 modules/persistence @@ -118,4 +117,4 @@ RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "--inspect-brk=0.0.0.0", "--enable-source-maps", "dist/entrypoints/onDemandApp.js"] +CMD ["node", "--inspect-brk=0.0.0.0", "--enable-source-maps", "onDemandApp.js"] diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a761e742b..adfb4a53e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -42,7 +42,7 @@ async function prepGithubPushPayload( error: {}, result: null, payload: { - jobType: 'githubPush', + jobType: 'productionDeploy', source: 'github', action: 'push', repoName: githubEvent.repository.name, diff --git a/api/controllers/v2/test-deploy.ts b/api/controllers/v2/test-deploy.ts new file mode 100644 index 000000000..09a536417 --- /dev/null +++ b/api/controllers/v2/test-deploy.ts @@ -0,0 +1,52 @@ +import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; +import * as mongodb from 'mongodb'; +import c from 'config'; +import { RepoEntitlementsRepository } from '../../../src/repositories/repoEntitlementsRepository'; +import { ConsoleLogger } from '../../../src/services/logger'; +import { buildEntitledBranchList, isUserEntitled } from '../../handlers/slack'; +import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; + +interface TestDeployRequest { + githubUsername: string; + repoName: string; + repoOwner: string; + branchName?: string; + directory?: string; +} +export async function handleTestDeployRequest(event: APIGatewayEvent): Promise { + const { DB_NAME, JOBS_QUEUE_URL, MONGO_ATLAS_URL } = process.env; + + if (!event.body) { + return { + statusCode: 400, + body: 'Event body is undefined', + }; + } + const { githubUsername } = JSON.parse(event.body) as TestDeployRequest; + + if (!MONGO_ATLAS_URL) { + console.error('MONGO_ATLAS_URL is not defined!'); + return { statusCode: 500, body: 'internal server error' }; + } + + const client = new mongodb.MongoClient(MONGO_ATLAS_URL); + await client.connect(); + const db = client.db(DB_NAME); + + const consoleLogger = new ConsoleLogger(); + const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger); + const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); + + const entitlement = await repoEntitlementRepository.getRepoEntitlementsByGithubUsername(githubUsername); + if (!isUserEntitled(entitlement)) { + return { + statusCode: 401, + body: 'User is not entitled!', + }; + } + + return { + statusCode: 500, + body: 'not complete', + }; +} diff --git a/api/handlers/jobs.ts b/api/handlers/jobs.ts index 8aedd6ffd..d2e67a54c 100644 --- a/api/handlers/jobs.ts +++ b/api/handlers/jobs.ts @@ -7,7 +7,7 @@ import { JobRepository } from '../../src/repositories/jobRepository'; import { GithubCommenter } from '../../src/services/github'; import { SlackConnector } from '../../src/services/slack'; import { RepoEntitlementsRepository } from '../../src/repositories/repoEntitlementsRepository'; -import { EnhancedJob, Job, JobStatus, Payload } from '../../src/entities/job'; +import { EnhancedJob, EnhancedPayload, Job, JobStatus, Payload } from '../../src/entities/job'; // Although data in payload should always be present, it's not guaranteed from // external callers @@ -280,3 +280,80 @@ function getPreviewUrl(payload: Payload | undefined, env: string): string | unde const possibleStagingSuffix = env === 'stg' ? 'stg' : ''; return `https://preview-mongodb${githubUsernameNoHyphens}${possibleStagingSuffix}.gatsbyjs.io/${project}/${branchName}/`; } + +interface CreateJobProps { + payload: EnhancedPayload; + jobTitle: string; + jobUserName: string; + jobUserEmail?: string; +} + +interface CreatePayloadProps { + jobType: string; + repoOwner: string; + repoName: string; + branchName: string; + project: string; + prefix: string; + urlSlug; + source: string; + action: string; + aliased: boolean; + primaryAlias: boolean; + stable: boolean; + directory?: string; + newHead?: string; +} + +export function createPayload({ + jobType, + repoOwner, + repoName, + branchName, + newHead, + project, + prefix, + urlSlug, + source, + action, + aliased = false, + primaryAlias = false, + stable = false, + directory, +}: CreatePayloadProps): EnhancedPayload { + return { + jobType, + source, + action, + repoName, + branchName, + project, + prefix, + aliased, + urlSlug, + isFork: true, + repoOwner, + url: 'https://github.com/' + repoOwner + '/' + repoName, + newHead, + primaryAlias, + stable, + directory, + }; +} + +export function createJob({ jobTitle, jobUserName, payload, jobUserEmail }: CreateJobProps): Omit { + return { + title: jobTitle, + user: jobUserName, + email: jobUserEmail, + status: JobStatus.inQueue, + createdTime: new Date(), + startTime: null, + endTime: null, + priority: 1, + error: {}, + result: null, + payload, + logs: [], + }; +} diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 52c71b95f..c27ef7305 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -10,6 +10,8 @@ import { getFeatureName } from '../../../utils/env'; const HANDLERS_PATH = path.join(__dirname, '/../../../../api/controllers/v2'); +const memorySize = 1024; + const bundling: BundlingOptions = { sourceMap: true, minify: true, @@ -46,6 +48,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const slackDisplayRepoLambda = new NodejsFunction(this, 'slackDisplayRepoLambda', { @@ -55,6 +58,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const dochubTriggerUpsertLambda = new NodejsFunction(this, 'dochubTriggerUpsertLambda', { @@ -63,6 +67,7 @@ export class WebhookApiConstruct extends Construct { handler: 'UpsertEdgeDictionaryItem', environment, timeout, + memorySize, }); const githubTriggerLambda = new NodejsFunction(this, 'githubTriggerLambda', { @@ -72,6 +77,7 @@ export class WebhookApiConstruct extends Construct { bundling, environment, timeout, + memorySize, }); const githubDeleteArtifactsLambda = new NodejsFunction(this, 'githubDeleteArtifactsLambda', { @@ -81,6 +87,7 @@ export class WebhookApiConstruct extends Construct { bundling, environment, timeout, + memorySize, }); const triggerLocalBuildLambda = new NodejsFunction(this, 'triggerLocalBuildLambda', { @@ -90,6 +97,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const handleJobsLambda = new NodejsFunction(this, 'handleJobsLambda', { @@ -99,6 +107,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const snootyBuildCompleteLambda = new NodejsFunction(this, 'snootyBuildCompleteLambda', { @@ -108,6 +117,16 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, + }); + const testDeployLambda = new NodejsFunction(this, 'testDeployLambda', { + entry: `${HANDLERS_PATH}/test-deploy.ts`, + runtime, + handler: 'handleTestDeployRequest', + environment, + bundling, + timeout, + memorySize, }); // generic handler for the root endpoint @@ -116,6 +135,7 @@ export class WebhookApiConstruct extends Construct { runtime, handler: 'RootEndpointLambda', timeout, + memorySize, }); const apiName = `webhookHandlers-${getFeatureName()}`; @@ -125,6 +145,20 @@ export class WebhookApiConstruct extends Construct { proxy: false, }); + const usagePlan = restApi.addUsagePlan('cacheUpdaterUsagePlan', { + name: 'defaultPlan', + apiStages: [ + { + api: restApi, + stage: restApi.deploymentStage, + }, + ], + }); + + const apiKey = restApi.addApiKey('cacheUpdaterApiKey'); + + usagePlan.addApiKey(apiKey); + const webhookEndpoint = restApi.root.addResource('webhook'); const slackEndpoint = webhookEndpoint.addResource('slack'); @@ -132,6 +166,7 @@ export class WebhookApiConstruct extends Construct { const githubEndpoint = webhookEndpoint.addResource('githubEndpoint'); const localEndpoint = webhookEndpoint.addResource('local'); const snootyEndpoint = webhookEndpoint.addResource('snooty'); + const testDeployEndpoint = webhookEndpoint.addResource('test-deploy'); // Shared /githubEndpoint/trigger endpoint const githubEndpointTrigger = githubEndpoint.addResource('trigger'); @@ -174,6 +209,11 @@ export class WebhookApiConstruct extends Construct { .addResource('complete', { defaultCorsPreflightOptions }) .addMethod('POST', new LambdaIntegration(snootyBuildCompleteLambda)); + testDeployEndpoint + .addResource('trigger') + .addResource('build', { defaultCorsPreflightOptions }) + .addMethod('POST', new LambdaIntegration(testDeployLambda), { apiKeyRequired: true }); + // grant permission for lambdas to enqueue messages to the jobs queue jobsQueue.grantSendMessages(slackTriggerLambda); jobsQueue.grantSendMessages(githubTriggerLambda); diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 8c00607dc..fa59671ad 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -12,7 +12,8 @@ import { LogGroup } from 'aws-cdk-lib/aws-logs'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import path from 'path'; -import { getEnv, isEnhanced } from '../../../utils/env'; +import { getEnv } from '../../../utils/env'; +import { Platform } from 'aws-cdk-lib/aws-ecr-assets'; interface WorkerConstructProps { dockerEnvironment: Record; @@ -69,12 +70,15 @@ export class WorkerConstruct extends Construct { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, }, + cacheFrom: [{ type: 'gha' }], + cacheTo: { type: 'gha', params: { mode: 'max' } }, + platform: Platform.LINUX_AMD64, }; const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); const taskDefinition = new FargateTaskDefinition(this, 'workerTaskDefinition', { - cpu: 4096, - memoryLimitMiB: 8192, + cpu: 16384, + memoryLimitMiB: 65536, taskRole, executionRole, }); @@ -95,6 +99,7 @@ export class WorkerConstruct extends Construct { taskDefinition.addContainer('workerImage', { image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), containerProps), environment: dockerEnvironment, + command: ['node', '--enable-source-maps', 'enhanced/enhancedApp.js'], logging: LogDrivers.awsLogs({ streamPrefix: 'autobuilderworker', diff --git a/cdk-infra/package-lock.json b/cdk-infra/package-lock.json index d2462491b..1e3bc19e1 100644 --- a/cdk-infra/package-lock.json +++ b/cdk-infra/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@aws-sdk/client-ssm": "^3.342.0", - "aws-cdk-lib": "2.73.0", + "aws-cdk-lib": "2.130.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" }, @@ -21,7 +21,7 @@ "@swc/helpers": "^0.5.1", "@types/jest": "^29.4.0", "@types/node": "18.14.6", - "aws-cdk": "2.73.0", + "aws-cdk": "2.130.0", "esbuild": "^0.18.3", "jest": "^29.5.0", "regenerator-runtime": "^0.13.11", @@ -44,19 +44,19 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.138", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.138.tgz", - "integrity": "sha512-1GPAkidoyOFPhOZbkaxnclNBSBxxcN8wejpSMCixifbPvPFMvJXjMd19Eq4WDPeDDHUEjuJU9tEtvzq3OFE7Gw==" + "version": "2.2.202", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.202.tgz", + "integrity": "sha512-JqlF0D4+EVugnG5dAsNZMqhu3HW7ehOXm5SDMxMbXNDMdsF0pxtQKNHRl52z1U9igsHmaFpUgSGjbhAJ+0JONg==" }, "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.1.tgz", - "integrity": "sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", + "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" }, - "node_modules/@aws-cdk/asset-node-proxy-agent-v5": { - "version": "2.0.114", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.114.tgz", - "integrity": "sha512-xXSptpTYIlxQyTpVud2N6/vzHaAsUSWtDrPxdEZoIJESuZJZKLP69PhELwG/NsD6WtxpWYa6LO6s2qvJhRUjew==" + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", + "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" }, "node_modules/@aws-crypto/crc32": { "version": "3.0.0", @@ -2823,9 +2823,9 @@ } }, "node_modules/aws-cdk": { - "version": "2.73.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.73.0.tgz", - "integrity": "sha512-4ZnY+OS83goCzv+1sCEpNTNiXWjY6KBzic2RNUObzpHjUskRSwUCtaeiv6OyZ55DZoP0tneAmWIBXHfixJ7iQw==", + "version": "2.130.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.130.0.tgz", + "integrity": "sha512-MsjGzQ2kZv0FEfXvpW7FTJRnefew0GrYt9M2SMN2Yn45+yjugGl2X8to416kABeFz1OFqW56hq8Y5BiLuFDVLQ==", "dev": true, "bin": { "cdk": "bin/cdk" @@ -2838,9 +2838,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.73.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.73.0.tgz", - "integrity": "sha512-r9CUe3R7EThr9U0Eb7kQCK4Ee34TDeMH+bonvGD9rNRRTYDauvAgNCsx4DZYYksPrXLRzWjzVbuXAHaDDzWt+A==", + "version": "2.130.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.130.0.tgz", + "integrity": "sha512-yK7ibePipdjlI4AFM94fwwtsCkmpWJ0JFZTMPahahC/3Pxe/BA/nnI/4Namvl5QPxW5QlU0xQYU7cywioq3RQg==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -2851,20 +2851,22 @@ "punycode", "semver", "table", - "yaml" + "yaml", + "mime-types" ], "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.97", - "@aws-cdk/asset-kubectl-v20": "^2.1.1", - "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.77", + "@aws-cdk/asset-awscli-v1": "^2.2.202", + "@aws-cdk/asset-kubectl-v20": "^2.1.2", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^9.1.0", - "ignore": "^5.2.4", + "fs-extra": "^11.2.0", + "ignore": "^5.3.1", "jsonschema": "^1.4.1", + "mime-types": "^2.1.35", "minimatch": "^3.1.2", - "punycode": "^2.3.0", - "semver": "^7.3.8", + "punycode": "^2.3.1", + "semver": "^7.6.0", "table": "^6.8.1", "yaml": "1.10.2" }, @@ -2925,14 +2927,6 @@ "node": ">=8" } }, - "node_modules/aws-cdk-lib/node_modules/at-least-node": { - "version": "1.0.0", - "inBundle": true, - "license": "ISC", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { "version": "1.0.2", "inBundle": true, @@ -2987,26 +2981,25 @@ "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "9.1.0", + "version": "11.2.0", "inBundle": true, "license": "MIT", "dependencies": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=14.14" } }, "node_modules/aws-cdk-lib/node_modules/graceful-fs": { - "version": "4.2.10", + "version": "4.2.11", "inBundle": true, "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -3061,6 +3054,25 @@ "node": ">=10" } }, + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/aws-cdk-lib/node_modules/minimatch": { "version": "3.1.2", "inBundle": true, @@ -3073,7 +3085,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.0", + "version": "2.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -3089,7 +3101,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.3.8", + "version": "7.6.0", "inBundle": true, "license": "ISC", "dependencies": { @@ -3158,7 +3170,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", "inBundle": true, "license": "MIT", "engines": { diff --git a/cdk-infra/package.json b/cdk-infra/package.json index 71416e9a9..dc46de026 100644 --- a/cdk-infra/package.json +++ b/cdk-infra/package.json @@ -21,7 +21,7 @@ "@swc/helpers": "^0.5.1", "@types/jest": "^29.4.0", "@types/node": "18.14.6", - "aws-cdk": "2.73.0", + "aws-cdk": "2.130.0", "esbuild": "^0.18.3", "jest": "^29.5.0", "regenerator-runtime": "^0.13.11", @@ -31,7 +31,7 @@ }, "dependencies": { "@aws-sdk/client-ssm": "^3.342.0", - "aws-cdk-lib": "2.73.0", + "aws-cdk-lib": "2.130.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" } diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index 2fdad6e1d..39816b924 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -24,6 +24,7 @@ Resources: Memory: ${self:custom.ecs.containerMemory.${self:provider.stage}} ExecutionRoleArn: !Ref ExecutionRole TaskRoleArn: !Ref TaskRole + EphemeralStorage: 100 ContainerDefinitions: - Name: ${self:service}-${self:provider.stage} Image: ${self:custom.ecs.imageUrl} diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index 1186632cc..615a219cc 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -45,14 +45,14 @@ custom: dev: '2048' stg: '2048' prd: '4096' - dotcomstg: '2048' - dotcomprd: '4096' + dotcomstg: '16384' + dotcomprd: '16384' containerMemory: dev: '8192' stg: '8192' prd: '24576' - dotcomstg: '8192' - dotcomprd: '24576' + dotcomstg: '65536' + dotcomprd: '65536' desiredCount: dev: '4' diff --git a/modules/persistence/src/services/connector/index.ts b/modules/persistence/src/services/connector/index.ts index e06c7890f..feaa37f8e 100644 --- a/modules/persistence/src/services/connector/index.ts +++ b/modules/persistence/src/services/connector/index.ts @@ -65,7 +65,10 @@ export const bulkWrite = async (operations: mongodb.AnyBulkWriteOperation[], col if (!operations || !operations.length) { return; } - return dbSession.collection(collection).bulkWrite(operations, { ordered: false }); + const result = await dbSession + .collection(collection) + .bulkWrite(operations, { ordered: false, maxTimeMS: 10000000 }); + return result; } catch (error) { console.error(`Error at bulk write time for ${collection}: ${error}`); throw error; diff --git a/package.json b/package.json index d78e8845b..663754b7c 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,9 @@ "clean": "node maintain.js", "build": "tsc", "build:esbuild:cacheUpdater": "esbuild src/cache-updater/index.ts --bundle --platform=node --outdir=./dist/ --allow-overwrite --sourcemap", - "build:esbuild": "esbuild src/onDemandApp.ts --bundle --platform=node --outdir=./dist/entrypoints --allow-overwrite --sourcemap", + "build:esbuild": "npm run build:esbuild:enhanced && npm run build:esbuild:onDemand", + "build:esbuild:onDemand": "esbuild src/onDemandApp.ts --bundle --platform=node --outdir=./ --allow-overwrite --sourcemap", + "build:esbuild:enhanced": "esbuild src/enhanced/enhancedApp.ts --bundle --platform=node --outdir=./enhanced --allow-overwrite --sourcemap", "format": "npm run prettier -- --check", "format:fix": "npm run prettier -- --write", "lint": "eslint --ext .ts .", diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index 7745326ba..08d2fb935 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -51,10 +51,11 @@ export async function downloadBuildDependencies( directory?: string ) { const commands: string[] = []; + const repoDir = getRepoDir(repoName, directory); await Promise.all( buildDependencies.map(async (dependencyInfo) => { - const repoDir = getRepoDir(repoName, directory); const targetDir = dependencyInfo.targetDir ?? repoDir; + let options = {}; if (targetDir != repoDir) { options = { cwd: repoDir }; @@ -91,19 +92,29 @@ export async function downloadBuildDependencies( commands.push(...responseSync); }) ); + + try { + await executeCliCommand({ command: 'ls', args: ['source/driver-examples'], options: { cwd: repoDir } }); + } catch (e) { + console.error(e); + } return commands; } -export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) { +export const checkRedirects = async (repoName: string, directory?: string) => + existsAsync(path.join(process.cwd(), getRepoDir(repoName, directory), 'config/redirects')); + +export async function prepareBuild( + repoName: string, + projectName: string, + baseUrl: string, + prefix: string, + directory?: string +) { const repoDir = getRepoDir(repoName, directory); // doing these in parallel - const commandPromises = [ - getCommitHash(repoDir), - getCommitBranch(repoDir), - getPatchId(repoDir), - existsAsync(path.join(process.cwd(), 'config/redirects')), - ]; + const commandPromises = [getCommitHash(repoDir), getCommitBranch(repoDir), getPatchId(repoDir)]; try { const dependencies = await Promise.all(commandPromises); @@ -111,6 +122,7 @@ export async function prepareBuild(repoName: string, projectName: string, baseUr repoDir, projectName, baseUrl, + prefix, commitHash: dependencies[1] as string | undefined, patchId: dependencies[2] as string | undefined, }); @@ -119,7 +131,6 @@ export async function prepareBuild(repoName: string, projectName: string, baseUr commitHash: dependencies[0] as string, commitBranch: dependencies[1] as string, patchId: dependencies[2] as string | undefined, - hasRedirects: dependencies[3] as boolean, bundlePath: `${repoDir}/bundle.zip`, repoDir, }; diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index ba79ca626..488661ccc 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -110,7 +110,7 @@ export async function executeAndPipeCommands( reject(new ExecuteCommandError('The second command failed', 1, err)); }); - cmdTo.on('exit', (exitCode) => { + cmdTo.on('close', (exitCode) => { // previous command errored out, return so we don't // accidentally resolve if the second command somehow still // exits without error diff --git a/src/commands/src/scripts/local-build/index.ts b/src/commands/src/scripts/local-build/index.ts index fc5b2c846..b7e4881b4 100644 --- a/src/commands/src/scripts/local-build/index.ts +++ b/src/commands/src/scripts/local-build/index.ts @@ -43,9 +43,9 @@ async function runDockerContainer(env: Record) { } async function main() { - const { repoName, repoOwner, directory, branchName } = getArgs(); + const { repoName, repoOwner, directory, branchName, jobType } = getArgs(); - const env = await getWorkerEnv('stg'); + const env = await getWorkerEnv('dotcomstg'); const octokitClient = getOctokitClient(env.GITHUB_BOT_PASSWORD); const commitPromise = octokitClient.request('GET /repos/{owner}/{repo}/commits/{ref}', { @@ -71,7 +71,7 @@ async function main() { const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); const project = (await docsetsRepository.getProjectByRepoName(repoName, directory)) as string; - const job = createLocalJob({ commit: commit.data, branchName, repoName, repoOwner, project, directory }); + const job = createLocalJob({ commit: commit.data, branchName, repoName, repoOwner, project, directory, jobType }); console.log('inserting job into queue collection'); const { insertedId: jobId } = await collection.insertOne(job); diff --git a/src/commands/src/scripts/local-build/utils/create-job.ts b/src/commands/src/scripts/local-build/utils/create-job.ts index 2a06538a5..16bb0f047 100644 --- a/src/commands/src/scripts/local-build/utils/create-job.ts +++ b/src/commands/src/scripts/local-build/utils/create-job.ts @@ -1,12 +1,17 @@ import { EnhancedJob, JobStatus } from '../../../../../entities/job'; import { CommitGetResponse } from './types'; +export const JOB_TYPES = ['githubPush', 'productionDeploy', 'manifestGeneration', 'regression'] as const; +export type JobType = (typeof JOB_TYPES)[number]; + +export const isValidJobType = (val: string): val is JobType => JOB_TYPES.includes(val as JobType); interface Props { branchName: string; repoOwner: string; repoName: string; commit: CommitGetResponse; project: string; + jobType: JobType; directory?: string; } @@ -17,11 +22,12 @@ export function createLocalJob({ commit, project, directory, + jobType, }: Props): Omit { return { isLocal: true, title: `${repoOwner}/${repoName}`, - user: commit.author?.name ?? '', + user: commit.author?.name ?? 'branberry', email: commit.author?.email ?? '', status: JobStatus.inQueue, createdTime: new Date(), @@ -32,9 +38,9 @@ export function createLocalJob({ result: null, pathPrefix: `${project}/docsworker-xlarge/${branchName}`, payload: { - jobType: 'githubPush', - source: 'github', - action: 'push', + jobType: jobType, + source: 'local', + action: 'debug', repoName, branchName, isFork: repoOwner !== '10gen' && repoOwner !== 'mongodb', @@ -42,7 +48,7 @@ export function createLocalJob({ url: commit.url, newHead: commit.sha, urlSlug: branchName, - prefix: '', + prefix: 'docs-qa', project: project, pathPrefix: `${project}/docsworker-xlarge/${branchName}`, mutPrefix: project, diff --git a/src/commands/src/scripts/local-build/utils/get-args.ts b/src/commands/src/scripts/local-build/utils/get-args.ts index ac840ac28..e630d2831 100644 --- a/src/commands/src/scripts/local-build/utils/get-args.ts +++ b/src/commands/src/scripts/local-build/utils/get-args.ts @@ -1,3 +1,5 @@ +import { isValidJobType } from './create-job'; + export function getArgs() { const helpIdx = process.argv.findIndex((str) => str === '--help' || str === '-h'); @@ -10,6 +12,7 @@ export function getArgs() { --repo-name, -n (required) The name of the repo e.g. docs-java or cloud-docs. --directory, -d (optional) The project directory path for a monorepo project. A slash is not needed at the beginning. For example, cloud-docs in the monorepo would just be cloud-docs for the argument. --branch-name, -b (optional) The branch name we want to parse. If not provided, the value 'master' is used by default. + --job-type -t (optional) The job type. Can be githubPush (uses the StagingJobHandler), productionDeploy (uses the ProductionJobHandler), manifestGeneration (uses the ManifestJobHandler), and regression (uses the RegressionJobHandler). Default value is githubPush. `); process.exit(0); @@ -20,6 +23,7 @@ export function getArgs() { // optional args const directoryIdx = process.argv.findIndex((str) => str === '--directory' || str === '-d'); const branchNameIdx = process.argv.findIndex((str) => str === '--branch-name' || str === '-b'); + const jobTypeIdx = process.argv.findIndex((str) => str === '--job-type' || str === '-t'); if (repoOwnerIdx === -1) throw new Error( @@ -48,15 +52,26 @@ export function getArgs() { if (process.argv[branchNameIdx + 1].startsWith('-')) throw new Error(`Please provide a valid branch name value. Value provided: ${process.argv[branchNameIdx + 1]}`); + if (jobTypeIdx + 1 === process.argv.length) throw new Error('Please provide a value for the branch name flag'); + if (process.argv[jobTypeIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid branch name value. Value provided: ${process.argv[jobTypeIdx + 1]}`); + const repoOwner = process.argv[repoOwnerIdx + 1]; const repoName = process.argv[repoNameIdx + 1]; const directory = directoryIdx !== -1 ? process.argv[directoryIdx + 1] : undefined; const branchName = branchNameIdx !== -1 ? process.argv[branchNameIdx + 1] : 'master'; + const jobType = jobTypeIdx !== -1 ? process.argv[jobTypeIdx + 1] : 'githubPush'; + + if (!isValidJobType(jobType)) + throw new Error( + `Error! Invalid option provided for jobType. Value provided: ${jobType}. Please run the command with the --help flag to view the valid options for this parameter.` + ); return { repoOwner, repoName, directory, branchName, + jobType, }; } diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index dcc3aef46..e473faaf4 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -158,6 +158,7 @@ export async function getWorkerEnv(env: AutoBuilderEnv): Promise { + return; +} diff --git a/src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts b/src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts new file mode 100644 index 000000000..dc9af1162 --- /dev/null +++ b/src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts @@ -0,0 +1,63 @@ +export function getTestDeployArgs() { + const helpIdx = process.argv.findIndex((str) => str === '--help' || str === '-h'); + + if (helpIdx !== -1) { + console.log(` + This command builds and runs the Autobuilder for local debugging. + + Flags: + --repo-owner, -o (required) The owner of the repo. Typically this is 'mongodb' or '10gen'. This should be your username for a fork. + --repo-name, -n (required) The name of the repo e.g. docs-java or cloud-docs. + --directory, -d (optional) The project directory path for a monorepo project. A slash is not needed at the beginning. For example, cloud-docs in the monorepo would just be cloud-docs for the argument. + --branch-name, -b (optional) The branch name we want to parse. If not provided, the value 'master' is used by default. + --job-type -t (optional) The job type. Can be githubPush (uses the StagingJobHandler), productionDeploy (uses the ProductionJobHandler), manifestGeneration (uses the ManifestJobHandler), and regression (uses the RegressionJobHandler). Default value is githubPush. + `); + + process.exit(0); + } + const repoOwnerIdx = process.argv.findIndex((str) => str === '--repo-owner' || str === '-o'); + const repoNameIdx = process.argv.findIndex((str) => str === '--repo-name' || str === '-n'); + + // optional args + const directoryIdx = process.argv.findIndex((str) => str === '--directory' || str === '-d'); + const branchNameIdx = process.argv.findIndex((str) => str === '--branch-name' || str === '-b'); + + if (repoOwnerIdx === -1) + throw new Error( + 'Please provide a repo owner. The flag to provide this is --repo-owner, or you can use -o as well.' + ); + + if (repoNameIdx === -1) + throw new Error('Please provide a repo name. The flag to provide this is --repo-name, or you can use -n as well.'); + + if (repoOwnerIdx + 1 === process.argv.length) throw new Error('Please provide a value for the repo owner flag'); + if (process.argv[repoOwnerIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid repo owner value. Value provided: ${process.argv[repoOwnerIdx + 1]}`); + + if (repoNameIdx + 1 === process.argv.length) throw new Error('Please provide a value for the repo name flag'); + if (process.argv[repoNameIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid repo name value. Value provided: ${process.argv[repoNameIdx + 1]}`); + + if (directoryIdx + 1 === process.argv.length) + throw new Error('Please provide a value for the monorepo directory flag'); + if (process.argv[directoryIdx + 1].startsWith('-')) + throw new Error( + `Please provide a valid monorepo directory value. Value provided: ${process.argv[directoryIdx + 1]}` + ); + + if (branchNameIdx + 1 === process.argv.length) throw new Error('Please provide a value for the branch name flag'); + if (process.argv[branchNameIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid branch name value. Value provided: ${process.argv[branchNameIdx + 1]}`); + + const repoOwner = process.argv[repoOwnerIdx + 1]; + const repoName = process.argv[repoNameIdx + 1]; + const directory = directoryIdx !== -1 ? process.argv[directoryIdx + 1] : undefined; + const branchName = branchNameIdx !== -1 ? process.argv[branchNameIdx + 1] : 'master'; + + return { + repoOwner, + repoName, + directory, + branchName, + }; +} diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 5d86baf73..781c6da65 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -2,35 +2,63 @@ import { CommandExecutorResponse, CommandExecutorResponseStatus } from '../../.. import { executeAndPipeCommands, executeCliCommand } from '../helpers'; interface NextGenDeployParams { - mutPrefix: string; - gitBranch: string; + branchName: string; hasConfigRedirects: boolean; - bucket: string; - url: string; + repoDir: string; + mutPrefix?: string | null; + bucket?: string; + url?: string; } /* This is still in development - use with caution */ /* Logs here for future debugging purposes */ export async function nextGenDeploy({ mutPrefix, - gitBranch, + branchName, hasConfigRedirects, + repoDir, bucket, url, }: NextGenDeployParams): Promise { + console.log('In next gen deploy'); + console.log('branchName', branchName); try { - if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { + if (hasConfigRedirects && (branchName === 'main' || branchName === 'master' || branchName === 'current')) { // equivalent to: mut-redirects config/redirects -o public/.htaccess - await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); + await executeCliCommand({ + command: 'mut-redirects', + args: ['config/redirects', '-o', 'public/.htaccess'], + options: { cwd: repoDir }, + }); console.log(`COMMAND: mut-redirects config/redirects -o public/.htaccess`); } + console.log('REDIRECTS COMPLETE!'); + if (!bucket) { + console.log(`nextGenStage has failed. Variable for S3 bucket address was undefined.`); + return { + status: CommandExecutorResponseStatus.failed, + output: 'Failed in nextGenStage: No value present for S3 bucket', + error: 'ERROR: No value present for S3 bucket.', + }; + } + if (!url) { + console.log(`nextGenStage has failed. Variable for URL address was undefined.`); + return { + status: CommandExecutorResponseStatus.failed, + output: 'Failed in nextGenStage: No value present for target url.', + error: 'ERROR: No value present for URL.', + }; + } + // equivalent to: yes | mut-publish public ${BUCKET} --prefix=${MUT_PREFIX} --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; - const { outputText } = await executeAndPipeCommands( + const { outputText, errorText } = await executeAndPipeCommands( { command: 'yes' }, { - command: 'mut-publish', + command: 'time', args: [ + '-v', + 'mut-publish', 'public', bucket, `--prefix=${mutPrefix}`, @@ -38,20 +66,20 @@ export async function nextGenDeploy({ `--deployed-url-prefix=${url}`, '--json', '--all-subdirectories', - '--dry-run', ], options: { cwd: `${process.cwd()}/snooty`, }, } ); - console.log( - `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run` - ); - console.log(`${outputText}\n Hosted at ${url}/${mutPrefix}`); + const output = `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories + \n${outputText} \n ${errorText} \n Hosted at ${url}${mutPrefix} + `; + + console.log(output); return { status: CommandExecutorResponseStatus.success, - output: outputText, + output, error: '', }; } catch (error) { diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index fc790f41b..b8b56a77c 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -3,8 +3,8 @@ import { CliCommandResponse, executeCliCommand } from '../helpers'; export async function nextGenHtml(): Promise { try { const result = await executeCliCommand({ - command: 'npm', - args: ['run', 'build'], + command: 'time', + args: ['-v', 'npm', 'run', 'build'], options: { cwd: `${process.cwd()}/snooty` }, }); return result; diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 70bba49d7..763d36e0b 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -31,8 +31,8 @@ export async function nextGenParse({ job, patchId, isProd }: NextGenParseParams) } try { const result = await executeCliCommand({ - command: 'snooty', - args: commandArgs, + command: 'time', + args: ['-v', 'snooty', ...commandArgs], options: { cwd: repoDir }, }); return result; diff --git a/src/commands/src/shared/persistence-module.ts b/src/commands/src/shared/persistence-module.ts index 2498a2b18..9712277bc 100644 --- a/src/commands/src/shared/persistence-module.ts +++ b/src/commands/src/shared/persistence-module.ts @@ -10,6 +10,8 @@ export async function persistenceModule({ job }: PersistenceModuleParams): Promi const bundlePath = `${repoDir}/bundle.zip`; const args = [ + '-v', + 'node', `${process.cwd()}/modules/persistence/index.js`, '--unhandled-rejections=strict', '--path', @@ -25,12 +27,12 @@ export async function persistenceModule({ job }: PersistenceModuleParams): Promi try { const result = await executeCliCommand({ - command: 'node', + command: 'time', args, }); return result; } catch (error) { - throw new Error(`oas-page-build failed. \n ${error}`); + throw new Error(`persistence-module failed. \n ${error}`); } } diff --git a/src/entities/job.ts b/src/entities/job.ts index eaceaba59..b6c8a78af 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -75,9 +75,9 @@ export type EnhancedPayload = { pathPrefix?: string | null; mutPrefix?: string | null; aliased?: boolean | null; - primaryAlias?: string | null; + primaryAlias?: boolean | null; stable?: boolean | null; - repoBranches?: any; + repoBranches?: unknown; regression?: boolean | null; urlSlug?: string | null; prefix: string; @@ -112,15 +112,21 @@ export type Job = { shouldGenerateSearchManifest: boolean; }; -export type EnhancedJob = { +/** + * NOTE: This is just the Job type, but we kept them separate + * to minimize any risk of interfering with existing code. This should be + * re-examined at some point so that we can replace the Job type with this one + * to reduce confusion. + */ +export interface EnhancedJob { _id: string; payload: EnhancedPayload; createdTime: Date; endTime: Date | null | undefined; - error?: any; + error?: unknown; logs: string[] | null | undefined; priority: number | null | undefined; - result?: any; + result?: unknown; startTime: Date | null; status: JobStatus | null; title: string; @@ -131,8 +137,8 @@ export type EnhancedJob = { buildCommands?: string[]; deployCommands?: string[]; invalidationStatusURL?: string | null; - email: string | null; // probably can be removed + email?: string | null; // probably can be removed comMessage?: string[] | null; purgedUrls?: string[] | null; shouldGenerateSearchManifest?: boolean; -}; +} diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 5b854fefa..cce1e45af 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -147,15 +147,15 @@ export abstract class JobHandler { } @throwIfJobInterupted() - private async constructPrefix(): Promise { + private constructPrefix(): void { const server_user = this._config.get('GATSBY_PARSER_USER'); - const pathPrefix = await this.getPathPrefix(); - // TODO: Can empty string check be removed? - if (pathPrefix || pathPrefix === '') { - this.currJob.payload.pathPrefix = pathPrefix; - const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; - this.currJob.payload.mutPrefix = mutPrefix; - } + const pathPrefix = this.getPathPrefix(); + + if (!pathPrefix) return; + + this.currJob.payload.pathPrefix = pathPrefix; + const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; + this.currJob.payload.mutPrefix = mutPrefix; } @throwIfJobInterupted() @@ -204,7 +204,6 @@ export abstract class JobHandler { try { await this._repoConnector.pullRepo(this.currJob); } catch (error) { - await error; throw error; } } @@ -268,6 +267,7 @@ export abstract class JobHandler { this.prepStageSpecificNextGenCommands(); this.constructEnvVars(); + return; if (this._currJob.payload.jobType === 'productionDeploy') { this._validator.throwIfNotPublishable(this._currJob); } @@ -552,10 +552,14 @@ export abstract class JobHandler { await this.cloneRepo(this._config.get('repo_dir')); this._logger.save(job._id, 'Cloned Repo'); - await this.commitCheck(); + this._logger.save(job._id, 'Checked Commit'); await this.pullRepo(); + + this._logger.save(job._id, 'Next gen build'); + await this.prepNextGenBuild(); + this._logger.save(job._id, 'Pulled Repo'); await this.setEnvironmentVariables(); this.logger.save(job._id, 'Prepared Environment variables'); @@ -570,7 +574,13 @@ export abstract class JobHandler { } const baseUrl = docset?.url?.[env] || 'https://mongodbcom-cdn.website.staging.corp.mongodb.com'; - const { patchId } = await prepareBuild(job.payload.repoName, job.payload.project, baseUrl, job.payload.directory); + const { patchId } = await prepareBuild( + job.payload.repoName, + job.payload.project, + baseUrl, + job.payload.prefix, + job.payload.directory + ); // Set patchId on payload for use in nextGenStage this._currJob.payload.patchId = patchId; @@ -607,7 +617,7 @@ export abstract class JobHandler { } @throwIfJobInterupted() - protected async deployGeneric(): Promise { + protected async deployWithMakefiles(): Promise { this.prepDeployCommands(); await this._logger.save(this.currJob._id, `${this._config.get('stage').padEnd(15)}Pushing to ${this.name}`); diff --git a/src/job/jobValidator.ts b/src/job/jobValidator.ts index 9434bdb48..fa2cef185 100644 --- a/src/job/jobValidator.ts +++ b/src/job/jobValidator.ts @@ -36,7 +36,11 @@ export class JobValidator implements IJobValidator { const entitlementToFind = `${job.payload.repoOwner}/${job.payload.repoName}${ job.payload.repoName === MONOREPO_NAME ? `/${job.payload.directory}` : `` }`; - if (!entitlementsObject?.repos?.includes(entitlementToFind)) { + return; + if ( + !entitlementsObject?.repos?.includes(entitlementToFind) && + (job.payload.repoOwner === '10gen' || job.payload.repoOwner === 'mongodb') + ) { throw new AuthorizationError(`${job.user} is not entitled for repo ${entitlementToFind}`); } } diff --git a/src/job/manifestJobHandler.ts b/src/job/manifestJobHandler.ts index 3f755cbd1..34493cabd 100644 --- a/src/job/manifestJobHandler.ts +++ b/src/job/manifestJobHandler.ts @@ -116,7 +116,7 @@ export class ManifestJobHandler extends JobHandler { async deploy(): Promise { try { - const resp = await this.deployGeneric(); // runs prepDeployCommands + const resp = await this.deployWithMakefiles(); // runs prepDeployCommands await this.logger.save(this.currJob._id, `(generate manifest) Manifest generation details:\n\n${resp?.output}`); return resp; } catch (errResult) { diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index a7ed1ac28..db5344a30 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -1,6 +1,5 @@ import { IConfig } from 'config'; import type { Job } from '../entities/job'; -import { InvalidJobError } from '../errors/errors'; import { DocsetsRepository } from '../repositories/docsetsRepository'; import { JobRepository } from '../repositories/jobRepository'; import { RepoBranchesRepository } from '../repositories/repoBranchesRepository'; @@ -13,6 +12,10 @@ import { IRepoConnector } from '../services/repo'; import { getDirectory, JobHandler } from './jobHandler'; import { IJobValidator } from './jobValidator'; import { joinUrlAndPrefix } from './manifestJobHandler'; +import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants'; +import { nextGenDeploy } from '../commands'; +import { checkRedirects } from '../commands/src/helpers/dependency-helpers'; +import path from 'path'; export class ProductionJobHandler extends JobHandler { constructor( @@ -141,22 +144,10 @@ export class ProductionJobHandler extends JobHandler { } getPathPrefix(): string { - try { - if (this.currJob.payload.prefix && this.currJob.payload.prefix === '') { - return this.currJob.payload.urlSlug ?? ''; - } - if (this.currJob.payload.urlSlug) { - if (this.currJob.payload.urlSlug === '') { - return this.currJob.payload.prefix; - } else { - return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; - } - } - return this.currJob.payload.prefix; - } catch (error) { - this.logger.save(this.currJob._id, error).then(); - throw new InvalidJobError(error.message); + if (this.currJob.payload.urlSlug) { + return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; } + return this.currJob.payload.prefix; } private async purgePublishedContent(makefileOutput: Array): Promise { @@ -186,7 +177,32 @@ export class ProductionJobHandler extends JobHandler { } async deploy(): Promise { - const resp = await this.deployGeneric(); + let resp; + if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && this.currJob.payload.repoName === MONOREPO_NAME) { + const { mutPrefix, branchName } = this.currJob.payload; + await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Begin Deploy without makefiles`); + + // using this as a way to test deploy with feature branches in dotcomstg (preprod) + const finalMutPrefix = + process.env.IS_FEATURE_BRANCH === 'true' && process.env.FEATURE_NAME + ? `${mutPrefix}${process.env.FEATURE_NAME}` + : mutPrefix; + + const { bucket, url } = await this.getEnvironmentVariables(); + + await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Check redirects`); + const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); + await this.logger.save( + this.currJob._id, + `${'(prod-monorepo)'.padEnd(15)} Redirects checked ${hasConfigRedirects}` + ); + console.log('Generic log to see that it actually logs stuff in CloudWatch (which it should)'); + const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`); + + resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects, repoDir }); + } else { + resp = await this.deployWithMakefiles(); + } try { if (resp?.output) { const makefileOutput = resp.output.replace(/\r/g, '').split(/\n/); diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index e543da9fb..b41a625a5 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -90,7 +90,7 @@ export class StagingJobHandler extends JobHandler { resp = await nextGenStage({ job: this.currJob, bucket, url }); await this.logger.save(this.currJob._id, `${'(stage)'.padEnd(15)} ${resp.output}`); } else { - resp = await this.deployGeneric(); + resp = await this.deployWithMakefiles(); } const summary = ''; if (resp.output?.includes('Summary')) {