DOP-4414: Lambda function to handle automated test deploys (#1011)

* DOP-4414 pushing to preprd * DOP-4414 logging more * DOP-4414 logging * DOP-4414 logging * DOP-4414 logging * DOP-4414 logging * DOP-4414 initial changes * DOP-4414 majority of ticket * DOP-4414 cleaning up * DOP-4414 refactoring * DOP-4414 webhook api construct * DOP-4414 checks build on main * DOP-4414 update validation * DOP-4414 deploy to preprd * DOP-4414 testing webhook * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 more changes * DOP-4414 cleaning up * DOP-4414 cleaning up * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 remove validation * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing jobupdates queue * DOP-4414 testing refactor * DOP-4414 testing refactor * DOP-4414 testing jobupdatesqueue * DOP-4414 cleaning up * DOP-4414 webhook update and cleaning * DOP-4414 secret * DOP-4414 checking main * DOP-4414 checking main * DOP-4414 types and env vars * DOP-4414 comment out check for main branch * DOP-4414 entitlements repos * DOP-4414 testing * revert "DOP-4414 testing" This reverts commit c4ac837. reverting changes * Revert "DOP-4414 entitlements repos" This reverts commit c86c466. reverting commit * Revert "DOP-4414 comment out check for main branch" This reverts commit bce69c3. reverting commit * reverting commit Revert "DOP-4414 types and env vars" This reverts commit aee5cd4. reverting commit * DOP-4414 comment out main check for testing * DOP-4414 testing * DOP-4414 add branch to deploy enhanced worker wq * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 nits * DOP-4414 fixing small error * DOP-4414 refactoring promises * DOP-4414 testing new promise format * DOP-4414 testing new promise format * DOP-4414 testing new promise format * DOP-4414 refactoring and cleaning * DOP-4414 refactoring and cleaning * DOP-4414 adding projects collection as constant * DOP-4414 add docs_metadata as constant * DOP-4414 remove unecessary vars from the configs * DOP-4414 readded name * DOP-4414 testing * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars removing from yamls * DOP-4414 testing env vars removing adding back to yamll * DOP-4414 testing env vars removing adding back to yamll * DOP-4414 testing env vars removing from get-env-vars * DOP-4414 testing env vars removing from get-env-vars * DOP-4414 testing env vars removing jsons * DOP-4414 ssm putparam * DOP-4414 removing env vars * DOP-4414 removing env vars * DOP-4414 removing more env vars * DOP-4414 removing more env vars * DOP-4414 uncomment main branch check * DOP-4414 cleaning up * DOP-4414 last cleaning * DOP-4414 removing from preprd * DOP-4414 playing with prefixes, all changes in this commit should not persist * DOP-4414 push to preprd * DOP-4414 trying to get jobhandler lambda to update and reflect changes * Fix workflow cache * Fix workflow paths * DOP-4414 nit changing how returnVal is declared * Fix workflow paths * Fix workflow paths * DOP-4414 changing prefix again * DOP-4414 adding code for newHead and changing mut prefix path * DOP-4414 removing newHead and passing in hash through title * DOP-4414 fixing conditional for amending mut prefix so that it executes * DOP-4414 placing commitHash at end of mutprefix instead of beginngin * DOP-4414 adding extra part to mutprefix path * DOP-4414 reversing changes made for testing purposes * DOP-4414 reverting more changes that were made for testing * DOP-4414 conditional nit --------- Co-authored-by: branberry <[email protected]>
mongodb · Apr 11, 2024 · bb57846 · bb57846
1 parent 0cb298a
commit bb57846
Show file tree

Hide file tree

Showing 17 changed files with 299 additions and 53 deletions.
diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml
@@ -1,9 +1,9 @@
 on:
   push:
-    paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"]
+    paths: ['api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**']
     branches:
-      - "main"
-      - "integration"
+      - 'main'
+      - 'integration'
 
 concurrency:
   group: environment-stg-enhanced-webhooks-${{ github.ref }}
@@ -30,4 +30,3 @@ jobs:
           npm ci
           npm run deploy:feature:stack -- -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg auto-builder-stack-enhancedApp-dotcomstg-webhooks
           npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg auto-builder-stack-enhancedApp-stg-webhooks
-
diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml
@@ -4,7 +4,6 @@ on:
     branches:
       - 'main'
       - 'integration'
-
 concurrency:
   group: environment-stg-enhanced-worker-${{ github.ref }}
   cancel-in-progress: true

diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml
@@ -61,6 +61,12 @@ jobs:
             node_modules
             cdk-infra/node_modules
           key: ${{ github.head_ref }}
+      - name: Install Dependencies
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -96,6 +102,12 @@ jobs:
             node_modules
             cdk-infra/node_modules
           key: ${{ github.head_ref }}
+      - name: Install Dependencies
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
       - uses: dorny/paths-filter@v2
         id: filter
         with:

diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts
@@ -1,71 +1,125 @@
 import * as c from 'config';
 import * as mongodb from 'mongodb';
 import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
-import { PushEvent } from '@octokit/webhooks-types';
+import { PushEvent, WorkflowRunCompletedEvent } from '@octokit/webhooks-types';
 
 import { JobRepository } from '../../../src/repositories/jobRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
-import { EnhancedJob, JobStatus } from '../../../src/entities/job';
+import { ProjectsRepository } from '../../../src/repositories/projectsRepository';
+import { EnhancedJob, EnhancedPayload, JobStatus } from '../../../src/entities/job';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
 import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 import { getMonorepoPaths } from '../../../src/monorepo';
 import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
 import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches';
 import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants';
 
+const SMOKETEST_SITES = [
+  'docs-landing',
+  'cloud-docs',
+  'docs-realm',
+  'docs',
+  'docs-atlas-cli',
+  'docs-node',
+  'docs-app-services',
+];
+
+//EnhancedPayload and EnhancedJob are used here for both githubPush (feature branch) events as well as productionDeploy(smoke test deploy) events for typing purposes
 async function prepGithubPushPayload(
-  githubEvent: PushEvent,
-  repoBranchesRepository: RepoBranchesRepository,
-  prefix: string,
-  repoInfo: ReposBranchesDocsetsDocument,
-  directory?: string
+  githubEvent: PushEvent | WorkflowRunCompletedEvent,
+  payload: EnhancedPayload,
+  title: string
 ): Promise<Omit<EnhancedJob, '_id'>> {
-  const branch_name = githubEvent.ref.split('/')[2];
-  const branch_info = await repoBranchesRepository.getRepoBranchAliases(
-    githubEvent.repository.name,
-    branch_name,
-    repoInfo.project
-  );
-  const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
-  const project = repoInfo?.project ?? githubEvent.repository.name;
-
   return {
-    title: githubEvent.repository.full_name,
-    user: githubEvent.pusher.name,
-    email: githubEvent.pusher.email ?? '',
+    title: title,
+    user: githubEvent.sender.login,
     status: JobStatus.inQueue,
     createdTime: new Date(),
     startTime: null,
     endTime: null,
     priority: 1,
     error: {},
     result: null,
-    payload: {
-      jobType: 'githubPush',
-      source: 'github',
-      action: 'push',
-      repoName: githubEvent.repository.name,
-      branchName: githubEvent.ref.split('/')[2],
-      isFork: githubEvent.repository.fork,
-      repoOwner: githubEvent.repository.owner.login,
-      url: githubEvent.repository.clone_url,
-      newHead: githubEvent.after,
-      urlSlug: urlSlug,
-      prefix: prefix,
-      project: project,
-      directory: directory,
-    },
+    payload,
     logs: [],
   };
 }
 
-export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
+interface CreatePayloadProps {
+  repoName: string;
+  isSmokeTestDeploy?: boolean;
+  prefix: string;
+  repoBranchesRepository: RepoBranchesRepository;
+  repoInfo: ReposBranchesDocsetsDocument;
+  newHead?: string;
+  repoOwner?: string;
+  githubEvent?: PushEvent;
+  directory?: string;
+}
+
+async function createPayload({
+  repoName,
+  isSmokeTestDeploy = false,
+  prefix,
+  repoBranchesRepository,
+  repoInfo,
+  newHead,
+  repoOwner = '',
+  githubEvent,
+  directory,
+}: CreatePayloadProps): Promise<EnhancedPayload> {
+  const source = 'github';
+  const project = repoInfo?.project ?? repoName;
+
+  let branchName: string;
+  let jobType: string;
+  let action: string;
+  let url: string;
+
+  if (isSmokeTestDeploy) {
+    url = 'https://github.com/' + repoOwner + '/' + repoName;
+    action = 'automatedTest';
+    jobType = 'productionDeploy';
+    branchName = 'master';
+  } else {
+    if (!githubEvent) {
+      throw new Error(`Non SmokeTest Deploy jobs must have a github Event`);
+    }
+    action = 'push';
+    jobType = 'githubPush';
+    branchName = githubEvent.ref.split('/')[2];
+    url = githubEvent.repository?.clone_url;
+    newHead = githubEvent.after;
+    repoOwner = githubEvent.repository?.owner?.login;
+  }
+
+  const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project);
+  const urlSlug = branchInfo.aliasObject?.urlSlug ?? branchName;
+
+  return {
+    jobType,
+    source,
+    action,
+    repoName,
+    repoOwner,
+    branchName,
+    project,
+    prefix,
+    urlSlug,
+    url,
+    newHead,
+    directory,
+  };
+}
+
+export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult | null> => {
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
+  const projectsRepository = new ProjectsRepository(client.db(process.env.METADATA_DB_NAME), c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
   const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
 
@@ -79,6 +133,132 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
     };
   }
 
+  if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
+    const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
+    return {
+      statusCode: 401,
+      headers: { 'Content-Type': 'text/plain' },
+      body: errMsg,
+    };
+  }
+
+  let body: WorkflowRunCompletedEvent;
+  try {
+    body = JSON.parse(event.body) as WorkflowRunCompletedEvent;
+  } catch (e) {
+    console.log('[TriggerBuild]: ERROR! Could not parse event.body', e);
+    return {
+      statusCode: 502,
+      headers: { 'Content-Type': 'text/plain' },
+      body: ' ERROR! Could not parse event.body',
+    };
+  }
+
+  if (body.workflow_run.conclusion != 'success')
+    return {
+      statusCode: 202,
+      headers: { 'Content-Type': 'text/plain' },
+      body: `Build on branch ${body.workflow_run.head_branch} is not complete and will not trigger smoke test site deployments`,
+    };
+
+  if (body.workflow_run.name != 'Deploy Staging ECS')
+    return {
+      statusCode: 202,
+      headers: { 'Content-Type': 'text/plain' },
+      body: `Workflow ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`,
+    };
+
+  // if the build was not building main branch, no need for smoke test sites
+  if (body.workflow_run.head_branch != 'main' || body.repository.fork) {
+    console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed');
+    return {
+      statusCode: 202,
+      headers: { 'Content-Type': 'text/plain' },
+      body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`,
+    };
+  }
+
+  //automated test builds will always deploy in dotcomstg
+  const env = 'dotcomstg';
+
+  async function createAndInsertJob() {
+    return await Promise.all(
+      SMOKETEST_SITES.map(async (repoName): Promise<string> => {
+        const jobTitle = `Smoke Test ${repoName} site for commit ${body.workflow_run.head_sha} on docs-worker-pool main branch`;
+        let repoInfo, projectEntry, repoOwner;
+        try {
+          repoInfo = await docsetsRepository.getRepo(repoName);
+          projectEntry = await projectsRepository.getProjectEntry(repoInfo.project);
+          repoOwner = projectEntry.github.organization;
+        } catch (err) {
+          consoleLogger.error(
+            `Atlas Repo Information Error`,
+            `RepoInfo, projectEntry, or repoOwner not found for docs site ${repoName}. RepoInfo: ${repoInfo}, projectEntry: ${projectEntry}, repoOwner: ${repoOwner}`
+          );
+          return err;
+        }
+
+        const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
+        const payload = await createPayload({
+          repoName,
+          isSmokeTestDeploy: true,
+          prefix: jobPrefix,
+          repoBranchesRepository,
+          repoInfo,
+          repoOwner,
+        });
+
+        //add logic for getting master branch, latest stable branch
+        const job = await prepGithubPushPayload(body, payload, jobTitle);
+
+        try {
+          consoleLogger.info(job.title, 'Creating Job');
+          const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));
+          jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0);
+          consoleLogger.info(job.title, `Created Job ${jobId}`);
+          return jobId;
+        } catch (err) {
+          consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`);
+          return err;
+        }
+      })
+    );
+  }
+
+  try {
+    const returnVal = await createAndInsertJob();
+    return {
+      statusCode: 202,
+      headers: { 'Content-Type': 'text/plain' },
+      body: 'Smoke Test Jobs Queued with the following Job Ids ' + returnVal,
+    };
+  } catch (err) {
+    return {
+      statusCode: 500,
+      headers: { 'Content-Type': 'text/plain' },
+      body: err,
+    };
+  }
+};
+
+export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
+  const client = new mongodb.MongoClient(c.get('dbUrl'));
+  await client.connect();
+  const db = client.db(c.get('dbName'));
+  const consoleLogger = new ConsoleLogger();
+  const jobRepository = new JobRepository(db, c, consoleLogger);
+  const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
+
+  if (!event.body) {
+    const err = 'Trigger build does not have a body in event payload';
+    return {
+      statusCode: 400,
+      headers: { 'Content-Type': 'text/plain' },
+      body: err,
+    };
+  }
+
   if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
     const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
     return {
@@ -110,9 +290,19 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   const env = c.get<string>('env');
 
   async function createAndInsertJob(path?: string) {
-    const repoInfo = await docsetsRepository.getRepo(body.repository.name, path);
+    const repo = body.repository;
+    const repoInfo = await docsetsRepository.getRepo(repo.name, path);
     const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
-    const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, path);
+    const jobTitle = repo.full_name;
+    const payload = await createPayload({
+      repoName: repo.name,
+      prefix: jobPrefix,
+      repoBranchesRepository,
+      repoInfo,
+      githubEvent: body,
+    });
+
+    const job = await prepGithubPushPayload(body, payload, jobTitle);
 
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts
@@ -6,7 +6,7 @@ import { ConsoleLogger, ILogger } from '../../../src/services/logger';
 import { SlackConnector } from '../../../src/services/slack';
 import { JobRepository } from '../../../src/repositories/jobRepository';
 import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
-import { JobStatus } from '../../../src/entities/job';
+import { EnhancedPayload, JobStatus } from '../../../src/entities/job';
 import {
   buildEntitledBranchList,
   getQSString,
@@ -256,7 +256,7 @@ function createPayload(
   };
 }
 
-function createJob(payload: any, jobTitle: string, jobUserName: string, jobUserEmail: string) {
+function createJob(payload: EnhancedPayload, jobTitle: string, jobUserName: string, jobUserEmail: string) {
   return {
     title: jobTitle,
     user: jobUserName,