diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..62c614441 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4401-b" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a761e742b..518543d91 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -1,12 +1,13 @@ import * as c from 'config'; import * as mongodb from 'mongodb'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; -import { PushEvent } from '@octokit/webhooks-types'; +import { PushEvent, WorkflowRunCompletedEvent } from '@octokit/webhooks-types'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; -import { EnhancedJob, JobStatus } from '../../../src/entities/job'; +import { ProjectsRepository } from '../../../src/repositories/projectsRepository'; +import { EnhancedJob, EnhancedPayload, JobStatus } from '../../../src/entities/job'; import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github'; import { DocsetsRepository } from '../../../src/repositories/docsetsRepository'; import { getMonorepoPaths } from '../../../src/monorepo'; @@ -14,26 +15,25 @@ import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils'; import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches'; import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; +const SMOKETEST_SITES = [ + 'docs-landing', + 'cloud-docs', + 'docs-realm', + 'docs', + 'docs-atlas-cli', + 'docs-node', + 'docs-app-services', +]; + +//EnhancedPayload and EnhancedJob are used here for both githubPush (feature branch) events as well as productionDeploy(smoke test deploy) events for typing purposes async function prepGithubPushPayload( - githubEvent: PushEvent, - repoBranchesRepository: RepoBranchesRepository, - prefix: string, - repoInfo: ReposBranchesDocsetsDocument, - directory?: string + githubEvent: PushEvent | WorkflowRunCompletedEvent, + payload: EnhancedPayload, + title: string ): Promise> { - const branch_name = githubEvent.ref.split('/')[2]; - const branch_info = await repoBranchesRepository.getRepoBranchAliases( - githubEvent.repository.name, - branch_name, - repoInfo.project - ); - const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; - const project = repoInfo?.project ?? githubEvent.repository.name; - return { - title: githubEvent.repository.full_name, - user: githubEvent.pusher.name, - email: githubEvent.pusher.email ?? '', + title: title, + user: githubEvent.sender.login, status: JobStatus.inQueue, createdTime: new Date(), startTime: null, @@ -41,31 +41,85 @@ async function prepGithubPushPayload( priority: 1, error: {}, result: null, - payload: { - jobType: 'githubPush', - source: 'github', - action: 'push', - repoName: githubEvent.repository.name, - branchName: githubEvent.ref.split('/')[2], - isFork: githubEvent.repository.fork, - repoOwner: githubEvent.repository.owner.login, - url: githubEvent.repository.clone_url, - newHead: githubEvent.after, - urlSlug: urlSlug, - prefix: prefix, - project: project, - directory: directory, - }, + payload, logs: [], }; } -export const TriggerBuild = async (event: APIGatewayEvent): Promise => { +interface CreatePayloadProps { + repoName: string; + isSmokeTestDeploy?: boolean; + prefix: string; + repoBranchesRepository: RepoBranchesRepository; + repoInfo: ReposBranchesDocsetsDocument; + newHead?: string; + repoOwner?: string; + directory?: string; + githubEvent?: PushEvent; +} + +async function createPayload({ + repoName, + isSmokeTestDeploy = false, + prefix, + repoBranchesRepository, + repoInfo, + newHead, + repoOwner = '', + githubEvent, + directory, +}: CreatePayloadProps): Promise { + const source = 'github'; + const project = repoInfo?.project ?? repoName; + + let branchName: string; + let jobType: string; + let action: string; + let url: string; + + if (isSmokeTestDeploy) { + url = 'https://github.com/' + repoOwner + '/' + repoName; + action = 'automatedTest'; + jobType = 'productionDeploy'; + branchName = 'master'; + } else { + if (!githubEvent) { + throw new Error(`Non SmokeTest Deploy jobs must have a github Event`); + } + action = 'push'; + jobType = 'githubPush'; + branchName = githubEvent.ref.split('/')[2]; + url = githubEvent.repository?.clone_url; + newHead = githubEvent.after; + repoOwner = githubEvent.repository?.owner?.login; + } + + const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); + const urlSlug = branchInfo.aliasObject?.urlSlug ?? branchName; + + return { + jobType, + source, + action, + repoName, + repoOwner, + branchName, + project, + prefix, + urlSlug, + url, + newHead, + directory, + }; +} + +export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); + const projectsRepository = new ProjectsRepository(client.db(process.env.METADATA_DB_NAME), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); @@ -79,6 +133,133 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('githubSecret'))) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } + + let body: WorkflowRunCompletedEvent; + try { + body = JSON.parse(event.body) as WorkflowRunCompletedEvent; + } catch (e) { + console.log('[TriggerBuild]: ERROR! Could not parse event.body', e); + return { + statusCode: 502, + headers: { 'Content-Type': 'text/plain' }, + body: ' ERROR! Could not parse event.body', + }; + } + + if (body.workflow_run.conclusion != 'success') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Build on branch ${body.workflow_run.head_branch} is not complete and will not trigger smoke test site deployments`, + }; + + if (body.workflow_run.name != 'Deploy Staging ECS') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Workflow ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, + }; + + // if the build was not building main branch, no need for smoke test sites + if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, + }; + } + + //automated test builds will always deploy in dotcomstg + const env = 'dotcomstg'; + + async function createAndInsertJob() { + return await Promise.all( + SMOKETEST_SITES.map(async (repoName): Promise => { + const jobTitle = 'Smoke Test ' + repoName; + let repoInfo, projectEntry, repoOwner; + try { + repoInfo = await docsetsRepository.getRepo(repoName); + projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); + repoOwner = projectEntry.github.organization; + } catch (err) { + consoleLogger.error( + `Atlas Repo Information Error`, + `RepoInfo, projectEntry, or repoOwner not found for docs site ${repoName}. RepoInfo: ${repoInfo}, projectEntry: ${projectEntry}, repoOwner: ${repoOwner}` + ); + return err; + } + + const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + const payload = await createPayload({ + repoName, + isSmokeTestDeploy: true, + prefix: jobPrefix, + repoBranchesRepository, + repoInfo, + repoOwner, + }); + + //add logic for getting master branch, latest stable branch + const job = await prepGithubPushPayload(body, payload, jobTitle); + + try { + consoleLogger.info(job.title, 'Creating Job'); + const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); + jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + consoleLogger.info(job.title, `Created Job ${jobId}`); + return jobId; + } catch (err) { + consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`); + return err; + } + }) + ); + } + + let returnVal; + try { + returnVal = await createAndInsertJob(); + } catch (err) { + return { + statusCode: 500, + headers: { 'Content-Type': 'text/plain' }, + body: err, + }; + } + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Smoke Test Jobs Queued with the following Job Ids ' + returnVal, + }; +}; + +export const TriggerBuild = async (event: APIGatewayEvent): Promise => { + const client = new mongodb.MongoClient(c.get('dbUrl')); + await client.connect(); + const db = client.db(c.get('dbName')); + const consoleLogger = new ConsoleLogger(); + const jobRepository = new JobRepository(db, c, consoleLogger); + const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); + const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); + + if (!event.body) { + const err = 'Trigger build does not have a body in event payload'; + return { + statusCode: 400, + headers: { 'Content-Type': 'text/plain' }, + body: err, + }; + } + if (!validateJsonWebhook(event, c.get('githubSecret'))) { const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; return { @@ -110,9 +291,19 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('env'); async function createAndInsertJob(path?: string) { - const repoInfo = await docsetsRepository.getRepo(body.repository.name, path); + const repo = body.repository; + const repoInfo = await docsetsRepository.getRepo(repo.name, path); const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, path); + const jobTitle = repo.full_name; + const payload = await createPayload({ + repoName: repo.name, + prefix: jobPrefix, + repoBranchesRepository, + repoInfo, + githubEvent: body, + }); + + const job = await prepGithubPushPayload(body, payload, jobTitle); consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts index 2cb5307bc..b6d76f74a 100644 --- a/api/controllers/v2/slack.ts +++ b/api/controllers/v2/slack.ts @@ -6,7 +6,7 @@ import { ConsoleLogger, ILogger } from '../../../src/services/logger'; import { SlackConnector } from '../../../src/services/slack'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; -import { JobStatus } from '../../../src/entities/job'; +import { EnhancedPayload, JobStatus } from '../../../src/entities/job'; import { buildEntitledBranchList, getQSString, @@ -256,7 +256,7 @@ function createPayload( }; } -function createJob(payload: any, jobTitle: string, jobUserName: string, jobUserEmail: string) { +function createJob(payload: EnhancedPayload, jobTitle: string, jobUserName: string, jobUserEmail: string) { return { title: jobTitle, user: jobUserName, diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 52c71b95f..10862f6b8 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -74,6 +74,15 @@ export class WebhookApiConstruct extends Construct { timeout, }); + const githubSmokeTestBuildLambda = new NodejsFunction(this, 'githubSmokeTestBuildLambda', { + entry: `${HANDLERS_PATH}/github.ts`, + runtime, + handler: 'triggerSmokeTestAutomatedBuild', + bundling, + environment, + timeout, + }); + const githubDeleteArtifactsLambda = new NodejsFunction(this, 'githubDeleteArtifactsLambda', { entry: `${HANDLERS_PATH}/github.ts`, runtime, @@ -160,6 +169,11 @@ export class WebhookApiConstruct extends Construct { .addResource('build', { defaultCorsPreflightOptions }) .addMethod('POST', new LambdaIntegration(githubTriggerLambda)); + // add endpoint for automated testing + githubEndpointTrigger + .addResource('smoke-test-build', { defaultCorsPreflightOptions }) + .addMethod('POST', new LambdaIntegration(githubSmokeTestBuildLambda)); + githubEndpointTrigger .addResource('delete', { defaultCorsPreflightOptions }) .addMethod('POST', new LambdaIntegration(githubDeleteArtifactsLambda)); @@ -176,11 +190,13 @@ export class WebhookApiConstruct extends Construct { // grant permission for lambdas to enqueue messages to the jobs queue jobsQueue.grantSendMessages(slackTriggerLambda); + jobsQueue.grantSendMessages(githubSmokeTestBuildLambda); jobsQueue.grantSendMessages(githubTriggerLambda); jobsQueue.grantSendMessages(triggerLocalBuildLambda); // grant permission for lambdas to enqueue messages to the job updates queue jobUpdatesQueue.grantSendMessages(slackTriggerLambda); + jobUpdatesQueue.grantSendMessages(githubSmokeTestBuildLambda); jobUpdatesQueue.grantSendMessages(githubTriggerLambda); jobUpdatesQueue.grantSendMessages(triggerLocalBuildLambda); diff --git a/cdk-infra/lib/constructs/api/webhook-env-construct.ts b/cdk-infra/lib/constructs/api/webhook-env-construct.ts index 800b9277d..0c53cb4a7 100644 --- a/cdk-infra/lib/constructs/api/webhook-env-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-env-construct.ts @@ -40,8 +40,10 @@ export class WebhookEnvConstruct extends Construct { MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`, DB_NAME: dbName, SNOOTY_DB_NAME: snootyDbName, + METADATA_DB_NAME: 'docs_metadata', REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, + PROJECTS_COL_NAME: 'projects', JOB_QUEUE_COL_NAME: jobCollection, NODE_CONFIG_DIR: './config', JOBS_QUEUE_URL: jobsQueue.queueUrl, diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 8c00607dc..63f550305 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -39,7 +39,7 @@ export class WorkerConstruct extends Construct { const taskRoleSsmPolicyStatement = new PolicyStatement({ effect: Effect.ALLOW, - actions: ['ssm:GetParameter'], + actions: ['ssm:GetParameter', 'ssm:PutParameter'], resources: ['*'], }); diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index ff3e1f8e4..de91138ad 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -77,6 +77,7 @@ export class WorkerEnvConstruct extends Construct { MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`, DB_NAME: dbName, SNOOTY_DB_NAME: snootyDbName, + METADATA_DB_NAME: 'docs_metadata', JOBS_QUEUE_URL: jobsQueue.queueUrl, JOB_UPDATES_QUEUE_URL: jobUpdatesQueue.queueUrl, GITHUB_BOT_USERNAME: githubBotUsername, @@ -89,6 +90,7 @@ export class WorkerEnvConstruct extends Construct { REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, JOB_QUEUE_COL_NAME: jobCollection, + PROJECTS_COL_NAME: 'projects', CDN_INVALIDATOR_SERVICE_URL: getCdnInvalidatorUrl(env), SEARCH_INDEX_BUCKET: 'docs-search-indexes-test', SEARCH_INDEX_FOLDER: getSearchIndexFolder(env), diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 5d86baf73..d41cac516 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -48,7 +48,7 @@ export async function nextGenDeploy({ console.log( `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run` ); - console.log(`${outputText}\n Hosted at ${url}/${mutPrefix}`); + console.log(`${outputText}\n Hosted at ${url}${mutPrefix}`); return { status: CommandExecutorResponseStatus.success, output: outputText, diff --git a/src/entities/job.ts b/src/entities/job.ts index eaceaba59..fcbbd5233 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -64,7 +64,7 @@ export type EnhancedPayload = { action: string; repoName: string; branchName: string; - isFork: boolean; + isFork?: boolean; isXlarge?: boolean | null; repoOwner: string; url: string; @@ -131,7 +131,7 @@ export type EnhancedJob = { buildCommands?: string[]; deployCommands?: string[]; invalidationStatusURL?: string | null; - email: string | null; // probably can be removed + email?: string | null; // probably can be removed comMessage?: string[] | null; purgedUrls?: string[] | null; shouldGenerateSearchManifest?: boolean; diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index a7ed1ac28..554f38cee 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -142,21 +142,34 @@ export class ProductionJobHandler extends JobHandler { getPathPrefix(): string { try { - if (this.currJob.payload.prefix && this.currJob.payload.prefix === '') { - return this.currJob.payload.urlSlug ?? ''; + const prefix = this.currJob.payload.urlSlug + ? `${this.currJob.payload.urlSlug}/${this.currJob.payload.prefix}` + : this.currJob.payload.prefix; + if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { + return `${prefix}/${this.currJob.payload.newHead}`; } - if (this.currJob.payload.urlSlug) { - if (this.currJob.payload.urlSlug === '') { - return this.currJob.payload.prefix; - } else { - return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; - } - } - return this.currJob.payload.prefix; + return prefix; } catch (error) { this.logger.save(this.currJob._id, error).then(); throw new InvalidJobError(error.message); } + // try { + // if (this.currJob.payload.prefix && this.currJob.payload.prefix === '') { + // return this.currJob.payload.urlSlug ?? ''; + // } + // if (this.currJob.payload.urlSlug) { + // if (this.currJob.payload.urlSlug === '') { + // return this.currJob.payload.prefix; + // } else { + // return this.currJob.payload.prefix + `/${this.currJob.payload.urlSlug}`; + // } + // } + + // return prefix; + // } catch (error) { + // this.logger.save(this.currJob._id, error).then(); + // throw new InvalidJobError(error.message); + // } } private async purgePublishedContent(makefileOutput: Array): Promise { @@ -181,7 +194,7 @@ export class ProductionJobHandler extends JobHandler { await this.jobRepository.insertInvalidationRequestStatusUrl(this.currJob._id, 'Invalidation Failed'); } } catch (error) { - await this.logger.save(this.currJob._id, error); + await this.logger.save(this.currJob._id, error.message); } } diff --git a/src/repositories/baseRepository.ts b/src/repositories/baseRepository.ts index 750a1f83f..564e9b268 100644 --- a/src/repositories/baseRepository.ts +++ b/src/repositories/baseRepository.ts @@ -71,7 +71,11 @@ export abstract class BaseRepository { ); } - protected async findOne(query: any, errorMsg: string, options: mongodb.FindOptions = {}): Promise { + protected async findOne( + query: any, + errorMsg: string, + options: mongodb.FindOptions = {} + ): Promise | null> { try { return await this.promiseTimeoutS( this._config.get('MONGO_TIMEOUT_S'), diff --git a/src/repositories/jobRepository.ts b/src/repositories/jobRepository.ts index 7712f15b3..d2e27da19 100644 --- a/src/repositories/jobRepository.ts +++ b/src/repositories/jobRepository.ts @@ -79,7 +79,7 @@ export class JobRepository extends BaseRepository { return jobIds; } - async getJobById(id: string): Promise { + async getJobById(id: string): Promise { const query = { _id: new objectId(id), }; diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts new file mode 100644 index 000000000..08ebb4e19 --- /dev/null +++ b/src/repositories/projectsRepository.ts @@ -0,0 +1,22 @@ +import mongodb from 'mongodb'; +import { IConfig } from 'config'; +import { BaseRepository } from './baseRepository'; +import { ILogger } from '../services/logger'; + +//Project information from docs_metadata.projects for parser builds. + +export class ProjectsRepository extends BaseRepository { + constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { + super(config, logger, 'ProjectsRepository', db.collection(process.env.PROJECTS_COL_NAME || '')); + } + + async getProjectEntry(name: string): Promise | null> { + const query = { name: name }; + const projectEntry = await this.findOne( + query, + `Mongo Timeout Error: Timedout while retrieving branches for ${name} + }` + ); + return projectEntry; + } +} diff --git a/src/repositories/repoEntitlementsRepository.ts b/src/repositories/repoEntitlementsRepository.ts index f9cdcde26..e878b7458 100644 --- a/src/repositories/repoEntitlementsRepository.ts +++ b/src/repositories/repoEntitlementsRepository.ts @@ -73,7 +73,7 @@ export class RepoEntitlementsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving entitlements for ${slackUserId}` ); // if user has specific entitlements - if ((entitlementsObject?.repos?.length ?? 0) > 0) { + if (entitlementsObject?.repos && entitlementsObject.repos.length > 0) { return { repos: entitlementsObject.repos, github_username: entitlementsObject.github_username,