From c4ab4bcd5f09297916e79ef527d27dd2d7dca05e Mon Sep 17 00:00:00 2001 From: Cassidy Schaufele Date: Mon, 18 Jul 2022 22:36:31 -0600 Subject: [PATCH 1/3] Add scheduled deploys for docs-landing --- api/controllers/v1/scheduled/index.ts | 14 ++ .../v1/scheduled/repos/docs-landing.ts | 7 + api/controllers/v1/scheduled/utils/index.ts | 160 ++++++++++++++++++ serverless.yml | 18 ++ 4 files changed, 199 insertions(+) create mode 100644 api/controllers/v1/scheduled/index.ts create mode 100644 api/controllers/v1/scheduled/repos/docs-landing.ts create mode 100644 api/controllers/v1/scheduled/utils/index.ts diff --git a/api/controllers/v1/scheduled/index.ts b/api/controllers/v1/scheduled/index.ts new file mode 100644 index 000000000..67c362ca7 --- /dev/null +++ b/api/controllers/v1/scheduled/index.ts @@ -0,0 +1,14 @@ +import { DeployRepo } from "./utils"; +import { options as docsLandingOptions } from './repos/docs-landing'; + +export const DeployDocsLanding = async () => { + const { branches, repoName, repoOwner } = docsLandingOptions; + for (const branch of branches) { + await DeployRepo(repoOwner, repoName, branch) + } + + return { + statusCode: 200, + headers: { 'Content-Type': 'application/json' }, + }; +} \ No newline at end of file diff --git a/api/controllers/v1/scheduled/repos/docs-landing.ts b/api/controllers/v1/scheduled/repos/docs-landing.ts new file mode 100644 index 000000000..c31c0f0c8 --- /dev/null +++ b/api/controllers/v1/scheduled/repos/docs-landing.ts @@ -0,0 +1,7 @@ +export const options = { + repoOwner : 'mongodb', + repoName : 'docs-landing', + branches : [ + "master" + ] +}; \ No newline at end of file diff --git a/api/controllers/v1/scheduled/utils/index.ts b/api/controllers/v1/scheduled/utils/index.ts new file mode 100644 index 000000000..de040e261 --- /dev/null +++ b/api/controllers/v1/scheduled/utils/index.ts @@ -0,0 +1,160 @@ +import * as c from 'config'; +import * as mongodb from 'mongodb'; +import { BranchRepository } from '../../../../../src/repositories/branchRepository'; +import { ConsoleLogger, ILogger } from '../../../../../src/services/logger'; +import { JobRepository } from '../../../../../src/repositories/jobRepository'; +// Used solely for adding parallel deploy jobs to another array +const deployHelper = (deployable, payload, jobTitle, jobUserName) => { + deployable.push(createJob({ ...payload }, jobTitle, jobUserName)); +}; + +function createJob(payload: any, jobTitle: string, jobUserName: string) { + return { + title: jobTitle, + user: jobUserName, + email: '', + status: 'inQueue', + createdTime: new Date(), + startTime: null, + endTime: null, + priority: 1, + error: {}, + result: null, + payload: payload, + logs: [], + }; +} + +function createPayload( + jobType: string, + repoOwner: string, + repoName: string, + branchName: string, + newHead: string, + project: string, + prefix: string, + urlSlug, + aliased = false, + primaryAlias = false, + stable = '' +) { + return { + jobType, + source: 'github', + action: 'push', + repoName, + branchName, + project, + prefix, + aliased, + urlSlug, + isFork: true, + private: repoOwner === '10gen', + isXlarge: true, + repoOwner, + url: 'https://github.com/' + repoOwner + '/' + repoName, + newHead, + primaryAlias, + stable, + }; +} +// For every repo/branch selected to be deployed, return an array of jobs with the payload data +// needed for a successful build. +const getDeployableJobs = async (repoOwner:string , repoName:string , branchName: string, branchRepository: BranchRepository) => { + const deployable = []; + const hashOption = null; + const jobUserName = 'scheduled_deploy'; + const jobTitle = `Scheduled Deploy Action` + + const repoInfo = await branchRepository.getRepo(repoName); + const non_versioned = repoInfo.branches.length === 1; + + const branchObject = await branchRepository.getRepoBranchAliases(repoName, branchName); + if (!branchObject?.aliasObject) return; + + const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; //bool + let aliases = branchObject.aliasObject.urlAliases; // array or null + let urlSlug = branchObject.aliasObject.urlSlug; // string or null, string must match value in urlAliases or gitBranchName + const isStableBranch = branchObject.aliasObject.isStableBranch; // bool or Falsey + aliases = aliases?.filter((a) => a); + if (!urlSlug || !urlSlug.trim()) { + urlSlug = branchName; + } + + // Generic payload, will be conditionally modified appropriately + const newPayload = createPayload( + 'productionDeploy', + repoOwner, + repoName, + branchName, + hashOption, + repoInfo.project, + repoInfo.prefix[c.get('env')], + urlSlug, + false, + false, + '-g' + ); + + if (!aliases || aliases.length === 0) { + if (non_versioned) { + newPayload.urlSlug = ''; + } + deployHelper(deployable, newPayload, jobTitle, jobUserName); + return; + } + + // if this is stable branch, we want autobuilder to know this is unaliased branch and therefore can reindex for search + newPayload.stable = isStableBranch ? '-g' : ''; + newPayload.aliased = true; + + // we use the primary alias for indexing search, not the original branch name (ie 'master'), for aliased repos + if (urlSlug) { + newPayload.urlSlug = urlSlug; + newPayload.primaryAlias = true; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } + + // handle non-versioned repos AND repos where only 1 version is active + if (non_versioned || (!publishOriginalBranchName && urlSlug === null)) { + newPayload.urlSlug = ''; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } else if (publishOriginalBranchName && urlSlug !== branchName) { + newPayload.urlSlug = branchName; + newPayload.primaryAlias = false; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } + + aliases.forEach(async (alias: string) => { + if (alias !== urlSlug) { + newPayload.stable = ''; + newPayload.urlSlug = alias; + newPayload.primaryAlias = false; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } + }); + + return deployable; + }; + +async function deployRepo(deployable: Array, logger: ILogger, jobRepository: JobRepository, jobQueueUrl) { + try { + await jobRepository.insertBulkJobs(deployable, jobQueueUrl); + } catch (err) { + logger.error('deployRepo', err);x + } +} + +export const DeployRepo = async (repoOwner, repoName, branchName): Promise => { + const consoleLogger = new ConsoleLogger(); + const client = new mongodb.MongoClient(c.get('dbUrl')); + await client.connect(); + const db = client.db(c.get('dbName')); + const branchRepository = new BranchRepository(db, c, consoleLogger); + const jobRepository = new JobRepository(db, c, consoleLogger); + + const deployable = await getDeployableJobs(repoOwner, repoName, branchName, branchRepository); + if (deployable.length > 0) { + await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); + } + }; \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index bb393f416..de968072b 100644 --- a/serverless.yml +++ b/serverless.yml @@ -168,6 +168,24 @@ functions: environment: <<: *webhook-env-core +# SCHEDULED DEPLOYS START + + v1ScheduledDeployDocsLanding: + handler: api/controllers/v1/scheduled/index.DeployDocsLanding + events: + - schedule: cron(0/2 * ? * MON-FRI *) + iamRoleStatementsName: 'dwpapi-v1ScheduledDeployDocsLanding-lr-${self:provider.region}-${self:provider.stage}' + iamRoleStatements: + - Effect: Allow + Action: + - "sqs:SendMessage" + Resource: + - "*" + environment: + <<: *webhook-env-core + +# SCHEDULED DEPLOYS END + v1TriggerLocalBuild: handler: api/controllers/v1/jobs.TriggerLocalBuild events: From 686cc368496e4fad5194d03c848fd07d9f4bb480 Mon Sep 17 00:00:00 2001 From: Cassidy Schaufele Date: Tue, 19 Jul 2022 08:40:12 -0600 Subject: [PATCH 2/3] Promise.all()ify scheduled deploy routine --- api/controllers/v1/scheduled/index.ts | 7 ++++--- api/controllers/v1/scheduled/utils/index.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api/controllers/v1/scheduled/index.ts b/api/controllers/v1/scheduled/index.ts index 67c362ca7..631efc3cd 100644 --- a/api/controllers/v1/scheduled/index.ts +++ b/api/controllers/v1/scheduled/index.ts @@ -3,9 +3,10 @@ import { options as docsLandingOptions } from './repos/docs-landing'; export const DeployDocsLanding = async () => { const { branches, repoName, repoOwner } = docsLandingOptions; - for (const branch of branches) { - await DeployRepo(repoOwner, repoName, branch) - } + Promise.all(branches.map(branch => { + return DeployRepo(repoOwner, repoName, branch) + }) + ) return { statusCode: 200, diff --git a/api/controllers/v1/scheduled/utils/index.ts b/api/controllers/v1/scheduled/utils/index.ts index de040e261..9e4d8cbeb 100644 --- a/api/controllers/v1/scheduled/utils/index.ts +++ b/api/controllers/v1/scheduled/utils/index.ts @@ -139,9 +139,9 @@ const getDeployableJobs = async (repoOwner:string , repoName:string , branchName async function deployRepo(deployable: Array, logger: ILogger, jobRepository: JobRepository, jobQueueUrl) { try { - await jobRepository.insertBulkJobs(deployable, jobQueueUrl); + return jobRepository.insertBulkJobs(deployable, jobQueueUrl); } catch (err) { - logger.error('deployRepo', err);x + logger.error('deployRepo', err); } } @@ -155,6 +155,6 @@ export const DeployRepo = async (repoOwner, repoName, branchName): Promise const deployable = await getDeployableJobs(repoOwner, repoName, branchName, branchRepository); if (deployable.length > 0) { - await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); + return deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); } }; \ No newline at end of file From 8a23d686e5c988b0804374ca0d2aad0574c97226 Mon Sep 17 00:00:00 2001 From: Cassidy Schaufele Date: Tue, 19 Jul 2022 10:33:38 -0600 Subject: [PATCH 3/3] fix linting --- .nvmrc | 1 + api/controllers/v1/scheduled/index.ts | 23 +- .../v1/scheduled/repos/docs-landing.ts | 10 +- api/controllers/v1/scheduled/utils/index.ts | 231 +++++++++--------- 4 files changed, 135 insertions(+), 130 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..9f4189900 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v14.17.6 \ No newline at end of file diff --git a/api/controllers/v1/scheduled/index.ts b/api/controllers/v1/scheduled/index.ts index 631efc3cd..bf9375836 100644 --- a/api/controllers/v1/scheduled/index.ts +++ b/api/controllers/v1/scheduled/index.ts @@ -1,15 +1,16 @@ -import { DeployRepo } from "./utils"; +import { DeployRepo } from './utils'; import { options as docsLandingOptions } from './repos/docs-landing'; export const DeployDocsLanding = async () => { - const { branches, repoName, repoOwner } = docsLandingOptions; - Promise.all(branches.map(branch => { - return DeployRepo(repoOwner, repoName, branch) - }) - ) + const { branches, repoName, repoOwner } = docsLandingOptions; + Promise.all( + branches.map((branch) => { + return DeployRepo(repoOwner, repoName, branch); + }) + ); - return { - statusCode: 200, - headers: { 'Content-Type': 'application/json' }, - }; -} \ No newline at end of file + return { + statusCode: 200, + headers: { 'Content-Type': 'application/json' }, + }; +}; diff --git a/api/controllers/v1/scheduled/repos/docs-landing.ts b/api/controllers/v1/scheduled/repos/docs-landing.ts index c31c0f0c8..2ae4e876f 100644 --- a/api/controllers/v1/scheduled/repos/docs-landing.ts +++ b/api/controllers/v1/scheduled/repos/docs-landing.ts @@ -1,7 +1,5 @@ export const options = { - repoOwner : 'mongodb', - repoName : 'docs-landing', - branches : [ - "master" - ] -}; \ No newline at end of file + repoOwner: 'mongodb', + repoName: 'docs-landing', + branches: ['master'], +}; diff --git a/api/controllers/v1/scheduled/utils/index.ts b/api/controllers/v1/scheduled/utils/index.ts index 9e4d8cbeb..472d0ee62 100644 --- a/api/controllers/v1/scheduled/utils/index.ts +++ b/api/controllers/v1/scheduled/utils/index.ts @@ -5,84 +5,89 @@ import { ConsoleLogger, ILogger } from '../../../../../src/services/logger'; import { JobRepository } from '../../../../../src/repositories/jobRepository'; // Used solely for adding parallel deploy jobs to another array const deployHelper = (deployable, payload, jobTitle, jobUserName) => { - deployable.push(createJob({ ...payload }, jobTitle, jobUserName)); + deployable.push(createJob({ ...payload }, jobTitle, jobUserName)); }; function createJob(payload: any, jobTitle: string, jobUserName: string) { - return { - title: jobTitle, - user: jobUserName, - email: '', - status: 'inQueue', - createdTime: new Date(), - startTime: null, - endTime: null, - priority: 1, - error: {}, - result: null, - payload: payload, - logs: [], - }; + return { + title: jobTitle, + user: jobUserName, + email: '', + status: 'inQueue', + createdTime: new Date(), + startTime: null, + endTime: null, + priority: 1, + error: {}, + result: null, + payload: payload, + logs: [], + }; } function createPayload( - jobType: string, - repoOwner: string, - repoName: string, - branchName: string, - newHead: string, - project: string, - prefix: string, - urlSlug, - aliased = false, - primaryAlias = false, - stable = '' + jobType: string, + repoOwner: string, + repoName: string, + branchName: string, + newHead: string, + project: string, + prefix: string, + urlSlug, + aliased = false, + primaryAlias = false, + stable = '' ) { - return { - jobType, - source: 'github', - action: 'push', - repoName, - branchName, - project, - prefix, - aliased, - urlSlug, - isFork: true, - private: repoOwner === '10gen', - isXlarge: true, - repoOwner, - url: 'https://github.com/' + repoOwner + '/' + repoName, - newHead, - primaryAlias, - stable, - }; + return { + jobType, + source: 'github', + action: 'push', + repoName, + branchName, + project, + prefix, + aliased, + urlSlug, + isFork: true, + private: repoOwner === '10gen', + isXlarge: true, + repoOwner, + url: 'https://github.com/' + repoOwner + '/' + repoName, + newHead, + primaryAlias, + stable, + }; } // For every repo/branch selected to be deployed, return an array of jobs with the payload data // needed for a successful build. -const getDeployableJobs = async (repoOwner:string , repoName:string , branchName: string, branchRepository: BranchRepository) => { - const deployable = []; - const hashOption = null; - const jobUserName = 'scheduled_deploy'; - const jobTitle = `Scheduled Deploy Action` +const getDeployableJobs = async ( + repoOwner: string, + repoName: string, + branchName: string, + branchRepository: BranchRepository +) => { + const deployable = []; + const hashOption = null; + const jobUserName = 'scheduled_deploy'; + const jobTitle = `Scheduled Deploy Action`; - const repoInfo = await branchRepository.getRepo(repoName); - const non_versioned = repoInfo.branches.length === 1; + const repoInfo = await branchRepository.getRepo(repoName); + const non_versioned = repoInfo.branches.length === 1; - const branchObject = await branchRepository.getRepoBranchAliases(repoName, branchName); - if (!branchObject?.aliasObject) return; + const branchObject = await branchRepository.getRepoBranchAliases(repoName, branchName); + if (!branchObject?.aliasObject) return; - const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; //bool - let aliases = branchObject.aliasObject.urlAliases; // array or null - let urlSlug = branchObject.aliasObject.urlSlug; // string or null, string must match value in urlAliases or gitBranchName - const isStableBranch = branchObject.aliasObject.isStableBranch; // bool or Falsey - aliases = aliases?.filter((a) => a); - if (!urlSlug || !urlSlug.trim()) { + const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; //bool + let aliases = branchObject.aliasObject.urlAliases; // array or null + let urlSlug = branchObject.aliasObject.urlSlug; // string or null, string must match value in urlAliases or gitBranchName + const isStableBranch = branchObject.aliasObject.isStableBranch; // bool or Falsey + aliases = aliases?.filter((a) => a); + if (!urlSlug || !urlSlug.trim()) { urlSlug = branchName; - } + } - // Generic payload, will be conditionally modified appropriately - const newPayload = createPayload( + // Generic payload, will be conditionally modified appropriately + const newPayload = createPayload( 'productionDeploy', repoOwner, repoName, @@ -94,67 +99,67 @@ const getDeployableJobs = async (repoOwner:string , repoName:string , branchName false, false, '-g' - ); + ); - if (!aliases || aliases.length === 0) { - if (non_versioned) { - newPayload.urlSlug = ''; - } - deployHelper(deployable, newPayload, jobTitle, jobUserName); - return; + if (!aliases || aliases.length === 0) { + if (non_versioned) { + newPayload.urlSlug = ''; } + deployHelper(deployable, newPayload, jobTitle, jobUserName); + return; + } - // if this is stable branch, we want autobuilder to know this is unaliased branch and therefore can reindex for search - newPayload.stable = isStableBranch ? '-g' : ''; - newPayload.aliased = true; + // if this is stable branch, we want autobuilder to know this is unaliased branch and therefore can reindex for search + newPayload.stable = isStableBranch ? '-g' : ''; + newPayload.aliased = true; - // we use the primary alias for indexing search, not the original branch name (ie 'master'), for aliased repos - if (urlSlug) { - newPayload.urlSlug = urlSlug; - newPayload.primaryAlias = true; - deployHelper(deployable, newPayload, jobTitle, jobUserName); - } + // we use the primary alias for indexing search, not the original branch name (ie 'master'), for aliased repos + if (urlSlug) { + newPayload.urlSlug = urlSlug; + newPayload.primaryAlias = true; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } - // handle non-versioned repos AND repos where only 1 version is active - if (non_versioned || (!publishOriginalBranchName && urlSlug === null)) { - newPayload.urlSlug = ''; - deployHelper(deployable, newPayload, jobTitle, jobUserName); - } else if (publishOriginalBranchName && urlSlug !== branchName) { - newPayload.urlSlug = branchName; - newPayload.primaryAlias = false; - deployHelper(deployable, newPayload, jobTitle, jobUserName); - } + // handle non-versioned repos AND repos where only 1 version is active + if (non_versioned || (!publishOriginalBranchName && urlSlug === null)) { + newPayload.urlSlug = ''; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } else if (publishOriginalBranchName && urlSlug !== branchName) { + newPayload.urlSlug = branchName; + newPayload.primaryAlias = false; + deployHelper(deployable, newPayload, jobTitle, jobUserName); + } - aliases.forEach(async (alias: string) => { + aliases.forEach(async (alias: string) => { if (alias !== urlSlug) { - newPayload.stable = ''; - newPayload.urlSlug = alias; - newPayload.primaryAlias = false; - deployHelper(deployable, newPayload, jobTitle, jobUserName); + newPayload.stable = ''; + newPayload.urlSlug = alias; + newPayload.primaryAlias = false; + deployHelper(deployable, newPayload, jobTitle, jobUserName); } - }); + }); - return deployable; - }; + return deployable; +}; async function deployRepo(deployable: Array, logger: ILogger, jobRepository: JobRepository, jobQueueUrl) { - try { - return jobRepository.insertBulkJobs(deployable, jobQueueUrl); - } catch (err) { - logger.error('deployRepo', err); - } + try { + return jobRepository.insertBulkJobs(deployable, jobQueueUrl); + } catch (err) { + logger.error('deployRepo', err); + } } export const DeployRepo = async (repoOwner, repoName, branchName): Promise => { - const consoleLogger = new ConsoleLogger(); - const client = new mongodb.MongoClient(c.get('dbUrl')); - await client.connect(); - const db = client.db(c.get('dbName')); - const branchRepository = new BranchRepository(db, c, consoleLogger); - const jobRepository = new JobRepository(db, c, consoleLogger); - - const deployable = await getDeployableJobs(repoOwner, repoName, branchName, branchRepository); - if (deployable.length > 0) { - return deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); - } - }; \ No newline at end of file + const consoleLogger = new ConsoleLogger(); + const client = new mongodb.MongoClient(c.get('dbUrl')); + await client.connect(); + const db = client.db(c.get('dbName')); + const branchRepository = new BranchRepository(db, c, consoleLogger); + const jobRepository = new JobRepository(db, c, consoleLogger); + + const deployable = await getDeployableJobs(repoOwner, repoName, branchName, branchRepository); + if (deployable.length > 0) { + return deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); + } +};