Merge branch 'master' into DOP-4010

.github/scripts/webhook-pr-comment.js

@@ -0,0 +1,14 @@
+const fs = require('fs');
+const path = require('path');
+module.exports = () => {
+  try {
+    const outputsFile = fs.readFileSync('cdk-infra/outputs.json').toString();
+    const outputs = JSON.parse(outputsFile);
+
+    const webhook = Object.values(outputs[`auto-builder-stack-enhancedApp-stg-${process.env.GIT_BRANCH}-webhooks`])[0];
+    return webhook;
+  } catch (error) {
+    console.log('Error occurred when retrieving Webhook URL', error);
+    return '';
+  }
+};

.github/workflows/deploy-feature-branch.yml

@@ -5,12 +5,11 @@ on:
     types:
       - opened
       - reopened
-
 name: Initial Feature Branch Deploy
 jobs: 
   deploy:
+    permissions: write-all
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
@@ -28,5 +27,18 @@ jobs:
           cd cdk-infra/
           npm ci
           npm run deploy:feature -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --outputs-file outputs.json
-      - name: Display resource URLs
-        run: cat cdk-infra/outputs.json
+      - name: Get Webhook URL 
+        uses: actions/github-script@v6
+        id: webhook
+        env: 
+          GIT_BRANCH: ${{github.head_ref}}
+        with:
+          script: |
+            const script = require('./.github/scripts/webhook-pr-comment.js');
+            return script()
+          result-encoding: string
+      - name: Comment on PR
+        if: steps.webhook.outputs.result != ''
+        run: gh pr comment ${{github.event.number}} --body "The URL for your feature branch webhook is ${{steps.webhook.outputs.result}}"
+        env: 
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update-feature-branch.yml

@@ -46,7 +46,7 @@ jobs:
           cd cdk-infra/
           npm ci
           npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhook
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks
   
       - name: Update Worker Stack
         if: steps.filter.outputs.worker == 'true'

Dockerfile

@@ -22,9 +22,9 @@ RUN cd ./modules/oas-page-builder \
 # where repo work will happen
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
-ARG SNOOTY_PARSER_VERSION=0.14.6
-ARG SNOOTY_FRONTEND_VERSION=0.14.13
-ARG MUT_VERSION=0.10.3
+ARG SNOOTY_PARSER_VERSION=0.14.9
+ARG SNOOTY_FRONTEND_VERSION=0.14.18
+ARG MUT_VERSION=0.10.7
 ARG REDOC_CLI_VERSION=1.2.2
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL

Dockerfile.enhanced

@@ -22,10 +22,10 @@ RUN cd ./modules/oas-page-builder \
 # where repo work will happen
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
-ARG SNOOTY_PARSER_VERSION=0.14.6
-ARG SNOOTY_FRONTEND_VERSION=0.14.13
-ARG MUT_VERSION=0.10.5
-ARG REDOC_CLI_VERSION=1.2.0
+ARG SNOOTY_PARSER_VERSION=0.14.9
+ARG SNOOTY_FRONTEND_VERSION=0.14.18
+ARG MUT_VERSION=0.10.7
+ARG REDOC_CLI_VERSION=1.2.2
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL
 ENV DEBIAN_FRONTEND=noninteractive

Dockerfile.legacy

@@ -2,7 +2,7 @@
 # The solution is probably to switch to Ubuntu 22.04 at that point, which is supported until 2027.
 FROM node:14-bullseye-slim
 
-ARG MUT_VERSION=0.10.3
+ARG MUT_VERSION=0.10.6
 
 ENV PATH="${PATH}:/opt/mut"
 

api/config/custom-environment-variables.json

@@ -20,6 +20,7 @@
   "entitlementCollection": "USER_ENTITLEMENT_COL_NAME",
   "dashboardUrl": "DASHBOARD_URL",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
+  "docsetsCollection": "DOCSETS_COL_NAME",
   "taskDefinitionFamily": "TASK_DEFINITION_FAMILY",
   "jobsQueueUrl": "JOBS_QUEUE_URL",
   "jobUpdatesQueueUrl": "JOB_UPDATES_QUEUE_URL",

api/config/default.json

@@ -19,6 +19,7 @@
   "jobQueueCollection": "JOB_QUEUE_COL_NAME",
   "entitlementCollection": "USER_ENTITLEMENT_COL_NAME",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
+  "docsetsCollection": "DOCSETS_COL_NAME",
   "MONGO_TIMEOUT_S": 15,
   "JOB_TIMEOUT_S": 900,
   "RETRY_TIMEOUT_MS": 5000,

api/controllers/v1/github.ts

@@ -4,13 +4,19 @@ import { JobRepository } from '../../../src/repositories/jobRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
+import { ReposBranchesDocument } from '../../../modules/persistence/src/services/metadata/repos_branches';
 
-async function prepGithubPushPayload(githubEvent: any, repoBranchesRepository: RepoBranchesRepository, prefix: string) {
+async function prepGithubPushPayload(
+  githubEvent: any,
+  repoBranchesRepository: RepoBranchesRepository,
+  prefix: string,
+  repoInfo: ReposBranchesDocument
+) {
   const branch_name = githubEvent.ref.split('/')[2];
   const branch_info = await repoBranchesRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
-  const repo_info = await repoBranchesRepository.getRepo(githubEvent.repository.name);
-  const project = repo_info?.project ?? githubEvent.repository.name;
+  const project = repoInfo?.project ?? githubEvent.repository.name;
 
   return {
     title: githubEvent.repository.full_name,
@@ -57,6 +63,7 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
 
   if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
     const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
@@ -77,10 +84,10 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
   }
 
   const env = c.get<string>('env');
-  const repoInfo = await repoBranchesRepository.getRepo(body.repository.name);
+  const repoInfo = await docsetsRepository.getRepo(body.repository.name);
   const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
   // TODO: Make job be of type Job
-  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix);
+  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
   try {
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

api/controllers/v1/jobs.ts

@@ -2,7 +2,6 @@ import * as c from 'config';
 import * as mongodb from 'mongodb';
 import { IConfig } from 'config';
 import { RepoEntitlementsRepository } from '../../../src/repositories/repoEntitlementsRepository';
-import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { SlackConnector } from '../../../src/services/slack';
 import { JobRepository } from '../../../src/repositories/jobRepository';
@@ -12,6 +11,7 @@ import { ECSContainer } from '../../../src/services/containerServices';
 import { SQSConnector } from '../../../src/services/queue';
 import { Batch } from '../../../src/services/batch';
 import { notifyBuildSummary, snootyBuildComplete } from '../../handlers/jobs';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 
 export const TriggerLocalBuild = async (event: any = {}, context: any = {}): Promise<any> => {
   const client = new mongodb.MongoClient(c.get('dbUrl'));
@@ -166,13 +166,22 @@ function prepProgressMessage(
   jobId: string,
   jobTitle: string,
   status: string,
-  errorReason: string
+  errorReason: string,
+  jobType?: string
 ): string {
   const msg = `Your Job (<${jobUrl}${jobId}|${jobTitle}>) `;
   const env = c.get<string>('env');
   switch (status) {
     case 'inQueue':
-      return msg + 'has successfully been added to the ' + env + ' queue.';
+      // Encourage writers to update to new webhook on githubPush jobs
+      let inQueueMsg = msg;
+      if (jobType == 'githubPush') {
+        const webhookWikiUrl =
+          'https://wiki.corp.mongodb.com/display/DE/How-To%3A+Use+Snooty%27s+Autobuilder+to+Build+Your+Content';
+        const updatePlease = `:exclamation: You used the old webhook for this build. <${webhookWikiUrl}|Update to the new webhook> in your fork of this repo to save 90s per build.`;
+        inQueueMsg = updatePlease + '\n\n' + msg;
+      }
+      return inQueueMsg + 'has successfully been added to the ' + env + ' queue.';
     case 'inProgress':
       return msg + 'is now being processed.';
     case 'completed':
@@ -213,7 +222,8 @@ async function NotifyBuildProgress(jobId: string): Promise<any> {
       jobId,
       jobTitle,
       fullDocument.status as string,
-      fullDocument?.error?.reason || ''
+      fullDocument?.error?.reason || '',
+      fullDocument?.payload.jobType
     ),
     entitlement['slack_user_id']
   );
@@ -248,7 +258,7 @@ async function SubmitArchiveJob(jobId: string) {
   const db = client.db(c.get('dbName'));
   const models = {
     jobs: new JobRepository(db, c, consoleLogger),
-    repoBranches: new RepoBranchesRepository(db, c, consoleLogger),
+    repoBranches: new DocsetsRepository(db, c, consoleLogger),
   };
   const job = await models.jobs.getJobById(jobId);
   const repo = await models.repoBranches.getRepo(job.payload.repoName);

api/controllers/v1/slack.ts

@@ -5,53 +5,14 @@ import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRe
 import { ConsoleLogger, ILogger } from '../../../src/services/logger';
 import { SlackConnector } from '../../../src/services/slack';
 import { JobRepository } from '../../../src/repositories/jobRepository';
-
-function isUserEntitled(entitlementsObject: any): boolean {
-  return (entitlementsObject?.repos?.length ?? 0) > 0;
-}
-
-function isRestrictedToDeploy(userId: string): boolean {
-  const { restrictedProdDeploy, entitledSlackUsers } = c.get<any>('prodDeploy');
-  return restrictedProdDeploy && !entitledSlackUsers.includes(userId);
-}
-
-function prepResponse(statusCode, contentType, body) {
-  return {
-    statusCode: statusCode,
-    headers: { 'Content-Type': contentType },
-    body: body,
-  };
-}
-
-async function buildEntitledBranchList(entitlement: any, repoBranchesRepository: RepoBranchesRepository) {
-  const entitledBranches: string[] = [];
-  for (const repo of entitlement.repos) {
-    const [repoOwner, repoName] = repo.split('/');
-    const branches = await repoBranchesRepository.getRepoBranches(repoName);
-    for (const branch of branches) {
-      let buildWithSnooty = true;
-      if ('buildsWithSnooty' in branch) {
-        buildWithSnooty = branch['buildsWithSnooty'];
-      }
-      if (buildWithSnooty) {
-        entitledBranches.push(`${repoOwner}/${repoName}/${branch['gitBranchName']}`);
-      }
-    }
-  }
-  return entitledBranches.sort();
-}
-
-function getQSString(qs: string) {
-  const key_val = {};
-  const arr = qs.split('&');
-  if (arr) {
-    arr.forEach((keyval) => {
-      const kvpair = keyval.split('=');
-      key_val[kvpair[0]] = kvpair[1];
-    });
-  }
-  return key_val;
-}
+import {
+  buildEntitledBranchList,
+  getQSString,
+  isRestrictedToDeploy,
+  isUserEntitled,
+  prepResponse,
+} from '../../handlers/slack';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 
 export const DisplayRepoOptions = async (event: any = {}, context: any = {}): Promise<any> => {
   const consoleLogger = new ConsoleLogger();
@@ -102,7 +63,12 @@ const deployHelper = (deployable, payload, jobTitle, jobUserName, jobUserEmail)
 
 // For every repo/branch selected to be deployed, return an array of jobs with the payload data
 // needed for a successful build.
-export const getDeployableJobs = async (values, entitlement, repoBranchesRepository: RepoBranchesRepository) => {
+export const getDeployableJobs = async (
+  values,
+  entitlement,
+  repoBranchesRepository: RepoBranchesRepository,
+  docsetsRepository: DocsetsRepository
+) => {
   const deployable = [];
 
   for (let i = 0; i < values.repo_option.length; i++) {
@@ -113,7 +79,7 @@ export const getDeployableJobs = async (values, entitlement, repoBranchesReposit
     const jobUserName = entitlement.github_username;
     const jobUserEmail = entitlement?.email ?? '';
 
-    const repoInfo = await repoBranchesRepository.getRepo(repoName);
+    const repoInfo = await docsetsRepository.getRepo(repoName);
     const non_versioned = repoInfo.branches.length === 1;
 
     const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName);
@@ -196,6 +162,7 @@ export const DeployRepo = async (event: any = {}, context: any = {}): Promise<an
   const db = client.db(c.get('dbName'));
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
   const jobRepository = new JobRepository(db, c, consoleLogger);
 
   // This is coming in as urlencoded string, need to decode before parsing
@@ -210,7 +177,7 @@ export const DeployRepo = async (event: any = {}, context: any = {}): Promise<an
 
   const values = slackConnector.parseSelection(stateValues);
 
-  const deployable = await getDeployableJobs(values, entitlement, repoBranchesRepository);
+  const deployable = await getDeployableJobs(values, entitlement, repoBranchesRepository, docsetsRepository);
   if (deployable.length > 0) {
     await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl'));
   }

api/controllers/v2/github.ts

@@ -8,17 +8,21 @@ import { ConsoleLogger } from '../../../src/services/logger';
 import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { EnhancedJob, JobStatus } from '../../../src/entities/job';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
+import { getMonorepoPaths } from '../../../src/monorepo';
+import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
+import { ReposBranchesDocument } from '../../../modules/persistence/src/services/metadata/associated_products';
 
 async function prepGithubPushPayload(
   githubEvent: PushEvent,
   repoBranchesRepository: RepoBranchesRepository,
-  prefix: string
+  prefix: string,
+  repoInfo: ReposBranchesDocument
 ): Promise<Omit<EnhancedJob, '_id'>> {
   const branch_name = githubEvent.ref.split('/')[2];
   const branch_info = await repoBranchesRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
-  const repo_info = await repoBranchesRepository.getRepo(githubEvent.repository.name);
-  const project = repo_info?.project ?? githubEvent.repository.name;
+  const project = repoInfo?.project ?? githubEvent.repository.name;
 
   return {
     title: githubEvent.repository.full_name,
@@ -56,6 +60,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
 
   if (!event.body) {
     const err = 'Trigger build does not have a body in event payload';
@@ -75,9 +80,9 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
       body: errMsg,
     };
   }
-  let body;
+  let body: PushEvent;
   try {
-    body = JSON.parse(event.body);
+    body = JSON.parse(event.body) as PushEvent;
   } catch (e) {
     console.log('[TriggerBuild]: ERROR! Could not parse event.body', e);
     return {
@@ -96,10 +101,26 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   }
 
   const env = c.get<string>('env');
-  const repoInfo = await repoBranchesRepository.getRepo(body.repository.name);
+  const repoInfo = await docsetsRepository.getRepo(body.repository.name);
   const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
 
-  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix);
+  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
+
+  if (process.env.MONOREPO_PATH_FEATURE === 'true') {
+    try {
+      if (body.head_commit && body.repository.owner.name) {
+        const monorepoPaths = await getMonorepoPaths({
+          commitSha: body.head_commit.id,
+          repoName: body.repository.name,
+          ownerName: body.repository.owner.name,
+          updatedFilePaths: getUpdatedFilePaths(body.head_commit),
+        });
+        console.log('monorepoPaths: ', monorepoPaths);
+      }
+    } catch (error) {
+      console.warn('Warning, attempting to get repo paths caused an error', error);
+    }
+  }
 
   try {
     consoleLogger.info(job.title, 'Creating Job');