DOP-4079: Slack Deploy Monorepo (#929)

* enable monorepo entitlements to flow from test-deploy to job creation

* add to preprd staging

* account for monorepo format in jobValidator

* log

* more logs

* further integration

* use simple path to repos/project

* clone to new dir

* cd further into project

* getDirectory used everywhere in v1

* add second argument

* more getDirectory

* source new meta branch

* ref monorepo meta

* changed path

* Empty-Commit

* github and slack v2, split out jobs per monorepoDir

* v2 github, use monorepoPaths on github push to kick off multiple jobs

* clean up

* use constant, consolidate conditional

* feature flag

* ensure feature flag in all needed files

* stage ecs

* await

* feature flag

* in main serverless

* use meta branch of my own

* fix tests

* clean

* delete comment

* final clean

* entitlement validation clean, monorepoDir => directory

* fix slack message

* fix test

* PR feedback

* consolidate env vars

* delete feature flag from config, unnecessary

* PR feedback

Dockerfile

@@ -74,7 +74,7 @@ USER docsworker-xlarge
 WORKDIR ${WORK_DIRECTORY}
 
 # get shared.mk
-RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
+RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/monorepo-pub-branches/makefiles/shared.mk -o shared.mk
 
 # install snooty frontend and docs-tools
 RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \

Dockerfile.enhanced

@@ -74,7 +74,7 @@ USER docsworker-xlarge
 WORKDIR ${WORK_DIRECTORY}
 
 # get shared.mk
-RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
+RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/monorepo-pub-branches/makefiles/shared.mk -o shared.mk
 
 # install snooty frontend and docs-tools
 RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \

api/controllers/v1/github.ts

@@ -14,7 +14,11 @@ async function prepGithubPushPayload(
   repoInfo: ReposBranchesDocument
 ) {
   const branch_name = githubEvent.ref.split('/')[2];
-  const branch_info = await repoBranchesRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
+  const branch_info = await repoBranchesRepository.getRepoBranchAliases(
+    githubEvent.repository.name,
+    branch_name,
+    repoInfo.project
+  );
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
   const project = repoInfo?.project ?? githubEvent.repository.name;
 

api/controllers/v1/jobs.ts

@@ -261,7 +261,7 @@ async function SubmitArchiveJob(jobId: string) {
     repoBranches: new DocsetsRepository(db, c, consoleLogger),
   };
   const job = await models.jobs.getJobById(jobId);
-  const repo = await models.repoBranches.getRepo(job.payload.repoName);
+  const repo = await models.repoBranches.getRepo(job.payload.repoName, job?.payload.directory);
 
   /* NOTE
    * we don't archive landing for two reasons:

api/controllers/v1/slack.ts

@@ -72,20 +72,32 @@ export const getDeployableJobs = async (
   const deployable = [];
 
   for (let i = 0; i < values.repo_option.length; i++) {
-    // e.g. mongodb/docs-realm/master => (site/repo/branch)
-    const [repoOwner, repoName, branchName] = values.repo_option[i].value.split('/');
+    let repoOwner: string, repoName: string, branchName: string, directory: string | undefined;
+    const splitValues = values.repo_option[i].value.split('/');
+
+    if (splitValues.length === 3) {
+      // e.g. mongodb/docs-realm/master => (owner/repo/branch)
+      [repoOwner, repoName, branchName] = splitValues;
+    } else if (splitValues.length === 4 && process.env.FEATURE_FLAG_MONOREPO_PATH === 'true') {
+      // e.g. 10gen/docs-monorepo/cloud-docs/master => (owner/monorepo/repoDirectory/branch)
+      [repoOwner, repoName, directory, branchName] = splitValues;
+    } else {
+      throw Error('Selected entitlement value is configured incorrectly. Check user entitlements!');
+    }
+
     const hashOption = values?.hash_option ?? null;
     const jobTitle = `Slack deploy: ${values.repo_option[i].value}, by ${entitlement.github_username}`;
     const jobUserName = entitlement.github_username;
     const jobUserEmail = entitlement?.email ?? '';
 
-    const repoInfo = await docsetsRepository.getRepo(repoName);
+    const repoInfo = await docsetsRepository.getRepo(repoName, directory);
     const non_versioned = repoInfo.branches.length === 1;
 
-    const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName);
+    const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project);
     if (!branchObject?.aliasObject) continue;
 
-    const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; //bool
+    // TODO: Create strong typing for these rather than comments
+    const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; // bool
     let aliases = branchObject.aliasObject.urlAliases; // array or null
     let urlSlug = branchObject.aliasObject.urlSlug; // string or null, string must match value in urlAliases or gitBranchName
     const isStableBranch = branchObject.aliasObject.isStableBranch; // bool or Falsey
@@ -106,7 +118,8 @@ export const getDeployableJobs = async (
       urlSlug,
       false,
       false,
-      false
+      false,
+      directory
     );
 
     newPayload.stable = !!isStableBranch;
@@ -198,7 +211,8 @@ function createPayload(
   urlSlug,
   aliased = false,
   primaryAlias = false,
-  stable = false
+  stable = false,
+  directory?: string
 ) {
   return {
     jobType,
@@ -217,6 +231,7 @@ function createPayload(
     newHead,
     primaryAlias,
     stable,
+    directory,
   };
 }
 

api/controllers/v2/github.ts

@@ -12,6 +12,7 @@ import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 import { getMonorepoPaths } from '../../../src/monorepo';
 import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
 import { ReposBranchesDocument } from '../../../modules/persistence/src/services/metadata/associated_products';
+import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants';
 
 async function prepGithubPushPayload(
   githubEvent: PushEvent,
@@ -20,7 +21,11 @@ async function prepGithubPushPayload(
   repoInfo: ReposBranchesDocument
 ): Promise<Omit<EnhancedJob, '_id'>> {
   const branch_name = githubEvent.ref.split('/')[2];
-  const branch_info = await repoBranchesRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
+  const branch_info = await repoBranchesRepository.getRepoBranchAliases(
+    githubEvent.repository.name,
+    branch_name,
+    repoInfo.project
+  );
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
   const project = repoInfo?.project ?? githubEvent.repository.name;
 
@@ -101,32 +106,60 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   }
 
   const env = c.get<string>('env');
-  const repoInfo = await docsetsRepository.getRepo(body.repository.name);
-  const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
 
-  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
+  async function createAndInsertJob(path?: string) {
+    const repoInfo = await docsetsRepository.getRepo(body.repository.name, path);
+    const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
+    const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
 
-  if (process.env.MONOREPO_PATH_FEATURE === 'true') {
+    consoleLogger.info(job.title, 'Creating Job');
+    const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));
+    jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0);
+    consoleLogger.info(job.title, `Created Job ${jobId}`);
+  }
+
+  if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && body.repository.name === MONOREPO_NAME) {
+    let monorepoPaths: string[] = [];
     try {
       if (body.head_commit && body.repository.owner.name) {
-        const monorepoPaths = await getMonorepoPaths({
+        monorepoPaths = await getMonorepoPaths({
           commitSha: body.head_commit.id,
           repoName: body.repository.name,
           ownerName: body.repository.owner.name,
           updatedFilePaths: getUpdatedFilePaths(body.head_commit),
         });
-        console.log('monorepoPaths: ', monorepoPaths);
+        consoleLogger.info('monoRepoPaths', `Monorepo Paths with new changes: ${monorepoPaths}`);
       }
     } catch (error) {
       console.warn('Warning, attempting to get repo paths caused an error', error);
     }
+
+    /* Create and insert Job for each monorepo project that has changes */
+    for (const path of monorepoPaths) {
+      // TODO: Deal with nested monorepo projects
+      /* For now, we will ignore nested monorepo projects until necessary */
+      if (path.split('/').length > 1) continue;
+
+      try {
+        await createAndInsertJob(`/${path}`);
+      } catch (err) {
+        return {
+          statusCode: 500,
+          headers: { 'Content-Type': 'text/plain' },
+          body: err,
+        };
+      }
+    }
+
+    return {
+      statusCode: 202,
+      headers: { 'Content-Type': 'text/plain' },
+      body: 'Jobs Queued',
+    };
   }
 
   try {
-    consoleLogger.info(job.title, 'Creating Job');
-    const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));
-    jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0);
-    consoleLogger.info(job.title, `Created Job ${jobId}`);
+    await createAndInsertJob();
   } catch (err) {
     return {
       statusCode: 500,

api/controllers/v2/slack.ts

@@ -89,17 +89,28 @@ export const getDeployableJobs = async (
   const deployable = [];
 
   for (let i = 0; i < values.repo_option.length; i++) {
-    // e.g. mongodb/docs-realm/master => (site/repo/branch)
-    const [repoOwner, repoName, branchName] = values.repo_option[i].value.split('/');
+    let repoOwner: string, repoName: string, branchName: string, directory: string | undefined;
+    const splitValues = values.repo_option[i].value.split('/');
+
+    if (splitValues.length === 3) {
+      // e.g. mongodb/docs-realm/master => (owner/repo/branch)
+      [repoOwner, repoName, branchName] = splitValues;
+    } else if (splitValues.length === 4 && process.env.FEATURE_FLAG_MONOREPO_PATH === 'true') {
+      // e.g. 10gen/docs-monorepo/cloud-docs/master => (owner/monorepo/repoDirectory/branch)
+      [repoOwner, repoName, directory, branchName] = splitValues;
+    } else {
+      throw Error('Selected entitlement value is configured incorrectly. Check user entitlements!');
+    }
+
     const hashOption = values?.hash_option ?? null;
     const jobTitle = `Slack deploy: ${values.repo_option[i].value}, by ${entitlement.github_username}`;
     const jobUserName = entitlement.github_username;
     const jobUserEmail = entitlement?.email ?? '';
 
-    const repoInfo = await docsetsRepository.getRepo(repoName);
+    const repoInfo = await docsetsRepository.getRepo(repoName, directory);
     const non_versioned = repoInfo.branches.length === 1;
 
-    const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName);
+    const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project);
     if (!branchObject?.aliasObject) continue;
 
     const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; //bool
@@ -123,7 +134,8 @@ export const getDeployableJobs = async (
       urlSlug,
       false,
       false,
-      false
+      false,
+      directory
     );
 
     newPayload.stable = !!isStableBranch;
@@ -223,7 +235,8 @@ function createPayload(
   urlSlug,
   aliased = false,
   primaryAlias = false,
-  stable = false
+  stable = false,
+  directory?: string
 ) {
   return {
     jobType,
@@ -241,6 +254,7 @@ function createPayload(
     newHead,
     primaryAlias,
     stable,
+    directory,
   };
 }
 

api/handlers/jobs.ts

@@ -82,7 +82,13 @@ async function prepSummaryMessage(
     if (repoName == 'mms-docs') {
       msg = `Your Job <${jobUrl}${jobId}|Completed>! \n- Repo: *${repoName}*\n- Branch: *${fullDocument.payload.branchName}*\n- Commit: *${fullDocument.payload.newHead}*\n- urlSlug: *${fullDocument.payload.urlSlug}*\n- Env: *${env}*\n*Urls*\n   *CM*: <${mms_urls[0]}|Cloud Manager> \n   *OPM*: <${mms_urls[1]}|OPS Manager>\n- InvalidationStatus: <${fullDocument.invalidationStatusURL}|Status> \nEnjoy  :smile:!`;
     } else {
-      msg = `Your Job <${jobUrl}${jobId}|Completed>! \n- Repo: *${repoName}*\n- Branch: *${fullDocument.payload.branchName}*\n- Commit: *${fullDocument.payload.newHead}*\n- urlSlug: *${fullDocument.payload.urlSlug}*\n- Env: *${env}*\n- Url: <${url}|${repoName}>\n- InvalidationStatus: <${fullDocument.invalidationStatusURL}|Status> \nEnjoy  :smile:!`;
+      msg = `Your Job <${jobUrl}${jobId}|Completed>! \n- Repo: *${repoName}${
+        fullDocument.payload.directory ? `/${fullDocument.payload.directory}` : ``
+      }*\n- Branch: *${fullDocument.payload.branchName}*\n- Commit: *${fullDocument.payload.newHead}*\n- urlSlug: *${
+        fullDocument.payload.urlSlug
+      }*\n- Env: *${env}*\n- Url: <${url}|${fullDocument.payload.directory ?? repoName}>\n- InvalidationStatus: <${
+        fullDocument.invalidationStatusURL
+      }|Status> \nEnjoy  :smile:!`;
     }
   }
   // Remove instances of two or more periods

api/handlers/slack.ts

@@ -21,15 +21,17 @@ export function prepResponse(statusCode, contentType, body) {
 export async function buildEntitledBranchList(entitlement: any, repoBranchesRepository: RepoBranchesRepository) {
   const entitledBranches: string[] = [];
   for (const repo of entitlement.repos) {
-    const [repoOwner, repoName] = repo.split('/');
-    const branches = await repoBranchesRepository.getRepoBranches(repoName);
+    const [repoOwner, repoName, directoryPath] = repo.split('/');
+    const branches = await repoBranchesRepository.getRepoBranches(repoName, directoryPath);
     for (const branch of branches) {
       let buildWithSnooty = true;
       if ('buildsWithSnooty' in branch) {
         buildWithSnooty = branch['buildsWithSnooty'];
       }
       if (buildWithSnooty) {
-        entitledBranches.push(`${repoOwner}/${repoName}/${branch['gitBranchName']}`);
+        entitledBranches.push(
+          `${repoOwner}/${repoName}${directoryPath ? '/' + directoryPath : ''}/${branch['gitBranchName']}`
+        );
       }
     }
   }

cdk-infra/lib/constructs/api/webhook-env-construct.ts

@@ -3,7 +3,7 @@ import { StringParameter } from 'aws-cdk-lib/aws-ssm';
 import { Construct } from 'constructs';
 import { getSsmPathPrefix } from '../../../utils/ssm';
 import { getDashboardUrl } from '../../../utils/slack';
-import { getEnv, getFeatureName } from '../../../utils/env';
+import { getEnv } from '../../../utils/env';
 
 interface WebhookEnvConstructProps {
   jobsQueue: IQueue;
@@ -18,14 +18,8 @@ export class WebhookEnvConstruct extends Construct {
 
     const ssmPrefix = getSsmPathPrefix();
     const env = getEnv();
-    const featureName = getFeatureName();
-
-    // Create configurable feature flag that lives in parameter store.
-    const monorepoPathFeature = new StringParameter(this, 'monorepoPathFeature', {
-      parameterName: `${ssmPrefix}/${featureName}/monorepo/path_feature`,
-      stringValue: env === 'dotcomstg' || env === 'stg' ? 'true' : 'false',
-    });
 
+    const featureFlagMonorepoPath = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/monorepo_path`);
     const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
     const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`);
     const repoBranchesCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/repo`);
@@ -56,7 +50,7 @@ export class WebhookEnvConstruct extends Construct {
       USER_ENTITLEMENT_COL_NAME: entitlementCollection,
       DASHBOARD_URL: getDashboardUrl(env, jobCollection),
       STAGE: env,
-      MONOREPO_PATH_FEATURE: monorepoPathFeature.stringValue,
+      FEATURE_FLAG_MONOREPO_PATH: featureFlagMonorepoPath.stringValue,
     };
   }
 }

cdk-infra/lib/constructs/worker/worker-env-construct.ts

@@ -50,6 +50,8 @@ export class WorkerEnvConstruct extends Construct {
     // doing this for the time being, but I think we don't need to necessarily retrieve this from ssm for feature branches, nor would we want to in that case
     const previewBuildEnabled = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/preview_build/enabled`);
     const featureFlagUpdatePages = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/update_pages`);
+    const featureFlagMonorepoPath = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/monorepo_path`);
+
     const entitlementCollection = StringParameter.valueFromLookup(
       this,
       `${ssmPrefix}/atlas/collections/user/entitlements`
@@ -76,6 +78,7 @@ export class WorkerEnvConstruct extends Construct {
       GATSBY_BASE_URL: gatsbyBaseUrl,
       PREVIEW_BUILD_ENABLED: previewBuildEnabled,
       FEATURE_FLAG_UPDATE_PAGES: featureFlagUpdatePages,
+      FEATURE_FLAG_MONOREPO_PATH: featureFlagMonorepoPath,
       USER_ENTITLEMENT_COL_NAME: entitlementCollection,
       NPM_EMAIL: npmEmail,
       REPO_BRANCHES_COL_NAME: repoBranchesCollection,

infrastructure/ecs-main/ecs_service.yml

@@ -62,6 +62,8 @@ Resources:
               Value: ${self:custom.previewBuildEnabled}
             - Name: FEATURE_FLAG_UPDATE_PAGES
               Value: ${self:custom.featureFlagUpdatePages}
+            - Name: FEATURE_FLAG_MONOREPO_PATH
+              Value: ${self:custom.featureFlagMonorepoPath}
             - Name: GATSBY_TEST_SEARCH_UI
               Value: ${self:custom.featureFlagSearchUI}
             - Name: GATSBY_SHOW_CHATBOT

infrastructure/ecs-main/serverless.yml

@@ -115,6 +115,7 @@ custom:
   gatsbyCloudPreviewWebhookEnabled: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/preview_webhook_enable}
   previewBuildEnabled: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/preview_build/enabled}
   featureFlagUpdatePages: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/update_pages}
+  featureFlagMonorepoPath: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/monorepo_path}
   featureFlagSearchUI: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/search_ui}
   gatsbyTestEmbedVersions: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/embedded_versions}
   gatsbyUseChatbot: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/use_chatbot}

serverless.yml

@@ -69,6 +69,7 @@ custom:
   slackSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/slack/webhook/secret}
   slackAuthToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/slack/auth/token}
   snootySecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/snooty/webhook/secret}
+  featureFlagMonorepoPath: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/monorepo_path}
   JobsQueueName: autobuilder-jobs-queue-${self:provider.stage}
   JobsDLQueueName: autobuilder-jobs-dlqueue-${self:provider.stage}
   JobUpdatesQueueName: autobuilder-job-updates-queue-${self:provider.stage}
@@ -120,6 +121,7 @@ webhook-env-core: &webhook-env-core
   SLACK_SECRET: ${self:custom.slackSecret}
   SLACK_TOKEN: ${self:custom.slackAuthToken}
   SNOOTY_SECRET: ${self:custom.snootySecret}
+  FEATURE_FLAG_MONOREPO_PATH: ${self:custom.featureFlagMonorepoPath}
   DASHBOARD_URL: ${self:custom.dashboardUrl.${self:provider.stage}}
   NODE_CONFIG_DIR: './api/config'
   TASK_DEFINITION_FAMILY: docs-worker-pool-${self:provider.stage}

src/enhanced/job/enhancedJobHandlers.ts

@@ -10,7 +10,10 @@ import { StagingJobHandler } from '../../job/stagingJobHandler';
  * @param this reference to current object
  */
 async function setEnvironmentVariablesEnhanced(this: JobHandler) {
-  const repo_info = await this._docsetsRepo.getRepoBranchesByRepoName(this.currJob.payload.repoName);
+  const repo_info = await this._docsetsRepo.getRepoBranchesByRepoName(
+    this.currJob.payload.repoName,
+    this.currJob.payload.project
+  );
 
   let env = this._config.get<string>('env');
   this.logger.info(

src/entities/job.ts

@@ -43,6 +43,7 @@ export type Payload = {
   prefix: string;
   project: string;
   includeInGlobalSearch: boolean;
+  directory?: string;
 };
 
 export type EnhancedPayload = {