Merge branch 'master' into DOP-4020-docsets

.github/workflows/update-feature-branch.yml

@@ -46,7 +46,7 @@ jobs:
           cd cdk-infra/
           npm ci
           npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhook
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks
   
       - name: Update Worker Stack
         if: steps.filter.outputs.worker == 'true'

Dockerfile

@@ -22,8 +22,8 @@ RUN cd ./modules/oas-page-builder \
 # where repo work will happen
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
-ARG SNOOTY_PARSER_VERSION=0.14.6
-ARG SNOOTY_FRONTEND_VERSION=0.14.13
+ARG SNOOTY_PARSER_VERSION=0.14.8
+ARG SNOOTY_FRONTEND_VERSION=0.14.14
 ARG MUT_VERSION=0.10.3
 ARG REDOC_CLI_VERSION=1.2.2
 ARG NPM_BASE_64_AUTH

api/controllers/v1/jobs.ts

@@ -166,13 +166,22 @@ function prepProgressMessage(
   jobId: string,
   jobTitle: string,
   status: string,
-  errorReason: string
+  errorReason: string,
+  jobType?: string
 ): string {
   const msg = `Your Job (<${jobUrl}${jobId}|${jobTitle}>) `;
   const env = c.get<string>('env');
   switch (status) {
     case 'inQueue':
-      return msg + 'has successfully been added to the ' + env + ' queue.';
+      // Encourage writers to update to new webhook on githubPush jobs
+      let inQueueMsg = msg;
+      if (jobType == 'githubPush') {
+        const webhookWikiUrl =
+          'https://wiki.corp.mongodb.com/display/DE/How-To%3A+Use+Snooty%27s+Autobuilder+to+Build+Your+Content';
+        const updatePlease = `:exclamation: You used the old webhook for this build. <${webhookWikiUrl}|Update to the new webhook> in your fork of this repo to save 90s per build.`;
+        inQueueMsg = updatePlease + '\n\n' + msg;
+      }
+      return inQueueMsg + 'has successfully been added to the ' + env + ' queue.';
     case 'inProgress':
       return msg + 'is now being processed.';
     case 'completed':
@@ -213,7 +222,8 @@ async function NotifyBuildProgress(jobId: string): Promise<any> {
       jobId,
       jobTitle,
       fullDocument.status as string,
-      fullDocument?.error?.reason || ''
+      fullDocument?.error?.reason || '',
+      fullDocument?.payload.jobType
     ),
     entitlement['slack_user_id']
   );

api/controllers/v2/github.ts

@@ -9,6 +9,8 @@ import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRe
 import { EnhancedJob, JobStatus } from '../../../src/entities/job';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
 import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
+import { getMonorepoPaths } from '../../../src/monorepo';
+import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
 
 async function prepGithubPushPayload(
   githubEvent: PushEvent,
@@ -78,9 +80,9 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
       body: errMsg,
     };
   }
-  let body;
+  let body: PushEvent;
   try {
-    body = JSON.parse(event.body);
+    body = JSON.parse(event.body) as PushEvent;
   } catch (e) {
     console.log('[TriggerBuild]: ERROR! Could not parse event.body', e);
     return {
@@ -104,6 +106,22 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
 
   const job = await prepGithubPushPayload(body, repoBranchesRepository, docsetsRepository, jobPrefix);
 
+  if (process.env.MONOREPO_PATH_FEATURE === 'true') {
+    try {
+      if (body.head_commit && body.repository.owner.name) {
+        const monorepoPaths = await getMonorepoPaths({
+          commitSha: body.head_commit.id,
+          repoName: body.repository.name,
+          ownerName: body.repository.owner.name,
+          updatedFilePaths: getUpdatedFilePaths(body.head_commit),
+        });
+        console.log('monorepoPaths: ', monorepoPaths);
+      }
+    } catch (error) {
+      console.warn('Warning, attempting to get repo paths caused an error', error);
+    }
+  }
+
   try {
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

cdk-infra/lib/constructs/api/webhook-env-construct.ts

@@ -3,7 +3,7 @@ import { StringParameter } from 'aws-cdk-lib/aws-ssm';
 import { Construct } from 'constructs';
 import { getSsmPathPrefix } from '../../../utils/ssm';
 import { getDashboardUrl } from '../../../utils/slack';
-import { getEnv } from '../../../utils/env';
+import { getEnv, getFeatureName } from '../../../utils/env';
 
 interface WebhookEnvConstructProps {
   jobsQueue: IQueue;
@@ -18,6 +18,13 @@ export class WebhookEnvConstruct extends Construct {
 
     const ssmPrefix = getSsmPathPrefix();
     const env = getEnv();
+    const featureName = getFeatureName();
+
+    // Create configurable feature flag that lives in parameter store.
+    const monorepoPathFeature = new StringParameter(this, 'monorepoPathFeature', {
+      parameterName: `${ssmPrefix}/${featureName}/monorepo/path_feature`,
+      stringValue: env === 'dotcomstg' || env === 'stg' ? 'true' : 'false',
+    });
 
     const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
     const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`);
@@ -49,6 +56,7 @@ export class WebhookEnvConstruct extends Construct {
       USER_ENTITLEMENT_COL_NAME: entitlementCollection,
       DASHBOARD_URL: getDashboardUrl(env, jobCollection),
       STAGE: env,
+      MONOREPO_PATH_FEATURE: monorepoPathFeature.stringValue,
     };
   }
 }

cdk-infra/lib/constructs/worker/worker-env-construct.ts

@@ -36,6 +36,9 @@ export class WorkerEnvConstruct extends Construct {
       `/docs/worker_pool/preview_webhook/snooty_gatsby_cloud_test/data_source`
     );
 
+    // front end feature flag for chatbot UI
+    const gatsbyUseChatbot = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/use_chatbot`);
+
     const githubBotUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/github/bot/username`);
 
     const npmEmail = StringParameter.valueFromLookup(this, `${ssmPrefix}/npm/email`);
@@ -82,6 +85,7 @@ export class WorkerEnvConstruct extends Construct {
       USE_CUSTOM_BUCKETS: `${getUseCustomBuckets()}`,
       FEATURE_NAME: `${getFeatureName()}`,
       GATSBY_TEST_SEARCH_UI: 'false',
+      GATSBY_SHOW_CHATBOT: gatsbyUseChatbot,
     };
   }
 }

config/custom-environment-variables.json

@@ -23,6 +23,7 @@
   "previewBuildEnabled": "PREVIEW_BUILD_ENABLED",
   "featureFlagUpdatePages": "FEATURE_FLAG_UPDATE_PAGES",
   "featureFlagSearchUI": "GATSBY_TEST_SEARCH_UI",
+  "gatsbyUseChatbot": "GATSBY_SHOW_CHATBOT",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
   "docsetsCollection": "DOCSETS_COL_NAME",
   "repo_dir": "repos",

config/default.json

@@ -32,6 +32,7 @@
   "previewBuildEnabled": "false",
   "featureFlagUpdatePages": "false",
   "featureFlagSearchUI": "false",
+  "gatsbyUseChatbot": "false",
   "parallel": {
     "enabled": true,
     "stg": {

infrastructure/ecs-main/ecs_service.yml

@@ -64,6 +64,8 @@ Resources:
               Value: ${self:custom.featureFlagUpdatePages}
             - Name: GATSBY_TEST_SEARCH_UI
               Value: ${self:custom.featureFlagSearchUI}
+            - Name: GATSBY_SHOW_CHATBOT
+              Value: ${self:custom.gatsbyUseChatbot}
             - Name: FASTLY_MAIN_TOKEN
               Value: ${self:custom.fastlyMainToken}
             - Name: FASTLY_MAIN_SERVICE_ID

infrastructure/ecs-main/serverless.yml

@@ -117,6 +117,7 @@ custom:
   featureFlagUpdatePages: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/update_pages}
   featureFlagSearchUI: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/search_ui}
   gatsbyTestEmbedVersions: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/embedded_versions}
+  gatsbyUseChatbot: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/use_chatbot}
   fastlyMainToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/main/token}
   fastlyMainServiceId: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/main/service_id}
   fastlyCloudManagerToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/cloudmanager/token}

src/clients/githubClient.ts

@@ -0,0 +1,19 @@
+import { Octokit } from '@octokit/rest';
+
+let client: Octokit;
+
+export function getOctokitClient(): Octokit {
+  if (client) return client;
+
+  try {
+    const { GITHUB_BOT_PASSWORD } = process.env;
+
+    if (!GITHUB_BOT_PASSWORD) throw new Error('GITHUB_BOT_PASSWORD is not defined');
+
+    client = new Octokit({ auth: GITHUB_BOT_PASSWORD });
+    return client;
+  } catch (error) {
+    console.error('ERROR! Failed to create Octokit client. Is GITHUB_BOT_PASSWORD defined?', error);
+    throw error;
+  }
+}

src/job/jobHandler.ts

@@ -109,7 +109,7 @@ export abstract class JobHandler {
         // completed after the Gatsby Cloud build via the SnootyBuildComplete lambda.
         const { _id: jobId, user } = this.currJob;
         const gatsbyCloudSiteId = await this._repoEntitlementsRepo.getGatsbySiteIdByGithubUsername(user);
-        if (gatsbyCloudSiteId && this.currJob.payload.jobType === 'githubPush') {
+        if (this.currJob.payload.isNextGen && gatsbyCloudSiteId && this.currJob.payload.jobType === 'githubPush') {
           this.logger.info(
             jobId,
             `User ${user} has a Gatsby Cloud site. The Autobuilder will not mark the build as completed right now.`
@@ -373,6 +373,7 @@ export abstract class JobHandler {
       GATSBY_BASE_URL: this._config.get<string>('gatsbyBaseUrl'),
       PREVIEW_BUILD_ENABLED: this._config.get<string>('previewBuildEnabled'),
       GATSBY_TEST_SEARCH_UI: this._config.get<string>('featureFlagSearchUI'),
+      GATSBY_SHOW_CHATBOT: this._config.get<string>('gatsbyUseChatbot'),
     };
 
     for (const [envName, envValue] of Object.entries(snootyFrontEndVars)) {

src/monorepo/index.ts

@@ -0,0 +1,42 @@
+import { getSnootyDirSet } from './utils/path-utils';
+import { GitCommitInfo } from './types/github-types';
+import { getProjectDirFromPath } from './services/get-paths';
+
+interface FileUpdatePayload {
+  ownerName: string;
+  repoName: string;
+  commitSha: string;
+  updatedFilePaths: string[];
+}
+
+/**
+ * Retrieves the path of project directories. This is determined
+ * by finding the nearest parent directory that has a snooty.toml file
+ * for a given updated file path from a commit.
+ * @param repoName Name of the repository to check.
+ * @param ownerName Name of the owner of the repository.
+ * @param commitSha The Git commit SHA that contains the changed files.
+ * @param updatedFilePaths An array of all of the changed files (added, removed, modified)
+ * from the commit. The method `getUpdatedFilePaths` in the `src/monorepo/utils/path-utils.ts
+ * can be used to parse these paths from a GitHub `Commit` object.
+ * @returns An array of all the project paths that need to be built.
+ */
+export async function getMonorepoPaths({
+  repoName,
+  ownerName,
+  commitSha,
+  updatedFilePaths,
+}: FileUpdatePayload): Promise<string[]> {
+  const commitInfo: GitCommitInfo = {
+    ownerName,
+    repoName,
+    commitSha,
+  };
+
+  const snootyDirSet = await getSnootyDirSet(commitInfo);
+
+  const projects = updatedFilePaths.map((path) => getProjectDirFromPath(path, snootyDirSet));
+
+  // remove empty strings and remove duplicated values
+  return Array.from(new Set(projects.filter((dir) => !!dir)));
+}

src/monorepo/services/get-paths.ts

@@ -0,0 +1,36 @@
+import { SNOOTY_TOML_FILENAME } from '../utils/monorepo-constants';
+
+/**
+ * This function returns the project path for a given file change from a docs repository
+ * within the monorepo. This function supports nested projects.
+ * @param path An added/modified/removed file path from a commit e.g. server-docs/source/index.rst
+ * @param commitInfo Contains information
+ * @returns The closest file path that contains a snooty.toml, relative to the path parameter.
+ */
+export function getProjectDirFromPath(path: string, snootyDirSet: Set<string>): string {
+  const pathArray = path.split('/');
+  if (pathArray.length === 0) {
+    console.warn('WARNING! Empty path found: ', path);
+    return '';
+  }
+
+  /**
+   * If the changed file is the snooty.toml file, we know that we
+   * are in the project's root directory. We can join the original
+   * pathArray to get the project path since the snooty.toml has been removed.
+   */
+  const changedFile = pathArray.pop();
+
+  if (changedFile === SNOOTY_TOML_FILENAME) return pathArray.join('/');
+
+  while (pathArray.length > 0) {
+    const currDir = pathArray.join('/');
+
+    if (snootyDirSet.has(currDir)) return currDir;
+
+    pathArray.pop();
+  }
+
+  console.warn(`WARNING! No snooty.toml found for the given path: ${path}`);
+  return '';
+}

src/monorepo/types/atlas-types.ts

@@ -0,0 +1,25 @@
+interface DirectoryConfig {
+  snooty_toml?: string;
+  source?: string;
+}
+
+interface RepoConfig {
+  repoName: string;
+  deployable: boolean;
+  branches: BranchConfig[];
+}
+
+interface BranchConfig {
+  gitBranchName: string;
+}
+
+// TODO: Populate these more. For DOP-3911, they are
+// being added for testing purposes.
+export interface DocSetEntry {
+  project: string;
+  prefix: string;
+  bucket: string;
+  url: string;
+  directories?: DirectoryConfig;
+  repos?: RepoConfig[];
+}

src/monorepo/types/github-types.ts

@@ -0,0 +1,5 @@
+export interface GitCommitInfo {
+  commitSha: string;
+  ownerName: string;
+  repoName: string;
+}

src/monorepo/utils/monorepo-constants.ts

@@ -0,0 +1,2 @@
+export const SNOOTY_TOML_FILENAME = 'snooty.toml';
+export const MONOREPO_NAME = 'docs-monorepo';

src/monorepo/utils/path-utils.ts

@@ -0,0 +1,50 @@
+import { Commit } from '@octokit/webhooks-types';
+import { getOctokitClient } from '../../clients/githubClient';
+import { GitCommitInfo } from '../types/github-types';
+import { SNOOTY_TOML_FILENAME } from './monorepo-constants';
+
+/**
+ * Creates a `Set` of all `snooty.toml` paths within the monorepo.
+ * The function retrieves the monorepo's tree structure from GitHub.
+ */
+export async function getSnootyDirSet({ commitSha, ownerName, repoName }: GitCommitInfo): Promise<Set<string>> {
+  try {
+    const client = getOctokitClient();
+
+    // getting the repository tree for a given commit SHA. This returns an object
+    // with the property `tree` that is a flat array of all files in the repository.
+    // The tree array contains objects that hold the file path.
+    // Unlike the contents API for repositories, the actual file content is not returned.
+    const { data } = await client.request('GET /repos/{owner}/{repo}/git/trees/{tree_sha}', {
+      owner: ownerName,
+      repo: repoName,
+      tree_sha: commitSha,
+      recursive: 'true',
+    });
+
+    const snootyTomlDirs = data.tree
+      .filter((treeNode) => !!treeNode.path?.includes(SNOOTY_TOML_FILENAME))
+      .map((treeNode) => {
+        // casting the `treeNode.path` from `(string | undefined)` to `string` since the filter will ensure that the result
+        // only includes treeNode.path values that are defined and include snooty.toml
+        // in the path i.e. we will not have `undefined` as a value in the resulting array.
+        const path = treeNode.path as string;
+
+        // the - 1 is to remove the trailing slash
+        return path.slice(0, path.length - SNOOTY_TOML_FILENAME.length - 1);
+      });
+
+    const snootyDirSet = new Set(snootyTomlDirs);
+
+    return snootyDirSet;
+  } catch (error) {
+    console.error(
+      `ERROR! Unable to retrieve tree for SHA: ${commitSha} owner name: ${ownerName} repo name: ${repoName}`,
+      error
+    );
+    throw error;
+  }
+}
+
+export const getUpdatedFilePaths = (commit: Commit): string[] =>
+  commit.modified.concat(commit.added).concat(commit.removed);