[DOP-3911]: Check project path for monorepo (#894)

* [DOP-3911]: Add way to check file changes and determine what projects have been changed if head_commit is included

* [DOP-3911]: Add some comments

* [DOP-3911]: Add checks for snooty.toml in monorepo

* [DOP-3911]: Add commit info and simplify files

* [DOP-3911]: Add unit tests

* [DOP-3911]: Update tests

* [DOP-3911]: Update comments

* [DOP-3911]: Remove extra comment

* [DOP-3911]: Remove unused code

* [DOP-3911]: Remove unused property in job payload

* [DOP-3911]: Remove unused import

* [DOP-3911]: Remove log and irrelevant comments

* [DOP-3911]: Refactor to start using tree approach and create new directories for organization

* [DOP-3911]: Refactor set to be called from parent function

* [DOP-3911]: More refactoring, update tests to use set

* [DOP-3911]: Clean up

* [DOP-3911]: Resolve merge conflicts

* [DOP-3911]: Resolve merge conflicts

* [DOP-3911]: Add types and add comments

* [DOP-3911]: Add feature

* [DOP-3911]: Add feature flag for testing monorepo path

* [DOP-3911]: Fix typo

* [DOP-3911]: Remove get snooty.toml

* [DOP-3911]: Remove use of config library, and clean up code

* [DOP-3911]: Add parameter store flag for ease of testing

* [DOP-3911]: Comment update

* [DOP-3911]: Properly name test

.github/workflows/update-feature-branch.yml

@@ -46,7 +46,7 @@ jobs:
           cd cdk-infra/
           npm ci
           npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
-            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhook
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks
   
       - name: Update Worker Stack
         if: steps.filter.outputs.worker == 'true'

api/controllers/v2/github.ts

@@ -8,6 +8,8 @@ import { ConsoleLogger } from '../../../src/services/logger';
 import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { EnhancedJob, JobStatus } from '../../../src/entities/job';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
+import { getMonorepoPaths } from '../../../src/monorepo';
+import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
 
 async function prepGithubPushPayload(
   githubEvent: PushEvent,
@@ -75,9 +77,9 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
       body: errMsg,
     };
   }
-  let body;
+  let body: PushEvent;
   try {
-    body = JSON.parse(event.body);
+    body = JSON.parse(event.body) as PushEvent;
   } catch (e) {
     console.log('[TriggerBuild]: ERROR! Could not parse event.body', e);
     return {
@@ -101,6 +103,22 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
 
   const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix);
 
+  if (process.env.MONOREPO_PATH_FEATURE === 'true') {
+    try {
+      if (body.head_commit && body.repository.owner.name) {
+        const monorepoPaths = await getMonorepoPaths({
+          commitSha: body.head_commit.id,
+          repoName: body.repository.name,
+          ownerName: body.repository.owner.name,
+          updatedFilePaths: getUpdatedFilePaths(body.head_commit),
+        });
+        console.log('monorepoPaths: ', monorepoPaths);
+      }
+    } catch (error) {
+      console.warn('Warning, attempting to get repo paths caused an error', error);
+    }
+  }
+
   try {
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

cdk-infra/lib/constructs/api/webhook-env-construct.ts

@@ -19,6 +19,12 @@ export class WebhookEnvConstruct extends Construct {
     const ssmPrefix = getSsmPathPrefix();
     const env = getEnv();
 
+    // Create configurable feature flag that lives in parameter store.
+    const monorepoPathFeature = new StringParameter(this, 'monorepoPathFeature', {
+      parameterName: `${ssmPrefix}/monorepo/path_feature`,
+      stringValue: env === 'dotcomstg' || env === 'stg' ? 'true' : 'false',
+    });
+
     const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
     const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`);
     const repoBranchesCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/repo`);
@@ -47,6 +53,7 @@ export class WebhookEnvConstruct extends Construct {
       USER_ENTITLEMENT_COL_NAME: entitlementCollection,
       DASHBOARD_URL: getDashboardUrl(env, jobCollection),
       STAGE: env,
+      MONOREPO_PATH_FEATURE: monorepoPathFeature.stringValue,
     };
   }
 }

src/clients/githubClient.ts

@@ -0,0 +1,19 @@
+import { Octokit } from '@octokit/rest';
+
+let client: Octokit;
+
+export function getOctokitClient(): Octokit {
+  if (client) return client;
+
+  try {
+    const { GITHUB_BOT_PASSWORD } = process.env;
+
+    if (!GITHUB_BOT_PASSWORD) throw new Error('GITHUB_BOT_PASSWORD is not defined');
+
+    client = new Octokit({ auth: GITHUB_BOT_PASSWORD });
+    return client;
+  } catch (error) {
+    console.error('ERROR! Failed to create Octokit client. Is GITHUB_BOT_PASSWORD defined?', error);
+    throw error;
+  }
+}

src/monorepo/index.ts

@@ -0,0 +1,42 @@
+import { getSnootyDirSet } from './utils/path-utils';
+import { GitCommitInfo } from './types/github-types';
+import { getProjectDirFromPath } from './services/get-paths';
+
+interface FileUpdatePayload {
+  ownerName: string;
+  repoName: string;
+  commitSha: string;
+  updatedFilePaths: string[];
+}
+
+/**
+ * Retrieves the path of project directories. This is determined
+ * by finding the nearest parent directory that has a snooty.toml file
+ * for a given updated file path from a commit.
+ * @param repoName Name of the repository to check.
+ * @param ownerName Name of the owner of the repository.
+ * @param commitSha The Git commit SHA that contains the changed files.
+ * @param updatedFilePaths An array of all of the changed files (added, removed, modified)
+ * from the commit. The method `getUpdatedFilePaths` in the `src/monorepo/utils/path-utils.ts
+ * can be used to parse these paths from a GitHub `Commit` object.
+ * @returns An array of all the project paths that need to be built.
+ */
+export async function getMonorepoPaths({
+  repoName,
+  ownerName,
+  commitSha,
+  updatedFilePaths,
+}: FileUpdatePayload): Promise<string[]> {
+  const commitInfo: GitCommitInfo = {
+    ownerName,
+    repoName,
+    commitSha,
+  };
+
+  const snootyDirSet = await getSnootyDirSet(commitInfo);
+
+  const projects = updatedFilePaths.map((path) => getProjectDirFromPath(path, snootyDirSet));
+
+  // remove empty strings and remove duplicated values
+  return Array.from(new Set(projects.filter((dir) => !!dir)));
+}

src/monorepo/services/get-paths.ts

@@ -0,0 +1,36 @@
+import { SNOOTY_TOML_FILENAME } from '../utils/monorepo-constants';
+
+/**
+ * This function returns the project path for a given file change from a docs repository
+ * within the monorepo. This function supports nested projects.
+ * @param path An added/modified/removed file path from a commit e.g. server-docs/source/index.rst
+ * @param commitInfo Contains information
+ * @returns The closest file path that contains a snooty.toml, relative to the path parameter.
+ */
+export function getProjectDirFromPath(path: string, snootyDirSet: Set<string>): string {
+  const pathArray = path.split('/');
+  if (pathArray.length === 0) {
+    console.warn('WARNING! Empty path found: ', path);
+    return '';
+  }
+
+  /**
+   * If the changed file is the snooty.toml file, we know that we
+   * are in the project's root directory. We can join the original
+   * pathArray to get the project path since the snooty.toml has been removed.
+   */
+  const changedFile = pathArray.pop();
+
+  if (changedFile === SNOOTY_TOML_FILENAME) return pathArray.join('/');
+
+  while (pathArray.length > 0) {
+    const currDir = pathArray.join('/');
+
+    if (snootyDirSet.has(currDir)) return currDir;
+
+    pathArray.pop();
+  }
+
+  console.warn(`WARNING! No snooty.toml found for the given path: ${path}`);
+  return '';
+}

src/monorepo/types/atlas-types.ts

@@ -0,0 +1,25 @@
+interface DirectoryConfig {
+  snooty_toml?: string;
+  source?: string;
+}
+
+interface RepoConfig {
+  repoName: string;
+  deployable: boolean;
+  branches: BranchConfig[];
+}
+
+interface BranchConfig {
+  gitBranchName: string;
+}
+
+// TODO: Populate these more. For DOP-3911, they are
+// being added for testing purposes.
+export interface DocSetEntry {
+  project: string;
+  prefix: string;
+  bucket: string;
+  url: string;
+  directories?: DirectoryConfig;
+  repos?: RepoConfig[];
+}

src/monorepo/types/github-types.ts

@@ -0,0 +1,5 @@
+export interface GitCommitInfo {
+  commitSha: string;
+  ownerName: string;
+  repoName: string;
+}

src/monorepo/utils/monorepo-constants.ts

@@ -0,0 +1,2 @@
+export const SNOOTY_TOML_FILENAME = 'snooty.toml';
+export const MONOREPO_NAME = 'docs-monorepo';

src/monorepo/utils/path-utils.ts

@@ -0,0 +1,50 @@
+import { Commit } from '@octokit/webhooks-types';
+import { getOctokitClient } from '../../clients/githubClient';
+import { GitCommitInfo } from '../types/github-types';
+import { SNOOTY_TOML_FILENAME } from './monorepo-constants';
+
+/**
+ * Creates a `Set` of all `snooty.toml` paths within the monorepo.
+ * The function retrieves the monorepo's tree structure from GitHub.
+ */
+export async function getSnootyDirSet({ commitSha, ownerName, repoName }: GitCommitInfo): Promise<Set<string>> {
+  try {
+    const client = getOctokitClient();
+
+    // getting the repository tree for a given commit SHA. This returns an object
+    // with the property `tree` that is a flat array of all files in the repository.
+    // The tree array contains objects that hold the file path.
+    // Unlike the contents API for repositories, the actual file content is not returned.
+    const { data } = await client.request('GET /repos/{owner}/{repo}/git/trees/{tree_sha}', {
+      owner: ownerName,
+      repo: repoName,
+      tree_sha: commitSha,
+      recursive: 'true',
+    });
+
+    const snootyTomlDirs = data.tree
+      .filter((treeNode) => !!treeNode.path?.includes(SNOOTY_TOML_FILENAME))
+      .map((treeNode) => {
+        // casting the `treeNode.path` from `(string | undefined)` to `string` since the filter will ensure that the result
+        // only includes treeNode.path values that are defined and include snooty.toml
+        // in the path i.e. we will not have `undefined` as a value in the resulting array.
+        const path = treeNode.path as string;
+
+        // the - 1 is to remove the trailing slash
+        return path.slice(0, path.length - SNOOTY_TOML_FILENAME.length - 1);
+      });
+
+    const snootyDirSet = new Set(snootyTomlDirs);
+
+    return snootyDirSet;
+  } catch (error) {
+    console.error(
+      `ERROR! Unable to retrieve tree for SHA: ${commitSha} owner name: ${ownerName} repo name: ${repoName}`,
+      error
+    );
+    throw error;
+  }
+}
+
+export const getUpdatedFilePaths = (commit: Commit): string[] =>
+  commit.modified.concat(commit.added).concat(commit.removed);

src/repositories/docSetRepository.ts

@@ -0,0 +1,45 @@
+import { IConfig } from 'config';
+import { Db } from 'mongodb';
+import { ILogger } from '../services/logger';
+import { BaseRepository } from './baseRepository';
+
+const docSetCollectionName = process.env.DOCS_SET_COLLECTION_NAME || 'docset';
+
+export class DocSetRepository extends BaseRepository {
+  constructor(db: Db, config: IConfig, logger: ILogger) {
+    super(config, logger, 'DocSetRepository', db.collection(docSetCollectionName));
+  }
+
+  /**
+   * Compares the project path from a monorepo push event, and compares it with
+   * what is configured in the docset entry in Atlas.
+   * @param path The project path where the snooty.toml file exists from the monorepo.
+   * This path will reflect the current project path from a given commit.
+   * @param projectName The project name for the docset entry.
+   * @returns A boolean representing whether or not the configured docset entry snooty_toml path
+   * matches the path found in GitHub.
+   */
+  async checkSnootyTomlPath(path: string, projectName: string) {
+    const query = { project: projectName };
+    try {
+      const docSetObject = await this.findOne(
+        query,
+        `Mongo Timeout Error: Timedout while retrieving repos entry for ${path}`
+      );
+
+      if (!docSetObject) {
+        console.warn(`WARNING: The docset does not exist for the following project: ${projectName} \n path: ${path}`);
+
+        return false;
+      }
+
+      return docSetObject.directories.snooty_toml === path;
+    } catch (error) {
+      console.warn(
+        `WARNING: Error occurred when retrieving project path for ${projectName}. The following path was provided: ${path}`,
+        error
+      );
+      return false;
+    }
+  }
+}