Merge branch 'master' into DOP-4274

.github/workflows/test-persistence.yml

@@ -34,7 +34,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-node@v1
       with:
-        node-version: '14.x'
+        node-version: '18.x'
     - name: Install dependencies
       run: npm install --dev
     - name: Lint

Dockerfile.enhanced

@@ -117,4 +117,4 @@ ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js
 
 RUN mkdir repos && chmod 755 repos
 EXPOSE 3000
-CMD ["node", "enhanced/enhancedApp.js"]
+CMD ["node", "--enable-source-maps", "enhanced/enhancedApp.js"]

api/controllers/v2/github.ts

@@ -138,6 +138,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
 
     /* Create and insert Job for each monorepo project that has changes */
     for (const path of monorepoPaths) {
+      consoleLogger.info(body.repository.full_name, `Create Job for Monorepo directory: /${path}`);
       // TODO: Deal with nested monorepo projects
       /* For now, we will ignore nested monorepo projects until necessary */
       if (path.split('/').length > 1) continue;

modules/persistence/.nvmrc

@@ -1 +1 @@
-v14.17.6
+v18.12.0

modules/persistence/src/services/metadata/ToC/index.ts

@@ -1,4 +1,3 @@
-import { AssociatedProduct, hasAssociations } from '../associated_products';
 import { Metadata } from '..';
 import { convertSlugToUrl } from './utils/convertSlugToUrl';
 
@@ -93,15 +92,19 @@ const mergeTocTreeOrder = (metadata: Metadata, node, insertions: TocOrderInserti
 // contains an associated_products entry
 export const traverseAndMerge = (
   metadata: Metadata,
-  associated_products: AssociatedProduct[],
+  umbrellaMetadata: Metadata,
   umbrellaToCs: ToCCopies,
   tocInsertions: ToCInsertions,
   tocOrderInsertions: TocOrderInsertions
 ) => {
   const { project } = metadata;
+  const associatedProducts = umbrellaMetadata.associated_products || [];
 
-  const toctree = hasAssociations(metadata) ? umbrellaToCs.original : umbrellaToCs.urlified;
-  const toBeInserted = new Set(associated_products.map((p) => p.name));
+  const associatedProductNames = associatedProducts.map((p) => p.name);
+  const toctree = structuredClone(
+    metadata.project === umbrellaMetadata.project ? umbrellaToCs.original : umbrellaToCs.urlified
+  );
+  const toBeInserted = new Set(associatedProductNames);
   let queue = [toctree];
   while (queue?.length && toBeInserted.size) {
     let next = queue.shift();

modules/persistence/src/services/metadata/associated_products/index.ts

@@ -26,12 +26,6 @@ interface AggregatedMetadata {
   most_recent: Metadata;
 }
 
-type EnvKeyedObject = {
-  prd: any;
-  preprd: any;
-  dotcomstg: any;
-  dotcomprd: any;
-};
 // TODO: move the branch/repobranch interfaces into their own file, or into a seperate abstraction?
 export interface BranchEntry {
   name: string;
@@ -69,8 +63,8 @@ const umbrellaMetadataEntry = async (project: string): Promise<Metadata> => {
       return null as unknown as Metadata;
     }
 
-    const repoDoc = await getRepoBranchesEntry(project);
-    const branchNames = repoDoc.branches.map((branchEntry) => branchEntry.gitBranchName);
+    const umbrellaRepos = await getRepoBranchesEntry(umbrella.project);
+    const branchNames = umbrellaRepos.branches.map((branchEntry) => branchEntry.gitBranchName);
     const entry = await snooty
       .collection('metadata')
       .find({
@@ -198,7 +192,7 @@ export const mergeAssociatedToCs = async (metadata: Metadata) => {
     const mergedMetadataEntries = [umbrellaMetadata, ...associatedMetadataEntries].map((metadataEntry) => {
       const mergedMetadataEntry = traverseAndMerge(
         metadataEntry,
-        umbrellaMetadata.associated_products || [],
+        umbrellaMetadata,
         umbrellaToCs,
         tocInsertions,
         tocOrderInsertions

modules/persistence/src/services/metadata/repos_branches/index.ts

@@ -65,6 +65,11 @@ const getAggregationPipeline = (matchCondition: any) => {
         repo: 0,
       },
     },
+    {
+      $sort: {
+        prodDeployable: -1,
+      },
+    },
   ];
 };
 

modules/persistence/tests/metadata/ToC.test.ts

@@ -122,7 +122,7 @@ describe('ToC module', () => {
         expect(
           traverseAndMerge(
             umbrellaMetadata as unknown as Metadata,
-            umbrellaMetadata.associated_products || [],
+            umbrellaMetadata,
             umbrellaToCs,
             tocInsertions,
             tocOrderInsertions
@@ -135,7 +135,7 @@ describe('ToC module', () => {
         expect(
           traverseAndMerge(
             metadata[0] as unknown as Metadata,
-            umbrellaMetadata.associated_products || [],
+            umbrellaMetadata,
             umbrellaToCs,
             tocInsertions,
             tocOrderInsertions

modules/persistence/tests/setupAfterEnv.ts

@@ -1,4 +1,5 @@
 beforeAll(() => {
   // Disable console.time from crowding test output
   console.time = jest.fn();
+  global.structuredClone = (val) => JSON.parse(JSON.stringify(val));
 });

src/commands/index.ts

@@ -1,17 +1,9 @@
-import { prepareBuildAndGetDependencies } from './src/helpers/dependency-helpers';
+import { prepareBuild } from './src/helpers/dependency-helpers';
 import { nextGenDeploy } from './src/shared/next-gen-deploy';
 import { nextGenHtml } from './src/shared/next-gen-html';
 import { nextGenParse } from './src/shared/next-gen-parse';
 import { nextGenStage } from './src/shared/next-gen-stage';
 import { oasPageBuild } from './src/shared/oas-page-build';
 import { persistenceModule } from './src/shared/persistence-module';
 
-export {
-  nextGenParse,
-  nextGenHtml,
-  nextGenStage,
-  persistenceModule,
-  oasPageBuild,
-  nextGenDeploy,
-  prepareBuildAndGetDependencies,
-};
+export { nextGenParse, nextGenHtml, nextGenStage, persistenceModule, oasPageBuild, nextGenDeploy, prepareBuild };

src/commands/src/helpers/dependency-helpers.ts

@@ -43,16 +43,25 @@ async function createEnvProdFile({
   }
 }
 
-export async function downloadBuildDependencies(buildDependencies: BuildDependencies, repoName: string) {
+export async function downloadBuildDependencies(
+  buildDependencies: BuildDependencies,
+  repoName: string,
+  directory?: string
+) {
   const commands: string[] = [];
   await Promise.all(
     buildDependencies.map(async (dependencyInfo) => {
-      const repoDir = getRepoDir(repoName);
+      const repoDir = getRepoDir(repoName, directory);
       const targetDir = dependencyInfo.targetDir ?? repoDir;
+      let options = {};
+      if (targetDir != repoDir) {
+        options = { cwd: repoDir };
+      }
       try {
         await executeCliCommand({
           command: 'mkdir',
           args: ['-p', targetDir],
+          options: options,
         });
       } catch (error) {
         console.error(
@@ -63,36 +72,29 @@ export async function downloadBuildDependencies(buildDependencies: BuildDependen
       }
       commands.push(`mkdir -p ${targetDir}`);
       await Promise.all(
-        dependencyInfo.dependencies.map((dep) => {
+        dependencyInfo.dependencies.map(async (dep) => {
+          commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`);
           try {
-            executeCliCommand({
+            return await executeCliCommand({
               command: 'curl',
-              args: ['-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`],
+              args: ['--max-time', '10', '-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`],
+              options: options,
             });
           } catch (error) {
             console.error(
               `ERROR! Could not curl ${dep.url} into ${targetDir}/${dep.filename}. Dependency information: `,
               dependencyInfo
             );
           }
-          commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`);
         })
       );
     })
   );
   return commands;
 }
 
-export async function prepareBuildAndGetDependencies(
-  repoName: string,
-  projectName: string,
-  baseUrl: string,
-  buildDependencies: BuildDependencies,
-  directory?: string
-) {
+export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) {
   const repoDir = getRepoDir(repoName, directory);
-  await downloadBuildDependencies(buildDependencies, repoName);
-  console.log('Downloaded Build dependencies');
 
   // doing these in parallel
   const commandPromises = [

src/commands/src/helpers/index.ts

@@ -14,9 +14,11 @@ const EPIPE_SYSCALL = 'write';
 
 export class ExecuteCommandError extends Error {
   data: unknown;
-  constructor(message: string, data: unknown) {
+  exitCode: number | null;
+  constructor(message: string, exitCode: number | null, data?: unknown) {
     super(message);
     this.data = data;
+    this.exitCode = exitCode;
   }
 }
 
@@ -80,7 +82,7 @@ export async function executeAndPipeCommands(
         return;
       }
 
-      reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err));
+      reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err.errno, err));
       hasRejected = true;
     });
 
@@ -89,7 +91,7 @@ export async function executeAndPipeCommands(
     });
 
     cmdFrom.on('error', (err) => {
-      reject(new ExecuteCommandError('The first command (cmdTo) failed', err));
+      reject(new ExecuteCommandError('The first command (cmdTo) failed', 1, err));
       hasRejected = true;
     });
 
@@ -105,7 +107,7 @@ export async function executeAndPipeCommands(
     });
 
     cmdTo.on('error', (err) => {
-      reject(new ExecuteCommandError('The second command failed', err));
+      reject(new ExecuteCommandError('The second command failed', 1, err));
     });
 
     cmdTo.on('exit', (exitCode) => {
@@ -127,7 +129,13 @@ export async function executeAndPipeCommands(
           console.error('error', errorText.join(''));
         }
 
-        reject(new ExecuteCommandError('The command failed', { exitCode, outputText, errorText }));
+        reject(
+          new ExecuteCommandError('The command failed', exitCode, {
+            exitCode,
+            outputText: outputText.join(''),
+            errorText: errorText.join(''),
+          })
+        );
         return;
       }
 
@@ -177,7 +185,7 @@ export async function executeCliCommand({
 
     executedCommand.on('error', (err) => {
       console.log(`ERROR in executeCliCommand.\nCommand: ${command} ${args.join(' ')}\nError: ${err}`);
-      reject(new ExecuteCommandError('The command failed', err));
+      reject(new ExecuteCommandError('The command failed', 1, err));
     });
 
     executedCommand.on('close', (exitCode) => {
@@ -204,7 +212,11 @@ export async function executeCliCommand({
             Options provided: ${JSON.stringify(options, null, 4)}\n
             Stdout: ${outputText.join('')} \n
             Error: ${errorText.join('')}`,
-            exitCode
+            exitCode,
+            {
+              outputText: outputText.join(''),
+              errorText: errorText.join(''),
+            }
           )
         );
         return;

src/commands/src/shared/next-gen-parse.ts

@@ -1,7 +1,7 @@
 import path from 'path';
 import { Job } from '../../../entities/job';
 import { getDirectory } from '../../../job/jobHandler';
-import { CliCommandResponse, executeCliCommand } from '../helpers';
+import { CliCommandResponse, ExecuteCommandError, executeCliCommand } from '../helpers';
 
 const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml';
 interface NextGenParseParams {
@@ -37,6 +37,9 @@ export async function nextGenParse({ job, patchId, isProd }: NextGenParseParams)
     });
     return result;
   } catch (error) {
+    if (error instanceof ExecuteCommandError && error.exitCode !== 1) {
+      return error.data as CliCommandResponse;
+    }
     throw new Error(`next-gen-parse failed. \n ${error}`);
   }
 }

src/job/jobHandler.ts

@@ -14,13 +14,7 @@ import { IJobValidator } from './jobValidator';
 import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository';
 import { DocsetsRepository } from '../repositories/docsetsRepository';
 import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants';
-import {
-  nextGenHtml,
-  nextGenParse,
-  oasPageBuild,
-  persistenceModule,
-  prepareBuildAndGetDependencies,
-} from '../commands';
+import { nextGenHtml, nextGenParse, oasPageBuild, persistenceModule, prepareBuild } from '../commands';
 import { downloadBuildDependencies } from '../commands/src/helpers/dependency-helpers';
 import { CliCommandResponse } from '../commands/src/helpers';
 require('fs');
@@ -213,17 +207,14 @@ export abstract class JobHandler {
   }
 
   @throwIfJobInterupted()
-  private async getBuildDependencies() {
-    const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(this.currJob.payload.repoName);
-    if (!buildDependencies) return [];
-    return buildDependencies;
-  }
-
-  @throwIfJobInterupted()
-  private async getAndBuildDependencies() {
-    const buildDependencies = await this.getBuildDependencies();
-    const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName);
-    this._logger.save(this._currJob._id, commands.join('\n'));
+  private async getAndDownloadBuildDependencies() {
+    const repoName = this.currJob.payload.repoName;
+    const directory = this.currJob.payload.repoName === MONOREPO_NAME ? this.currJob.payload.directory : undefined;
+    const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(repoName, directory);
+    if (!buildDependencies) return;
+    await this._logger.save(this._currJob._id, 'Identified Build dependencies');
+    const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName, directory);
+    await this._logger.save(this._currJob._id, `${commands.join('\n')}`);
   }
 
   @throwIfJobInterupted()
@@ -550,7 +541,7 @@ export abstract class JobHandler {
     this._logger.save(this._currJob._id, 'Checked Commit');
     await this.pullRepo();
     this._logger.save(this._currJob._id, 'Pulled Repo');
-    await this.getAndBuildDependencies();
+    await this.getAndDownloadBuildDependencies();
     this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
     this.prepBuildCommands();
     this._logger.save(this._currJob._id, 'Prepared Build commands');
@@ -581,8 +572,8 @@ export abstract class JobHandler {
     await this.setEnvironmentVariables();
     this.logger.save(job._id, 'Prepared Environment variables');
 
-    const buildDependencies = await this.getBuildDependencies();
-    this._logger.save(this._currJob._id, 'Identified Build dependencies');
+    await this.getAndDownloadBuildDependencies();
+    this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
 
     const docset = await this._docsetsRepo.getRepo(this._currJob.payload.repoName, this._currJob.payload.directory);
     let env = this._config.get<string>('env');
@@ -591,16 +582,9 @@ export abstract class JobHandler {
     }
     const baseUrl = docset?.url?.[env] || 'https://mongodbcom-cdn.website.staging.corp.mongodb.com';
 
-    const { patchId } = await prepareBuildAndGetDependencies(
-      job.payload.repoName,
-      job.payload.project,
-      baseUrl,
-      buildDependencies,
-      job.payload.directory
-    );
+    const { patchId } = await prepareBuild(job.payload.repoName, job.payload.project, baseUrl, job.payload.directory);
     // Set patchId on payload for use in nextGenStage
     this._currJob.payload.patchId = patchId;
-    this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
 
     let buildStepOutput: CliCommandResponse;