DOP-4073: Adding support for curling dependencies stored in repos_branches

src/commands/src/helpers/dependency-helpers.ts

@@ -2,6 +2,7 @@ import path from 'path';
 import fs from 'fs';
 import { executeCliCommand, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '.';
 import { promisify } from 'util';
+import { BuildDependencies } from '../../../entities/job';
 
 const existsAsync = promisify(fs.exists);
 const writeFileAsync = promisify(fs.writeFile);
@@ -33,12 +34,59 @@ async function createEnvProdFile(repoDir: string, projectName: string, baseUrl:
   }
 }
 
-export async function prepareBuildAndGetDependencies(repoName: string, projectName: string, baseUrl: string) {
+export async function downloadBuildDependencies(buildDependencies: BuildDependencies, repoName: string) {
+  const commands: string[] = [];
+  await Promise.all(
+    buildDependencies.map(async (dependencyInfo) => {
+      const repoDir = getRepoDir(repoName);
+      const targetDir = dependencyInfo.targetDir ?? repoDir;
+      try {
+        await executeCliCommand({
+          command: 'mkdir',
+          args: ['-p', targetDir],
+        });
+      } catch (error) {
+        console.error(
+          `ERROR! Could not create target directory ${targetDir}. Dependency information: `,
+          dependencyInfo
+        );
+        throw error;
+      }
+      commands.push(`mkdir -p ${targetDir}`);
+      await Promise.all(
+        dependencyInfo.dependencies.map((dep) => {
+          try {
+            executeCliCommand({
+              command: 'curl',
+              args: ['-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`],
+            });
+          } catch (error) {
+            console.error(
+              `ERROR! Could not curl ${dep.url} into ${targetDir}/${dep.filename}. Dependency information: `,
+              dependencyInfo
+            );
+          }
+          commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`);
+        })
+      );
+    })
+  );
+  return commands;
+}
+
+export async function prepareBuildAndGetDependencies(
+  repoName: string,
+  projectName: string,
+  baseUrl: string,
+  buildDependencies: BuildDependencies
+) {
   // before we get build dependencies, we need to clone the repo
   await cloneRepo(repoName);
 
   const repoDir = getRepoDir(repoName);
 
+  await downloadBuildDependencies(buildDependencies, repoName);
+
   // doing these in parallel
   const commandPromises = [
     getCommitHash(repoDir),

src/entities/job.ts

@@ -17,6 +17,18 @@ export enum JobStatus {
 //   regression = 'regression',
 // }
 
+export type BuildDependencies = DependencyInfo[];
+
+type Dependency = {
+  url: string;
+  filename: string;
+};
+
+type DependencyInfo = {
+  targetDir?: string;
+  dependencies: Dependency[];
+};
+
 export type Payload = {
   jobType: string;
   source: string;

src/entrypoints/localApp.ts

@@ -15,11 +15,35 @@ async function localApp() {
   const baseUrl = 'https://www.mongodb.com';
   const bucket = 'docs-java-dotcomstg';
   const mutPrefix = 'docs/drivers/java/sync';
+  const buildDependencies = [
+    {
+      dependencies: [
+        {
+          url: 'https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/publishedbranches/docs-mongodb-internal.yaml',
+          filename: 'published-branches.yaml',
+        },
+      ],
+    },
+    {
+      buildDir: 'source/driver-examples',
+      dependencies: [
+        {
+          url: 'https://raw.githubusercontent.com/mongodb/mongo-python-driver/master/test/test_examples.py',
+          filename: 'test_examples.py',
+        },
+        {
+          url: 'https://raw.githubusercontent.com/mongodb/motor/master/test/asyncio_tests/test_examples.py',
+          filename: 'test_examples_motor.py',
+        },
+      ],
+    },
+  ];
 
   const { commitHash, patchId, bundlePath, commitBranch, hasRedirects, repoDir } = await prepareBuildAndGetDependencies(
     repoName,
     projectName,
-    baseUrl
+    baseUrl,
+    buildDependencies
   );
 
   console.log('Begin snooty build...');

src/job/jobHandler.ts

@@ -13,6 +13,7 @@ import { IJobValidator } from './jobValidator';
 import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository';
 import { DocsetsRepository } from '../repositories/docsetsRepository';
 import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants';
+import { downloadBuildDependencies } from '../commands/src/helpers/dependency-helpers';
 require('fs');
 
 export abstract class JobHandler {
@@ -202,6 +203,14 @@ export abstract class JobHandler {
     }
   }
 
+  @throwIfJobInterupted()
+  private async getAndBuildDependencies() {
+    const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(this.currJob.payload.repoName);
+    if (!buildDependencies) return;
+    const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName);
+    this._logger.save(this._currJob._id, commands.join('\n'));
+  }
+
   @throwIfJobInterupted()
   private async downloadMakeFile(): Promise<void> {
     try {
@@ -480,6 +489,8 @@ export abstract class JobHandler {
     this._logger.save(this._currJob._id, 'Checked Commit');
     await this.pullRepo();
     this._logger.save(this._currJob._id, 'Pulled Repo');
+    await this.getAndBuildDependencies();
+    this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
     this.prepBuildCommands();
     this._logger.save(this._currJob._id, 'Prepared Build commands');
     await this.prepNextGenBuild();

src/repositories/repoBranchesRepository.ts

@@ -2,12 +2,22 @@ import { Db } from 'mongodb';
 import { BaseRepository } from './baseRepository';
 import { ILogger } from '../services/logger';
 import { IConfig } from 'config';
+import { BuildDependencies } from '../entities/job';
 
 export class RepoBranchesRepository extends BaseRepository {
   constructor(db: Db, config: IConfig, logger: ILogger) {
     super(config, logger, 'RepoBranchesRepository', db.collection(config.get('repoBranchesCollection')));
   }
 
+  async getBuildDependencies(repoName: string): Promise<BuildDependencies> {
+    const query = { repoName: repoName };
+    const repo = await this.findOne(
+      query,
+      `Mongo Timeout Error: Timedout while retrieving build dependencies for ${repoName}`
+    );
+    return repo?.optionalBuildSteps?.buildDependencies;
+  }
+
   async getRepoBranches(repoName: string, directoryPath?: string): Promise<any> {
     const query = { repoName: repoName };
     if (directoryPath) query['directories.snooty_toml'] = `/${directoryPath}`;