From 834e691b0e3f46ff59853d769d2be1e2d43bad60 Mon Sep 17 00:00:00 2001 From: Maya Raman Date: Mon, 22 Jan 2024 15:14:10 -0500 Subject: [PATCH] DOP-4209 (#958) * experiments, added nextGenParse call to jobHandler * stage ecs - no makefiles * alter snooty version * next gen html * command key * parse * catch * logger * remove error * more logging * cwd * repoDir correct * quotations * successful parse? odd address * oas page build * xlarge * persistence module * quote * no parse * try catch parse * type error * no parse, use persistence and build * log errors * localApp working * remove /dist * add build deps & nextgenhtml * add logging to build deps * log html * remove build deps for envs * env vars * build deps before executeBuild * staging monorepo jobs * comment * localApp type cleanup * stage * deploy * clean fs * status * no event body * throw new error * commented out all unnecessary steps * explicitly call build steps * remove steps that should not be used * include job inqueue for error * log status * Empty-Commit * comment * used process.env to find URL and BUCKET * save localApp * uncomment prepNextGenBuild * remove throw error * not build on preprd * buildCommands array * log publish * use normal deploy * deploy * add cp commands * cd snooty * log outputs * ref repoDir rather than cwd * correct reposDir vs repoDir * add slash * add repos to prodFileName * log build deps * change prodFIleName to cwd/snooty * run parse * remove deploy * override build in stagingJobHandler * override build, set up debugger * vscode launch * remove dist * remove dist * use project * readd build deps * add cp and cd commands to nextGenHtml * cp correct paths * remove first cp and cd * remove cp * ref snooty filepath correctly * dockerfile snooty branch mm-log * remove first cp * cd .. * commands * stringify, logs * use chdir * cp second * log in clicommand * pass logger to mut publish * dotcomstg -> stg * checkout and pull branch * add pull repo and change clone * commented out incorrect code * local run works for both cloud-docs and monorepo/cloud-docs * log event and boyd * log out trigger build * log repo name and feat flag * feature flag * ssmprefix * env * dist * remove feature flag for feature branch build * log why no paths * change slash of path * clean * organize code * clean * redoc * fixes from merge * comment out builddeps, use redoc rc * duplicate clone * clean, get bucket and url * log env vars * pass logger to getEnvVar * takeover preprd * add to v1? * log which build * force directory * add directory to debug command and local build * remove from preprd * clean up * keep conditionals for buildCommands in normal build * further cleaning * remove logs * number of logs * curl into repoDir/targetDir * try new flow of logging * remove comments * allow output and error text to be returned from nextGenStage * clean logs * revert targetDir for downloadBuildDependencies * clean, wrapWithBenchmark * PR feedback * [DOP-4127]: Update dockerfile.local to have redoc installed properly * [DOP-4127]: Use new SQS queue URL * [DOP-4127]: Install redoc bundle * [DOP-4127]: Revert how redoc is installed * PR feedback, second round * replace useWithBenchmarks with isNextGen * source patchId from getBuildAndGetDependencies * [DOP-4204]: Update README for local Autobuilder (#954) * [DOP-4204]: Add help command * [DOP-4204]: Typo * [DOP-4204]: Formatting * [DOP-4024]: Update README.md * [DOP-4204]: Troubleshooting * [DOP-4204]: Fix typos * [DOP-4204]: Rename images and move into shared folder * [DOP-4204]: Update troubleshooting * added logging and error throwing in wrapWithBenchmark * conditionally write patchId and commitHash env vars * seeing if we have build dependencies * adding await * log * strringify * adding monorepo handling * snooty toml * logging * Adding slash * adding timeout to curl * adding logging * finally * bruh * L * oh well * adding repodir * adding monorepo support * log * cwd * adding to both curl and mkdir * adding logging * testing with executeclicommand not throwing * avoiding merge conflict * cleaning up * restructuring * [DOP-4269]: Refactor error handling * [DOP-4269]: Add text to error * consolidating * fixing merge stuff in readme * merge * async in dep helper * error message * nvm removing error msg * trying * seems like the best way for await idk * one more attempt * Nvm * not working * trying this out * nvm --------- Co-authored-by: Matt Meigs Co-authored-by: branberry --- Dockerfile.enhanced | 2 +- api/controllers/v2/github.ts | 1 + src/commands/index.ts | 12 +----- .../src/helpers/dependency-helpers.ts | 32 +++++++------- src/commands/src/helpers/index.ts | 26 ++++++++---- src/commands/src/shared/next-gen-parse.ts | 5 ++- src/job/jobHandler.ts | 42 ++++++------------- src/repositories/repoBranchesRepository.ts | 3 +- 8 files changed, 59 insertions(+), 64 deletions(-) diff --git a/Dockerfile.enhanced b/Dockerfile.enhanced index e85cb3624..5eedb5d8c 100644 --- a/Dockerfile.enhanced +++ b/Dockerfile.enhanced @@ -117,4 +117,4 @@ ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "enhanced/enhancedApp.js"] +CMD ["node", "--enable-source-maps", "enhanced/enhancedApp.js"] diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e562d2a2f..a761e742b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -138,6 +138,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise 1) continue; diff --git a/src/commands/index.ts b/src/commands/index.ts index 77b25f7a6..7c32ac6ac 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -1,4 +1,4 @@ -import { prepareBuildAndGetDependencies } from './src/helpers/dependency-helpers'; +import { prepareBuild } from './src/helpers/dependency-helpers'; import { nextGenDeploy } from './src/shared/next-gen-deploy'; import { nextGenHtml } from './src/shared/next-gen-html'; import { nextGenParse } from './src/shared/next-gen-parse'; @@ -6,12 +6,4 @@ import { nextGenStage } from './src/shared/next-gen-stage'; import { oasPageBuild } from './src/shared/oas-page-build'; import { persistenceModule } from './src/shared/persistence-module'; -export { - nextGenParse, - nextGenHtml, - nextGenStage, - persistenceModule, - oasPageBuild, - nextGenDeploy, - prepareBuildAndGetDependencies, -}; +export { nextGenParse, nextGenHtml, nextGenStage, persistenceModule, oasPageBuild, nextGenDeploy, prepareBuild }; diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index 17da25acd..3e2f20ae6 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -43,16 +43,25 @@ async function createEnvProdFile({ } } -export async function downloadBuildDependencies(buildDependencies: BuildDependencies, repoName: string) { +export async function downloadBuildDependencies( + buildDependencies: BuildDependencies, + repoName: string, + directory?: string +) { const commands: string[] = []; await Promise.all( buildDependencies.map(async (dependencyInfo) => { - const repoDir = getRepoDir(repoName); + const repoDir = getRepoDir(repoName, directory); const targetDir = dependencyInfo.targetDir ?? repoDir; + let options = {}; + if (targetDir != repoDir) { + options = { cwd: repoDir }; + } try { await executeCliCommand({ command: 'mkdir', args: ['-p', targetDir], + options: options, }); } catch (error) { console.error( @@ -63,11 +72,13 @@ export async function downloadBuildDependencies(buildDependencies: BuildDependen } commands.push(`mkdir -p ${targetDir}`); await Promise.all( - dependencyInfo.dependencies.map((dep) => { + dependencyInfo.dependencies.map(async (dep) => { + commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`); try { - executeCliCommand({ + return await executeCliCommand({ command: 'curl', - args: ['-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`], + args: ['--max-time', '10', '-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`], + options: options, }); } catch (error) { console.error( @@ -75,7 +86,6 @@ export async function downloadBuildDependencies(buildDependencies: BuildDependen dependencyInfo ); } - commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`); }) ); }) @@ -83,16 +93,8 @@ export async function downloadBuildDependencies(buildDependencies: BuildDependen return commands; } -export async function prepareBuildAndGetDependencies( - repoName: string, - projectName: string, - baseUrl: string, - buildDependencies: BuildDependencies, - directory?: string -) { +export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) { const repoDir = getRepoDir(repoName, directory); - await downloadBuildDependencies(buildDependencies, repoName); - console.log('Downloaded Build dependencies'); // doing these in parallel const commandPromises = [ diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index dd22ad760..ba79ca626 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -14,9 +14,11 @@ const EPIPE_SYSCALL = 'write'; export class ExecuteCommandError extends Error { data: unknown; - constructor(message: string, data: unknown) { + exitCode: number | null; + constructor(message: string, exitCode: number | null, data?: unknown) { super(message); this.data = data; + this.exitCode = exitCode; } } @@ -80,7 +82,7 @@ export async function executeAndPipeCommands( return; } - reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err)); + reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err.errno, err)); hasRejected = true; }); @@ -89,7 +91,7 @@ export async function executeAndPipeCommands( }); cmdFrom.on('error', (err) => { - reject(new ExecuteCommandError('The first command (cmdTo) failed', err)); + reject(new ExecuteCommandError('The first command (cmdTo) failed', 1, err)); hasRejected = true; }); @@ -105,7 +107,7 @@ export async function executeAndPipeCommands( }); cmdTo.on('error', (err) => { - reject(new ExecuteCommandError('The second command failed', err)); + reject(new ExecuteCommandError('The second command failed', 1, err)); }); cmdTo.on('exit', (exitCode) => { @@ -127,7 +129,13 @@ export async function executeAndPipeCommands( console.error('error', errorText.join('')); } - reject(new ExecuteCommandError('The command failed', { exitCode, outputText, errorText })); + reject( + new ExecuteCommandError('The command failed', exitCode, { + exitCode, + outputText: outputText.join(''), + errorText: errorText.join(''), + }) + ); return; } @@ -177,7 +185,7 @@ export async function executeCliCommand({ executedCommand.on('error', (err) => { console.log(`ERROR in executeCliCommand.\nCommand: ${command} ${args.join(' ')}\nError: ${err}`); - reject(new ExecuteCommandError('The command failed', err)); + reject(new ExecuteCommandError('The command failed', 1, err)); }); executedCommand.on('close', (exitCode) => { @@ -204,7 +212,11 @@ export async function executeCliCommand({ Options provided: ${JSON.stringify(options, null, 4)}\n Stdout: ${outputText.join('')} \n Error: ${errorText.join('')}`, - exitCode + exitCode, + { + outputText: outputText.join(''), + errorText: errorText.join(''), + } ) ); return; diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 12467d04c..70bba49d7 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,7 +1,7 @@ import path from 'path'; import { Job } from '../../../entities/job'; import { getDirectory } from '../../../job/jobHandler'; -import { CliCommandResponse, executeCliCommand } from '../helpers'; +import { CliCommandResponse, ExecuteCommandError, executeCliCommand } from '../helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; interface NextGenParseParams { @@ -37,6 +37,9 @@ export async function nextGenParse({ job, patchId, isProd }: NextGenParseParams) }); return result; } catch (error) { + if (error instanceof ExecuteCommandError && error.exitCode !== 1) { + return error.data as CliCommandResponse; + } throw new Error(`next-gen-parse failed. \n ${error}`); } } diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 947a5692d..73a50a1c3 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -14,13 +14,7 @@ import { IJobValidator } from './jobValidator'; import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository'; import { DocsetsRepository } from '../repositories/docsetsRepository'; import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants'; -import { - nextGenHtml, - nextGenParse, - oasPageBuild, - persistenceModule, - prepareBuildAndGetDependencies, -} from '../commands'; +import { nextGenHtml, nextGenParse, oasPageBuild, persistenceModule, prepareBuild } from '../commands'; import { downloadBuildDependencies } from '../commands/src/helpers/dependency-helpers'; import { CliCommandResponse } from '../commands/src/helpers'; require('fs'); @@ -213,17 +207,14 @@ export abstract class JobHandler { } @throwIfJobInterupted() - private async getBuildDependencies() { - const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(this.currJob.payload.repoName); - if (!buildDependencies) return []; - return buildDependencies; - } - - @throwIfJobInterupted() - private async getAndBuildDependencies() { - const buildDependencies = await this.getBuildDependencies(); - const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName); - this._logger.save(this._currJob._id, commands.join('\n')); + private async getAndDownloadBuildDependencies() { + const repoName = this.currJob.payload.repoName; + const directory = this.currJob.payload.repoName === MONOREPO_NAME ? this.currJob.payload.directory : undefined; + const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(repoName, directory); + if (!buildDependencies) return; + await this._logger.save(this._currJob._id, 'Identified Build dependencies'); + const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName, directory); + await this._logger.save(this._currJob._id, `${commands.join('\n')}`); } @throwIfJobInterupted() @@ -550,7 +541,7 @@ export abstract class JobHandler { this._logger.save(this._currJob._id, 'Checked Commit'); await this.pullRepo(); this._logger.save(this._currJob._id, 'Pulled Repo'); - await this.getAndBuildDependencies(); + await this.getAndDownloadBuildDependencies(); this._logger.save(this._currJob._id, 'Downloaded Build dependencies'); this.prepBuildCommands(); this._logger.save(this._currJob._id, 'Prepared Build commands'); @@ -581,8 +572,8 @@ export abstract class JobHandler { await this.setEnvironmentVariables(); this.logger.save(job._id, 'Prepared Environment variables'); - const buildDependencies = await this.getBuildDependencies(); - this._logger.save(this._currJob._id, 'Identified Build dependencies'); + await this.getAndDownloadBuildDependencies(); + this._logger.save(this._currJob._id, 'Downloaded Build dependencies'); const docset = await this._docsetsRepo.getRepo(this._currJob.payload.repoName, this._currJob.payload.directory); let env = this._config.get('env'); @@ -591,16 +582,9 @@ export abstract class JobHandler { } const baseUrl = docset?.url?.[env] || 'https://mongodbcom-cdn.website.staging.corp.mongodb.com'; - const { patchId } = await prepareBuildAndGetDependencies( - job.payload.repoName, - job.payload.project, - baseUrl, - buildDependencies, - job.payload.directory - ); + const { patchId } = await prepareBuild(job.payload.repoName, job.payload.project, baseUrl, job.payload.directory); // Set patchId on payload for use in nextGenStage this._currJob.payload.patchId = patchId; - this._logger.save(this._currJob._id, 'Downloaded Build dependencies'); let buildStepOutput: CliCommandResponse; diff --git a/src/repositories/repoBranchesRepository.ts b/src/repositories/repoBranchesRepository.ts index cf8cb1cd4..f2764c3ef 100644 --- a/src/repositories/repoBranchesRepository.ts +++ b/src/repositories/repoBranchesRepository.ts @@ -9,8 +9,9 @@ export class RepoBranchesRepository extends BaseRepository { super(config, logger, 'RepoBranchesRepository', db.collection(config.get('repoBranchesCollection'))); } - async getBuildDependencies(repoName: string): Promise { + async getBuildDependencies(repoName: string, directoryName?: string): Promise { const query = { repoName: repoName }; + if (directoryName) query['directories.snooty_toml'] = `/${directoryName}`; const repo = await this.findOne( query, `Mongo Timeout Error: Timedout while retrieving build dependencies for ${repoName}`