From 987824284e702dc921bfbcf0d7711d694a30a82b Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 21 Sep 2023 13:17:59 -0500 Subject: [PATCH 01/51] [DOP-4033]: Add initial commands --- src/enhanced/utils/commands/index.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/enhanced/utils/commands/index.ts diff --git a/src/enhanced/utils/commands/index.ts b/src/enhanced/utils/commands/index.ts new file mode 100644 index 000000000..9e0fe8f2e --- /dev/null +++ b/src/enhanced/utils/commands/index.ts @@ -0,0 +1,11 @@ +import simpleGit from 'simple-git'; +import fs from 'fs'; +import path from 'path'; + +function getPatchId(): string | undefined { + if (!fs.existsSync(path.join(__dirname, 'myPatch.patch'))) return; +} + +export function nextGenParse() { + getPatchId(); +} From 902a357725a38dfac4397cbe42c8b26721543aaf Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 21 Sep 2023 14:40:53 -0500 Subject: [PATCH 02/51] [DOP-4033]: Create helpers for cli commands --- src/enhanced/utils/commands/helpers.ts | 21 +++++++++++++++++++++ src/enhanced/utils/commands/index.ts | 7 +++++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 src/enhanced/utils/commands/helpers.ts diff --git a/src/enhanced/utils/commands/helpers.ts b/src/enhanced/utils/commands/helpers.ts new file mode 100644 index 000000000..6cc024d31 --- /dev/null +++ b/src/enhanced/utils/commands/helpers.ts @@ -0,0 +1,21 @@ +import { spawn } from 'child_process'; + +export class ExecuteCommandError extends Error { + data: unknown; + constructor(message: string, data: unknown) { + super(message); + this.data = data; + } +} +export async function executeCliCommand(command: string, args?: readonly string[]): Promise { + const executedCommand = spawn(command, args); + return new Promise((resolve, reject) => { + executedCommand.stdout.on('data', (data) => { + resolve(data as T); + }); + + executedCommand.stderr.on('data', (data) => { + reject(new ExecuteCommandError('The command failed', data)); + }); + }); +} diff --git a/src/enhanced/utils/commands/index.ts b/src/enhanced/utils/commands/index.ts index 9e0fe8f2e..924ce03fe 100644 --- a/src/enhanced/utils/commands/index.ts +++ b/src/enhanced/utils/commands/index.ts @@ -1,11 +1,14 @@ import simpleGit from 'simple-git'; import fs from 'fs'; import path from 'path'; +import { executeCliCommand } from './helpers'; -function getPatchId(): string | undefined { +async function getPatchId(): Promise { if (!fs.existsSync(path.join(__dirname, 'myPatch.patch'))) return; + + const gitPatchId = await executeCliCommand('git', ['patch-id']); } -export function nextGenParse() { +export async function nextGenParse() { getPatchId(); } From 534084b0e2c0f1fce69750b6cb39a1db02a5f328 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 21 Sep 2023 15:03:54 -0500 Subject: [PATCH 03/51] [DOP-4033]: Add default args --- src/enhanced/utils/commands/index.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/enhanced/utils/commands/index.ts b/src/enhanced/utils/commands/index.ts index 924ce03fe..3d3390560 100644 --- a/src/enhanced/utils/commands/index.ts +++ b/src/enhanced/utils/commands/index.ts @@ -3,12 +3,21 @@ import fs from 'fs'; import path from 'path'; import { executeCliCommand } from './helpers'; -async function getPatchId(): Promise { - if (!fs.existsSync(path.join(__dirname, 'myPatch.patch'))) return; +const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; - const gitPatchId = await executeCliCommand('git', ['patch-id']); -} +async function getPatchId(): Promise { + const gitPatchId = await executeCliCommand('git', ['patch-id']); -export async function nextGenParse() { - getPatchId(); + return gitPatchId; +} +async function getPatchClause() { + throw new Error('not implemented'); +} +export async function nextGenParse(): Promise { + const defaultParseArgs = []; + const isPatch = !fs.existsSync(path.join(__dirname, 'myPatch.patch')); + const commandArgs = isPatch ? [...defaultParseArgs, '--commit'] : defaultParseArgs; + if (!fs.existsSync(path.join(__dirname, 'myPatch.patch'))) { + await getPatchId(); + } } From 8995249d6ad02ee41700b5d8a31aae7760674941 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 21 Sep 2023 17:59:07 -0500 Subject: [PATCH 04/51] [DOP-4033]: Move directories --- src/enhanced/commands/index.ts | 3 +++ src/enhanced/{utils/commands => commands/src}/helpers.ts | 0 .../commands/index.ts => commands/src/next-gen-parse.ts} | 9 +++++---- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 src/enhanced/commands/index.ts rename src/enhanced/{utils/commands => commands/src}/helpers.ts (100%) rename src/enhanced/{utils/commands/index.ts => commands/src/next-gen-parse.ts} (84%) diff --git a/src/enhanced/commands/index.ts b/src/enhanced/commands/index.ts new file mode 100644 index 000000000..8e62631d2 --- /dev/null +++ b/src/enhanced/commands/index.ts @@ -0,0 +1,3 @@ +import { nextGenParse } from './src/next-gen-parse'; + +export { nextGenParse }; diff --git a/src/enhanced/utils/commands/helpers.ts b/src/enhanced/commands/src/helpers.ts similarity index 100% rename from src/enhanced/utils/commands/helpers.ts rename to src/enhanced/commands/src/helpers.ts diff --git a/src/enhanced/utils/commands/index.ts b/src/enhanced/commands/src/next-gen-parse.ts similarity index 84% rename from src/enhanced/utils/commands/index.ts rename to src/enhanced/commands/src/next-gen-parse.ts index 3d3390560..72368d54c 100644 --- a/src/enhanced/utils/commands/index.ts +++ b/src/enhanced/commands/src/next-gen-parse.ts @@ -1,4 +1,3 @@ -import simpleGit from 'simple-git'; import fs from 'fs'; import path from 'path'; import { executeCliCommand } from './helpers'; @@ -13,11 +12,13 @@ async function getPatchId(): Promise { async function getPatchClause() { throw new Error('not implemented'); } + +async function snootyBuild() { + throw new Error('not implemented'); +} + export async function nextGenParse(): Promise { const defaultParseArgs = []; const isPatch = !fs.existsSync(path.join(__dirname, 'myPatch.patch')); const commandArgs = isPatch ? [...defaultParseArgs, '--commit'] : defaultParseArgs; - if (!fs.existsSync(path.join(__dirname, 'myPatch.patch'))) { - await getPatchId(); - } } From 1ccbad7512f9b7d9c7b1bedb3571691fe2ebae3b Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 22 Sep 2023 10:30:42 -0500 Subject: [PATCH 05/51] [DOP-4033]: Update spawn to work async and add code for getPatchId command --- src/enhanced/commands/src/helpers.ts | 43 +++++++++++++++++---- src/enhanced/commands/src/next-gen-parse.ts | 13 ++++++- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/enhanced/commands/src/helpers.ts b/src/enhanced/commands/src/helpers.ts index 6cc024d31..43f90c25a 100644 --- a/src/enhanced/commands/src/helpers.ts +++ b/src/enhanced/commands/src/helpers.ts @@ -1,4 +1,11 @@ -import { spawn } from 'child_process'; +import { + SpawnOptions, + SpawnOptionsWithStdioTuple, + SpawnOptionsWithoutStdio, + StdioNull, + StdioPipe, + spawn, +} from 'child_process'; export class ExecuteCommandError extends Error { data: unknown; @@ -7,15 +14,37 @@ export class ExecuteCommandError extends Error { this.data = data; } } -export async function executeCliCommand(command: string, args?: readonly string[]): Promise { - const executedCommand = spawn(command, args); + +interface CliCommandResponse { + stdout: string; + stderr: string; +} + +export async function executeCliCommand( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {} +): Promise { + const executedCommand = spawn(command, args, options); return new Promise((resolve, reject) => { - executedCommand.stdout.on('data', (data) => { - resolve(data as T); + const stdout: string[] = []; + const stderr: string[] = []; + + executedCommand.stdout?.on('data', (data: Buffer) => { + stdout.push(data.toString()); + }); + + executedCommand.stderr?.on('data', (data: Buffer) => { + stderr.push(data.toString()); + }); + + executedCommand.on('error', (err) => { + reject(new ExecuteCommandError('The command failed', err)); }); - executedCommand.stderr.on('data', (data) => { - reject(new ExecuteCommandError('The command failed', data)); + resolve({ + stdout: stdout.join(), + stderr: stderr.join(), }); }); } diff --git a/src/enhanced/commands/src/next-gen-parse.ts b/src/enhanced/commands/src/next-gen-parse.ts index 72368d54c..78f57b463 100644 --- a/src/enhanced/commands/src/next-gen-parse.ts +++ b/src/enhanced/commands/src/next-gen-parse.ts @@ -1,11 +1,22 @@ import fs from 'fs'; import path from 'path'; import { executeCliCommand } from './helpers'; +import { promisify } from 'util'; +const openAsync = promisify(fs.open); const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; async function getPatchId(): Promise { - const gitPatchId = await executeCliCommand('git', ['patch-id']); + const fileId = await openAsync(path.join(__dirname, 'myPatch.patch'), 'r'); + const { stdout: gitPatchId } = await executeCliCommand('git', ['patch-id'], { + stdio: [fileId, process.stdout, process.stderr], + }); + + fs.close(fileId, (err) => { + if (err) { + console.error('error when closing myPatch.patch', err); + } + }); return gitPatchId; } From 5281b1390b534b38934aa02e27ff961530d7f91f Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 22 Sep 2023 12:03:57 -0500 Subject: [PATCH 06/51] [DOP-4033]: Refactor executeCliCommand --- src/enhanced/commands/src/helpers.ts | 26 +++++++++++---------- src/enhanced/commands/src/next-gen-parse.ts | 7 ++++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/enhanced/commands/src/helpers.ts b/src/enhanced/commands/src/helpers.ts index 43f90c25a..c663a58c7 100644 --- a/src/enhanced/commands/src/helpers.ts +++ b/src/enhanced/commands/src/helpers.ts @@ -1,11 +1,4 @@ -import { - SpawnOptions, - SpawnOptionsWithStdioTuple, - SpawnOptionsWithoutStdio, - StdioNull, - StdioPipe, - spawn, -} from 'child_process'; +import { SpawnOptions, spawn } from 'child_process'; export class ExecuteCommandError extends Error { data: unknown; @@ -25,11 +18,12 @@ export async function executeCliCommand( args: readonly string[] = [], options: SpawnOptions = {} ): Promise { - const executedCommand = spawn(command, args, options); return new Promise((resolve, reject) => { const stdout: string[] = []; const stderr: string[] = []; + const executedCommand = spawn(command, args, options); + executedCommand.stdout?.on('data', (data: Buffer) => { stdout.push(data.toString()); }); @@ -42,9 +36,17 @@ export async function executeCliCommand( reject(new ExecuteCommandError('The command failed', err)); }); - resolve({ - stdout: stdout.join(), - stderr: stderr.join(), + executedCommand.on('close', (exitCode) => { + if (exitCode !== 0) { + console.error(`ERROR! The command ${command} closed with an exit code other than 0: ${exitCode}.`); + console.error('Arguments provided: ', args); + console.error('Options provided: ', options); + } + + resolve({ + stdout: stdout.join(), + stderr: stderr.join(), + }); }); }); } diff --git a/src/enhanced/commands/src/next-gen-parse.ts b/src/enhanced/commands/src/next-gen-parse.ts index 78f57b463..041ebcc38 100644 --- a/src/enhanced/commands/src/next-gen-parse.ts +++ b/src/enhanced/commands/src/next-gen-parse.ts @@ -4,6 +4,7 @@ import { executeCliCommand } from './helpers'; import { promisify } from 'util'; const openAsync = promisify(fs.open); +const existsAsync = promisify(fs.exists); const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; async function getPatchId(): Promise { @@ -28,8 +29,10 @@ async function snootyBuild() { throw new Error('not implemented'); } -export async function nextGenParse(): Promise { +export async function nextGenParse(repoName: string): Promise { + const repoDir = path.join(__dirname, `repos/${repoName}`); const defaultParseArgs = []; - const isPatch = !fs.existsSync(path.join(__dirname, 'myPatch.patch')); + const isPatch = !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); + const commandArgs = isPatch ? [...defaultParseArgs, '--commit'] : defaultParseArgs; } From dce8dd7888aa8e9a9ccefba2a48015fa8ea1a01b Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 22 Sep 2023 12:52:01 -0500 Subject: [PATCH 07/51] [DOP-4033]: Wrap up nextgenparse refactor --- src/enhanced/commands/src/helpers.ts | 14 ++++++++ src/enhanced/commands/src/next-gen-parse.ts | 38 +++++++++------------ 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/enhanced/commands/src/helpers.ts b/src/enhanced/commands/src/helpers.ts index c663a58c7..96023b91c 100644 --- a/src/enhanced/commands/src/helpers.ts +++ b/src/enhanced/commands/src/helpers.ts @@ -1,4 +1,9 @@ import { SpawnOptions, spawn } from 'child_process'; +import { promisify } from 'util'; +import fs from 'fs'; + +const openAsync = promisify(fs.open); +const closeAsync = promisify(fs.close); export class ExecuteCommandError extends Error { data: unknown; @@ -50,3 +55,12 @@ export async function executeCliCommand( }); }); } + +export async function readFileAndExec(command: string, filePath: string, args?: string[]): Promise { + const fileId = await openAsync(filePath, 'r'); + const response = await executeCliCommand(command, args, { stdio: [fileId, process.stdout, process.stderr] }); + + await closeAsync(fileId); + + return response; +} diff --git a/src/enhanced/commands/src/next-gen-parse.ts b/src/enhanced/commands/src/next-gen-parse.ts index 041ebcc38..966563156 100644 --- a/src/enhanced/commands/src/next-gen-parse.ts +++ b/src/enhanced/commands/src/next-gen-parse.ts @@ -1,38 +1,32 @@ import fs from 'fs'; import path from 'path'; -import { executeCliCommand } from './helpers'; +import { executeCliCommand, readFileAndExec } from './helpers'; import { promisify } from 'util'; -const openAsync = promisify(fs.open); const existsAsync = promisify(fs.exists); const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; -async function getPatchId(): Promise { - const fileId = await openAsync(path.join(__dirname, 'myPatch.patch'), 'r'); - const { stdout: gitPatchId } = await executeCliCommand('git', ['patch-id'], { - stdio: [fileId, process.stdout, process.stderr], - }); +async function getPatchId(repoDir: string): Promise { + const filePath = path.join(repoDir, 'myPatch.patch'); - fs.close(fileId, (err) => { - if (err) { - console.error('error when closing myPatch.patch', err); - } - }); + const { stdout: gitPatchId } = await readFileAndExec('git', filePath, ['patch-id']); return gitPatchId; } -async function getPatchClause() { - throw new Error('not implemented'); -} - -async function snootyBuild() { - throw new Error('not implemented'); -} export async function nextGenParse(repoName: string): Promise { const repoDir = path.join(__dirname, `repos/${repoName}`); - const defaultParseArgs = []; - const isPatch = !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); - const commandArgs = isPatch ? [...defaultParseArgs, '--commit'] : defaultParseArgs; + const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; + + const hasPatch = !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); + + if (hasPatch) { + const patchId = await getPatchId(repoDir); + + commandArgs.push('--patch'); + commandArgs.push(patchId); + } + + await executeCliCommand('snooty', commandArgs); } From 7324870a3fae6c1a96a58b7a05d6a54b14201002 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 22 Sep 2023 14:42:53 -0500 Subject: [PATCH 08/51] [DOP-4033]: Move commands directory out of enhanced --- src/{enhanced => }/commands/index.ts | 0 src/{enhanced => }/commands/src/helpers.ts | 0 src/{enhanced => }/commands/src/next-gen-parse.ts | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename src/{enhanced => }/commands/index.ts (100%) rename src/{enhanced => }/commands/src/helpers.ts (100%) rename src/{enhanced => }/commands/src/next-gen-parse.ts (96%) diff --git a/src/enhanced/commands/index.ts b/src/commands/index.ts similarity index 100% rename from src/enhanced/commands/index.ts rename to src/commands/index.ts diff --git a/src/enhanced/commands/src/helpers.ts b/src/commands/src/helpers.ts similarity index 100% rename from src/enhanced/commands/src/helpers.ts rename to src/commands/src/helpers.ts diff --git a/src/enhanced/commands/src/next-gen-parse.ts b/src/commands/src/next-gen-parse.ts similarity index 96% rename from src/enhanced/commands/src/next-gen-parse.ts rename to src/commands/src/next-gen-parse.ts index 966563156..32dfe3079 100644 --- a/src/enhanced/commands/src/next-gen-parse.ts +++ b/src/commands/src/next-gen-parse.ts @@ -11,7 +11,7 @@ async function getPatchId(repoDir: string): Promise { const { stdout: gitPatchId } = await readFileAndExec('git', filePath, ['patch-id']); - return gitPatchId; + return gitPatchId.slice(0, 7); } export async function nextGenParse(repoName: string): Promise { From ab09876cff3f3f36dcf4e6f29ec157007dff3c71 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 22 Sep 2023 15:01:16 -0500 Subject: [PATCH 09/51] [DOP-4033]: Add next gen html command --- src/commands/src/helpers.ts | 3 +++ src/commands/src/next-gen-html.ts | 13 +++++++++++++ src/commands/src/next-gen-parse.ts | 4 ++-- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 src/commands/src/next-gen-html.ts diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers.ts index 96023b91c..a630fc3b7 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers.ts @@ -1,6 +1,7 @@ import { SpawnOptions, spawn } from 'child_process'; import { promisify } from 'util'; import fs from 'fs'; +import path from 'path'; const openAsync = promisify(fs.open); const closeAsync = promisify(fs.close); @@ -64,3 +65,5 @@ export async function readFileAndExec(command: string, filePath: string, args?: return response; } + +export const getRepoDir = (repoName: string) => path.join(__dirname, `repos/${repoName}`); diff --git a/src/commands/src/next-gen-html.ts b/src/commands/src/next-gen-html.ts new file mode 100644 index 000000000..36d43d29c --- /dev/null +++ b/src/commands/src/next-gen-html.ts @@ -0,0 +1,13 @@ +import fs from 'fs'; +import { promisify } from 'util'; +import { getRepoDir } from './helpers'; + +const writeFileAsync = promisify(fs.writeFile); + +async function createEnvProdFile(repoDir: string) { + await writeFileAsync(repoDir, '', 'utf8'); +} +export async function nextGenHtml(repoName: string) { + const repoDir = getRepoDir(repoName); + await createEnvProdFile(repoName); +} diff --git a/src/commands/src/next-gen-parse.ts b/src/commands/src/next-gen-parse.ts index 32dfe3079..2a441442f 100644 --- a/src/commands/src/next-gen-parse.ts +++ b/src/commands/src/next-gen-parse.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import { executeCliCommand, readFileAndExec } from './helpers'; +import { executeCliCommand, getRepoDir, readFileAndExec } from './helpers'; import { promisify } from 'util'; const existsAsync = promisify(fs.exists); @@ -15,7 +15,7 @@ async function getPatchId(repoDir: string): Promise { } export async function nextGenParse(repoName: string): Promise { - const repoDir = path.join(__dirname, `repos/${repoName}`); + const repoDir = getRepoDir(repoName); const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; From 18ff8d8ad6475413c166fde4fefc7d50fbf1adf6 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 22 Sep 2023 15:01:30 -0500 Subject: [PATCH 10/51] [DOP-4033]: Use correct arg for createEnvProdFile --- src/commands/src/next-gen-html.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/next-gen-html.ts b/src/commands/src/next-gen-html.ts index 36d43d29c..45747a56e 100644 --- a/src/commands/src/next-gen-html.ts +++ b/src/commands/src/next-gen-html.ts @@ -9,5 +9,5 @@ async function createEnvProdFile(repoDir: string) { } export async function nextGenHtml(repoName: string) { const repoDir = getRepoDir(repoName); - await createEnvProdFile(repoName); + await createEnvProdFile(repoDir); } From 8e03ca29718ce7463a84bcd1b8fedcf13c45ff9e Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 25 Sep 2023 11:17:28 -0500 Subject: [PATCH 11/51] [DOP-4033]: Add git hash --- src/commands/src/helpers.ts | 7 +++++++ src/commands/src/next-gen-html.ts | 2 ++ src/commands/src/next-gen-parse.ts | 7 +++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers.ts index a630fc3b7..f78231dc9 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers.ts @@ -66,4 +66,11 @@ export async function readFileAndExec(command: string, filePath: string, args?: return response; } +export async function getCommitHash(): Promise { + // git rev-parse --short HEAD + const response = await executeCliCommand('git', ['rev-parse', '--short', 'HEAD']); + + return response.stdout; +} + export const getRepoDir = (repoName: string) => path.join(__dirname, `repos/${repoName}`); diff --git a/src/commands/src/next-gen-html.ts b/src/commands/src/next-gen-html.ts index 45747a56e..524f09e9c 100644 --- a/src/commands/src/next-gen-html.ts +++ b/src/commands/src/next-gen-html.ts @@ -9,5 +9,7 @@ async function createEnvProdFile(repoDir: string) { } export async function nextGenHtml(repoName: string) { const repoDir = getRepoDir(repoName); + + // might move this since technically next-gen-html doesn't create the file await createEnvProdFile(repoDir); } diff --git a/src/commands/src/next-gen-parse.ts b/src/commands/src/next-gen-parse.ts index 2a441442f..d85d6e0f2 100644 --- a/src/commands/src/next-gen-parse.ts +++ b/src/commands/src/next-gen-parse.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import { executeCliCommand, getRepoDir, readFileAndExec } from './helpers'; +import { executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from './helpers'; import { promisify } from 'util'; const existsAsync = promisify(fs.exists); @@ -22,7 +22,10 @@ export async function nextGenParse(repoName: string): Promise { const hasPatch = !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); if (hasPatch) { - const patchId = await getPatchId(repoDir); + const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash()]); + + commandArgs.push('--commit'); + commandArgs.push(commitHash); commandArgs.push('--patch'); commandArgs.push(patchId); From 3566e9abc351cc799653ccf172812cbd68df9255 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 26 Sep 2023 05:15:32 -0500 Subject: [PATCH 12/51] [DOP-4033]: Move commands into shared directory --- src/commands/index.ts | 2 +- src/commands/src/{ => shared}/next-gen-html.ts | 2 +- src/commands/src/{ => shared}/next-gen-parse.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/commands/src/{ => shared}/next-gen-html.ts (90%) rename src/commands/src/{ => shared}/next-gen-parse.ts (97%) diff --git a/src/commands/index.ts b/src/commands/index.ts index 8e62631d2..d0a02b903 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -1,3 +1,3 @@ -import { nextGenParse } from './src/next-gen-parse'; +import { nextGenParse } from './src/shared/next-gen-parse'; export { nextGenParse }; diff --git a/src/commands/src/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts similarity index 90% rename from src/commands/src/next-gen-html.ts rename to src/commands/src/shared/next-gen-html.ts index 524f09e9c..8f3a69fd9 100644 --- a/src/commands/src/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import { promisify } from 'util'; -import { getRepoDir } from './helpers'; +import { getRepoDir } from '../helpers'; const writeFileAsync = promisify(fs.writeFile); diff --git a/src/commands/src/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts similarity index 97% rename from src/commands/src/next-gen-parse.ts rename to src/commands/src/shared/next-gen-parse.ts index d85d6e0f2..f1455461e 100644 --- a/src/commands/src/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import { executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from './helpers'; +import { executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from '../helpers'; import { promisify } from 'util'; const existsAsync = promisify(fs.exists); From 7d4dca652b9d1fc6500479f88342defb66c6b569 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 26 Sep 2023 05:24:23 -0500 Subject: [PATCH 13/51] [DOP-4033]: Refactor --- src/commands/src/helpers.ts | 2 ++ src/commands/src/shared/next-gen-parse.ts | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers.ts index f78231dc9..6b9ce3733 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers.ts @@ -5,6 +5,7 @@ import path from 'path'; const openAsync = promisify(fs.open); const closeAsync = promisify(fs.close); +const existsAsync = promisify(fs.exists); export class ExecuteCommandError extends Error { data: unknown; @@ -73,4 +74,5 @@ export async function getCommitHash(): Promise { return response.stdout; } +export const checkIfPatched = async (repoDir: string) => !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); export const getRepoDir = (repoName: string) => path.join(__dirname, `repos/${repoName}`); diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index f1455461e..a2028881a 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,9 +1,7 @@ -import fs from 'fs'; import path from 'path'; -import { executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from '../helpers'; -import { promisify } from 'util'; -const existsAsync = promisify(fs.exists); +import { checkIfPatched, executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from '../helpers'; + const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; async function getPatchId(repoDir: string): Promise { @@ -19,7 +17,7 @@ export async function nextGenParse(repoName: string): Promise { const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; - const hasPatch = !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); + const hasPatch = await checkIfPatched(repoDir); if (hasPatch) { const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash()]); From 43efb54cd621e363dfac2f7019809fe4b1f68690 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 26 Sep 2023 05:56:32 -0500 Subject: [PATCH 14/51] [DOP-4033]: Create base job for standard builds ie jobs with makefiles that don't override shared,mk --- src/commands/src/helpers.ts | 11 +++++++ src/commands/src/shared/base-job.ts | 36 +++++++++++++++++++++++ src/commands/src/shared/next-gen-parse.ts | 10 ------- 3 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 src/commands/src/shared/base-job.ts diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers.ts index 6b9ce3733..81aa28658 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers.ts @@ -67,6 +67,14 @@ export async function readFileAndExec(command: string, filePath: string, args?: return response; } +export async function getPatchId(repoDir: string): Promise { + const filePath = path.join(repoDir, 'myPatch.patch'); + + const { stdout: gitPatchId } = await readFileAndExec('git', filePath, ['patch-id']); + + return gitPatchId.slice(0, 7); +} + export async function getCommitHash(): Promise { // git rev-parse --short HEAD const response = await executeCliCommand('git', ['rev-parse', '--short', 'HEAD']); @@ -76,3 +84,6 @@ export async function getCommitHash(): Promise { export const checkIfPatched = async (repoDir: string) => !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); export const getRepoDir = (repoName: string) => path.join(__dirname, `repos/${repoName}`); + +export const RSTSPEC_FLAG = + '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; diff --git a/src/commands/src/shared/base-job.ts b/src/commands/src/shared/base-job.ts new file mode 100644 index 000000000..49c787cb5 --- /dev/null +++ b/src/commands/src/shared/base-job.ts @@ -0,0 +1,36 @@ +import { getRepoDir, checkIfPatched, getCommitHash, executeCliCommand, getPatchId, RSTSPEC_FLAG } from '../helpers'; + +export abstract class BaseJob { + repoName: string; + + constructor(repoName: string) { + this.repoName = repoName; + } + + async nextGenParse(repoName: string): Promise { + const repoDir = getRepoDir(repoName); + + const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; + + const hasPatch = await checkIfPatched(repoDir); + + if (hasPatch) { + const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash()]); + + commandArgs.push('--commit'); + commandArgs.push(commitHash); + + commandArgs.push('--patch'); + commandArgs.push(patchId); + } + + await executeCliCommand('snooty', commandArgs); + } + async nextGenHtml(repoName: string) { + getRepoDir(repoName); + } + + async nextGenDeploy() { + throw new Error('Not implemented'); + } +} diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index a2028881a..cb470db07 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,17 +1,7 @@ -import path from 'path'; - import { checkIfPatched, executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from '../helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; -async function getPatchId(repoDir: string): Promise { - const filePath = path.join(repoDir, 'myPatch.patch'); - - const { stdout: gitPatchId } = await readFileAndExec('git', filePath, ['patch-id']); - - return gitPatchId.slice(0, 7); -} - export async function nextGenParse(repoName: string): Promise { const repoDir = getRepoDir(repoName); From 6a39606132f01fb0a62ed84b3dfd2fa8d1e2aac6 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 26 Sep 2023 10:09:11 -0500 Subject: [PATCH 15/51] [DOP-4033]: Add the ability to append to file using executeCliCommand --- src/commands/src/helpers.ts | 34 +++++++++++---- src/commands/src/shared/base-job.ts | 36 ---------------- .../src/shared/standard-job-runner.ts | 43 +++++++++++++++++++ 3 files changed, 69 insertions(+), 44 deletions(-) delete mode 100644 src/commands/src/shared/base-job.ts create mode 100644 src/commands/src/shared/standard-job-runner.ts diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers.ts index 81aa28658..74dd2b06f 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers.ts @@ -15,22 +15,32 @@ export class ExecuteCommandError extends Error { } } +interface CliCommandParams { + command: string; + args?: readonly string[]; + options?: SpawnOptions; + writeStream?: fs.WriteStream; +} + interface CliCommandResponse { stdout: string; stderr: string; } -export async function executeCliCommand( - command: string, - args: readonly string[] = [], - options: SpawnOptions = {} -): Promise { +export async function executeCliCommand({ + command, + args = [], + options = {}, + writeStream, +}: CliCommandParams): Promise { return new Promise((resolve, reject) => { const stdout: string[] = []; const stderr: string[] = []; const executedCommand = spawn(command, args, options); + if (writeStream) executedCommand.stdout?.pipe(writeStream); + executedCommand.stdout?.on('data', (data: Buffer) => { stdout.push(data.toString()); }); @@ -58,9 +68,17 @@ export async function executeCliCommand( }); } +export async function executeAndWriteToFile() { + return null; +} + export async function readFileAndExec(command: string, filePath: string, args?: string[]): Promise { const fileId = await openAsync(filePath, 'r'); - const response = await executeCliCommand(command, args, { stdio: [fileId, process.stdout, process.stderr] }); + const response = await executeCliCommand({ + command, + args, + options: { stdio: [fileId, process.stdout, process.stderr] }, + }); await closeAsync(fileId); @@ -76,8 +94,8 @@ export async function getPatchId(repoDir: string): Promise { } export async function getCommitHash(): Promise { - // git rev-parse --short HEAD - const response = await executeCliCommand('git', ['rev-parse', '--short', 'HEAD']); + // equivalent to git rev-parse --short HEAD + const response = await executeCliCommand({ command: 'git', args: ['rev-parse', '--short', 'HEAD'] }); return response.stdout; } diff --git a/src/commands/src/shared/base-job.ts b/src/commands/src/shared/base-job.ts deleted file mode 100644 index 49c787cb5..000000000 --- a/src/commands/src/shared/base-job.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getRepoDir, checkIfPatched, getCommitHash, executeCliCommand, getPatchId, RSTSPEC_FLAG } from '../helpers'; - -export abstract class BaseJob { - repoName: string; - - constructor(repoName: string) { - this.repoName = repoName; - } - - async nextGenParse(repoName: string): Promise { - const repoDir = getRepoDir(repoName); - - const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; - - const hasPatch = await checkIfPatched(repoDir); - - if (hasPatch) { - const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash()]); - - commandArgs.push('--commit'); - commandArgs.push(commitHash); - - commandArgs.push('--patch'); - commandArgs.push(patchId); - } - - await executeCliCommand('snooty', commandArgs); - } - async nextGenHtml(repoName: string) { - getRepoDir(repoName); - } - - async nextGenDeploy() { - throw new Error('Not implemented'); - } -} diff --git a/src/commands/src/shared/standard-job-runner.ts b/src/commands/src/shared/standard-job-runner.ts new file mode 100644 index 000000000..e1c010c37 --- /dev/null +++ b/src/commands/src/shared/standard-job-runner.ts @@ -0,0 +1,43 @@ +import { getRepoDir, checkIfPatched, getCommitHash, executeCliCommand, getPatchId, RSTSPEC_FLAG } from '../helpers'; + +export class StandardJobRunner { + repoName: string; + repoDir: string; + project: string; + + constructor(repoName: string, project: string) { + this.repoName = repoName; + this.project = project; + + this.repoDir = getRepoDir(repoName); + } + + async nextGenParse(): Promise { + const commandArgs = ['build', this.repoDir, '--output', `${this.repoDir}/bundle.zip`, RSTSPEC_FLAG]; + + const hasPatch = await checkIfPatched(this.repoDir); + + if (hasPatch) { + const [patchId, commitHash] = await Promise.all([getPatchId(this.repoDir), getCommitHash()]); + + commandArgs.push('--commit'); + commandArgs.push(commitHash); + + commandArgs.push('--patch'); + commandArgs.push(patchId); + } + + await executeCliCommand({ command: 'snooty', args: commandArgs }); + } + + async nextGenHtml() { + // copy .env.production to the snooty directory + await executeCliCommand({ command: 'cp', args: [`${this.repoDir}/.env.production`, 'snooty'] }); + + await executeCliCommand({ command: 'echo', args: [`"GATSBY_SITE=${this.project}"`] }); + } + + async nextGenDeploy() { + throw new Error('Not implemented'); + } +} From e27bae39b60d7509061f93e02a68e2adcb9e20a8 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 26 Sep 2023 10:22:23 -0500 Subject: [PATCH 16/51] [DOP-4033]: Add logic to write to .env.production file --- src/commands/src/helpers.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers.ts index 74dd2b06f..3e9af1fa6 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers.ts @@ -93,6 +93,20 @@ export async function getPatchId(repoDir: string): Promise { return gitPatchId.slice(0, 7); } +export async function addProjectToEnv(project: string) { + return new Promise((resolve, reject) => { + const stream = fs.createWriteStream(path.join(__dirname, 'snooty/.env.production'), { flags: 'a+' }); + + stream.write(project); + stream.close(); + stream.on('error', (error) => { + console.log('Error when writing to file!', error); + reject(error); + }); + stream.on('close', resolve); + }); +} + export async function getCommitHash(): Promise { // equivalent to git rev-parse --short HEAD const response = await executeCliCommand({ command: 'git', args: ['rev-parse', '--short', 'HEAD'] }); From 6b9df1169eedea436b6fb29ea9bc63094c2b3dbe Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 27 Sep 2023 09:46:57 -0500 Subject: [PATCH 17/51] [DOP-4033]: Add more logic to write to files and start working creating test env --- Dockerfile.local | 56 +++++++++++++++++++ src/commands/src/{ => helpers}/helpers.ts | 32 +++++++++-- src/commands/src/shared/next-gen-html.ts | 2 +- src/commands/src/shared/next-gen-parse.ts | 4 +- .../src/shared/standard-job-runner.ts | 9 ++- src/entrypoints/localApp.ts | 3 + 6 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 Dockerfile.local rename src/commands/src/{ => helpers}/helpers.ts (79%) create mode 100644 src/entrypoints/localApp.ts diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 000000000..a0b347474 --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,56 @@ +FROM node:18.16.0-alpine as ts-compiler +ARG WORK_DIRECTORY=/home/docsworker-xlarge +ARG SNOOTY_PARSER_VERSION=0.14.9 +ARG SNOOTY_FRONTEND_VERSION=0.14.16 +ARG MUT_VERSION=0.10.7 +ARG REDOC_CLI_VERSION=1.2.2 +ARG NPM_BASE_64_AUTH +ARG NPM_EMAIL +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /app + +RUN apk add curl + +# install snooty parser +RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ + && unzip -d /opt/ snooty-parser.zip + +# install mut +RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ + && unzip -d /opt/ mut.zip + +# install snooty frontend and docs-tools +RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ + && cd snooty \ + && npm ci --legacy-peer-deps --omit=dev + +COPY package*.json ./ +RUN npm ci --legacy-peer-deps + +COPY tsconfig*.json ./ +COPY config config/ +COPY src src/ +COPY api api/ + +COPY modules/persistence ./modules/persistence/ +COPY modules/oas-page-builder ./modules/oas-page-builder/ + +RUN npm run build + +# install persistence module +RUN cd ./modules/persistence \ + && npm ci --legacy-peer-deps \ + && npm run build + +# Build modules +# OAS Page Builder +RUN cd ./modules/oas-page-builder \ + && npm ci --legacy-peer-deps \ + && npm run build + +ENV PATH="${PATH}:/opt/snooty:/opt/mut" + +EXPOSE 3000 + +CMD ["node", "entrypoints/localApp.js"] \ No newline at end of file diff --git a/src/commands/src/helpers.ts b/src/commands/src/helpers/helpers.ts similarity index 79% rename from src/commands/src/helpers.ts rename to src/commands/src/helpers/helpers.ts index 3e9af1fa6..044b247e6 100644 --- a/src/commands/src/helpers.ts +++ b/src/commands/src/helpers/helpers.ts @@ -58,6 +58,9 @@ export async function executeCliCommand({ console.error(`ERROR! The command ${command} closed with an exit code other than 0: ${exitCode}.`); console.error('Arguments provided: ', args); console.error('Options provided: ', options); + + reject(new ExecuteCommandError('The command failed', exitCode)); + return; } resolve({ @@ -68,11 +71,32 @@ export async function executeCliCommand({ }); } -export async function executeAndWriteToFile() { - return null; +export interface ExecuteIOCommandParams { + command: string; + filePath: string; + args?: string[]; +} + +/** + * This function is equivalent to a double redirect + * e.g. echo "Hello!" >> hello.txt + * @param param0 + */ +export async function executeAndWriteToFile({ command, filePath, args }: ExecuteIOCommandParams) { + const writeStream = fs.createWriteStream(filePath, { + flags: 'a+', + }); + + const result = await executeCliCommand({ command, args, writeStream }); + + return result; } -export async function readFileAndExec(command: string, filePath: string, args?: string[]): Promise { +export async function readFileAndExec({ + command, + filePath, + args, +}: ExecuteIOCommandParams): Promise { const fileId = await openAsync(filePath, 'r'); const response = await executeCliCommand({ command, @@ -88,7 +112,7 @@ export async function readFileAndExec(command: string, filePath: string, args?: export async function getPatchId(repoDir: string): Promise { const filePath = path.join(repoDir, 'myPatch.patch'); - const { stdout: gitPatchId } = await readFileAndExec('git', filePath, ['patch-id']); + const { stdout: gitPatchId } = await readFileAndExec({ command: 'git', filePath, args: ['patch-id'] }); return gitPatchId.slice(0, 7); } diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index 8f3a69fd9..bbb25594d 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import { promisify } from 'util'; -import { getRepoDir } from '../helpers'; +import { getRepoDir } from '../helpers/helpers'; const writeFileAsync = promisify(fs.writeFile); diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index cb470db07..5ba2d72c8 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,4 +1,4 @@ -import { checkIfPatched, executeCliCommand, getCommitHash, getRepoDir, readFileAndExec } from '../helpers'; +import { checkIfPatched, executeCliCommand, getCommitHash, getPatchId, getRepoDir } from '../helpers/helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; @@ -19,5 +19,5 @@ export async function nextGenParse(repoName: string): Promise { commandArgs.push(patchId); } - await executeCliCommand('snooty', commandArgs); + await executeCliCommand({ command: 'snooty', args: commandArgs }); } diff --git a/src/commands/src/shared/standard-job-runner.ts b/src/commands/src/shared/standard-job-runner.ts index e1c010c37..7a8c229b5 100644 --- a/src/commands/src/shared/standard-job-runner.ts +++ b/src/commands/src/shared/standard-job-runner.ts @@ -1,4 +1,11 @@ -import { getRepoDir, checkIfPatched, getCommitHash, executeCliCommand, getPatchId, RSTSPEC_FLAG } from '../helpers'; +import { + getRepoDir, + checkIfPatched, + getCommitHash, + executeCliCommand, + getPatchId, + RSTSPEC_FLAG, +} from '../helpers/helpers'; export class StandardJobRunner { repoName: string; diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts new file mode 100644 index 000000000..b809d77a5 --- /dev/null +++ b/src/entrypoints/localApp.ts @@ -0,0 +1,3 @@ +async function localApp() {} + +localApp(); From 3ae00e38d6fc1e02139008e936cdad9c549586a6 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 27 Sep 2023 13:45:26 -0500 Subject: [PATCH 18/51] [DOP-4033]: Add local testing app and make test build faster --- Dockerfile.base | 31 ++++++++++ Dockerfile.local | 57 +++++++++---------- package.json | 1 + .../src/helpers/{helpers.ts => index.ts} | 8 ++- src/commands/src/shared/next-gen-html.ts | 2 +- src/commands/src/shared/next-gen-parse.ts | 4 +- .../src/shared/standard-job-runner.ts | 11 +--- src/entrypoints/localApp.ts | 17 +++++- 8 files changed, 87 insertions(+), 44 deletions(-) create mode 100644 Dockerfile.base rename src/commands/src/helpers/{helpers.ts => index.ts} (94%) diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 000000000..ec0d01c47 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,31 @@ +# Builds the base dependencies needed +FROM node:18.16.0-alpine +ARG SNOOTY_PARSER_VERSION=0.14.9 +ARG SNOOTY_FRONTEND_VERSION=0.14.16 +ARG MUT_VERSION=0.10.7 +ARG REDOC_CLI_VERSION=1.2.2 +ARG NPM_BASE_64_AUTH +ARG NPM_EMAIL +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /app + +RUN apk update && apk add --no-cache python3 make g++ curl git +RUN apk add --update --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/community --repository http://dl-3.alpinelinux.org/alpine/edge/main vips-dev + +# install snooty parser +RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ + && unzip -d /opt/ snooty-parser.zip + +# install mut +RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ + && unzip -d /opt/ mut.zip + +RUN curl -L -o redoc.zip https://github.com/mongodb-forks/redoc/archive/refs/tags/v${REDOC_CLI_VERSION}.zip \ + && unzip redoc.zip \ + && mv redoc-${REDOC_CLI_VERSION} redoc/ + +# install snooty frontend and docs-tools +RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ + && cd snooty \ + && npm ci --legacy-peer-deps --omit=dev diff --git a/Dockerfile.local b/Dockerfile.local index a0b347474..3357e7900 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,56 +1,55 @@ -FROM node:18.16.0-alpine as ts-compiler -ARG WORK_DIRECTORY=/home/docsworker-xlarge +FROM autobuilder/base:latest ARG SNOOTY_PARSER_VERSION=0.14.9 ARG SNOOTY_FRONTEND_VERSION=0.14.16 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL -ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app -RUN apk add curl - -# install snooty parser -RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ snooty-parser.zip - -# install mut -RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ mut.zip - -# install snooty frontend and docs-tools -RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ - && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev - COPY package*.json ./ RUN npm ci --legacy-peer-deps -COPY tsconfig*.json ./ -COPY config config/ -COPY src src/ -COPY api api/ - -COPY modules/persistence ./modules/persistence/ -COPY modules/oas-page-builder ./modules/oas-page-builder/ +COPY modules/persistence/package*.json ./modules/persistence/ +RUN cd ./modules/persistence \ + && npm ci --legacy-peer-deps -RUN npm run build +COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/ +RUN cd ./modules/oas-page-builder \ + && npm ci --legacy-peer-deps # install persistence module +COPY modules/persistence/tsconfig*.json ./modules/persistence +COPY modules/persistence/src ./modules/persistence/src/ +COPY modules/persistence/index.ts ./modules/persistence + RUN cd ./modules/persistence \ - && npm ci --legacy-peer-deps \ && npm run build # Build modules # OAS Page Builder +COPY modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder +COPY modules/oas-page-builder/src ./modules/oas-page-builder/src/ +COPY modules/oas-page-builder/index.ts ./modules/oas-page-builder + RUN cd ./modules/oas-page-builder \ - && npm ci --legacy-peer-deps \ && npm run build +# Root project build +COPY tsconfig*.json ./ +COPY config config/ +COPY api api/ +COPY src src/ + +RUN npm run build + ENV PATH="${PATH}:/opt/snooty:/opt/mut" +RUN mkdir repos/ + +RUN git --version + EXPOSE 3000 -CMD ["node", "entrypoints/localApp.js"] \ No newline at end of file +CMD ["node", "build/entrypoints/localApp.js"] \ No newline at end of file diff --git a/package.json b/package.json index 524693ade..276d1ebcc 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "node index.js", "clean": "node maintain.js", "build": "tsc", + "build:esbuild": "esbuild", "format": "npm run prettier -- --check", "format:fix": "npm run prettier -- --write", "lint": "eslint --ext .ts .", diff --git a/src/commands/src/helpers/helpers.ts b/src/commands/src/helpers/index.ts similarity index 94% rename from src/commands/src/helpers/helpers.ts rename to src/commands/src/helpers/index.ts index 044b247e6..1d9869d09 100644 --- a/src/commands/src/helpers/helpers.ts +++ b/src/commands/src/helpers/index.ts @@ -131,9 +131,13 @@ export async function addProjectToEnv(project: string) { }); } -export async function getCommitHash(): Promise { +export async function getCommitHash(repoDir: string): Promise { // equivalent to git rev-parse --short HEAD - const response = await executeCliCommand({ command: 'git', args: ['rev-parse', '--short', 'HEAD'] }); + const response = await executeCliCommand({ + command: 'git', + args: ['rev-parse', '--short', 'HEAD'], + options: { cwd: repoDir }, + }); return response.stdout; } diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index bbb25594d..8f3a69fd9 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import { promisify } from 'util'; -import { getRepoDir } from '../helpers/helpers'; +import { getRepoDir } from '../helpers'; const writeFileAsync = promisify(fs.writeFile); diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 5ba2d72c8..4dbb7fe0f 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,4 +1,4 @@ -import { checkIfPatched, executeCliCommand, getCommitHash, getPatchId, getRepoDir } from '../helpers/helpers'; +import { checkIfPatched, executeCliCommand, getCommitHash, getPatchId, getRepoDir } from '../helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; @@ -10,7 +10,7 @@ export async function nextGenParse(repoName: string): Promise { const hasPatch = await checkIfPatched(repoDir); if (hasPatch) { - const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash()]); + const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash(repoDir)]); commandArgs.push('--commit'); commandArgs.push(commitHash); diff --git a/src/commands/src/shared/standard-job-runner.ts b/src/commands/src/shared/standard-job-runner.ts index 7a8c229b5..cb0162172 100644 --- a/src/commands/src/shared/standard-job-runner.ts +++ b/src/commands/src/shared/standard-job-runner.ts @@ -1,11 +1,4 @@ -import { - getRepoDir, - checkIfPatched, - getCommitHash, - executeCliCommand, - getPatchId, - RSTSPEC_FLAG, -} from '../helpers/helpers'; +import { getRepoDir, checkIfPatched, getCommitHash, executeCliCommand, getPatchId, RSTSPEC_FLAG } from '../helpers'; export class StandardJobRunner { repoName: string; @@ -25,7 +18,7 @@ export class StandardJobRunner { const hasPatch = await checkIfPatched(this.repoDir); if (hasPatch) { - const [patchId, commitHash] = await Promise.all([getPatchId(this.repoDir), getCommitHash()]); + const [patchId, commitHash] = await Promise.all([getPatchId(this.repoDir), getCommitHash(this.repoDir)]); commandArgs.push('--commit'); commandArgs.push(commitHash); diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index b809d77a5..0209f6d52 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,3 +1,18 @@ -async function localApp() {} +import path from 'path'; +import { executeCliCommand, getCommitHash } from '../commands/src/helpers'; + +async function localApp() { + const repoDir = path.join(__dirname, '/repos'); + await executeCliCommand({ + command: 'git', + args: ['clone', 'https://github.com/mongodb/docs-java'], + options: { cwd: repoDir }, + }); + const gitHash = await getCommitHash(repoDir); + + console.log(gitHash); + + while (true) {} +} localApp(); From 32f7dc6a0b9da1d73e5396a5d225753e8eb17a2f Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 27 Sep 2023 17:37:27 -0500 Subject: [PATCH 19/51] [DOP-4033]: Add test code to confirm commands work, and remove base dockerfile --- Dockerfile.base | 31 ----------- Dockerfile.local | 87 +++++++++++++++++++++++++------ package.json | 2 +- src/commands/src/helpers/index.ts | 12 +++-- src/entrypoints/localApp.ts | 4 +- 5 files changed, 83 insertions(+), 53 deletions(-) delete mode 100644 Dockerfile.base diff --git a/Dockerfile.base b/Dockerfile.base deleted file mode 100644 index ec0d01c47..000000000 --- a/Dockerfile.base +++ /dev/null @@ -1,31 +0,0 @@ -# Builds the base dependencies needed -FROM node:18.16.0-alpine -ARG SNOOTY_PARSER_VERSION=0.14.9 -ARG SNOOTY_FRONTEND_VERSION=0.14.16 -ARG MUT_VERSION=0.10.7 -ARG REDOC_CLI_VERSION=1.2.2 -ARG NPM_BASE_64_AUTH -ARG NPM_EMAIL -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /app - -RUN apk update && apk add --no-cache python3 make g++ curl git -RUN apk add --update --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/community --repository http://dl-3.alpinelinux.org/alpine/edge/main vips-dev - -# install snooty parser -RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ snooty-parser.zip - -# install mut -RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ mut.zip - -RUN curl -L -o redoc.zip https://github.com/mongodb-forks/redoc/archive/refs/tags/v${REDOC_CLI_VERSION}.zip \ - && unzip redoc.zip \ - && mv redoc-${REDOC_CLI_VERSION} redoc/ - -# install snooty frontend and docs-tools -RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ - && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev diff --git a/Dockerfile.local b/Dockerfile.local index 3357e7900..c4e32c001 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,55 +1,110 @@ -FROM autobuilder/base:latest +FROM ubuntu:20.04 ARG SNOOTY_PARSER_VERSION=0.14.9 ARG SNOOTY_FRONTEND_VERSION=0.14.16 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL +ARG WORK_DIRECTORY=/app +ARG SNOOTY_PARSER_VERSION=0.14.9 +ARG SNOOTY_FRONTEND_VERSION=0.14.16 +ARG MUT_VERSION=0.10.7 +ARG REDOC_CLI_VERSION=1.2.2 +ARG NPM_BASE_64_AUTH +ARG NPM_EMAIL +ENV DEBIAN_FRONTEND=noninteractive + +# install legacy build environment for docs +RUN apt-get -o Acquire::Check-Valid-Until=false update +RUN apt-get -y install libpython2.7-dev python2.7 git rsync unzip curl +RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py +RUN python2.7 get-pip.py +RUN pip install requests virtualenv virtualenvwrapper py-dateutil +RUN python2.7 -m pip install python-dateutil +RUN virtualenv /venv +RUN /venv/bin/pip install --upgrade --force setuptools +RUN /venv/bin/pip install -r https://raw.githubusercontent.com/mongodb/docs-tools/master/giza/requirements.txt + +# helper libraries for docs builds +RUN apt-get update && apt-get install -y vim git + + +ENV PATH="${PATH}:/opt/snooty:/opt/mut:/app/.local/bin:/usr/local/lib/python2.7/dist-packages/virtualenv/bin" + +# get node 18 +# https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab +RUN apt-get install --yes curl +RUN curl --location https://deb.nodesource.com/setup_18.x | bash - +RUN apt-get install --yes nodejs +RUN apt-get install --yes build-essential WORKDIR /app -COPY package*.json ./ -RUN npm ci --legacy-peer-deps +# install snooty parser +RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ + && unzip -d /opt/ snooty-parser.zip + +# install mut +RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ + && unzip -d /opt/ mut.zip + +RUN curl -L -o redoc.zip https://github.com/mongodb-forks/redoc/archive/refs/tags/v${REDOC_CLI_VERSION}.zip \ + && unzip redoc.zip \ + && mv redoc-${REDOC_CLI_VERSION} redoc/ + +# setup user and root directory +RUN useradd -ms /bin/bash docsworker +RUN chmod 755 -R /app +RUN chown -Rv docsworker /app +USER docsworker + +# install snooty frontend and docs-tools +RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ + && cd snooty \ + && npm ci --legacy-peer-deps --omit=dev + +RUN mkdir -p modules/persistence && chmod 755 modules/persistence COPY modules/persistence/package*.json ./modules/persistence/ RUN cd ./modules/persistence \ && npm ci --legacy-peer-deps +RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/ RUN cd ./modules/oas-page-builder \ && npm ci --legacy-peer-deps # install persistence module -COPY modules/persistence/tsconfig*.json ./modules/persistence -COPY modules/persistence/src ./modules/persistence/src/ -COPY modules/persistence/index.ts ./modules/persistence + +COPY --chown=docsworker modules/persistence/tsconfig*.json ./modules/persistence +COPY --chown=docsworker modules/persistence/src ./modules/persistence/src/ +COPY --chown=docsworker modules/persistence/index.ts ./modules/persistence RUN cd ./modules/persistence \ && npm run build # Build modules # OAS Page Builder -COPY modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder -COPY modules/oas-page-builder/src ./modules/oas-page-builder/src/ -COPY modules/oas-page-builder/index.ts ./modules/oas-page-builder + +COPY --chown=docsworker modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder +COPY --chown=docsworker modules/oas-page-builder/src ./modules/oas-page-builder/src/ +COPY --chown=docsworker modules/oas-page-builder/index.ts ./modules/oas-page-builder RUN cd ./modules/oas-page-builder \ && npm run build # Root project build +COPY package*.json ./ +RUN npm ci --legacy-peer-deps COPY tsconfig*.json ./ COPY config config/ COPY api api/ COPY src src/ -RUN npm run build - -ENV PATH="${PATH}:/opt/snooty:/opt/mut" - -RUN mkdir repos/ +RUN npm run build:esbuild -RUN git --version +RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "build/entrypoints/localApp.js"] \ No newline at end of file +CMD ["node", "dist/entrypoints/localApp.js", "--enable-source-maps"] \ No newline at end of file diff --git a/package.json b/package.json index 276d1ebcc..4c349508a 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "start": "node index.js", "clean": "node maintain.js", "build": "tsc", - "build:esbuild": "esbuild", + "build:esbuild": "esbuild src/entrypoints/localApp.ts --bundle --platform=node --outdir=./dist/entrypoints --allow-overwrite --sourcemap", "format": "npm run prettier -- --check", "format:fix": "npm run prettier -- --write", "lint": "eslint --ext .ts .", diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 1d9869d09..0bd92310a 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -50,7 +50,9 @@ export async function executeCliCommand({ }); executedCommand.on('error', (err) => { - reject(new ExecuteCommandError('The command failed', err)); + if (err) { + reject(new ExecuteCommandError('The command failed', err)); + } }); executedCommand.on('close', (exitCode) => { @@ -59,6 +61,10 @@ export async function executeCliCommand({ console.error('Arguments provided: ', args); console.error('Options provided: ', options); + if (stderr) { + console.error(stderr.join()); + } + reject(new ExecuteCommandError('The command failed', exitCode)); return; } @@ -119,7 +125,7 @@ export async function getPatchId(repoDir: string): Promise { export async function addProjectToEnv(project: string) { return new Promise((resolve, reject) => { - const stream = fs.createWriteStream(path.join(__dirname, 'snooty/.env.production'), { flags: 'a+' }); + const stream = fs.createWriteStream(path.join(process.cwd(), 'snooty/.env.production'), { flags: 'a+' }); stream.write(project); stream.close(); @@ -143,7 +149,7 @@ export async function getCommitHash(repoDir: string): Promise { } export const checkIfPatched = async (repoDir: string) => !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); -export const getRepoDir = (repoName: string) => path.join(__dirname, `repos/${repoName}`); +export const getRepoDir = (repoName: string) => path.join(process.cwd(), `repos/${repoName}`); export const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index 0209f6d52..f7588675a 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -2,13 +2,13 @@ import path from 'path'; import { executeCliCommand, getCommitHash } from '../commands/src/helpers'; async function localApp() { - const repoDir = path.join(__dirname, '/repos'); + const repoDir = path.join(process.cwd(), '/repos'); await executeCliCommand({ command: 'git', args: ['clone', 'https://github.com/mongodb/docs-java'], options: { cwd: repoDir }, }); - const gitHash = await getCommitHash(repoDir); + const gitHash = await getCommitHash(`${repoDir}/docs-java`); console.log(gitHash); From 8b9519dc7cc3b0f58f95147977710758d47bb731 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 29 Sep 2023 10:46:26 -0500 Subject: [PATCH 20/51] [DOP-4033]: Add next-gen-stage, and add support for arm64 dockerfile --- Dockerfile.local | 41 +++++++++++------------ src/commands/src/helpers/index.ts | 18 ++++++++-- src/commands/src/shared/next-gen-html.ts | 27 ++++++++++++--- src/commands/src/shared/next-gen-parse.ts | 13 +++++-- src/commands/src/shared/next-gen-stage.ts | 25 ++++++++++++++ src/entrypoints/localApp.ts | 23 ++++++++++--- 6 files changed, 113 insertions(+), 34 deletions(-) create mode 100644 src/commands/src/shared/next-gen-stage.ts diff --git a/Dockerfile.local b/Dockerfile.local index c4e32c001..bef5c4080 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM arm64v8/ubuntu:20.04 ARG SNOOTY_PARSER_VERSION=0.14.9 ARG SNOOTY_FRONTEND_VERSION=0.14.16 ARG MUT_VERSION=0.10.7 @@ -14,22 +14,8 @@ ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ENV DEBIAN_FRONTEND=noninteractive -# install legacy build environment for docs -RUN apt-get -o Acquire::Check-Valid-Until=false update -RUN apt-get -y install libpython2.7-dev python2.7 git rsync unzip curl -RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py -RUN python2.7 get-pip.py -RUN pip install requests virtualenv virtualenvwrapper py-dateutil -RUN python2.7 -m pip install python-dateutil -RUN virtualenv /venv -RUN /venv/bin/pip install --upgrade --force setuptools -RUN /venv/bin/pip install -r https://raw.githubusercontent.com/mongodb/docs-tools/master/giza/requirements.txt - # helper libraries for docs builds -RUN apt-get update && apt-get install -y vim git - - -ENV PATH="${PATH}:/opt/snooty:/opt/mut:/app/.local/bin:/usr/local/lib/python2.7/dist-packages/virtualenv/bin" +RUN apt-get update && apt-get install -y vim git unzip zip # get node 18 # https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab @@ -37,22 +23,35 @@ RUN apt-get install --yes curl RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential +RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config WORKDIR /app +RUN python3 -m pip install poetry # install snooty parser -RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ snooty-parser.zip +RUN git clone -b v${SNOOTY_PARSER_VERSION} --depth 1 https://github.com/mongodb/snooty-parser.git \ + && cd snooty-parser \ + && python3 -m poetry install \ + && make package \ + && mv dist/snooty /opt/ +# RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ +# && unzip -d /opt/ snooty-parser.zip # install mut -RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ - && unzip -d /opt/ mut.zip + +RUN git clone -b v${MUT_VERSION} --depth 1 https://github.com/mongodb/mut.git \ + && cd mut \ + && python3 -m poetry install \ + && make package \ + && mv dist/mut /opt/ RUN curl -L -o redoc.zip https://github.com/mongodb-forks/redoc/archive/refs/tags/v${REDOC_CLI_VERSION}.zip \ && unzip redoc.zip \ && mv redoc-${REDOC_CLI_VERSION} redoc/ +ENV PATH="${PATH}:/opt/snooty:/opt/mut:/app/.local/bin" + # setup user and root directory RUN useradd -ms /bin/bash docsworker RUN chmod 755 -R /app @@ -107,4 +106,4 @@ RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "dist/entrypoints/localApp.js", "--enable-source-maps"] \ No newline at end of file +CMD ["node", "--enable-source-maps", "dist/entrypoints/localApp.js"] \ No newline at end of file diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 0bd92310a..603112d25 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -22,7 +22,7 @@ interface CliCommandParams { writeStream?: fs.WriteStream; } -interface CliCommandResponse { +export interface CliCommandResponse { stdout: string; stderr: string; } @@ -61,6 +61,10 @@ export async function executeCliCommand({ console.error('Arguments provided: ', args); console.error('Options provided: ', options); + if (stdout) { + console.error(stdout.join()); + } + if (stderr) { console.error(stderr.join()); } @@ -136,6 +140,16 @@ export async function addProjectToEnv(project: string) { stream.on('close', resolve); }); } +export async function getCommitBranch(repoDir: string): Promise { + // equivalent to git rev-parse --short HEAD + const response = await executeCliCommand({ + command: 'git', + args: ['rev-parse', '--abbrev-ref', 'HEAD'], + options: { cwd: repoDir }, + }); + + return response.stdout; +} export async function getCommitHash(repoDir: string): Promise { // equivalent to git rev-parse --short HEAD @@ -148,7 +162,7 @@ export async function getCommitHash(repoDir: string): Promise { return response.stdout; } -export const checkIfPatched = async (repoDir: string) => !(await existsAsync(path.join(repoDir, 'myPatch.patch'))); +export const checkIfPatched = async (repoDir: string) => !existsAsync(path.join(repoDir, 'myPatch.patch')); export const getRepoDir = (repoName: string) => path.join(process.cwd(), `repos/${repoName}`); export const RSTSPEC_FLAG = diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index 8f3a69fd9..bebca46af 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -1,15 +1,34 @@ import fs from 'fs'; import { promisify } from 'util'; -import { getRepoDir } from '../helpers'; +import { executeCliCommand, getRepoDir } from '../helpers'; const writeFileAsync = promisify(fs.writeFile); -async function createEnvProdFile(repoDir: string) { - await writeFileAsync(repoDir, '', 'utf8'); +async function createEnvProdFile(repoDir: string, projectName: string) { + try { + await writeFileAsync( + `${process.cwd()}/snooty/.env.production`, + `GATSBY_BASE_URL=docs.mongodb.com + GATSBY_SITE=${projectName} + GATSBY_MANIFEST_PATH=${repoDir}/bundle.zip`, + 'utf8' + ); + } catch (e) { + console.error(`ERROR! Could not write to .env.production`); + throw e; + } } export async function nextGenHtml(repoName: string) { const repoDir = getRepoDir(repoName); // might move this since technically next-gen-html doesn't create the file - await createEnvProdFile(repoDir); + await createEnvProdFile(repoDir, 'java'); + + const result = await executeCliCommand({ + command: 'npm', + args: ['run', 'build'], + options: { cwd: `${process.cwd()}/snooty` }, + }); + + return result; } diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 4dbb7fe0f..80f358c84 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,8 +1,15 @@ -import { checkIfPatched, executeCliCommand, getCommitHash, getPatchId, getRepoDir } from '../helpers'; +import { + CliCommandResponse, + checkIfPatched, + executeCliCommand, + getCommitHash, + getPatchId, + getRepoDir, +} from '../helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; -export async function nextGenParse(repoName: string): Promise { +export async function nextGenParse(repoName: string): Promise { const repoDir = getRepoDir(repoName); const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; @@ -19,5 +26,5 @@ export async function nextGenParse(repoName: string): Promise { commandArgs.push(patchId); } - await executeCliCommand({ command: 'snooty', args: commandArgs }); + return executeCliCommand({ command: 'snooty', args: commandArgs }); } diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts new file mode 100644 index 000000000..d9fe44cbf --- /dev/null +++ b/src/commands/src/shared/next-gen-stage.ts @@ -0,0 +1,25 @@ +import { checkIfPatched, executeCliCommand, getCommitHash, getPatchId, getRepoDir } from '../helpers'; + +interface StageParams { + repoName: string; + mutPrefix: string; + projectName: string; + bucketName: string; +} + +export async function nextGenStage({ repoName, mutPrefix, projectName, bucketName }: StageParams) { + const repoDir = getRepoDir(repoName); + + const hasPatch = await checkIfPatched(repoDir); + + const commandArgs = ['public', bucketName, '--stage']; + + if (hasPatch && projectName !== mutPrefix) { + const [commitHash, patchId] = await Promise.all([getCommitHash(repoDir), getPatchId(repoDir)]); + commandArgs.push(`--prefix="${commitHash}/${patchId}/${mutPrefix}"`); + } + + const result = await executeCliCommand({ command: 'mut-publish', args: commandArgs }); + + return result; +} diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index f7588675a..d3544e2d2 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,17 +1,32 @@ import path from 'path'; -import { executeCliCommand, getCommitHash } from '../commands/src/helpers'; +import { executeCliCommand } from '../commands/src/helpers'; +import { nextGenParse } from '../commands/src/shared/next-gen-parse'; +import { nextGenHtml } from '../commands/src/shared/next-gen-html'; async function localApp() { const repoDir = path.join(process.cwd(), '/repos'); + const repoName = 'docs-java'; + await executeCliCommand({ command: 'git', - args: ['clone', 'https://github.com/mongodb/docs-java'], + args: ['clone', `https://github.com/mongodb/${repoName}`], options: { cwd: repoDir }, }); - const gitHash = await getCommitHash(`${repoDir}/docs-java`); - console.log(gitHash); + console.log('Begin snooty build...'); + const snootyBuildRes = await nextGenParse(repoName); + + console.log(snootyBuildRes.stderr); + + console.log('snooty build complete'); + + console.log('Begin next-gen-html...'); + + const nextGenHtmlRes = await nextGenHtml(repoName); + + console.log(nextGenHtmlRes.stdout); + console.log('next-gen-html complete'); while (true) {} } From 4518c130789dceee2fcabc9b3933fb7ca42954b2 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 29 Sep 2023 10:49:33 -0500 Subject: [PATCH 21/51] [DOP-4033]: Remove extra args from dockerfile --- Dockerfile.local | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index bef5c4080..a2e72d57c 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,11 +1,6 @@ FROM arm64v8/ubuntu:20.04 -ARG SNOOTY_PARSER_VERSION=0.14.9 -ARG SNOOTY_FRONTEND_VERSION=0.14.16 -ARG MUT_VERSION=0.10.7 -ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL -ARG WORK_DIRECTORY=/app ARG SNOOTY_PARSER_VERSION=0.14.9 ARG SNOOTY_FRONTEND_VERSION=0.14.16 ARG MUT_VERSION=0.10.7 From a9ae14a1113393eb165efc7f2abd458a1e79b290 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 29 Sep 2023 12:32:45 -0500 Subject: [PATCH 22/51] [DOP-4033]: Add oas-page-build and persistence module --- Dockerfile.local | 4 +-- src/commands/src/shared/next-gen-deploy.ts | 0 src/commands/src/shared/next-gen-stage.ts | 16 +++++++---- src/commands/src/shared/oas-page-build.ts | 22 +++++++++++++++ src/commands/src/shared/persistence-module.ts | 28 +++++++++++++++++++ src/entrypoints/localApp.ts | 5 ++-- 6 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 src/commands/src/shared/next-gen-deploy.ts create mode 100644 src/commands/src/shared/oas-page-build.ts create mode 100644 src/commands/src/shared/persistence-module.ts diff --git a/Dockerfile.local b/Dockerfile.local index a2e72d57c..d707e1725 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -2,7 +2,7 @@ FROM arm64v8/ubuntu:20.04 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG SNOOTY_PARSER_VERSION=0.14.9 -ARG SNOOTY_FRONTEND_VERSION=0.14.16 +ARG SNOOTY_FRONTEND_VERSION=0.14.18 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH @@ -101,4 +101,4 @@ RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "--enable-source-maps", "dist/entrypoints/localApp.js"] \ No newline at end of file +CMD ["node", "dist/entrypoints/localApp.js"] \ No newline at end of file diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index d9fe44cbf..2beb2650e 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,25 +1,29 @@ -import { checkIfPatched, executeCliCommand, getCommitHash, getPatchId, getRepoDir } from '../helpers'; +import { checkIfPatched, executeCliCommand, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '../helpers'; interface StageParams { repoName: string; mutPrefix: string; projectName: string; bucketName: string; + url: string; } -export async function nextGenStage({ repoName, mutPrefix, projectName, bucketName }: StageParams) { +export async function nextGenStage({ repoName, mutPrefix, projectName, bucketName, url }: StageParams) { const repoDir = getRepoDir(repoName); - const hasPatch = await checkIfPatched(repoDir); + const [hasPatch, commitBranch] = await Promise.all([checkIfPatched(repoDir), getCommitBranch(repoDir)]); + + let hostedAtUrl = `${url}/${mutPrefix}/${process.env.USER}/${commitBranch}/`; const commandArgs = ['public', bucketName, '--stage']; if (hasPatch && projectName !== mutPrefix) { const [commitHash, patchId] = await Promise.all([getCommitHash(repoDir), getPatchId(repoDir)]); commandArgs.push(`--prefix="${commitHash}/${patchId}/${mutPrefix}"`); + hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/${process.env.USER}/${commitBranch}/`; } - const result = await executeCliCommand({ command: 'mut-publish', args: commandArgs }); - - return result; + const { stdout } = await executeCliCommand({ command: 'mut-publish', args: commandArgs }); + const resultMessage = `${stdout}\n Hosted at ${hostedAtUrl}`; + return resultMessage; } diff --git a/src/commands/src/shared/oas-page-build.ts b/src/commands/src/shared/oas-page-build.ts new file mode 100644 index 000000000..f90d1fbf2 --- /dev/null +++ b/src/commands/src/shared/oas-page-build.ts @@ -0,0 +1,22 @@ +import { executeCliCommand } from '../helpers'; + +interface OasPageBuildParams { + bundlePath: string; + repoDir: string; + siteUrl: string; +} + +export async function oasPageBuild({ bundlePath, repoDir, siteUrl }: OasPageBuildParams) { + const { stdout } = await executeCliCommand({ + command: 'node', + args: [ + `${process.cwd()}/modules/oas-page-builder/dist/index.js`, + '--output', + `${repoDir}/public`, + '--redoc', + `${process.cwd()}/redoc/cli/index.js`, + '--site-url', + siteUrl, + ], + }); +} diff --git a/src/commands/src/shared/persistence-module.ts b/src/commands/src/shared/persistence-module.ts new file mode 100644 index 000000000..292bb0ef7 --- /dev/null +++ b/src/commands/src/shared/persistence-module.ts @@ -0,0 +1,28 @@ +import { executeCliCommand } from '../helpers'; + +interface PersistenceModuleParams { + bundlePath: string; + jobId: string; + repoOwner?: string; +} +export async function persistenceModule({ + bundlePath, + jobId, + repoOwner = 'docs-builder-bot', +}: PersistenceModuleParams) { + const { stdout } = await executeCliCommand({ + command: 'node', + args: [ + `${process.cwd()}/modules/persistence/dist/index.js`, + '--unhandled-rejections=strict', + '--path', + bundlePath, + '--githubUser', + repoOwner, + '--jobId', + jobId, + ], + }); + + return stdout; +} diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index d3544e2d2..c33250efc 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -5,7 +5,7 @@ import { nextGenHtml } from '../commands/src/shared/next-gen-html'; async function localApp() { const repoDir = path.join(process.cwd(), '/repos'); - const repoName = 'docs-java'; + const repoName = 'docs-landing'; await executeCliCommand({ command: 'git', @@ -27,7 +27,8 @@ async function localApp() { console.log(nextGenHtmlRes.stdout); console.log('next-gen-html complete'); - while (true) {} + + console.log('Begin next-gen-stage...'); } localApp(); From c5037fd5f673a7c295b12ef4cc901413835ea122 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 29 Sep 2023 12:33:21 -0500 Subject: [PATCH 23/51] [DOP-4033]: Add bundle path for oaspagebuild --- src/commands/src/shared/oas-page-build.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/commands/src/shared/oas-page-build.ts b/src/commands/src/shared/oas-page-build.ts index f90d1fbf2..d892b823d 100644 --- a/src/commands/src/shared/oas-page-build.ts +++ b/src/commands/src/shared/oas-page-build.ts @@ -11,6 +11,8 @@ export async function oasPageBuild({ bundlePath, repoDir, siteUrl }: OasPageBuil command: 'node', args: [ `${process.cwd()}/modules/oas-page-builder/dist/index.js`, + '--bundle', + bundlePath, '--output', `${repoDir}/public`, '--redoc', @@ -19,4 +21,6 @@ export async function oasPageBuild({ bundlePath, repoDir, siteUrl }: OasPageBuil siteUrl, ], }); + + return stdout; } From 9f3cd88bc8b7efe3d517c0357675e8687d1eb736 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 2 Oct 2023 09:36:31 -0500 Subject: [PATCH 24/51] [DOP-4033]: Work on logic for deploy and ability to pipe output from one command to another --- Dockerfile.local | 20 ++++++------ src/commands/src/helpers/index.ts | 37 ++++++++++++---------- src/commands/src/shared/next-gen-deploy.ts | 1 + src/entrypoints/localApp.ts | 5 +-- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index d707e1725..9c8b6b15b 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -30,8 +30,6 @@ RUN git clone -b v${SNOOTY_PARSER_VERSION} --depth 1 https://github.com/mongodb/ && python3 -m poetry install \ && make package \ && mv dist/snooty /opt/ -# RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ -# && unzip -d /opt/ snooty-parser.zip # install mut @@ -68,7 +66,15 @@ COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/ RUN cd ./modules/oas-page-builder \ && npm ci --legacy-peer-deps -# install persistence module +# Root project build +COPY package*.json ./ +RUN npm ci --legacy-peer-deps +COPY tsconfig*.json ./ +COPY config config/ +COPY api api/ +COPY src src/ + +# Build persistence module COPY --chown=docsworker modules/persistence/tsconfig*.json ./modules/persistence COPY --chown=docsworker modules/persistence/src ./modules/persistence/src/ @@ -87,14 +93,6 @@ COPY --chown=docsworker modules/oas-page-builder/index.ts ./modules/oas-page-bui RUN cd ./modules/oas-page-builder \ && npm run build -# Root project build -COPY package*.json ./ -RUN npm ci --legacy-peer-deps -COPY tsconfig*.json ./ -COPY config config/ -COPY api api/ -COPY src src/ - RUN npm run build:esbuild RUN mkdir repos && chmod 755 repos diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 603112d25..981c6c338 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -2,6 +2,7 @@ import { SpawnOptions, spawn } from 'child_process'; import { promisify } from 'util'; import fs from 'fs'; import path from 'path'; +import { Writable } from 'stream'; const openAsync = promisify(fs.open); const closeAsync = promisify(fs.close); @@ -20,11 +21,12 @@ interface CliCommandParams { args?: readonly string[]; options?: SpawnOptions; writeStream?: fs.WriteStream; + writeTarget?: Writable; } export interface CliCommandResponse { - stdout: string; - stderr: string; + outputText: string; + errorText: string; } export async function executeCliCommand({ @@ -32,21 +34,23 @@ export async function executeCliCommand({ args = [], options = {}, writeStream, + writeTarget, }: CliCommandParams): Promise { return new Promise((resolve, reject) => { - const stdout: string[] = []; - const stderr: string[] = []; + const outputText: string[] = []; + const errorText: string[] = []; const executedCommand = spawn(command, args, options); if (writeStream) executedCommand.stdout?.pipe(writeStream); - executedCommand.stdout?.on('data', (data: Buffer) => { - stdout.push(data.toString()); + outputText.push(data.toString()); + + if (writeTarget) writeTarget.write(data); }); executedCommand.stderr?.on('data', (data: Buffer) => { - stderr.push(data.toString()); + errorText.push(data.toString()); }); executedCommand.on('error', (err) => { @@ -56,17 +60,18 @@ export async function executeCliCommand({ }); executedCommand.on('close', (exitCode) => { + if (writeTarget) writeTarget.end(); if (exitCode !== 0) { console.error(`ERROR! The command ${command} closed with an exit code other than 0: ${exitCode}.`); console.error('Arguments provided: ', args); console.error('Options provided: ', options); - if (stdout) { - console.error(stdout.join()); + if (outputText) { + console.error(outputText.join()); } - if (stderr) { - console.error(stderr.join()); + if (errorText) { + console.error(errorText.join()); } reject(new ExecuteCommandError('The command failed', exitCode)); @@ -74,8 +79,8 @@ export async function executeCliCommand({ } resolve({ - stdout: stdout.join(), - stderr: stderr.join(), + outputText: outputText.join(), + errorText: errorText.join(), }); }); }); @@ -122,7 +127,7 @@ export async function readFileAndExec({ export async function getPatchId(repoDir: string): Promise { const filePath = path.join(repoDir, 'myPatch.patch'); - const { stdout: gitPatchId } = await readFileAndExec({ command: 'git', filePath, args: ['patch-id'] }); + const { outputText: gitPatchId } = await readFileAndExec({ command: 'git', filePath, args: ['patch-id'] }); return gitPatchId.slice(0, 7); } @@ -148,7 +153,7 @@ export async function getCommitBranch(repoDir: string): Promise { options: { cwd: repoDir }, }); - return response.stdout; + return response.outputText; } export async function getCommitHash(repoDir: string): Promise { @@ -159,7 +164,7 @@ export async function getCommitHash(repoDir: string): Promise { options: { cwd: repoDir }, }); - return response.stdout; + return response.outputText; } export const checkIfPatched = async (repoDir: string) => !existsAsync(path.join(repoDir, 'myPatch.patch')); diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index e69de29bb..aca680a91 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -0,0 +1 @@ +export async function nextGenDeploy() {} diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index c33250efc..e70076735 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -13,10 +13,11 @@ async function localApp() { options: { cwd: repoDir }, }); + console.log('Hello'); console.log('Begin snooty build...'); const snootyBuildRes = await nextGenParse(repoName); - console.log(snootyBuildRes.stderr); + console.log(snootyBuildRes.errorText); console.log('snooty build complete'); @@ -24,7 +25,7 @@ async function localApp() { const nextGenHtmlRes = await nextGenHtml(repoName); - console.log(nextGenHtmlRes.stdout); + console.log(nextGenHtmlRes.outputText); console.log('next-gen-html complete'); From 4b5a432ca2cc9d63418255ddbd1d7e59ccdf9286 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 2 Oct 2023 15:37:27 -0500 Subject: [PATCH 25/51] [DOP-4033]: Work on logic for deploy and ability to pipe output from one command to another --- src/commands/src/helpers/index.ts | 12 ++++++++---- src/commands/src/shared/next-gen-stage.ts | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 981c6c338..2ffad2091 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -28,13 +28,19 @@ export interface CliCommandResponse { outputText: string; errorText: string; } +export async function executeAndPipeCommands(cmdFromParams: CliCommandParams, cmdToParams: CliCommandParams) { + const cmdFrom = spawn(cmdFromParams.command, cmdFromParams.args || [], cmdFromParams.options || {}); + const cmdTo = spawn(cmdToParams.command, cmdToParams.args || [], cmdToParams.options || {}); + cmdFrom.stdout?.on('data', (data) => { + cmdTo.stdin?.write(data); + }); +} export async function executeCliCommand({ command, args = [], options = {}, writeStream, - writeTarget, }: CliCommandParams): Promise { return new Promise((resolve, reject) => { const outputText: string[] = []; @@ -45,8 +51,6 @@ export async function executeCliCommand({ if (writeStream) executedCommand.stdout?.pipe(writeStream); executedCommand.stdout?.on('data', (data: Buffer) => { outputText.push(data.toString()); - - if (writeTarget) writeTarget.write(data); }); executedCommand.stderr?.on('data', (data: Buffer) => { @@ -60,7 +64,7 @@ export async function executeCliCommand({ }); executedCommand.on('close', (exitCode) => { - if (writeTarget) writeTarget.end(); + if (writeStream) writeStream.end(); if (exitCode !== 0) { console.error(`ERROR! The command ${command} closed with an exit code other than 0: ${exitCode}.`); console.error('Arguments provided: ', args); diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 2beb2650e..b6c46a24f 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -23,7 +23,7 @@ export async function nextGenStage({ repoName, mutPrefix, projectName, bucketNam hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/${process.env.USER}/${commitBranch}/`; } - const { stdout } = await executeCliCommand({ command: 'mut-publish', args: commandArgs }); - const resultMessage = `${stdout}\n Hosted at ${hostedAtUrl}`; + const { outputText } = await executeCliCommand({ command: 'mut-publish', args: commandArgs }); + const resultMessage = `${outputText}\n Hosted at ${hostedAtUrl}`; return resultMessage; } From 1a635d18d0ba5b84ade4cf4975b90201611ff770 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 06:23:51 -0500 Subject: [PATCH 26/51] [DOP-4033]: Work on logic for deploy and ability to pipe output from one command to another --- src/commands/src/helpers/index.ts | 32 ++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 2ffad2091..441169d55 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -28,12 +28,31 @@ export interface CliCommandResponse { outputText: string; errorText: string; } + export async function executeAndPipeCommands(cmdFromParams: CliCommandParams, cmdToParams: CliCommandParams) { - const cmdFrom = spawn(cmdFromParams.command, cmdFromParams.args || [], cmdFromParams.options || {}); - const cmdTo = spawn(cmdToParams.command, cmdToParams.args || [], cmdToParams.options || {}); + return new Promise((resolve, reject) => { + const cmdFrom = spawn(cmdFromParams.command, cmdFromParams.args || [], cmdFromParams.options || {}); + const cmdTo = spawn(cmdToParams.command, cmdToParams.args || [], cmdToParams.options || {}); + + const outputText: string[] = []; + + cmdFrom.stdout?.on('data', (data: Buffer) => { + cmdTo.stdin?.write(data); + }); + + cmdFrom.on('error', (err) => { + reject(new ExecuteCommandError('The first command failed', err)); + }); - cmdFrom.stdout?.on('data', (data) => { - cmdTo.stdin?.write(data); + cmdTo.stdout?.on('data', (data: Buffer) => { + outputText.push(data.toString()); + }); + + cmdTo.on('error', (err) => { + reject(new ExecuteCommandError('The second command failed', err)); + }); + + cmdTo.on('exit', (code) => {}); }); } export async function executeCliCommand({ @@ -58,13 +77,12 @@ export async function executeCliCommand({ }); executedCommand.on('error', (err) => { - if (err) { - reject(new ExecuteCommandError('The command failed', err)); - } + reject(new ExecuteCommandError('The command failed', err)); }); executedCommand.on('close', (exitCode) => { if (writeStream) writeStream.end(); + if (exitCode !== 0) { console.error(`ERROR! The command ${command} closed with an exit code other than 0: ${exitCode}.`); console.error('Arguments provided: ', args); From 82872a0e00c1656a45b3ca48f0361b0d9f6b4347 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 06:32:23 -0500 Subject: [PATCH 27/51] [DOP-4033]: Finish up pipe logic --- src/commands/src/helpers/index.ts | 41 ++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 441169d55..5d9c92035 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -31,28 +31,63 @@ export interface CliCommandResponse { export async function executeAndPipeCommands(cmdFromParams: CliCommandParams, cmdToParams: CliCommandParams) { return new Promise((resolve, reject) => { + let hasRejected = false; + const cmdFrom = spawn(cmdFromParams.command, cmdFromParams.args || [], cmdFromParams.options || {}); const cmdTo = spawn(cmdToParams.command, cmdToParams.args || [], cmdToParams.options || {}); - const outputText: string[] = []; - cmdFrom.stdout?.on('data', (data: Buffer) => { cmdTo.stdin?.write(data); }); cmdFrom.on('error', (err) => { reject(new ExecuteCommandError('The first command failed', err)); + hasRejected = true; }); + const outputText: string[] = []; + const errorText: string[] = []; + cmdTo.stdout?.on('data', (data: Buffer) => { outputText.push(data.toString()); }); + cmdTo.stderr?.on('data', (data: Buffer) => { + errorText.push(data.toString()); + }); + cmdTo.on('error', (err) => { reject(new ExecuteCommandError('The second command failed', err)); }); - cmdTo.on('exit', (code) => {}); + cmdTo.on('exit', (exitCode) => { + // previous command errored out, return so we don't + // accidentally resolve if the second command somehow still + // exits without error + if (hasRejected) return; + + if (exitCode !== 0) { + console.error(`ERROR! The command ${cmdToParams.command} closed with an exit code other than 0: ${exitCode}.`); + console.error('Arguments provided: ', cmdToParams.args); + console.error('Options provided: ', cmdToParams.options); + + if (outputText) { + console.error(outputText.join()); + } + + if (errorText) { + console.error(errorText.join()); + } + + reject(new ExecuteCommandError('The command failed', exitCode)); + return; + } + + resolve({ + outputText: outputText.join(), + errorText: errorText.join(), + }); + }); }); } export async function executeCliCommand({ From d6798e367d212e8700175c0f2cf63433612e2920 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 10:18:21 -0500 Subject: [PATCH 28/51] [DOP-4033]: Add next-gen-deploy --- src/commands/src/helpers/index.ts | 5 +++- src/commands/src/shared/next-gen-deploy.ts | 28 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 5d9c92035..3f61e64f6 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -29,7 +29,10 @@ export interface CliCommandResponse { errorText: string; } -export async function executeAndPipeCommands(cmdFromParams: CliCommandParams, cmdToParams: CliCommandParams) { +export async function executeAndPipeCommands( + cmdFromParams: CliCommandParams, + cmdToParams: CliCommandParams +): Promise { return new Promise((resolve, reject) => { let hasRejected = false; diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index aca680a91..c2295d501 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -1 +1,27 @@ -export async function nextGenDeploy() {} +import fs from 'fs'; +import path from 'path'; +import { promisify } from 'util'; +import { executeAndPipeCommands, executeCliCommand, getCommitBranch, getRepoDir } from '../helpers'; + +const existsAsync = promisify(fs.exists); + +export async function nextGenDeploy({ repoName }) { + const repoDir = getRepoDir(repoName); + + const [hasConfigRedirects, gitBranch] = await Promise.all([ + existsAsync(path.join(process.cwd(), 'config/redirects')), + getCommitBranch(repoDir), + ]); + + if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { + // mut-redirects config/redirects -o public/.htaccess + + await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); + } + + // yes | mut-publish public ${BUCKET} --prefix="${MUT_PREFIX}" --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; + await executeAndPipeCommands( + { command: 'yes' }, + { command: 'mut-publish', args: ['public', bucket, `--prefix=${mutPrefix}`, '--json', '--all-subdirectories'] } + ); +} From 66bd8203e67d3180a8ba0a7d10d1f1ef760b451a Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 10:21:03 -0500 Subject: [PATCH 29/51] [DOP-4033]: Remove unused imports --- src/commands/src/shared/next-gen-deploy.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index c2295d501..587f59753 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -1,21 +1,21 @@ import fs from 'fs'; import path from 'path'; import { promisify } from 'util'; -import { executeAndPipeCommands, executeCliCommand, getCommitBranch, getRepoDir } from '../helpers'; +import { executeAndPipeCommands, executeCliCommand } from '../helpers'; const existsAsync = promisify(fs.exists); -export async function nextGenDeploy({ repoName }) { - const repoDir = getRepoDir(repoName); +interface NextGenDeployParams { + bucket: string; + mutPrefix: string; + gitBranch: string; +} - const [hasConfigRedirects, gitBranch] = await Promise.all([ - existsAsync(path.join(process.cwd(), 'config/redirects')), - getCommitBranch(repoDir), - ]); +export async function nextGenDeploy({ bucket, mutPrefix, gitBranch }: NextGenDeployParams) { + const hasConfigRedirects = await existsAsync(path.join(process.cwd(), 'config/redirects')); if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { // mut-redirects config/redirects -o public/.htaccess - await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); } From dcc7e6281b4000d662e108f92814dff9c6639242 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 10:21:58 -0500 Subject: [PATCH 30/51] [DOP-4033]: Remove standard job handler --- .../src/shared/standard-job-runner.ts | 43 ------------------- 1 file changed, 43 deletions(-) delete mode 100644 src/commands/src/shared/standard-job-runner.ts diff --git a/src/commands/src/shared/standard-job-runner.ts b/src/commands/src/shared/standard-job-runner.ts deleted file mode 100644 index cb0162172..000000000 --- a/src/commands/src/shared/standard-job-runner.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { getRepoDir, checkIfPatched, getCommitHash, executeCliCommand, getPatchId, RSTSPEC_FLAG } from '../helpers'; - -export class StandardJobRunner { - repoName: string; - repoDir: string; - project: string; - - constructor(repoName: string, project: string) { - this.repoName = repoName; - this.project = project; - - this.repoDir = getRepoDir(repoName); - } - - async nextGenParse(): Promise { - const commandArgs = ['build', this.repoDir, '--output', `${this.repoDir}/bundle.zip`, RSTSPEC_FLAG]; - - const hasPatch = await checkIfPatched(this.repoDir); - - if (hasPatch) { - const [patchId, commitHash] = await Promise.all([getPatchId(this.repoDir), getCommitHash(this.repoDir)]); - - commandArgs.push('--commit'); - commandArgs.push(commitHash); - - commandArgs.push('--patch'); - commandArgs.push(patchId); - } - - await executeCliCommand({ command: 'snooty', args: commandArgs }); - } - - async nextGenHtml() { - // copy .env.production to the snooty directory - await executeCliCommand({ command: 'cp', args: [`${this.repoDir}/.env.production`, 'snooty'] }); - - await executeCliCommand({ command: 'echo', args: [`"GATSBY_SITE=${this.project}"`] }); - } - - async nextGenDeploy() { - throw new Error('Not implemented'); - } -} From 24b4c0098524084210e3f9f25c1b11a8d1b5e9fe Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 13:36:37 -0500 Subject: [PATCH 31/51] [DOP-4033]: Add execution helper --- src/commands/src/helpers/execution-helper.ts | 27 +++++++++++++++++++ src/commands/src/helpers/index.ts | 11 +++++--- src/commands/src/shared/next-gen-deploy.ts | 10 +++---- src/commands/src/shared/next-gen-html.ts | 6 ++--- src/commands/src/shared/next-gen-parse.ts | 24 ++++++----------- src/commands/src/shared/next-gen-stage.ts | 6 ++--- src/commands/src/shared/oas-page-build.ts | 4 +-- src/commands/src/shared/persistence-module.ts | 4 +-- 8 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 src/commands/src/helpers/execution-helper.ts diff --git a/src/commands/src/helpers/execution-helper.ts b/src/commands/src/helpers/execution-helper.ts new file mode 100644 index 000000000..058ebdb50 --- /dev/null +++ b/src/commands/src/helpers/execution-helper.ts @@ -0,0 +1,27 @@ +import path from 'path'; +import fs from 'fs'; +import { checkIfPatched, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '.'; +import { promisify } from 'util'; + +const existsAsync = promisify(fs.exists); + +export async function getCliBuildDependencies(repoName: string) { + const repoDir = getRepoDir(repoName); + const commandPromises = [ + checkIfPatched(repoDir), + getCommitHash(repoDir), + getCommitBranch(repoDir), + getPatchId(repoDir), + existsAsync(path.join(process.cwd(), 'config/redirects')), + ]; + + const deps = await Promise.all(commandPromises); + return { + hasPatch: deps[0] as string, + commitHash: deps[1] as string, + commitBranch: deps[2] as string, + patchId: deps[3] as string | undefined, + hasRedirects: deps[4] as boolean, + bundlePath: `${repoDir}/bundle.zip`, + }; +} diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 3f61e64f6..349631509 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -184,12 +184,15 @@ export async function readFileAndExec({ return response; } -export async function getPatchId(repoDir: string): Promise { +export async function getPatchId(repoDir: string): Promise { const filePath = path.join(repoDir, 'myPatch.patch'); + try { + const { outputText: gitPatchId } = await readFileAndExec({ command: 'git', filePath, args: ['patch-id'] }); - const { outputText: gitPatchId } = await readFileAndExec({ command: 'git', filePath, args: ['patch-id'] }); - - return gitPatchId.slice(0, 7); + return gitPatchId.slice(0, 7); + } catch (err) { + console.warn('No patch ID found'); + } } export async function addProjectToEnv(project: string) { diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 587f59753..d9ba13474 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -1,18 +1,14 @@ -import fs from 'fs'; -import path from 'path'; -import { promisify } from 'util'; import { executeAndPipeCommands, executeCliCommand } from '../helpers'; -const existsAsync = promisify(fs.exists); - interface NextGenDeployParams { bucket: string; mutPrefix: string; gitBranch: string; + hasConfigRedirects: boolean; } -export async function nextGenDeploy({ bucket, mutPrefix, gitBranch }: NextGenDeployParams) { - const hasConfigRedirects = await existsAsync(path.join(process.cwd(), 'config/redirects')); +export async function nextGenDeploy({ bucket, mutPrefix, gitBranch, hasConfigRedirects }: NextGenDeployParams) { + // const hasConfigRedirects = await existsAsync(path.join(process.cwd(), 'config/redirects')); if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { // mut-redirects config/redirects -o public/.htaccess diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index bebca46af..7ddd91456 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import { promisify } from 'util'; -import { executeCliCommand, getRepoDir } from '../helpers'; +import { executeCliCommand } from '../helpers'; const writeFileAsync = promisify(fs.writeFile); @@ -18,9 +18,7 @@ async function createEnvProdFile(repoDir: string, projectName: string) { throw e; } } -export async function nextGenHtml(repoName: string) { - const repoDir = getRepoDir(repoName); - +export async function nextGenHtml(repoDir: string) { // might move this since technically next-gen-html doesn't create the file await createEnvProdFile(repoDir, 'java'); diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 80f358c84..ca61313a6 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,24 +1,16 @@ -import { - CliCommandResponse, - checkIfPatched, - executeCliCommand, - getCommitHash, - getPatchId, - getRepoDir, -} from '../helpers'; +import { CliCommandResponse, executeCliCommand } from '../helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; -export async function nextGenParse(repoName: string): Promise { - const repoDir = getRepoDir(repoName); - +interface NextGenParseParams { + repoDir: string; + patchId?: string; + commitHash?: string; +} +export async function nextGenParse({ repoDir, patchId, commitHash }: NextGenParseParams): Promise { const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; - const hasPatch = await checkIfPatched(repoDir); - - if (hasPatch) { - const [patchId, commitHash] = await Promise.all([getPatchId(repoDir), getCommitHash(repoDir)]); - + if (patchId && commitHash) { commandArgs.push('--commit'); commandArgs.push(commitHash); diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index b6c46a24f..84bee695b 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,16 +1,14 @@ import { checkIfPatched, executeCliCommand, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '../helpers'; interface StageParams { - repoName: string; + repoDir: string; mutPrefix: string; projectName: string; bucketName: string; url: string; } -export async function nextGenStage({ repoName, mutPrefix, projectName, bucketName, url }: StageParams) { - const repoDir = getRepoDir(repoName); - +export async function nextGenStage({ repoDir, mutPrefix, projectName, bucketName, url }: StageParams) { const [hasPatch, commitBranch] = await Promise.all([checkIfPatched(repoDir), getCommitBranch(repoDir)]); let hostedAtUrl = `${url}/${mutPrefix}/${process.env.USER}/${commitBranch}/`; diff --git a/src/commands/src/shared/oas-page-build.ts b/src/commands/src/shared/oas-page-build.ts index d892b823d..2faf4b21a 100644 --- a/src/commands/src/shared/oas-page-build.ts +++ b/src/commands/src/shared/oas-page-build.ts @@ -7,7 +7,7 @@ interface OasPageBuildParams { } export async function oasPageBuild({ bundlePath, repoDir, siteUrl }: OasPageBuildParams) { - const { stdout } = await executeCliCommand({ + const { outputText } = await executeCliCommand({ command: 'node', args: [ `${process.cwd()}/modules/oas-page-builder/dist/index.js`, @@ -22,5 +22,5 @@ export async function oasPageBuild({ bundlePath, repoDir, siteUrl }: OasPageBuil ], }); - return stdout; + return outputText; } diff --git a/src/commands/src/shared/persistence-module.ts b/src/commands/src/shared/persistence-module.ts index 292bb0ef7..07fc956f6 100644 --- a/src/commands/src/shared/persistence-module.ts +++ b/src/commands/src/shared/persistence-module.ts @@ -10,7 +10,7 @@ export async function persistenceModule({ jobId, repoOwner = 'docs-builder-bot', }: PersistenceModuleParams) { - const { stdout } = await executeCliCommand({ + const { outputText } = await executeCliCommand({ command: 'node', args: [ `${process.cwd()}/modules/persistence/dist/index.js`, @@ -24,5 +24,5 @@ export async function persistenceModule({ ], }); - return stdout; + return outputText; } From c5d668084f9e46ca62b78b642b4cf327d06e5404 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 3 Oct 2023 13:38:37 -0500 Subject: [PATCH 32/51] [DOP-4033]: Refactor localApp --- src/commands/src/helpers/execution-helper.ts | 1 + src/commands/src/shared/next-gen-parse.ts | 4 ++-- src/entrypoints/localApp.ts | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/commands/src/helpers/execution-helper.ts b/src/commands/src/helpers/execution-helper.ts index 058ebdb50..4f78fe317 100644 --- a/src/commands/src/helpers/execution-helper.ts +++ b/src/commands/src/helpers/execution-helper.ts @@ -23,5 +23,6 @@ export async function getCliBuildDependencies(repoName: string) { patchId: deps[3] as string | undefined, hasRedirects: deps[4] as boolean, bundlePath: `${repoDir}/bundle.zip`, + repoDir, }; } diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index ca61313a6..fc1847ec9 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -4,13 +4,13 @@ const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty interface NextGenParseParams { repoDir: string; + commitHash: string; patchId?: string; - commitHash?: string; } export async function nextGenParse({ repoDir, patchId, commitHash }: NextGenParseParams): Promise { const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`, RSTSPEC_FLAG]; - if (patchId && commitHash) { + if (patchId) { commandArgs.push('--commit'); commandArgs.push(commitHash); diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index e70076735..c0941e476 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,11 +1,11 @@ -import path from 'path'; import { executeCliCommand } from '../commands/src/helpers'; import { nextGenParse } from '../commands/src/shared/next-gen-parse'; import { nextGenHtml } from '../commands/src/shared/next-gen-html'; +import { getCliBuildDependencies } from '../commands/src/helpers/execution-helper'; async function localApp() { - const repoDir = path.join(process.cwd(), '/repos'); const repoName = 'docs-landing'; + const { repoDir, commitHash, patchId } = await getCliBuildDependencies(repoName); await executeCliCommand({ command: 'git', @@ -15,7 +15,7 @@ async function localApp() { console.log('Hello'); console.log('Begin snooty build...'); - const snootyBuildRes = await nextGenParse(repoName); + const snootyBuildRes = await nextGenParse({ repoDir, commitHash, patchId }); console.log(snootyBuildRes.errorText); From 17830fb5f1d99d976b3d9946e455ac836583fa35 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 4 Oct 2023 14:26:43 -0500 Subject: [PATCH 33/51] [DOP-4033]: Add stage job --- Dockerfile.local | 13 ++++---- src/commands/src/helpers/execution-helper.ts | 26 ++++++++++++++-- src/commands/src/helpers/index.ts | 13 -------- src/commands/src/shared/next-gen-html.ts | 23 +------------- src/commands/src/shared/next-gen-stage.ts | 28 +++++++++++++---- src/entrypoints/localApp.ts | 32 ++++++++++++++++---- 6 files changed, 78 insertions(+), 57 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index 9c8b6b15b..944e81b29 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -69,11 +69,6 @@ RUN cd ./modules/oas-page-builder \ # Root project build COPY package*.json ./ RUN npm ci --legacy-peer-deps -COPY tsconfig*.json ./ -COPY config config/ -COPY api api/ -COPY src src/ - # Build persistence module COPY --chown=docsworker modules/persistence/tsconfig*.json ./modules/persistence @@ -85,7 +80,6 @@ RUN cd ./modules/persistence \ # Build modules # OAS Page Builder - COPY --chown=docsworker modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder COPY --chown=docsworker modules/oas-page-builder/src ./modules/oas-page-builder/src/ COPY --chown=docsworker modules/oas-page-builder/index.ts ./modules/oas-page-builder @@ -93,10 +87,15 @@ COPY --chown=docsworker modules/oas-page-builder/index.ts ./modules/oas-page-bui RUN cd ./modules/oas-page-builder \ && npm run build +COPY tsconfig*.json ./ +COPY config config/ +COPY api api/ +COPY src src/ + RUN npm run build:esbuild RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "dist/entrypoints/localApp.js"] \ No newline at end of file +CMD ["node", "--enable-source-maps", "dist/entrypoints/localApp.js"] \ No newline at end of file diff --git a/src/commands/src/helpers/execution-helper.ts b/src/commands/src/helpers/execution-helper.ts index 4f78fe317..5341d2879 100644 --- a/src/commands/src/helpers/execution-helper.ts +++ b/src/commands/src/helpers/execution-helper.ts @@ -4,15 +4,36 @@ import { checkIfPatched, getCommitBranch, getCommitHash, getPatchId, getRepoDir import { promisify } from 'util'; const existsAsync = promisify(fs.exists); +const writeFileAsync = promisify(fs.writeFile); -export async function getCliBuildDependencies(repoName: string) { - const repoDir = getRepoDir(repoName); +async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: string, prefix = '') { + const prodFileName = `${process.cwd()}/snooty/.env.production`; + + try { + await writeFileAsync( + prodFileName, + `GATSBY_BASE_URL=docs.mongodb.com + GATSBY_SITE=${projectName} + GATSBY_MANIFEST_PATH=${repoDir}/bundle.zip + GATSBY_PARSER_USER=${process.env.USER} + GATSBY_BASE_URL=${baseUrl} + GATSBY_PATH_PREFIX=${prefix}`, + 'utf8' + ); + } catch (e) { + console.error(`ERROR! Could not write to .env.production`); + throw e; + } +} + +export async function getCliBuildDependencies(repoDir: string, projectName: string, baseUrl: string) { const commandPromises = [ checkIfPatched(repoDir), getCommitHash(repoDir), getCommitBranch(repoDir), getPatchId(repoDir), existsAsync(path.join(process.cwd(), 'config/redirects')), + createEnvProdFile(repoDir, projectName, baseUrl), ]; const deps = await Promise.all(commandPromises); @@ -23,6 +44,5 @@ export async function getCliBuildDependencies(repoName: string) { patchId: deps[3] as string | undefined, hasRedirects: deps[4] as boolean, bundlePath: `${repoDir}/bundle.zip`, - repoDir, }; } diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 349631509..bbd936f08 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -195,19 +195,6 @@ export async function getPatchId(repoDir: string): Promise { } } -export async function addProjectToEnv(project: string) { - return new Promise((resolve, reject) => { - const stream = fs.createWriteStream(path.join(process.cwd(), 'snooty/.env.production'), { flags: 'a+' }); - - stream.write(project); - stream.close(); - stream.on('error', (error) => { - console.log('Error when writing to file!', error); - reject(error); - }); - stream.on('close', resolve); - }); -} export async function getCommitBranch(repoDir: string): Promise { // equivalent to git rev-parse --short HEAD const response = await executeCliCommand({ diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index 7ddd91456..226a14a97 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -1,27 +1,6 @@ -import fs from 'fs'; -import { promisify } from 'util'; import { executeCliCommand } from '../helpers'; -const writeFileAsync = promisify(fs.writeFile); - -async function createEnvProdFile(repoDir: string, projectName: string) { - try { - await writeFileAsync( - `${process.cwd()}/snooty/.env.production`, - `GATSBY_BASE_URL=docs.mongodb.com - GATSBY_SITE=${projectName} - GATSBY_MANIFEST_PATH=${repoDir}/bundle.zip`, - 'utf8' - ); - } catch (e) { - console.error(`ERROR! Could not write to .env.production`); - throw e; - } -} -export async function nextGenHtml(repoDir: string) { - // might move this since technically next-gen-html doesn't create the file - await createEnvProdFile(repoDir, 'java'); - +export async function nextGenHtml() { const result = await executeCliCommand({ command: 'npm', args: ['run', 'build'], diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 84bee695b..ac583454a 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,4 +1,11 @@ -import { checkIfPatched, executeCliCommand, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '../helpers'; +import { + checkIfPatched, + executeAndWriteToFile, + executeCliCommand, + getCommitBranch, + getCommitHash, + getPatchId, +} from '../helpers'; interface StageParams { repoDir: string; @@ -11,17 +18,26 @@ interface StageParams { export async function nextGenStage({ repoDir, mutPrefix, projectName, bucketName, url }: StageParams) { const [hasPatch, commitBranch] = await Promise.all([checkIfPatched(repoDir), getCommitBranch(repoDir)]); - let hostedAtUrl = `${url}/${mutPrefix}/${process.env.USER}/${commitBranch}/`; + let hostedAtUrl = `${url}/${mutPrefix}/docsworker/${commitBranch}/`; + let prefix = mutPrefix; const commandArgs = ['public', bucketName, '--stage']; - if (hasPatch && projectName !== mutPrefix) { + if (hasPatch && projectName === mutPrefix) { const [commitHash, patchId] = await Promise.all([getCommitHash(repoDir), getPatchId(repoDir)]); - commandArgs.push(`--prefix="${commitHash}/${patchId}/${mutPrefix}"`); - hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/${process.env.USER}/${commitBranch}/`; + prefix = `${commitHash}/${patchId}/${mutPrefix}`; + hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/docsworker/${commitBranch}/`; } - const { outputText } = await executeCliCommand({ command: 'mut-publish', args: commandArgs }); + commandArgs.push(`--prefix="${prefix}"`); + + const { outputText } = await executeCliCommand({ + command: 'mut-publish', + args: commandArgs, + options: { + cwd: `${process.cwd()}/snooty`, + }, + }); const resultMessage = `${outputText}\n Hosted at ${hostedAtUrl}`; return resultMessage; } diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index c0941e476..319b5e27a 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,19 +1,25 @@ -import { executeCliCommand } from '../commands/src/helpers'; +import { executeCliCommand, getRepoDir } from '../commands/src/helpers'; import { nextGenParse } from '../commands/src/shared/next-gen-parse'; import { nextGenHtml } from '../commands/src/shared/next-gen-html'; import { getCliBuildDependencies } from '../commands/src/helpers/execution-helper'; +import { nextGenStage } from '../commands/src/shared/next-gen-stage'; +import { oasPageBuild } from '../commands/src/shared/oas-page-build'; async function localApp() { - const repoName = 'docs-landing'; - const { repoDir, commitHash, patchId } = await getCliBuildDependencies(repoName); + const repoName = 'docs-java'; + const projectName = 'java'; + const baseUrl = 'https://www.mongodb.com'; + const repoDir = getRepoDir(repoName); + // mocking out the clone aspect of a job await executeCliCommand({ command: 'git', args: ['clone', `https://github.com/mongodb/${repoName}`], - options: { cwd: repoDir }, + options: { cwd: `${process.cwd()}/repos` }, }); - console.log('Hello'); + const { commitHash, patchId, bundlePath } = await getCliBuildDependencies(repoDir, projectName, baseUrl); + console.log('Begin snooty build...'); const snootyBuildRes = await nextGenParse({ repoDir, commitHash, patchId }); @@ -23,13 +29,27 @@ async function localApp() { console.log('Begin next-gen-html...'); - const nextGenHtmlRes = await nextGenHtml(repoName); + const nextGenHtmlRes = await nextGenHtml(); + console.log('Begin oas-page-build...'); + const mutPrefix = 'docs'; + const siteUrl = mutPrefix ? `${baseUrl}/${mutPrefix}` : `${baseUrl}`; + const oasPageBuildRes = await oasPageBuild({ repoDir, bundlePath, siteUrl }); console.log(nextGenHtmlRes.outputText); console.log('next-gen-html complete'); console.log('Begin next-gen-stage...'); + + const resultMessage = await nextGenStage({ + repoDir, + projectName, + bucketName: 'docs-mongodb-org-stg', + url: baseUrl, + mutPrefix, + }); + console.log(resultMessage); + console.log('Begin next-gen-stage complete'); } localApp(); From d2c6f132bf1cb44bc9ed8f0e1fe0f0a677d950f9 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 5 Oct 2023 12:03:12 -0500 Subject: [PATCH 34/51] [DOP-4033]: Update persistence module --- src/commands/src/shared/next-gen-stage.ts | 4 ++- src/commands/src/shared/oas-page-build.ts | 2 ++ src/commands/src/shared/persistence-module.ts | 27 +++++++++++-------- src/entrypoints/localApp.ts | 17 +++++++++--- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index ac583454a..49828c16a 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -31,13 +31,15 @@ export async function nextGenStage({ repoDir, mutPrefix, projectName, bucketName commandArgs.push(`--prefix="${prefix}"`); - const { outputText } = await executeCliCommand({ + const { outputText, errorText } = await executeCliCommand({ command: 'mut-publish', args: commandArgs, options: { cwd: `${process.cwd()}/snooty`, }, }); + + console.log(errorText); const resultMessage = `${outputText}\n Hosted at ${hostedAtUrl}`; return resultMessage; } diff --git a/src/commands/src/shared/oas-page-build.ts b/src/commands/src/shared/oas-page-build.ts index 2faf4b21a..5950a27f2 100644 --- a/src/commands/src/shared/oas-page-build.ts +++ b/src/commands/src/shared/oas-page-build.ts @@ -13,6 +13,8 @@ export async function oasPageBuild({ bundlePath, repoDir, siteUrl }: OasPageBuil `${process.cwd()}/modules/oas-page-builder/dist/index.js`, '--bundle', bundlePath, + '--repo', + repoDir, '--output', `${repoDir}/public`, '--redoc', diff --git a/src/commands/src/shared/persistence-module.ts b/src/commands/src/shared/persistence-module.ts index 07fc956f6..e1b7851e9 100644 --- a/src/commands/src/shared/persistence-module.ts +++ b/src/commands/src/shared/persistence-module.ts @@ -2,7 +2,7 @@ import { executeCliCommand } from '../helpers'; interface PersistenceModuleParams { bundlePath: string; - jobId: string; + jobId?: string; repoOwner?: string; } export async function persistenceModule({ @@ -10,18 +10,23 @@ export async function persistenceModule({ jobId, repoOwner = 'docs-builder-bot', }: PersistenceModuleParams) { + const args = [ + `${process.cwd()}/modules/persistence/dist/index.js`, + '--unhandled-rejections=strict', + '--path', + bundlePath, + '--githubUser', + repoOwner, + ]; + + if (jobId) { + args.push('--jobId'); + args.push(jobId); + } + const { outputText } = await executeCliCommand({ command: 'node', - args: [ - `${process.cwd()}/modules/persistence/dist/index.js`, - '--unhandled-rejections=strict', - '--path', - bundlePath, - '--githubUser', - repoOwner, - '--jobId', - jobId, - ], + args, }); return outputText; diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index 319b5e27a..d02ec5d0e 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -4,6 +4,7 @@ import { nextGenHtml } from '../commands/src/shared/next-gen-html'; import { getCliBuildDependencies } from '../commands/src/helpers/execution-helper'; import { nextGenStage } from '../commands/src/shared/next-gen-stage'; import { oasPageBuild } from '../commands/src/shared/oas-page-build'; +import { persistenceModule } from '../commands/src/shared/persistence-module'; async function localApp() { const repoName = 'docs-java'; @@ -20,6 +21,8 @@ async function localApp() { const { commitHash, patchId, bundlePath } = await getCliBuildDependencies(repoDir, projectName, baseUrl); + console.log('Hello'); + console.log('Begin snooty build...'); const snootyBuildRes = await nextGenParse({ repoDir, commitHash, patchId }); @@ -27,18 +30,25 @@ async function localApp() { console.log('snooty build complete'); + console.log('Begin persistence-module'); + const persistenceModuleRes = await persistenceModule({ bundlePath }); + console.log(persistenceModuleRes); + console.log('persistence-module complete'); + console.log('Begin next-gen-html...'); const nextGenHtmlRes = await nextGenHtml(); + console.log(nextGenHtmlRes.outputText); + + console.log('next-gen-html complete'); console.log('Begin oas-page-build...'); const mutPrefix = 'docs'; const siteUrl = mutPrefix ? `${baseUrl}/${mutPrefix}` : `${baseUrl}`; const oasPageBuildRes = await oasPageBuild({ repoDir, bundlePath, siteUrl }); - console.log(nextGenHtmlRes.outputText); - - console.log('next-gen-html complete'); + console.log('oas-page-build compelte'); + console.log(oasPageBuildRes); console.log('Begin next-gen-stage...'); const resultMessage = await nextGenStage({ @@ -50,6 +60,7 @@ async function localApp() { }); console.log(resultMessage); console.log('Begin next-gen-stage complete'); + console.log(process.env); } localApp(); From 4827b011f40536c4a0798937cc1cbd51452714b0 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 6 Oct 2023 10:56:29 -0500 Subject: [PATCH 35/51] [DOP-4033]: Add next-gen-deploy --- src/entrypoints/localApp.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index d02ec5d0e..051dfa858 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -60,7 +60,8 @@ async function localApp() { }); console.log(resultMessage); console.log('Begin next-gen-stage complete'); - console.log(process.env); + + console.log('Begin next-gen-deploy'); } localApp(); From 5c44115ffd961e04a4d8786f4c6705baa54dff83 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 6 Oct 2023 12:28:32 -0500 Subject: [PATCH 36/51] [DOP-4033]: Update next-gen-deploy to work --- src/commands/src/helpers/execution-helper.ts | 26 +++++++++----- src/commands/src/helpers/index.ts | 22 ++++++++++-- src/commands/src/shared/next-gen-deploy.ts | 23 ++++++++++-- src/commands/src/shared/next-gen-stage.ts | 29 +++++++-------- src/entrypoints/localApp.ts | 37 ++++++++++++-------- 5 files changed, 96 insertions(+), 41 deletions(-) diff --git a/src/commands/src/helpers/execution-helper.ts b/src/commands/src/helpers/execution-helper.ts index 5341d2879..fd5037b0e 100644 --- a/src/commands/src/helpers/execution-helper.ts +++ b/src/commands/src/helpers/execution-helper.ts @@ -1,11 +1,18 @@ import path from 'path'; import fs from 'fs'; -import { checkIfPatched, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '.'; +import { executeCliCommand, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '.'; import { promisify } from 'util'; const existsAsync = promisify(fs.exists); const writeFileAsync = promisify(fs.writeFile); +async function cloneRepo(repoName: string) { + await executeCliCommand({ + command: 'git', + args: ['clone', `https://github.com/mongodb/${repoName}`], + options: { cwd: `${process.cwd()}/repos` }, + }); +} async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: string, prefix = '') { const prodFileName = `${process.cwd()}/snooty/.env.production`; @@ -26,9 +33,12 @@ async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: } } -export async function getCliBuildDependencies(repoDir: string, projectName: string, baseUrl: string) { +export async function getCliBuildDependencies(repoName: string, projectName: string, baseUrl: string) { + await cloneRepo(repoName); + + const repoDir = getRepoDir(repoName); + const commandPromises = [ - checkIfPatched(repoDir), getCommitHash(repoDir), getCommitBranch(repoDir), getPatchId(repoDir), @@ -38,11 +48,11 @@ export async function getCliBuildDependencies(repoDir: string, projectName: stri const deps = await Promise.all(commandPromises); return { - hasPatch: deps[0] as string, - commitHash: deps[1] as string, - commitBranch: deps[2] as string, - patchId: deps[3] as string | undefined, - hasRedirects: deps[4] as boolean, + commitHash: deps[0] as string, + commitBranch: deps[1] as string, + patchId: deps[2] as string | undefined, + hasRedirects: deps[3] as boolean, bundlePath: `${repoDir}/bundle.zip`, + repoDir, }; } diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index bbd936f08..41076072d 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -40,9 +40,27 @@ export async function executeAndPipeCommands( const cmdTo = spawn(cmdToParams.command, cmdToParams.args || [], cmdToParams.options || {}); cmdFrom.stdout?.on('data', (data: Buffer) => { + if (!cmdTo.stdin?.writable) { + cmdFrom.stdin?.end(); + cmdFrom.kill(); + return; + } + cmdTo.stdin?.write(data); }); + cmdFrom.stdout?.on('error', (err) => { + console.log('error on cmdFrom out', err); + }); + + cmdTo.stdin?.on('finish', () => { + console.log('finished stdin'); + }); + + cmdTo.stdin?.on('error', () => { + console.log('stdin done'); + }); + cmdFrom.on('error', (err) => { reject(new ExecuteCommandError('The first command failed', err)); hasRejected = true; @@ -75,11 +93,11 @@ export async function executeAndPipeCommands( console.error('Options provided: ', cmdToParams.options); if (outputText) { - console.error(outputText.join()); + console.error('output', outputText.join()); } if (errorText) { - console.error(errorText.join()); + console.error('error', errorText.join()); } reject(new ExecuteCommandError('The command failed', exitCode)); diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index d9ba13474..9167f1599 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -5,9 +5,10 @@ interface NextGenDeployParams { mutPrefix: string; gitBranch: string; hasConfigRedirects: boolean; + url: string; } -export async function nextGenDeploy({ bucket, mutPrefix, gitBranch, hasConfigRedirects }: NextGenDeployParams) { +export async function nextGenDeploy({ bucket, mutPrefix, gitBranch, hasConfigRedirects, url }: NextGenDeployParams) { // const hasConfigRedirects = await existsAsync(path.join(process.cwd(), 'config/redirects')); if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { @@ -16,8 +17,24 @@ export async function nextGenDeploy({ bucket, mutPrefix, gitBranch, hasConfigRed } // yes | mut-publish public ${BUCKET} --prefix="${MUT_PREFIX}" --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; - await executeAndPipeCommands( + const { outputText } = await executeAndPipeCommands( { command: 'yes' }, - { command: 'mut-publish', args: ['public', bucket, `--prefix=${mutPrefix}`, '--json', '--all-subdirectories'] } + { + command: 'mut-publish', + args: [ + 'public', + bucket, + `--prefix=${mutPrefix}`, + '--deploy', + `--deployed-url-prefix=${url}`, + '--json', + '--all-subdirectories', + ], + options: { + cwd: `${process.cwd()}/snooty`, + }, + } ); + + return `${outputText}\n Hosted at ${url}/${mutPrefix}`; } diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 49828c16a..64fbd686c 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,29 +1,30 @@ -import { - checkIfPatched, - executeAndWriteToFile, - executeCliCommand, - getCommitBranch, - getCommitHash, - getPatchId, -} from '../helpers'; +import { executeCliCommand, getCommitHash, getPatchId } from '../helpers'; interface StageParams { repoDir: string; mutPrefix: string; projectName: string; - bucketName: string; + bucket: string; url: string; + patchId?: string; + commitBranch: string; } -export async function nextGenStage({ repoDir, mutPrefix, projectName, bucketName, url }: StageParams) { - const [hasPatch, commitBranch] = await Promise.all([checkIfPatched(repoDir), getCommitBranch(repoDir)]); - +export async function nextGenStage({ + repoDir, + mutPrefix, + projectName, + bucket, + url, + patchId, + commitBranch, +}: StageParams) { let hostedAtUrl = `${url}/${mutPrefix}/docsworker/${commitBranch}/`; let prefix = mutPrefix; - const commandArgs = ['public', bucketName, '--stage']; + const commandArgs = ['public', bucket, '--stage']; - if (hasPatch && projectName === mutPrefix) { + if (patchId && projectName === mutPrefix) { const [commitHash, patchId] = await Promise.all([getCommitHash(repoDir), getPatchId(repoDir)]); prefix = `${commitHash}/${patchId}/${mutPrefix}`; hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/docsworker/${commitBranch}/`; diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index 051dfa858..89cfdcbd8 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,25 +1,24 @@ -import { executeCliCommand, getRepoDir } from '../commands/src/helpers'; import { nextGenParse } from '../commands/src/shared/next-gen-parse'; import { nextGenHtml } from '../commands/src/shared/next-gen-html'; import { getCliBuildDependencies } from '../commands/src/helpers/execution-helper'; import { nextGenStage } from '../commands/src/shared/next-gen-stage'; import { oasPageBuild } from '../commands/src/shared/oas-page-build'; import { persistenceModule } from '../commands/src/shared/persistence-module'; +import { nextGenDeploy } from '../commands/src/shared/next-gen-deploy'; async function localApp() { + // TODO: Fetch this from repos_branches const repoName = 'docs-java'; const projectName = 'java'; const baseUrl = 'https://www.mongodb.com'; - const repoDir = getRepoDir(repoName); + const bucket = 'docs-java-dotcomstg'; + const mutPrefix = 'docs/drivers/java/sync'; - // mocking out the clone aspect of a job - await executeCliCommand({ - command: 'git', - args: ['clone', `https://github.com/mongodb/${repoName}`], - options: { cwd: `${process.cwd()}/repos` }, - }); - - const { commitHash, patchId, bundlePath } = await getCliBuildDependencies(repoDir, projectName, baseUrl); + const { commitHash, patchId, bundlePath, commitBranch, hasRedirects, repoDir } = await getCliBuildDependencies( + repoName, + projectName, + baseUrl + ); console.log('Hello'); @@ -43,7 +42,6 @@ async function localApp() { console.log('next-gen-html complete'); console.log('Begin oas-page-build...'); - const mutPrefix = 'docs'; const siteUrl = mutPrefix ? `${baseUrl}/${mutPrefix}` : `${baseUrl}`; const oasPageBuildRes = await oasPageBuild({ repoDir, bundlePath, siteUrl }); console.log('oas-page-build compelte'); @@ -52,16 +50,27 @@ async function localApp() { console.log('Begin next-gen-stage...'); const resultMessage = await nextGenStage({ + patchId, + commitBranch, repoDir, projectName, - bucketName: 'docs-mongodb-org-stg', + bucket, url: baseUrl, mutPrefix, }); console.log(resultMessage); - console.log('Begin next-gen-stage complete'); + console.log('next-gen-stage complete'); - console.log('Begin next-gen-deploy'); + console.log('Begin next-gen-deploy...'); + const deployRes = await nextGenDeploy({ + bucket, + hasConfigRedirects: hasRedirects, + gitBranch: commitBranch, + mutPrefix, + url: baseUrl, + }); + console.log(deployRes); + console.log('next-gen-deploy complete'); } localApp(); From 42753cfa9b2ca99be03b6ebb8a63acf4667e9018 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 11 Oct 2023 08:21:44 -0500 Subject: [PATCH 37/51] [DOP-4033]: Add comments --- config/default.json | 1 - .../{execution-helper.ts => dependency-helpers.ts} | 14 +++++++++----- src/commands/src/helpers/index.ts | 14 ++++++++++++++ src/commands/src/shared/next-gen-deploy.ts | 6 ++---- src/entrypoints/localApp.ts | 4 +--- 5 files changed, 26 insertions(+), 13 deletions(-) rename src/commands/src/helpers/{execution-helper.ts => dependency-helpers.ts} (81%) diff --git a/config/default.json b/config/default.json index 64539c7ee..9b0a73341 100644 --- a/config/default.json +++ b/config/default.json @@ -33,7 +33,6 @@ "featureFlagUpdatePages": "false", "featureFlagSearchUI": "false", "gatsbyUseChatbot": "false", - "gatsbyHideUnifiedFooterLocale": "true", "parallel": { "enabled": true, "stg": { diff --git a/src/commands/src/helpers/execution-helper.ts b/src/commands/src/helpers/dependency-helpers.ts similarity index 81% rename from src/commands/src/helpers/execution-helper.ts rename to src/commands/src/helpers/dependency-helpers.ts index fd5037b0e..86dcef4da 100644 --- a/src/commands/src/helpers/execution-helper.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -34,10 +34,13 @@ async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: } export async function getCliBuildDependencies(repoName: string, projectName: string, baseUrl: string) { + // before we get build dependencies, we need to clone + // the repo await cloneRepo(repoName); const repoDir = getRepoDir(repoName); + // doing these in parallel const commandPromises = [ getCommitHash(repoDir), getCommitBranch(repoDir), @@ -46,12 +49,13 @@ export async function getCliBuildDependencies(repoName: string, projectName: str createEnvProdFile(repoDir, projectName, baseUrl), ]; - const deps = await Promise.all(commandPromises); + const dependencies = await Promise.all(commandPromises); + return { - commitHash: deps[0] as string, - commitBranch: deps[1] as string, - patchId: deps[2] as string | undefined, - hasRedirects: deps[3] as boolean, + commitHash: dependencies[0] as string, + commitBranch: dependencies[1] as string, + patchId: dependencies[2] as string | undefined, + hasRedirects: dependencies[3] as boolean, bundlePath: `${repoDir}/bundle.zip`, repoDir, }; diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 41076072d..1e8c97122 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -111,6 +111,20 @@ export async function executeAndPipeCommands( }); }); } + +/** + * A promisified way to execute CLI commands. This approach uses spawn instead of exec, which + * is a safer way of executing CLI commands. Also, spawn allows us to stream output in real-time. + * @param {string} command: The CLI command we want to execute + * @param {string[] | undefined} args: Arguments we want to provide to the command + * @param {SpawnOptions | undefined} options: Options to configure the spawn process + * @param {fs.WriteStream | undefined} writeStream: A writable stream object to pipe output to. + * For example, we can mimic ls >> directory.txt by creating a WriteStream object to write to + * directory.txt, and then provide the WriteStream so that we can pipe the output from the ls + * command to the WriteStream. + * @returns {Promise} An object containing the CLI output from stdout and stderr. + * stdout is the outputText property, and stderr is the errorText property. + */ export async function executeCliCommand({ command, args = [], diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 9167f1599..d980e1d1f 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -9,14 +9,12 @@ interface NextGenDeployParams { } export async function nextGenDeploy({ bucket, mutPrefix, gitBranch, hasConfigRedirects, url }: NextGenDeployParams) { - // const hasConfigRedirects = await existsAsync(path.join(process.cwd(), 'config/redirects')); - if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { - // mut-redirects config/redirects -o public/.htaccess + // equivalent to: mut-redirects config/redirects -o public/.htaccess await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); } - // yes | mut-publish public ${BUCKET} --prefix="${MUT_PREFIX}" --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; + // equivalent to: yes | mut-publish public ${BUCKET} --prefix="${MUT_PREFIX}" --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; const { outputText } = await executeAndPipeCommands( { command: 'yes' }, { diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index 89cfdcbd8..daac509f9 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,6 +1,6 @@ import { nextGenParse } from '../commands/src/shared/next-gen-parse'; import { nextGenHtml } from '../commands/src/shared/next-gen-html'; -import { getCliBuildDependencies } from '../commands/src/helpers/execution-helper'; +import { getCliBuildDependencies } from '../commands/src/helpers/dependency-helpers'; import { nextGenStage } from '../commands/src/shared/next-gen-stage'; import { oasPageBuild } from '../commands/src/shared/oas-page-build'; import { persistenceModule } from '../commands/src/shared/persistence-module'; @@ -20,8 +20,6 @@ async function localApp() { baseUrl ); - console.log('Hello'); - console.log('Begin snooty build...'); const snootyBuildRes = await nextGenParse({ repoDir, commitHash, patchId }); From 6749b661f923677925a3ba16134468c4bd82f6d3 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 11 Oct 2023 12:22:38 -0500 Subject: [PATCH 38/51] [DOP-4033]: Add more comments --- .../src/helpers/dependency-helpers.ts | 5 +- src/commands/src/helpers/index.ts | 69 +++++++++++++------ src/entrypoints/localApp.ts | 4 +- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index 86dcef4da..c5043476b 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -33,9 +33,8 @@ async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: } } -export async function getCliBuildDependencies(repoName: string, projectName: string, baseUrl: string) { - // before we get build dependencies, we need to clone - // the repo +export async function prepareBuildAndGetDependencies(repoName: string, projectName: string, baseUrl: string) { + // before we get build dependencies, we need to clone the repo await cloneRepo(repoName); const repoDir = getRepoDir(repoName); diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 1e8c97122..d0bf0b4fd 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -8,6 +8,10 @@ const openAsync = promisify(fs.open); const closeAsync = promisify(fs.close); const existsAsync = promisify(fs.exists); +const EPIPE_CODE = 'EPIPE'; +const EPIPE_ERRNO = -32; +const EPIPE_SYSCALL = 'write'; + export class ExecuteCommandError extends Error { data: unknown; constructor(message: string, data: unknown) { @@ -29,6 +33,17 @@ export interface CliCommandResponse { errorText: string; } +interface StdinError { + errno: number; + code: string; + syscall: string; +} +/** + * Method to replicate piping output from one command to another e.g. `yes | mut-publish public` + * @param {CliCommandParams} cmdFromParams The command we want to pipe output from to another command + * @param {CliCommandParams} cmdToParams The command that receives input from another command + * @returns {CliCommandResponse} The `CliCommandResponse` from the cmdTo command + */ export async function executeAndPipeCommands( cmdFromParams: CliCommandParams, cmdToParams: CliCommandParams @@ -40,29 +55,41 @@ export async function executeAndPipeCommands( const cmdTo = spawn(cmdToParams.command, cmdToParams.args || [], cmdToParams.options || {}); cmdFrom.stdout?.on('data', (data: Buffer) => { + // For some commands, the command that is being written to + // will end before the first command finishes. In some cases, + // we do want this to happen. For example, the cli command `yes` will + // infinitely output yes to the terminal as a way of automatically responding + // to prompts from the subsequent command. Once the second command completes, + // we don't want `yes` to continue to run, so we kill the command. if (!cmdTo.stdin?.writable) { - cmdFrom.stdin?.end(); cmdFrom.kill(); return; } + // this is where we pipe data from the first command to the second command. cmdTo.stdin?.write(data); }); - cmdFrom.stdout?.on('error', (err) => { - console.log('error on cmdFrom out', err); - }); + cmdTo.stdin?.on('error', (err: StdinError) => { + // the error event for the cmdTo stdin gets called whenever it closes prematurely, + // but this is expected in certain situations e.g. when using the `yes` command. + // If this condition is met, we know that this expected, and ignore it otherwise we throw. + // If we don't check, we get an unhandled error exception. + if (err.code === EPIPE_CODE && err.syscall === EPIPE_SYSCALL && err.errno === EPIPE_ERRNO) { + console.log('stdin done'); + return; + } - cmdTo.stdin?.on('finish', () => { - console.log('finished stdin'); + reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err)); + hasRejected = true; }); - cmdTo.stdin?.on('error', () => { - console.log('stdin done'); + cmdFrom.stdout?.on('error', (err) => { + console.log('error on cmdFrom out', err); }); cmdFrom.on('error', (err) => { - reject(new ExecuteCommandError('The first command failed', err)); + reject(new ExecuteCommandError('The first command (cmdTo) failed', err)); hasRejected = true; }); @@ -100,7 +127,7 @@ export async function executeAndPipeCommands( console.error('error', errorText.join()); } - reject(new ExecuteCommandError('The command failed', exitCode)); + reject(new ExecuteCommandError('The command failed', { exitCode, outputText, errorText })); return; } @@ -114,16 +141,16 @@ export async function executeAndPipeCommands( /** * A promisified way to execute CLI commands. This approach uses spawn instead of exec, which - * is a safer way of executing CLI commands. Also, spawn allows us to stream output in real-time. - * @param {string} command: The CLI command we want to execute - * @param {string[] | undefined} args: Arguments we want to provide to the command - * @param {SpawnOptions | undefined} options: Options to configure the spawn process - * @param {fs.WriteStream | undefined} writeStream: A writable stream object to pipe output to. - * For example, we can mimic ls >> directory.txt by creating a WriteStream object to write to - * directory.txt, and then provide the WriteStream so that we can pipe the output from the ls - * command to the WriteStream. - * @returns {Promise} An object containing the CLI output from stdout and stderr. - * stdout is the outputText property, and stderr is the errorText property. + * is a safer way of executing CLI commands. Also, spawn allows us to stream input and output in real-time. + * @param {string} command The CLI command we want to execute + * @param {string[] | undefined} args Arguments we want to provide to the command + * @param {SpawnOptions | undefined} options Options to configure the spawn function + * @param {fs.WriteStream | undefined} writeStream A writable stream object to pipe output to. + * For example, we can `mimic ls >> directory.txt` by creating a `WriteStream` object to write to + * `directory.txt`, and then provide the `WriteStream` so that we can pipe the output from the `ls` + * command to the `WriteStream`. + * @returns {Promise} An object containing the CLI output from `stdout` and `stderr`. + * stdout is the `outputText` property, and `stderr` is the `errorText` property. */ export async function executeCliCommand({ command, @@ -228,7 +255,7 @@ export async function getPatchId(repoDir: string): Promise { } export async function getCommitBranch(repoDir: string): Promise { - // equivalent to git rev-parse --short HEAD + // equivalent to git rev-parse --abbrev-ref HEAD const response = await executeCliCommand({ command: 'git', args: ['rev-parse', '--abbrev-ref', 'HEAD'], diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index daac509f9..a22205334 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,6 +1,6 @@ import { nextGenParse } from '../commands/src/shared/next-gen-parse'; import { nextGenHtml } from '../commands/src/shared/next-gen-html'; -import { getCliBuildDependencies } from '../commands/src/helpers/dependency-helpers'; +import { prepareBuildAndGetDependencies } from '../commands/src/helpers/dependency-helpers'; import { nextGenStage } from '../commands/src/shared/next-gen-stage'; import { oasPageBuild } from '../commands/src/shared/oas-page-build'; import { persistenceModule } from '../commands/src/shared/persistence-module'; @@ -14,7 +14,7 @@ async function localApp() { const bucket = 'docs-java-dotcomstg'; const mutPrefix = 'docs/drivers/java/sync'; - const { commitHash, patchId, bundlePath, commitBranch, hasRedirects, repoDir } = await getCliBuildDependencies( + const { commitHash, patchId, bundlePath, commitBranch, hasRedirects, repoDir } = await prepareBuildAndGetDependencies( repoName, projectName, baseUrl From dc492ac1d21a80719ceb3b9600836dcc04f42a0c Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 12 Oct 2023 09:21:14 -0500 Subject: [PATCH 39/51] [DOP-4033]: Remove index.ts --- src/commands/index.ts | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 src/commands/index.ts diff --git a/src/commands/index.ts b/src/commands/index.ts deleted file mode 100644 index d0a02b903..000000000 --- a/src/commands/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { nextGenParse } from './src/shared/next-gen-parse'; - -export { nextGenParse }; From 3adf6337e5a26faa937c020a8ccf10661b6d4a51 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 12 Oct 2023 09:23:07 -0500 Subject: [PATCH 40/51] [DOP-4033]: Refactor to use index file --- src/commands/index.ts | 17 +++++++++++++++++ src/entrypoints/localApp.ts | 16 +++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 src/commands/index.ts diff --git a/src/commands/index.ts b/src/commands/index.ts new file mode 100644 index 000000000..77b25f7a6 --- /dev/null +++ b/src/commands/index.ts @@ -0,0 +1,17 @@ +import { prepareBuildAndGetDependencies } from './src/helpers/dependency-helpers'; +import { nextGenDeploy } from './src/shared/next-gen-deploy'; +import { nextGenHtml } from './src/shared/next-gen-html'; +import { nextGenParse } from './src/shared/next-gen-parse'; +import { nextGenStage } from './src/shared/next-gen-stage'; +import { oasPageBuild } from './src/shared/oas-page-build'; +import { persistenceModule } from './src/shared/persistence-module'; + +export { + nextGenParse, + nextGenHtml, + nextGenStage, + persistenceModule, + oasPageBuild, + nextGenDeploy, + prepareBuildAndGetDependencies, +}; diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index a22205334..a5dddb5b2 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -1,10 +1,12 @@ -import { nextGenParse } from '../commands/src/shared/next-gen-parse'; -import { nextGenHtml } from '../commands/src/shared/next-gen-html'; -import { prepareBuildAndGetDependencies } from '../commands/src/helpers/dependency-helpers'; -import { nextGenStage } from '../commands/src/shared/next-gen-stage'; -import { oasPageBuild } from '../commands/src/shared/oas-page-build'; -import { persistenceModule } from '../commands/src/shared/persistence-module'; -import { nextGenDeploy } from '../commands/src/shared/next-gen-deploy'; +import { + nextGenDeploy, + nextGenHtml, + nextGenParse, + nextGenStage, + oasPageBuild, + persistenceModule, + prepareBuildAndGetDependencies, +} from '../commands'; async function localApp() { // TODO: Fetch this from repos_branches From b2bc784bd682bc612767043b2239d4817d71b36a Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 16 Oct 2023 09:44:14 -0500 Subject: [PATCH 41/51] [DOP-4033]: Use correct values for nextgenstage --- src/commands/src/shared/next-gen-stage.ts | 4 ++-- src/entrypoints/localApp.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 64fbd686c..d0f67db75 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -8,16 +8,17 @@ interface StageParams { url: string; patchId?: string; commitBranch: string; + commitHash: string; } export async function nextGenStage({ - repoDir, mutPrefix, projectName, bucket, url, patchId, commitBranch, + commitHash, }: StageParams) { let hostedAtUrl = `${url}/${mutPrefix}/docsworker/${commitBranch}/`; let prefix = mutPrefix; @@ -25,7 +26,6 @@ export async function nextGenStage({ const commandArgs = ['public', bucket, '--stage']; if (patchId && projectName === mutPrefix) { - const [commitHash, patchId] = await Promise.all([getCommitHash(repoDir), getPatchId(repoDir)]); prefix = `${commitHash}/${patchId}/${mutPrefix}`; hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/docsworker/${commitBranch}/`; } diff --git a/src/entrypoints/localApp.ts b/src/entrypoints/localApp.ts index a5dddb5b2..46e32e530 100644 --- a/src/entrypoints/localApp.ts +++ b/src/entrypoints/localApp.ts @@ -57,6 +57,7 @@ async function localApp() { bucket, url: baseUrl, mutPrefix, + commitHash, }); console.log(resultMessage); console.log('next-gen-stage complete'); From bdacf2dfa27e7431c9fda80a68507e7101a23783 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 16 Oct 2023 10:16:47 -0500 Subject: [PATCH 42/51] [DOP-4033]: Respond to review comments --- src/commands/src/helpers/dependency-helpers.ts | 3 +-- src/commands/src/helpers/index.ts | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index c5043476b..cb2180bf5 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -19,8 +19,7 @@ async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: try { await writeFileAsync( prodFileName, - `GATSBY_BASE_URL=docs.mongodb.com - GATSBY_SITE=${projectName} + `GATSBY_SITE=${projectName} GATSBY_MANIFEST_PATH=${repoDir}/bundle.zip GATSBY_PARSER_USER=${process.env.USER} GATSBY_BASE_URL=${baseUrl} diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index d0bf0b4fd..cf65b8aac 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -278,6 +278,3 @@ export async function getCommitHash(repoDir: string): Promise { export const checkIfPatched = async (repoDir: string) => !existsAsync(path.join(repoDir, 'myPatch.patch')); export const getRepoDir = (repoName: string) => path.join(process.cwd(), `repos/${repoName}`); - -export const RSTSPEC_FLAG = - '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; From 1f0328bbcd8193446fb2f7aed3dc7b83999c190f Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 16 Oct 2023 10:37:07 -0500 Subject: [PATCH 43/51] [DOP-4033]: Use PATH_PREFIX --- src/commands/src/helpers/dependency-helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index cb2180bf5..c733d186a 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -23,7 +23,7 @@ async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: GATSBY_MANIFEST_PATH=${repoDir}/bundle.zip GATSBY_PARSER_USER=${process.env.USER} GATSBY_BASE_URL=${baseUrl} - GATSBY_PATH_PREFIX=${prefix}`, + PATH_PREFIX=${prefix}`, 'utf8' ); } catch (e) { From f5497f9272cc7edf652b48a7609900cc4cebdb15 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 16 Oct 2023 10:37:27 -0500 Subject: [PATCH 44/51] [DOP-4033]: Remove param0 --- src/commands/src/helpers/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index cf65b8aac..a18f7d894 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -214,7 +214,6 @@ export interface ExecuteIOCommandParams { /** * This function is equivalent to a double redirect * e.g. echo "Hello!" >> hello.txt - * @param param0 */ export async function executeAndWriteToFile({ command, filePath, args }: ExecuteIOCommandParams) { const writeStream = fs.createWriteStream(filePath, { From fbb025697d2dafce5896aee9d5ef1cc2101e7fe6 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 17 Oct 2023 08:40:49 -0500 Subject: [PATCH 45/51] [DOP-4033]: Respond to review feedback --- src/commands/src/shared/next-gen-stage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index d0f67db75..758d3a482 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,4 +1,4 @@ -import { executeCliCommand, getCommitHash, getPatchId } from '../helpers'; +import { executeCliCommand } from '../helpers'; interface StageParams { repoDir: string; From 80f4e2fcf0b790f0c6acf71b594b461b5b9352f8 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 17 Oct 2023 08:46:08 -0500 Subject: [PATCH 46/51] [DOP-4033]: Respond to review feedback --- src/commands/src/shared/next-gen-stage.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 758d3a482..464f2cc1a 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,5 +1,6 @@ import { executeCliCommand } from '../helpers'; +const DOCS_WORKER_USER = 'docsworker-xlarge'; interface StageParams { repoDir: string; mutPrefix: string; @@ -20,14 +21,14 @@ export async function nextGenStage({ commitBranch, commitHash, }: StageParams) { - let hostedAtUrl = `${url}/${mutPrefix}/docsworker/${commitBranch}/`; + let hostedAtUrl = `${url}/${mutPrefix}/${DOCS_WORKER_USER}/${commitBranch}/`; let prefix = mutPrefix; const commandArgs = ['public', bucket, '--stage']; if (patchId && projectName === mutPrefix) { prefix = `${commitHash}/${patchId}/${mutPrefix}`; - hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/docsworker/${commitBranch}/`; + hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/${DOCS_WORKER_USER}/${commitBranch}/`; } commandArgs.push(`--prefix="${prefix}"`); From a06ebe1225ed8bd490bfd8a14dba295e1c7ffe72 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 17 Oct 2023 08:58:11 -0500 Subject: [PATCH 47/51] [DOP-4033]: Remove extra log --- src/commands/src/shared/next-gen-stage.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 464f2cc1a..ff1acffdf 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -33,7 +33,7 @@ export async function nextGenStage({ commandArgs.push(`--prefix="${prefix}"`); - const { outputText, errorText } = await executeCliCommand({ + const { outputText } = await executeCliCommand({ command: 'mut-publish', args: commandArgs, options: { @@ -41,7 +41,6 @@ export async function nextGenStage({ }, }); - console.log(errorText); const resultMessage = `${outputText}\n Hosted at ${hostedAtUrl}`; return resultMessage; } From 99ad2e18fbd25ce927286d2c009099f600176a88 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 17 Oct 2023 08:59:31 -0500 Subject: [PATCH 48/51] [DOP-4033]: Add empty string to join to remove commas --- src/commands/src/helpers/index.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index a18f7d894..b4b9f901e 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -120,11 +120,11 @@ export async function executeAndPipeCommands( console.error('Options provided: ', cmdToParams.options); if (outputText) { - console.error('output', outputText.join()); + console.error('output', outputText.join('')); } if (errorText) { - console.error('error', errorText.join()); + console.error('error', errorText.join('')); } reject(new ExecuteCommandError('The command failed', { exitCode, outputText, errorText })); @@ -132,8 +132,8 @@ export async function executeAndPipeCommands( } resolve({ - outputText: outputText.join(), - errorText: errorText.join(), + outputText: outputText.join(''), + errorText: errorText.join(''), }); }); }); @@ -186,11 +186,11 @@ export async function executeCliCommand({ console.error('Options provided: ', options); if (outputText) { - console.error(outputText.join()); + console.error(outputText.join('')); } if (errorText) { - console.error(errorText.join()); + console.error(errorText.join('')); } reject(new ExecuteCommandError('The command failed', exitCode)); @@ -198,8 +198,8 @@ export async function executeCliCommand({ } resolve({ - outputText: outputText.join(), - errorText: errorText.join(), + outputText: outputText.join(''), + errorText: errorText.join(''), }); }); }); From 079fe47eb3eaa60d7c8145a698d26aaf9a7bd2fe Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 17 Oct 2023 14:03:59 -0500 Subject: [PATCH 49/51] [DOP-4033]: Use console.log --- src/commands/src/helpers/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index b4b9f901e..8682e1110 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -186,7 +186,7 @@ export async function executeCliCommand({ console.error('Options provided: ', options); if (outputText) { - console.error(outputText.join('')); + console.log(outputText.join('')); } if (errorText) { From 0d63308909f3995ec50c712f9e0b0651d0f8b4e0 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 19 Oct 2023 10:17:30 -0500 Subject: [PATCH 50/51] [DOP-4033]: Check array length --- src/commands/src/helpers/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index 8682e1110..4089b2178 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -73,7 +73,7 @@ export async function executeAndPipeCommands( cmdTo.stdin?.on('error', (err: StdinError) => { // the error event for the cmdTo stdin gets called whenever it closes prematurely, // but this is expected in certain situations e.g. when using the `yes` command. - // If this condition is met, we know that this expected, and ignore it otherwise we throw. + // If this condition is met, we know that this expected and ignore it otherwise we throw. // If we don't check, we get an unhandled error exception. if (err.code === EPIPE_CODE && err.syscall === EPIPE_SYSCALL && err.errno === EPIPE_ERRNO) { console.log('stdin done'); @@ -119,11 +119,11 @@ export async function executeAndPipeCommands( console.error('Arguments provided: ', cmdToParams.args); console.error('Options provided: ', cmdToParams.options); - if (outputText) { - console.error('output', outputText.join('')); + if (outputText.length) { + console.log('output', outputText.join('')); } - if (errorText) { + if (errorText.length) { console.error('error', errorText.join('')); } @@ -185,11 +185,11 @@ export async function executeCliCommand({ console.error('Arguments provided: ', args); console.error('Options provided: ', options); - if (outputText) { + if (outputText.length) { console.log(outputText.join('')); } - if (errorText) { + if (errorText.length) { console.error(errorText.join('')); } From 234c843e8a84324e60c3232a1b2e328870208388 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 19 Oct 2023 10:24:32 -0500 Subject: [PATCH 51/51] [DOP-4033]: Add error handling for dependency helper --- .../src/helpers/dependency-helpers.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index c733d186a..fce60dbf1 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -47,14 +47,19 @@ export async function prepareBuildAndGetDependencies(repoName: string, projectNa createEnvProdFile(repoDir, projectName, baseUrl), ]; - const dependencies = await Promise.all(commandPromises); - - return { - commitHash: dependencies[0] as string, - commitBranch: dependencies[1] as string, - patchId: dependencies[2] as string | undefined, - hasRedirects: dependencies[3] as boolean, - bundlePath: `${repoDir}/bundle.zip`, - repoDir, - }; + try { + const dependencies = await Promise.all(commandPromises); + + return { + commitHash: dependencies[0] as string, + commitBranch: dependencies[1] as string, + patchId: dependencies[2] as string | undefined, + hasRedirects: dependencies[3] as boolean, + bundlePath: `${repoDir}/bundle.zip`, + repoDir, + }; + } catch (error) { + console.error('ERROR! Could not get build dependencies'); + throw error; + } }