From c0d653bbd0198465afc74e961e9c5bdb7027b86a Mon Sep 17 00:00:00 2001 From: Anton Vikulov Date: Sun, 29 Oct 2023 21:34:09 +0500 Subject: [PATCH 1/2] add worker pool --- scripts/build.cli.js | 2 +- src/cmd/build/index.ts | 41 ++- src/constants.ts | 5 +- src/resolvers/lintPage.ts | 6 +- src/resolvers/processPage.ts | 250 ++++++++++++++ src/services/leading.ts | 8 +- src/services/preset.ts | 16 + src/services/tocs.ts | 32 +- src/steps/processLinter.ts | 104 ++---- src/steps/processPages.ts | 315 +++--------------- src/steps/processPool.ts | 102 ++++++ src/utils/file.ts | 17 +- src/utils/worker.ts | 14 + src/utils/workers.ts | 5 + src/vcs-connector/connector-models.ts | 12 + src/vcs-connector/github.ts | 89 ++--- src/vcs-connector/index.ts | 2 +- src/workers/linter/index.ts | 58 ---- src/workers/mainBridge.ts | 82 +++++ src/workers/pool/index.ts | 130 ++++++++ .../services/metadataAuthors.test.ts | 15 +- .../services/metadataContributors.test.ts | 11 + tests/units/services/authors.test.ts | 11 + tests/units/services/metadata.test.ts | 11 + 24 files changed, 859 insertions(+), 479 deletions(-) create mode 100644 src/resolvers/processPage.ts create mode 100644 src/steps/processPool.ts create mode 100644 src/utils/workers.ts delete mode 100644 src/workers/linter/index.ts create mode 100644 src/workers/mainBridge.ts create mode 100644 src/workers/pool/index.ts diff --git a/scripts/build.cli.js b/scripts/build.cli.js index 06beb979..be03ac20 100644 --- a/scripts/build.cli.js +++ b/scripts/build.cli.js @@ -23,7 +23,7 @@ const commonConfig = { const builds = [ [['src/index.ts'], 'build/index.js'], - [['src/workers/linter/index.ts'], 'build/linter.js'], + [['src/workers/pool/index.ts'], 'build/pool.js'], ]; Promise.all(builds.map(([entries, outfile]) => { diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index 6c82e9a2..d3a6a7a7 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -13,13 +13,13 @@ import {join, resolve} from 'path'; import {ArgvService, Includers} from '../../services'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; import { - initLinterWorkers, processAssets, processExcludedFiles, processLinter, processLogs, processPages, processServiceFiles, + saveSinglePages, } from '../../steps'; import {prepareMapFile} from '../../steps/processMapFile'; import shell from 'shelljs'; @@ -27,6 +27,13 @@ import {Resources} from '../../models'; import {copyFiles, logger} from '../../utils'; import {upload as publishFilesToS3} from '../publish/upload'; import glob from 'glob'; +import {createVCSConnector} from '../../vcs-connector'; +import { + finishProcessPool, + getPoolEnv, + initProcessPool, + terminateProcessPool, +} from '../../steps/processPool'; export const build = { command: ['build', '$0'], @@ -197,6 +204,7 @@ async function handler(args: Arguments) { addMapFile, allowCustomResources, resources, + singlePage, } = ArgvService.getConfig(); preparingTemporaryFolders(userOutputFolder); @@ -213,19 +221,29 @@ async function handler(args: Arguments) { const pathToRedirects = join(args.input, REDIRECTS_FILENAME); const pathToLintConfig = join(args.input, LINT_CONFIG_FILENAME); - if (!lintDisabled) { - /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ - await initLinterWorkers(); + const vcsConnector = createVCSConnector(); + if (vcsConnector) { + await vcsConnector.init(); } - const processes = [ - !lintDisabled && processLinter(), - !buildDisabled && processPages(outputBundlePath), - ].filter(Boolean) as Promise[]; + await initProcessPool({vcsConnector}); - await Promise.all(processes); + try { + await Promise.all([ + !lintDisabled && processLinter(), + !buildDisabled && processPages(vcsConnector), + ]); + await finishProcessPool(); + } finally { + await terminateProcessPool(); + } if (!buildDisabled) { + if (singlePage) { + const {singlePageResults} = getPoolEnv(); + await saveSinglePages(outputBundlePath, singlePageResults); + } + // process additional files switch (outputFormat) { case 'html': @@ -287,7 +305,7 @@ async function handler(args: Arguments) { } } } catch (err) { - logger.error('', err.message); + logger.error('', (err as Error).message); } finally { processLogs(tmpInputFolder); @@ -303,7 +321,6 @@ function preparingTemporaryFolders(userOutputFolder: string) { // Create temporary input/output folders shell.rm('-rf', args.input, args.output); shell.mkdir(args.input, args.output); - shell.chmod('-R', 'u+w', args.input); copyFiles( args.rootInput, @@ -315,4 +332,6 @@ function preparingTemporaryFolders(userOutputFolder: string) { ignore: ['node_modules/**', '*/node_modules/**'], }), ); + + shell.chmod('-R', 'u+w', args.input); } diff --git a/src/constants.ts b/src/constants.ts index 97dce0c1..5ddc7fec 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -116,7 +116,8 @@ export const REGEXP_INCLUDE_FILE_PATH = /(?<=[(]).+(?=[)])/g; // Regexp result: authorLogin export const REGEXP_AUTHOR = /(?<=author:\s).+(?=\r?\n)/g; -export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 1000; -export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || os.cpus().length - 1; +export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 10; +export const THREAD_PART_COUNT = Number(process.env.THREAD_PART_COUNT) || 8; +export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || os.cpus().length - 1 || 1; export const metadataBorder = '---'; diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index f795bb2e..88fdd169 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -5,13 +5,13 @@ import { PluginOptions, default as yfmlint, } from '@diplodoc/transform/lib/yfmlint'; -import {readFileSync} from 'fs'; import {bold} from 'chalk'; import {ArgvService, PluginService} from '../services'; import {getVarsPerFile, getVarsPerRelativeFile} from '../utils'; import {liquidMd2Html} from './md2html'; import {liquidMd2Md} from './md2md'; +import * as fs from 'fs'; interface FileTransformOptions { path: string; @@ -28,13 +28,13 @@ export interface ResolverLintOptions { onFinish?: () => void; } -export function lintPage(options: ResolverLintOptions) { +export async function lintPage(options: ResolverLintOptions) { const {inputPath, fileExtension, onFinish} = options; const {input} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); try { - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = await fs.promises.readFile(resolvedPath, 'utf8'); const lintFn: Function = FileLinter[fileExtension]; if (!lintFn) { diff --git a/src/resolvers/processPage.ts b/src/resolvers/processPage.ts new file mode 100644 index 00000000..14bb5541 --- /dev/null +++ b/src/resolvers/processPage.ts @@ -0,0 +1,250 @@ +import {logger} from '../utils'; +import {ArgvService, LeadingService, TocService} from '../services'; +import {basename, dirname, extname, join, resolve} from 'path'; +import {BUNDLE_FOLDER, ResourceType} from '../constants'; +import {VCSConnector} from '../vcs-connector/connector-models'; +import {LeadingPage, MetaDataOptions, PathData, Resources, SinglePageResult} from '../models'; +import {DocInnerProps} from '@diplodoc/client'; +import {bold} from 'chalk'; +import log from '@diplodoc/transform/lib/log'; +import * as fs from 'fs'; +import {dump, load} from 'js-yaml'; +import {resolveMd2Md} from './md2md'; +import {resolveMd2HTML} from './md2html'; +import shell from 'shelljs'; + +let singlePageResults: Record; +let singlePagePaths: Record>; + +export interface ProcessPageOptions { + pathToFile: string; + vcsConnector?: VCSConnector; + singlePageResults: Record; + singlePagePaths: Record>; +} + +export async function processPage(options: ProcessPageOptions) { + singlePageResults = options.singlePageResults; + singlePagePaths = options.singlePagePaths; + + const {pathToFile, vcsConnector} = options; + const { + input: inputFolderPath, + output: outputFolderPath, + outputFormat, + singlePage, + resolveConditions, + } = ArgvService.getConfig(); + + const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + + const pathData = getPathData( + pathToFile, + inputFolderPath, + outputFolderPath, + outputFormat, + outputBundlePath, + ); + + logger.proc(pathToFile); + + const metaDataOptions = getMetaDataOptions(pathData, inputFolderPath.length, vcsConnector); + + await preparingPagesByOutputFormat(pathData, metaDataOptions, resolveConditions, singlePage); +} + +function getPathData( + pathToFile: string, + inputFolderPath: string, + outputFolderPath: string, + outputFormat: string, + outputBundlePath: string, +): PathData { + const pathToDir: string = dirname(pathToFile); + const filename: string = basename(pathToFile); + const fileExtension: string = extname(pathToFile); + const fileBaseName: string = basename(filename, fileExtension); + const outputDir = resolve(outputFolderPath, pathToDir); + const outputFileName = `${fileBaseName}.${outputFormat}`; + const outputPath = resolve(outputDir, outputFileName); + const resolvedPathToFile = resolve(inputFolderPath, pathToFile); + const outputTocDir = TocService.getTocDir(resolvedPathToFile); + + const pathData: PathData = { + pathToFile, + resolvedPathToFile, + filename, + fileBaseName, + fileExtension, + outputDir, + outputPath, + outputFormat, + outputBundlePath, + outputTocDir, + inputFolderPath, + outputFolderPath, + }; + + return pathData; +} + +function savePageResultForSinglePage(pageProps: DocInnerProps, pathData: PathData): void { + const {pathToFile, outputTocDir} = pathData; + + if (pageProps.data.leading) { + return; + } + + singlePagePaths[outputTocDir] = singlePagePaths[outputTocDir] || new Set(); + + if (singlePagePaths[outputTocDir].has(pathToFile)) { + return; + } + + singlePagePaths[outputTocDir].add(pathToFile); + + singlePageResults[outputTocDir] = singlePageResults[outputTocDir] || []; + singlePageResults[outputTocDir].push({ + path: pathToFile, + content: pageProps.data.html, + title: pageProps.data.title, + }); +} + +function getMetaDataOptions( + pathData: PathData, + inputFolderPathLength: number, + vcsConnector?: VCSConnector, +): MetaDataOptions { + const {contributors, addSystemMeta, resources, allowCustomResources} = ArgvService.getConfig(); + + const metaDataOptions: MetaDataOptions = { + vcsConnector, + fileData: { + tmpInputFilePath: pathData.resolvedPathToFile, + inputFolderPathLength, + fileContent: '', + }, + isContributorsEnabled: Boolean(contributors && vcsConnector), + addSystemMeta, + }; + + if (allowCustomResources && resources) { + const allowedResources = Object.entries(resources).reduce((acc: Resources, [key, val]) => { + if (Object.keys(ResourceType).includes(key)) { + acc[key as keyof typeof ResourceType] = val; + } + return acc; + }, {}); + + metaDataOptions.resources = allowedResources; + } + + return metaDataOptions; +} + +async function preparingPagesByOutputFormat( + path: PathData, + metaDataOptions: MetaDataOptions, + resolveConditions: boolean, + singlePage: boolean, +): Promise { + const { + filename, + fileExtension, + fileBaseName, + outputDir, + resolvedPathToFile, + outputFormat, + pathToFile, + } = path; + const {allowCustomResources} = ArgvService.getConfig(); + + try { + await fs.promises.mkdir(outputDir, {recursive: true}); + + const isYamlFileExtension = fileExtension === '.yaml'; + + if (resolveConditions && fileBaseName === 'index' && isYamlFileExtension) { + await LeadingService.filterFile(pathToFile); + } + + if (outputFormat === 'md' && isYamlFileExtension && allowCustomResources) { + await processingYamlFile(path, metaDataOptions); + return; + } + + if ( + (outputFormat === 'md' && isYamlFileExtension) || + (outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') + ) { + copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); + return; + } + + switch (outputFormat) { + case 'md': + await processingFileToMd(path, metaDataOptions); + return; + case 'html': { + const resolvedFileProps = await processingFileToHtml(path, metaDataOptions); + + if (singlePage) { + savePageResultForSinglePage(resolvedFileProps, path); + } + + return; + } + } + } catch (e) { + const message = `No such file or has no access to ${bold(resolvedPathToFile)}`; + log.error(message); + } +} + +async function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptions) { + const {pathToFile, outputFolderPath, inputFolderPath} = path; + + const filePath = resolve(inputFolderPath, pathToFile); + const content = await fs.promises.readFile(filePath, 'utf8'); + const parsedContent = load(content) as LeadingPage; + + if (metaDataOptions.resources) { + parsedContent.meta = {...parsedContent.meta, ...metaDataOptions.resources}; + } + + await fs.promises.writeFile(resolve(outputFolderPath, pathToFile), dump(parsedContent)); +} + +function copyFileWithoutChanges(resolvedPathToFile: string, outputDir: string, filename: string) { + const from = resolvedPathToFile; + const to = resolve(outputDir, filename); + + shell.cp(from, to); +} + +async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptions): Promise { + const {outputPath, pathToFile} = path; + + await resolveMd2Md({ + inputPath: pathToFile, + outputPath, + metadata: metaDataOptions, + }); +} + +async function processingFileToHtml( + path: PathData, + metaDataOptions: MetaDataOptions, +): Promise { + const {outputBundlePath, filename, fileExtension, outputPath, pathToFile} = path; + + return resolveMd2HTML({ + inputPath: pathToFile, + outputBundlePath, + fileExtension, + outputPath, + filename, + metadata: metaDataOptions, + }); +} diff --git a/src/services/leading.ts b/src/services/leading.ts index d785a996..84015320 100644 --- a/src/services/leading.ts +++ b/src/services/leading.ts @@ -1,5 +1,4 @@ import {dirname, resolve} from 'path'; -import {readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; @@ -12,13 +11,14 @@ import { liquidField, liquidFields, } from './utils'; +import * as fs from 'fs'; -function filterFile(path: string) { +async function filterFile(path: string) { const {input: inputFolderPath, vars} = ArgvService.getConfig(); const pathToDir = dirname(path); const filePath = resolve(inputFolderPath, path); - const content = readFileSync(filePath, 'utf8'); + const content = await fs.promises.readFile(filePath, 'utf8'); const parsedIndex = load(content) as LeadingPage; const combinedVars = { @@ -74,7 +74,7 @@ function filterFile(path: string) { } }); - writeFileSync(filePath, dump(parsedIndex)); + await fs.promises.writeFile(filePath, dump(parsedIndex)); } catch (error) { log.error(`Error while filtering index file: ${path}. Error message: ${error}`); } diff --git a/src/services/preset.ts b/src/services/preset.ts index 78d126ca..fae8f631 100644 --- a/src/services/preset.ts +++ b/src/services/preset.ts @@ -1,9 +1,12 @@ import {dirname, normalize} from 'path'; import {DocPreset, YfmPreset} from '../models'; +import {mapToObject, objFillMap} from '../utils/worker'; export type PresetStorage = Map; +export type PresetStorageDump = ReturnType; + let presetStorage: PresetStorage = new Map(); function add(parsedPreset: DocPreset, path: string, varsPreset: string) { @@ -47,9 +50,22 @@ function setPresetStorage(preset: Map): void { presetStorage = preset; } +function dumpData() { + return { + presetStorageKeyValue: mapToObject(presetStorage), + }; +} + +function loadData({presetStorageKeyValue}: PresetStorageDump) { + presetStorage.clear(); + objFillMap(presetStorageKeyValue, presetStorage); +} + export default { add, get, getPresetStorage, setPresetStorage, + load: loadData, + dump: dumpData, }; diff --git a/src/services/tocs.ts b/src/services/tocs.ts index 2f02156b..648d6321 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -14,6 +14,7 @@ import {IncludeMode, Stage} from '../constants'; import {isExternalHref, logger} from '../utils'; import {filterFiles, firstFilterTextItems, liquidField} from './utils'; import {IncludersError, applyIncluders} from './includers'; +import {mapToObject, objFillMap} from '../utils/worker'; export interface TocServiceData { storage: Map; @@ -21,6 +22,8 @@ export interface TocServiceData { includedTocPaths: Set; } +export type TocServiceDataDump = ReturnType; + const storage: TocServiceData['storage'] = new Map(); let navigationPaths: TocServiceData['navigationPaths'] = []; const includedTocPaths: TocServiceData['includedTocPaths'] = new Set(); @@ -139,7 +142,9 @@ function prepareNavigationPaths(parsedToc: YfmToc, dirPath: string) { storage.set(href, parsedToc); const navigationPath = _normalizeHref(href); - navigationPaths.push(navigationPath); + if (!navigationPaths.includes(navigationPath)) { + navigationPaths.push(navigationPath); + } } }); } @@ -396,6 +401,29 @@ function setNavigationPaths(paths: TocServiceData['navigationPaths']) { navigationPaths = paths; } +function dumpData() { + return { + storageKeyValue: mapToObject(storage), + navigationPaths, + includedTocPathsArr: Array.from(includedTocPaths.keys()), + }; +} + +function loadData({ + storageKeyValue, + includedTocPathsArr, + navigationPaths: navigationPathsLocal, +}: TocServiceDataDump) { + navigationPaths.splice(0); + navigationPaths.push(...navigationPathsLocal); + + storage.clear(); + objFillMap(storageKeyValue, storage); + + includedTocPaths.clear(); + includedTocPathsArr.forEach((v) => includedTocPaths.add(v)); +} + export default { add, getForPath, @@ -403,4 +431,6 @@ export default { getTocDir, getIncludedTocPaths, setNavigationPaths, + dump: dumpData, + load: loadData, }; diff --git a/src/steps/processLinter.ts b/src/steps/processLinter.ts index 13c3c050..d27fd191 100644 --- a/src/steps/processLinter.ts +++ b/src/steps/processLinter.ts @@ -1,101 +1,41 @@ -import log from '@diplodoc/transform/lib/log'; -import {Thread, Worker, spawn} from 'threads'; +import {PluginService, TocService} from '../services'; +import {getChunkSize} from '../utils/workers'; +import {runPool} from './processPool'; +import {chunk} from 'lodash'; +import {lintPage} from '../resolvers'; import {extname} from 'path'; - -import {ArgvService, PluginService, PresetService, TocService} from '../services'; -import {ProcessLinterWorker} from '../workers/linter'; import {logger} from '../utils'; -import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '../constants'; -import {lintPage} from '../resolvers'; -import {splitOnChunks} from '../utils/worker'; - -let processLinterWorkers: (ProcessLinterWorker & Thread)[]; -let navigationPathsChunks: string[][]; +import {LINTING_FINISHED} from '../constants'; export async function processLinter(): Promise { - const argvConfig = ArgvService.getConfig(); - const navigationPaths = TocService.getNavigationPaths(); - if (!processLinterWorkers) { - lintPagesFallback(navigationPaths); - + if (process.env.DISABLE_PARALLEL_BUILD) { + await lintPagesFallback(navigationPaths); return; } - const presetStorage = PresetService.getPresetStorage(); - - /* Subscribe on the linted page event */ - processLinterWorkers.forEach((worker) => { - worker.getProcessedPages().subscribe((pathToFile) => { - logger.info(pathToFile as string, LINTING_FINISHED); - }); - }); + const navigationPathsChunks = chunk(navigationPaths, getChunkSize(navigationPaths)); - /* Run processing the linter */ await Promise.all( - processLinterWorkers.map((worker, i) => { - const navigationPathsChunk = navigationPathsChunks[i]; - - return worker.run({ - argvConfig, - presetStorage, - navigationPaths: navigationPathsChunk, - }); - }), - ); - - /* Unsubscribe from workers */ - await Promise.all( - processLinterWorkers.map((worker) => { - return worker.finish().then((logs) => { - log.add(logs); - }); - }), - ); - - /* Terminate workers */ - await Promise.all( - processLinterWorkers.map((worker) => { - return Thread.terminate(worker); + navigationPathsChunks.map(async (navigationPathsChunk) => { + await runPool('lint', navigationPathsChunk); }), ); } -export async function initLinterWorkers() { - const navigationPaths = TocService.getNavigationPaths(); - const chunkSize = getChunkSize(navigationPaths); - - if (process.env.DISABLE_PARALLEL_BUILD || chunkSize < MIN_CHUNK_SIZE || WORKERS_COUNT <= 0) { - return; - } - - navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize).filter((arr) => arr.length); - - const workersCount = navigationPathsChunks.length; +async function lintPagesFallback(navigationPaths: string[]) { + PluginService.setPlugins(); - processLinterWorkers = await Promise.all( - new Array(workersCount).fill(null).map(() => { - // TODO: get linter path from env - return spawn(new Worker('./linter'), {timeout: 60000}); + await Promise.all( + navigationPaths.map(async (pathToFile) => { + await lintPage({ + inputPath: pathToFile, + fileExtension: extname(pathToFile), + onFinish: () => { + logger.info(pathToFile, LINTING_FINISHED); + }, + }); }), ); } - -function getChunkSize(arr: string[]) { - return Math.ceil(arr.length / WORKERS_COUNT); -} - -function lintPagesFallback(navigationPaths: string[]) { - PluginService.setPlugins(); - - navigationPaths.forEach((pathToFile) => { - lintPage({ - inputPath: pathToFile, - fileExtension: extname(pathToFile), - onFinish: () => { - logger.info(pathToFile, LINTING_FINISHED); - }, - }); - }); -} diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index f80c1df7..f8dfcaf3 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -1,123 +1,39 @@ -import type {DocInnerProps} from '@diplodoc/client'; -import {basename, dirname, extname, join, relative, resolve} from 'path'; -import shell from 'shelljs'; -import {copyFileSync, readFileSync, writeFileSync} from 'fs'; -import {bold} from 'chalk'; -import {dump, load} from 'js-yaml'; -import {asyncify, mapLimit} from 'async'; - -import log from '@diplodoc/transform/lib/log'; - -import {ArgvService, LeadingService, PluginService, TocService} from '../services'; -import {resolveMd2HTML, resolveMd2Md} from '../resolvers'; -import { - generateStaticMarkup, - joinSinglePageResults, - logger, - transformTocForSinglePage, -} from '../utils'; -import { - LeadingPage, - MetaDataOptions, - PathData, - Resources, - SinglePageResult, - YfmToc, -} from '../models'; +import {join, relative, resolve} from 'path'; +import {writeFileSync} from 'fs'; + +import {ArgvService, PluginService, TocService} from '../services'; +import {generateStaticMarkup, joinSinglePageResults, transformTocForSinglePage} from '../utils'; +import {SinglePageResult, YfmToc} from '../models'; +import {Lang, SINGLE_PAGE_DATA_FILENAME, SINGLE_PAGE_FILENAME} from '../constants'; +import {getChunkSize} from '../utils/workers'; +import {getPoolEnv, runPool} from './processPool'; +import {chunk} from 'lodash'; import {VCSConnector} from '../vcs-connector/connector-models'; -import {getVCSConnector} from '../vcs-connector'; -import {Lang, ResourceType, SINGLE_PAGE_DATA_FILENAME, SINGLE_PAGE_FILENAME} from '../constants'; - -const singlePageResults: Record = {}; -const singlePagePaths: Record> = {}; +import {asyncify, mapLimit} from 'async'; +import {processPage} from '../resolvers/processPage'; // Processes files of documentation (like index.yaml, *.md) -export async function processPages(outputBundlePath: string): Promise { - const { - input: inputFolderPath, - output: outputFolderPath, - outputFormat, - singlePage, - resolveConditions, - } = ArgvService.getConfig(); - - const vcsConnector = await getVCSConnector(); - - PluginService.setPlugins(); - +export async function processPages(vcsConnector?: VCSConnector): Promise { const navigationPaths = TocService.getNavigationPaths(); - const concurrency = 500; - - await mapLimit( - navigationPaths, - concurrency, - asyncify(async (pathToFile: string) => { - const pathData = getPathData( - pathToFile, - inputFolderPath, - outputFolderPath, - outputFormat, - outputBundlePath, - ); - logger.proc(pathToFile); + if (process.env.DISABLE_PARALLEL_BUILD) { + await processPagesFallback(vcsConnector, navigationPaths); + return; + } - const metaDataOptions = getMetaDataOptions( - pathData, - inputFolderPath.length, - vcsConnector, - ); + const navigationPathsChunks = chunk(navigationPaths, getChunkSize(navigationPaths)); - await preparingPagesByOutputFormat( - pathData, - metaDataOptions, - resolveConditions, - singlePage, - ); + await Promise.all( + navigationPathsChunks.map(async (navigationPathsChunk) => { + await runPool('transform', navigationPathsChunk); }), ); - - if (singlePage) { - await saveSinglePages(outputBundlePath); - } } -function getPathData( - pathToFile: string, - inputFolderPath: string, - outputFolderPath: string, - outputFormat: string, +export async function saveSinglePages( outputBundlePath: string, -): PathData { - const pathToDir: string = dirname(pathToFile); - const filename: string = basename(pathToFile); - const fileExtension: string = extname(pathToFile); - const fileBaseName: string = basename(filename, fileExtension); - const outputDir = resolve(outputFolderPath, pathToDir); - const outputFileName = `${fileBaseName}.${outputFormat}`; - const outputPath = resolve(outputDir, outputFileName); - const resolvedPathToFile = resolve(inputFolderPath, pathToFile); - const outputTocDir = TocService.getTocDir(resolvedPathToFile); - - const pathData: PathData = { - pathToFile, - resolvedPathToFile, - filename, - fileBaseName, - fileExtension, - outputDir, - outputPath, - outputFormat, - outputBundlePath, - outputTocDir, - inputFolderPath, - outputFolderPath, - }; - - return pathData; -} - -async function saveSinglePages(outputBundlePath: string) { + singlePageResults: Record, +) { const { input: inputFolderPath, output: outputFolderPath, @@ -156,7 +72,7 @@ async function saveSinglePages(outputBundlePath: string) { pathname: SINGLE_PAGE_FILENAME, }, lang: lang || Lang.RU, - }; + } as unknown as Parameters[0]; const outputTocDir = resolve(outputFolderPath, relative(inputFolderPath, tocDir)); const relativeOutputBundlePath = relative(outputTocDir, outputBundlePath); @@ -175,168 +91,27 @@ async function saveSinglePages(outputBundlePath: string) { } } -function savePageResultForSinglePage(pageProps: DocInnerProps, pathData: PathData): void { - const {pathToFile, outputTocDir} = pathData; - - if (pageProps.data.leading) { - return; - } - - singlePagePaths[outputTocDir] = singlePagePaths[outputTocDir] || new Set(); - - if (singlePagePaths[outputTocDir].has(pathToFile)) { - return; - } - - singlePagePaths[outputTocDir].add(pathToFile); - - singlePageResults[outputTocDir] = singlePageResults[outputTocDir] || []; - singlePageResults[outputTocDir].push({ - path: pathToFile, - content: pageProps.data.html, - title: pageProps.data.title, - }); -} - -function getMetaDataOptions( - pathData: PathData, - inputFolderPathLength: number, - vcsConnector?: VCSConnector, -): MetaDataOptions { - const {contributors, addSystemMeta, resources, allowCustomResources} = ArgvService.getConfig(); - - const metaDataOptions: MetaDataOptions = { - vcsConnector, - fileData: { - tmpInputFilePath: pathData.resolvedPathToFile, - inputFolderPathLength, - fileContent: '', - }, - isContributorsEnabled: Boolean(contributors && vcsConnector), - addSystemMeta, - }; - - if (allowCustomResources && resources) { - const allowedResources = Object.entries(resources).reduce((acc: Resources, [key, val]) => { - if (Object.keys(ResourceType).includes(key)) { - acc[key as keyof typeof ResourceType] = val; - } - return acc; - }, {}); - - metaDataOptions.resources = allowedResources; - } - - return metaDataOptions; -} - -async function preparingPagesByOutputFormat( - path: PathData, - metaDataOptions: MetaDataOptions, - resolveConditions: boolean, - singlePage: boolean, -): Promise { - const { - filename, - fileExtension, - fileBaseName, - outputDir, - resolvedPathToFile, - outputFormat, - pathToFile, - } = path; - const {allowCustomResources} = ArgvService.getConfig(); - - try { - shell.mkdir('-p', outputDir); - - const isYamlFileExtension = fileExtension === '.yaml'; - - if (resolveConditions && fileBaseName === 'index' && isYamlFileExtension) { - LeadingService.filterFile(pathToFile); - } - - if (outputFormat === 'md' && isYamlFileExtension && allowCustomResources) { - processingYamlFile(path, metaDataOptions); - return; - } - - if ( - (outputFormat === 'md' && isYamlFileExtension) || - (outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') - ) { - copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); - return; - } - - switch (outputFormat) { - case 'md': - await processingFileToMd(path, metaDataOptions); - return; - case 'html': { - const resolvedFileProps = await processingFileToHtml(path, metaDataOptions); - - if (singlePage) { - savePageResultForSinglePage(resolvedFileProps, path); - } - - return; - } - } - } catch (e) { - const message = `No such file or has no access to ${bold(resolvedPathToFile)}`; - console.log(message, e); - log.error(message); - } -} -//@ts-ignore -function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptions) { - const {pathToFile, outputFolderPath, inputFolderPath} = path; - - const filePath = resolve(inputFolderPath, pathToFile); - const content = readFileSync(filePath, 'utf8'); - const parsedContent = load(content) as LeadingPage; - - if (metaDataOptions.resources) { - parsedContent.meta = {...parsedContent.meta, ...metaDataOptions.resources}; - } - - writeFileSync(resolve(outputFolderPath, pathToFile), dump(parsedContent)); -} - -function copyFileWithoutChanges( - resolvedPathToFile: string, - outputDir: string, - filename: string, -): void { - const from = resolvedPathToFile; - const to = resolve(outputDir, filename); - - copyFileSync(from, to); -} - -async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptions): Promise { - const {outputPath, pathToFile} = path; +async function processPagesFallback( + vcsConnector: VCSConnector | undefined, + navigationPaths: string[], +) { + PluginService.setPlugins(); - await resolveMd2Md({ - inputPath: pathToFile, - outputPath, - metadata: metaDataOptions, - }); -} + const {singlePageResults} = getPoolEnv(); + const singlePagePaths: Record> = {}; -async function processingFileToHtml( - path: PathData, - metaDataOptions: MetaDataOptions, -): Promise { - const {outputBundlePath, filename, fileExtension, outputPath, pathToFile} = path; + const concurrency = 500; - return resolveMd2HTML({ - inputPath: pathToFile, - outputBundlePath, - fileExtension, - outputPath, - filename, - metadata: metaDataOptions, - }); + await mapLimit( + navigationPaths, + concurrency, + asyncify(async (pathToFile: string) => { + await processPage({ + pathToFile, + vcsConnector, + singlePageResults, + singlePagePaths, + }); + }), + ); } diff --git a/src/steps/processPool.ts b/src/steps/processPool.ts new file mode 100644 index 00000000..96c2f0b8 --- /dev/null +++ b/src/steps/processPool.ts @@ -0,0 +1,102 @@ +import {ModuleThread, Pool, spawn, Thread, Worker} from 'threads'; +import {ProcessPoolWorker} from '../workers/pool'; +import {ArgvService, PresetService, TocService} from '../services'; +import {WORKERS_COUNT} from '../constants'; +import {VCSConnector} from '../vcs-connector/connector-models'; +import log from '@diplodoc/transform/lib/log'; +import {SinglePageResult} from '../models'; +import {logger} from '../utils'; +import {MainBridge} from '../workers/mainBridge'; + +const poolEnv = { + singlePageResults: {} as Record, +}; + +const workers: (ProcessPoolWorker & Thread)[] = []; +let pool: Pool> | undefined; + +interface InitProcessPoolProps { + vcsConnector?: VCSConnector; +} + +export async function initProcessPool({vcsConnector}: InitProcessPoolProps) { + const argvConfig = ArgvService.getConfig(); + const presetStorageDump = PresetService.dump(); + const tocServiceDataDump = TocService.dump(); + const vcsConnectorDump = vcsConnector?.dump(); + + const scope = { + vcsConnector: vcsConnector, + }; + + // eslint-disable-next-line new-cap + pool = Pool(async () => { + const worker = await spawn(new Worker('./pool'), {timeout: 60000}); + await worker.init({ + argvConfig, + presetStorageDump, + tocServiceDataDump, + vcsConnectorDump, + }); + worker.getSubject().subscribe(async (payload) => { + switch (payload.type) { + case 'path': { + logger.info(payload.path, payload.message); + break; + } + case 'call': { + await MainBridge.handleCall(worker, payload, scope); + break; + } + } + }); + workers.push(worker); + return worker; + }, WORKERS_COUNT); +} + +export async function runPool(type: 'lint' | 'transform', navigationPaths: string[]) { + if (!pool) { + throw new Error('Pool is not initiated'); + } + + return pool.queue(async (worker) => { + if (type === 'lint') { + await worker.lint({navigationPaths}); + } else { + await worker.transform({navigationPaths}); + } + }); +} + +export async function finishProcessPool() { + await Promise.all( + workers.map(async (worker) => { + const {logs, singlePageResults: singlePageResultsLocal} = await worker.finish(); + + Object.entries(singlePageResultsLocal).forEach(([key, values]) => { + let arr = poolEnv.singlePageResults[key]; + if (!arr) { + arr = poolEnv.singlePageResults[key] = []; + } + arr.push(...values); + }); + + log.add(logs); + }), + ); +} + +export async function terminateProcessPool() { + if (!pool) { + throw new Error('Pool is not initiated'); + } + + workers.splice(0); + await pool.terminate(true); + pool = undefined; +} + +export function getPoolEnv() { + return poolEnv; +} diff --git a/src/utils/file.ts b/src/utils/file.ts index 021f1dad..c599f84f 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -1,6 +1,5 @@ import {dirname, resolve} from 'path'; import shell from 'shelljs'; -import {copyFileSync} from 'fs'; import {logger} from './logger'; export function copyFiles( @@ -8,14 +7,20 @@ export function copyFiles( outputFolderPath: string, files: string[], ): void { - for (const pathToAsset of files) { - const outputDir: string = resolve(outputFolderPath, dirname(pathToAsset)); + const dirs = new Set(); + + files.forEach((pathToAsset) => { + const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); const from = resolve(inputFolderPath, pathToAsset); const to = resolve(outputFolderPath, pathToAsset); - shell.mkdir('-p', outputDir); - copyFileSync(from, to); + if (!dirs.has(outputDir)) { + dirs.add(outputDir); + shell.mkdir('-p', outputDir); + } + + shell.cp(from, to); logger.copy(pathToAsset); - } + }); } diff --git a/src/utils/worker.ts b/src/utils/worker.ts index bc97d028..bd5515f7 100644 --- a/src/utils/worker.ts +++ b/src/utils/worker.ts @@ -8,3 +8,17 @@ export function splitOnChunks(array: T[], chunkSize = 1000) { return chunks; } + +export function mapToObject(map: Map) { + const obj: Record = {}; + map.forEach((value, key) => { + obj[key] = value; + }); + return obj; +} + +export function objFillMap(obj: Record, map: Map) { + Object.entries(obj).forEach(([key, value]) => { + map.set(key, value); + }); +} diff --git a/src/utils/workers.ts b/src/utils/workers.ts new file mode 100644 index 00000000..5583d5ad --- /dev/null +++ b/src/utils/workers.ts @@ -0,0 +1,5 @@ +import {MIN_CHUNK_SIZE, THREAD_PART_COUNT, WORKERS_COUNT} from '../constants'; + +export function getChunkSize(arr: string[]) { + return Math.max(Math.ceil(arr.length / WORKERS_COUNT / THREAD_PART_COUNT), MIN_CHUNK_SIZE); +} diff --git a/src/vcs-connector/connector-models.ts b/src/vcs-connector/connector-models.ts index 2adb8cc5..20f7a973 100644 --- a/src/vcs-connector/connector-models.ts +++ b/src/vcs-connector/connector-models.ts @@ -1,4 +1,5 @@ import { + Contributor, Contributors, ContributorsByPathFunction, ExternalAuthorByPathFunction, @@ -26,11 +27,22 @@ export enum GitHubConnectorFields { ENDPOINT = 'endpoint', } +export type VCSConnectorDump = { + authorByGitEmail: Record; + authorByPath: Record; + contributorsByPath: Record; + contributorsData: Record; + userLoginGithubUserCache: Record; +}; + export interface VCSConnector { + init: () => Promise; getExternalAuthorByPath: ExternalAuthorByPathFunction; addNestedContributorsForPath: NestedContributorsForPathFunction; getContributorsByPath: ContributorsByPathFunction; getUserByLogin: UserByLoginFunction; + dump: () => VCSConnectorDump; + load: (dump: VCSConnectorDump) => void; } export interface VCSConnectorConfig { diff --git a/src/vcs-connector/github.ts b/src/vcs-connector/github.ts index f664eb65..7148ca8d 100644 --- a/src/vcs-connector/github.ts +++ b/src/vcs-connector/github.ts @@ -5,19 +5,13 @@ import {minimatch} from 'minimatch'; import github from './client/github'; import {ArgvService} from '../services'; -import { - CommitInfo, - Contributor, - Contributors, - ContributorsByPathFunction, - ExternalAuthorByPathFunction, - NestedContributorsForPathFunction, -} from '../models'; +import {CommitInfo, Contributor, Contributors} from '../models'; import { FileContributors, GitHubConnectorFields, SourceType, VCSConnector, + VCSConnectorDump, } from './connector-models'; import { ALL_CONTRIBUTORS_RECEIVED, @@ -27,41 +21,65 @@ import { import {addSlashPrefix, logger} from '../utils'; import {validateConnectorFields} from './connector-validator'; import process from 'process'; +import {mapToObject, objFillMap} from '../utils/worker'; const authorByGitEmail: Map = new Map(); const authorByPath: Map = new Map(); const contributorsByPath: Map = new Map(); const contributorsData: Map = new Map(); +const userLoginGithubUserCache = new Map(); -async function getGitHubVCSConnector(): Promise { +function getGitHubVCSConnector(): VCSConnector | undefined { const {contributors} = ArgvService.getConfig(); + if (!contributors) { + return undefined; + } const httpClientByToken = getHttpClientByToken(); if (!httpClientByToken) { return undefined; } - let addNestedContributorsForPath: NestedContributorsForPathFunction = () => {}; - let getContributorsByPath: ContributorsByPathFunction = () => - Promise.resolve({} as FileContributors); - const getExternalAuthorByPath: ExternalAuthorByPathFunction = (path: string) => - authorByPath.get(path) ?? null; - - if (contributors) { - await getAllContributorsTocFiles(httpClientByToken); - addNestedContributorsForPath = (path: string, nestedContributors: Contributors) => - addNestedContributorsForPathFunction(path, nestedContributors); - getContributorsByPath = async (path: string) => getFileContributorsByPath(path); - } - return { - getExternalAuthorByPath, - addNestedContributorsForPath, - getContributorsByPath, + init: async () => await getAllContributorsTocFiles(httpClientByToken), + getExternalAuthorByPath: (path: string) => authorByPath.get(path) ?? null, + addNestedContributorsForPath: (path: string, nestedContributors: Contributors) => { + addContributorForPath([path], nestedContributors, true); + }, + getContributorsByPath: async (path: string) => getFileContributorsByPath(path), getUserByLogin: (login: string) => getUserByLogin(httpClientByToken, login), + dump: dump, + load: load, }; } +function dump() { + return { + authorByGitEmail: mapToObject(authorByGitEmail), + authorByPath: mapToObject(authorByPath), + contributorsByPath: mapToObject(contributorsByPath), + contributorsData: mapToObject(contributorsData), + userLoginGithubUserCache: mapToObject(userLoginGithubUserCache), + }; +} + +function load(dumpData: VCSConnectorDump) { + authorByGitEmail.clear(); + objFillMap(dumpData.authorByGitEmail, authorByGitEmail); + + authorByPath.clear(); + objFillMap(dumpData.authorByPath, authorByPath); + + contributorsByPath.clear(); + objFillMap(dumpData.contributorsByPath, contributorsByPath); + + contributorsData.clear(); + objFillMap(dumpData.contributorsData, contributorsData); + + userLoginGithubUserCache.clear(); + objFillMap(dumpData.userLoginGithubUserCache, userLoginGithubUserCache); +} + function getHttpClientByToken(): Octokit | null { const {connector, contributors} = ArgvService.getConfig(); @@ -277,6 +295,11 @@ async function getFileContributorsByPath(path: string): Promise { + const fromCache = userLoginGithubUserCache.get(userLogin); + if (fromCache) { + return fromCache; + } + const user = await github.getRepoUser(octokit, userLogin); if (!user) { @@ -284,21 +307,11 @@ async function getUserByLogin(octokit: Octokit, userLogin: string): Promise { +export function createVCSConnector(): VCSConnector | undefined { const {connector} = ArgvService.getConfig(); const connectorType = process.env.VCS_CONNECTOR_TYPE || (connector && connector.type); diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts deleted file mode 100644 index 969ee487..00000000 --- a/src/workers/linter/index.ts +++ /dev/null @@ -1,58 +0,0 @@ -import log from '@diplodoc/transform/lib/log'; -import {extname} from 'path'; -import {Observable, Subject} from 'threads/observable'; -import {expose} from 'threads'; - -import {ArgvService, PluginService, PresetService, TocService} from '../../services'; -import {TocServiceData} from '../../services/tocs'; -import {PresetStorage} from '../../services/preset'; -import {YfmArgv} from '../../models'; -import {lintPage} from '../../resolvers'; - -let processedPages = new Subject(); - -interface ProcessLinterWorkerOptions { - argvConfig: YfmArgv; - navigationPaths: TocServiceData['navigationPaths']; - presetStorage: PresetStorage; -} - -async function run({argvConfig, presetStorage, navigationPaths}: ProcessLinterWorkerOptions) { - ArgvService.set(argvConfig); - PresetService.setPresetStorage(presetStorage); - TocService.setNavigationPaths(navigationPaths); - PluginService.setPlugins(); - - TocService.getNavigationPaths().forEach((pathToFile) => { - lintPage({ - inputPath: pathToFile, - fileExtension: extname(pathToFile), - onFinish: () => { - processedPages.next(pathToFile); - }, - }); - }); -} - -async function finish() { - processedPages.complete(); - processedPages = new Subject(); - - return log.get(); -} - -function getProcessedPages() { - return Observable.from(processedPages); -} - -export type ProcessLinterWorker = { - run: typeof run; - finish: typeof finish; - getProcessedPages: typeof getProcessedPages; -}; - -expose({ - run, - finish, - getProcessedPages, -}); diff --git a/src/workers/mainBridge.ts b/src/workers/mainBridge.ts new file mode 100644 index 00000000..e3432684 --- /dev/null +++ b/src/workers/mainBridge.ts @@ -0,0 +1,82 @@ +type EmitFn = (data: SendPayload) => void; +type Callback = (payload: ReceivePayload) => void; +type ReplyFn = (id: number, payload: ReceivePayload) => void; + +export type SendPayload = {type: 'call'; id: number; method: string; args: unknown[]}; +export type ReceivePayload = { + result?: unknown; + error?: {name: string; message: string; stack?: string}; +}; + +export class MainBridge { + static handleCall = async ( + worker: {reply: ReplyFn}, + data: SendPayload, + scope: Record, + ) => { + const {id, method, args} = data; + let result; + let error; + try { + result = await resolveMethod(scope, method)(...args); + } catch (ex) { + const {name, message, stack} = ex as Error; + error = {name, message, stack}; + } + worker.reply(id, {result, error}); + }; + + private index = 0; + private idCallback; + private readonly emitFn; + + constructor(emitFn: EmitFn) { + this.idCallback = new Map(); + this.emitFn = emitFn; + } + + createFn(method: string): T { + const wrappedFn = (...args: unknown[]) => { + return this.remoteCall(method, args); + }; + return wrappedFn as unknown as T; + } + + handleReply(id: number, payload: ReceivePayload) { + const cb = this.idCallback.get(id); + if (cb) { + this.idCallback.delete(id); + cb(payload); + } + return; + } + + private remoteCall(method: string, args: unknown[]) { + return new Promise((resolve, reject) => { + const id = ++this.index; + this.idCallback.set(id, ({result, error}) => { + if (error) { + const err = new Error(error.message); + Object.assign(err, error); + reject(err); + } else { + resolve(result); + } + }); + this.emitFn({type: 'call', id, method, args}); + }); + } +} + +function resolveMethod(scope: Record, method: string) { + let fn; + let fnProto; + let root = scope; + const parts = method.split('.'); + while (parts.length) { + fnProto = root; + fn = root[parts.shift() as string]; + root = fn as Record; + } + return (fn as Function).bind(fnProto); +} diff --git a/src/workers/pool/index.ts b/src/workers/pool/index.ts new file mode 100644 index 00000000..f1fd7cce --- /dev/null +++ b/src/workers/pool/index.ts @@ -0,0 +1,130 @@ +import {SinglePageResult, YfmArgv} from '../../models'; +import {TocServiceDataDump} from '../../services/tocs'; +import {PresetStorageDump} from '../../services/preset'; +import {VCSConnector, VCSConnectorDump} from '../../vcs-connector/connector-models'; +import {ArgvService, PluginService, PresetService, TocService} from '../../services'; +import {createVCSConnector} from '../../vcs-connector'; +import {Observable, Subject} from 'threads/observable'; +import {MainBridge, ReceivePayload, SendPayload} from '../mainBridge'; +import {expose} from 'threads'; +import {asyncify, mapLimit} from 'async'; +import {lintPage} from '../../resolvers'; +import {extname} from 'path'; +import log from '@diplodoc/transform/lib/log'; +import {processPage} from '../../resolvers/processPage'; +import {LINTING_FINISHED, PROCESSING_FINISHED} from '../../constants'; + +const concurrency = 500; + +export type PoolSubjectPayload = {type: 'path'; path: string; message: string} | SendPayload; + +let subject = new Subject(); + +const singlePageResults: Record = {}; +let vcsConnector: VCSConnector | undefined; + +const mainBridge = new MainBridge((payload) => subject.next(payload)); + +interface ProcessPoolWorkerOptions { + argvConfig: YfmArgv; + presetStorageDump: PresetStorageDump; + tocServiceDataDump: TocServiceDataDump; + vcsConnectorDump?: VCSConnectorDump; +} + +async function init({ + argvConfig, + vcsConnectorDump, + tocServiceDataDump, + presetStorageDump, +}: ProcessPoolWorkerOptions) { + ArgvService.set(argvConfig); + PresetService.load(presetStorageDump); + TocService.load(tocServiceDataDump); + + vcsConnector = createVCSConnector(); + if (vcsConnectorDump && vcsConnector) { + vcsConnector.load(vcsConnectorDump); + vcsConnector.getUserByLogin = mainBridge.createFn( + 'vcsConnector.getUserByLogin', + ); + } + + PluginService.setPlugins(); +} + +interface LintProps { + navigationPaths: string[]; +} + +async function lint({navigationPaths}: LintProps) { + await mapLimit( + navigationPaths, + concurrency, + asyncify(async (pathToFile: string) => { + await lintPage({ + inputPath: pathToFile, + fileExtension: extname(pathToFile), + onFinish: () => { + subject.next({type: 'path', path: pathToFile, message: LINTING_FINISHED}); + }, + }); + }), + ); +} + +interface TransformProps { + navigationPaths: string[]; +} + +async function transform({navigationPaths}: TransformProps) { + const singlePagePaths: Record> = {}; + + await mapLimit( + navigationPaths, + concurrency, + asyncify(async (pathToFile: string) => { + await processPage({ + pathToFile, + vcsConnector, + singlePageResults, + singlePagePaths, + }).finally(() => { + subject.next({type: 'path', path: pathToFile, message: PROCESSING_FINISHED}); + }); + }), + ); +} + +async function finish() { + subject.complete(); + subject = new Subject(); + + return {logs: log.get(), singlePageResults}; +} + +function getSubject() { + return Observable.from(subject); +} + +function reply(id: number, payload: ReceivePayload) { + mainBridge.handleReply(id, payload); +} + +export type ProcessPoolWorker = { + init: typeof init; + lint: typeof lint; + transform: typeof transform; + getSubject: typeof getSubject; + reply: typeof reply; + finish: typeof finish; +}; + +expose({ + init, + lint, + transform, + getSubject, + reply, + finish, +}); diff --git a/tests/integrations/services/metadataAuthors.test.ts b/tests/integrations/services/metadataAuthors.test.ts index 674beeae..c75971de 100644 --- a/tests/integrations/services/metadataAuthors.test.ts +++ b/tests/integrations/services/metadataAuthors.test.ts @@ -1,9 +1,9 @@ import {readFileSync} from 'fs'; import {REGEXP_AUTHOR} from '../../../src/constants'; import {replaceDoubleToSingleQuotes, сarriage} from '../../../src/utils/markup'; -import {MetaDataOptions} from 'models'; +import {Contributor, MetaDataOptions} from 'models'; import {getContentWithUpdatedMetadata} from 'services/metadata'; -import {VCSConnector} from 'vcs-connector/connector-models'; +import {FileContributors, VCSConnector} from 'vcs-connector/connector-models'; const authorAliasInMetadataFilePath = 'mocks/fileContent/metadata/authorAliasInMetadata.md'; const fullAuthorInMetadataFilePath = 'mocks/fileContent/metadata/fullAuthorInMetadata.md'; @@ -23,6 +23,17 @@ describe('getContentWithUpdatedMetadata (Authors)', () => { }; const defaultVCSConnector: VCSConnector = { + init: async () => {}, + load: () => {}, + dump: () => { + return { + authorByGitEmail: {}, + authorByPath: {}, + contributorsByPath: {}, + contributorsData: {}, + userLoginGithubUserCache: {}, + } + }, addNestedContributorsForPath: () => { }, getContributorsByPath: () => Promise.resolve(null), getUserByLogin: () => Promise.resolve(expectedAuthorData), diff --git a/tests/integrations/services/metadataContributors.test.ts b/tests/integrations/services/metadataContributors.test.ts index faaa3464..6f7e89da 100644 --- a/tests/integrations/services/metadataContributors.test.ts +++ b/tests/integrations/services/metadataContributors.test.ts @@ -22,6 +22,17 @@ describe('getContentWithUpdatedMetadata (Contributors)', () => { }; const defaultVCSConnector: VCSConnector = { + init: async () => {}, + load: () => {}, + dump: () => { + return { + authorByGitEmail: {}, + authorByPath: {}, + contributorsByPath: {}, + contributorsData: {}, + userLoginGithubUserCache: {}, + } + }, addNestedContributorsForPath: () => { }, getContributorsByPath: () => Promise.resolve(null), getUserByLogin: () => Promise.resolve(null), diff --git a/tests/units/services/authors.test.ts b/tests/units/services/authors.test.ts index 796fddfb..9169f957 100644 --- a/tests/units/services/authors.test.ts +++ b/tests/units/services/authors.test.ts @@ -20,6 +20,17 @@ const author = { const authorByPath: Map = new Map(); const defaultVCSConnector: VCSConnector = { + init: async () => {}, + load: () => {}, + dump: () => { + return { + authorByGitEmail: {}, + authorByPath: {}, + contributorsByPath: {}, + contributorsData: {}, + userLoginGithubUserCache: {}, + } + }, addNestedContributorsForPath: () => {}, getContributorsByPath: () => Promise.resolve(null), getUserByLogin: () => Promise.resolve(author), diff --git a/tests/units/services/metadata.test.ts b/tests/units/services/metadata.test.ts index ae64e770..bacc1ec5 100644 --- a/tests/units/services/metadata.test.ts +++ b/tests/units/services/metadata.test.ts @@ -34,6 +34,17 @@ jest.mock('services/authors', () => ({ })); const defaultVCSConnector: VCSConnector = { + init: async () => {}, + load: () => {}, + dump: () => { + return { + authorByGitEmail: {}, + authorByPath: {}, + contributorsByPath: {}, + contributorsData: {}, + userLoginGithubUserCache: {}, + } + }, addNestedContributorsForPath: () => { }, getContributorsByPath: () => Promise.resolve(null), getUserByLogin: () => Promise.resolve(null), From 9eb8060d215c6fe5d5f193571c840953281c27c0 Mon Sep 17 00:00:00 2001 From: Anton Vikulov Date: Mon, 30 Oct 2023 08:53:50 +0500 Subject: [PATCH 2/2] use shell cp in tocs --- src/services/tocs.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/tocs.ts b/src/services/tocs.ts index 648d6321..d97fefea 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -1,5 +1,5 @@ import {dirname, extname, join, normalize, parse, relative, resolve, sep} from 'path'; -import {copyFileSync, existsSync, readFileSync, writeFileSync} from 'fs'; +import {existsSync, readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import shell from 'shelljs'; import walkSync from 'walk-sync'; @@ -210,7 +210,7 @@ function _copyTocDir(tocPath: string, destDir: string) { writeFileSync(to, updatedFileContent); } else { - copyFileSync(from, to); + shell.cp(from, to); } }); }