Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(pages): Add workers in process pages #579

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat(pages): Add workers in process pages #579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0d653b
add worker pool
Feverqwe Oct 29, 2023
9eb8060
use shell cp in tocs
Feverqwe Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/build.cli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const commonConfig = {

const builds = [
[['src/index.ts'], 'build/index.js'],
[['src/workers/linter/index.ts'], 'build/linter.js'],
[['src/workers/pool/index.ts'], 'build/pool.js'],
];

Promise.all(builds.map(([entries, outfile]) => {
Expand Down
41 changes: 30 additions & 11 deletions src/cmd/build/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,27 @@
import {ArgvService, Includers} from '../../services';
import OpenapiIncluder from '@diplodoc/openapi-extension/includer';
import {
initLinterWorkers,
processAssets,
processExcludedFiles,
processLinter,
processLogs,
processPages,
processServiceFiles,
saveSinglePages,
} from '../../steps';
import {prepareMapFile} from '../../steps/processMapFile';
import shell from 'shelljs';
import {Resources} from '../../models';
import {copyFiles, logger} from '../../utils';
import {upload as publishFilesToS3} from '../publish/upload';
import glob from 'glob';
import {createVCSConnector} from '../../vcs-connector';
import {
finishProcessPool,
getPoolEnv,
initProcessPool,
terminateProcessPool,
} from '../../steps/processPool';

export const build = {
command: ['build', '$0'],
Expand Down Expand Up @@ -174,7 +181,7 @@
);
}

async function handler(args: Arguments<any>) {

Check warning on line 184 in src/cmd/build/index.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

Unexpected any. Specify a different type
const userOutputFolder = resolve(args.output);
const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
Expand All @@ -186,7 +193,7 @@
input: tmpInputFolder,
output: tmpOutputFolder,
});
Includers.init([OpenapiIncluder as any]);

Check warning on line 196 in src/cmd/build/index.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

Unexpected any. Specify a different type

const {
output: outputFolderPath,
Expand All @@ -197,6 +204,7 @@
addMapFile,
allowCustomResources,
resources,
singlePage,
} = ArgvService.getConfig();

preparingTemporaryFolders(userOutputFolder);
Expand All @@ -213,19 +221,29 @@
const pathToRedirects = join(args.input, REDIRECTS_FILENAME);
const pathToLintConfig = join(args.input, LINT_CONFIG_FILENAME);

if (!lintDisabled) {
/* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
await initLinterWorkers();
const vcsConnector = createVCSConnector();
if (vcsConnector) {
await vcsConnector.init();
}

const processes = [
!lintDisabled && processLinter(),
!buildDisabled && processPages(outputBundlePath),
].filter(Boolean) as Promise<void>[];
await initProcessPool({vcsConnector});

await Promise.all(processes);
try {
await Promise.all([
!lintDisabled && processLinter(),
!buildDisabled && processPages(vcsConnector),
]);
await finishProcessPool();
} finally {
await terminateProcessPool();
}

if (!buildDisabled) {
if (singlePage) {
const {singlePageResults} = getPoolEnv();
await saveSinglePages(outputBundlePath, singlePageResults);
}

// process additional files
switch (outputFormat) {
case 'html':
Expand Down Expand Up @@ -287,7 +305,7 @@
}
}
} catch (err) {
logger.error('', err.message);
logger.error('', (err as Error).message);
} finally {
processLogs(tmpInputFolder);

Expand All @@ -303,7 +321,6 @@
// Create temporary input/output folders
shell.rm('-rf', args.input, args.output);
shell.mkdir(args.input, args.output);
shell.chmod('-R', 'u+w', args.input);

copyFiles(
args.rootInput,
Expand All @@ -315,4 +332,6 @@
ignore: ['node_modules/**', '*/node_modules/**'],
}),
);

shell.chmod('-R', 'u+w', args.input);
}
5 changes: 3 additions & 2 deletions src/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,31 +38,31 @@
export const SINGLE_PAGE_DATA_FILENAME = 'single-page.json';
export const CUSTOM_STYLE = 'custom-style';

export enum Stage {

Check warning on line 41 in src/constants.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

'Stage' is already declared in the upper scope on line 41 column 13
NEW = 'new',
PREVIEW = 'preview',
TECH_PREVIEW = 'tech-preview',
SKIP = 'skip',
}

export enum Lang {

Check warning on line 48 in src/constants.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

'Lang' is already declared in the upper scope on line 48 column 13
RU = 'ru',
EN = 'en',
}

export enum Platforms {

Check warning on line 53 in src/constants.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

'Platforms' is already declared in the upper scope on line 53 column 13
WINDOWS = 'win32',
MAC = 'darwin',
LINUX = 'linux',
}

export enum IncludeMode {

Check warning on line 59 in src/constants.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

'IncludeMode' is already declared in the upper scope on line 59 column 13
ROOT_MERGE = 'root_merge',
MERGE = 'merge',
LINK = 'link',
}

export enum ResourceType {

Check warning on line 65 in src/constants.ts

View workflow job for this annotation

GitHub Actions / Verify Files 

'ResourceType' is already declared in the upper scope on line 65 column 13
style = 'style',
script = 'script',
}
Expand Down Expand Up @@ -116,7 +116,8 @@
// Regexp result: authorLogin
export const REGEXP_AUTHOR = /(?<=author:\s).+(?=\r?\n)/g;

export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 1000;
export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || os.cpus().length - 1;
export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 10;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Мммм. Это зафорсит включение воркеров на большинстве существующих сборок.
Так ведь?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Это скорее нижняя граница, сколько файлов попадет в один воркер, теперь это работает через пулл воркеров. Кол-во запущенных воркеров всегда будет равно WORKERS_COUNT , я чуть изменил функцию расчета чанка.

export const THREAD_PART_COUNT = Number(process.env.THREAD_PART_COUNT) || 8;
export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || os.cpus().length - 1 || 1;

export const metadataBorder = '---';
6 changes: 3 additions & 3 deletions src/resolvers/lintPage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ import {
PluginOptions,
default as yfmlint,
} from '@diplodoc/transform/lib/yfmlint';
import {readFileSync} from 'fs';
import {bold} from 'chalk';

import {ArgvService, PluginService} from '../services';
import {getVarsPerFile, getVarsPerRelativeFile} from '../utils';
import {liquidMd2Html} from './md2html';
import {liquidMd2Md} from './md2md';
import * as fs from 'fs';

interface FileTransformOptions {
path: string;
Expand All @@ -28,13 +28,13 @@ export interface ResolverLintOptions {
onFinish?: () => void;
}

export function lintPage(options: ResolverLintOptions) {
export async function lintPage(options: ResolverLintOptions) {
const {inputPath, fileExtension, onFinish} = options;
const {input} = ArgvService.getConfig();
const resolvedPath: string = resolve(input, inputPath);

try {
const content: string = readFileSync(resolvedPath, 'utf8');
const content: string = await fs.promises.readFile(resolvedPath, 'utf8');

const lintFn: Function = FileLinter[fileExtension];
if (!lintFn) {
Expand Down
Loading
Loading