From 091639dd98fed553b77b0ca0c07a31f9a5adb53c Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 12 Nov 2024 12:38:45 +0300 Subject: [PATCH 1/9] fix: Fix some lint and typecheck errors --- src/models.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/models.ts b/src/models.ts index 5711dda9..16680fd5 100644 --- a/src/models.ts +++ b/src/models.ts @@ -86,11 +86,10 @@ interface YfmConfig { changelogs: string | boolean; analytics?: DocAnalytics; useLegacyConditions?: boolean; - search?: - | true - | ({ - provider?: string; - } & {[prop: string]: unknown}); + search: { + enabled: boolean; + provider?: string; + } & {[prop: string]: unknown}; } export interface YfmArgv extends YfmConfig { From 1cebf9ba2256401f43338183337cc0ee99e636df Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 12 Nov 2024 12:40:47 +0300 Subject: [PATCH 2/9] feat: Implement new build configuration systemcon --- src/commands/build/__tests__/index.ts | 120 ++++++++ src/commands/build/config.ts | 148 ++++++++++ .../build/features/changelogs/config.ts | 10 + .../build/features/changelogs/index.ts | 26 ++ .../build/features/contributors/config.ts | 16 ++ .../build/features/contributors/index.ts | 55 ++++ src/commands/build/features/legacy/config.ts | 68 +++++ .../build/features/legacy/index.spec.ts | 270 ++++++++++++++++++ src/commands/build/features/legacy/index.ts | 115 ++++++++ src/commands/build/features/linter/config.ts | 15 + .../build/features/linter/index.spec.ts | 83 ++++++ src/commands/build/features/linter/index.ts | 97 +++++++ .../build/features/redirects/index.ts | 71 +++++ src/commands/build/features/search/config.ts | 15 + .../build/features/search/index.spec.ts | 72 +++++ src/commands/build/features/search/index.ts | 58 ++++ .../build/features/singlepage/config.ts | 10 + .../build/features/singlepage/index.ts | 26 ++ .../build/features/templating/config.ts | 55 ++++ .../build/features/templating/index.spec.ts | 206 +++++++++++++ .../build/features/templating/index.ts | 94 ++++++ src/commands/build/handler.ts | 107 +++++++ src/commands/build/index.spec.ts | 263 +++++++++++++++++ src/commands/build/run.ts | 108 +++++++ 24 files changed, 2108 insertions(+) create mode 100644 src/commands/build/__tests__/index.ts create mode 100644 src/commands/build/config.ts create mode 100644 src/commands/build/features/changelogs/config.ts create mode 100644 src/commands/build/features/changelogs/index.ts create mode 100644 src/commands/build/features/contributors/config.ts create mode 100644 src/commands/build/features/contributors/index.ts create mode 100644 src/commands/build/features/legacy/config.ts create mode 100644 src/commands/build/features/legacy/index.spec.ts create mode 100644 src/commands/build/features/legacy/index.ts create mode 100644 src/commands/build/features/linter/config.ts create mode 100644 src/commands/build/features/linter/index.spec.ts create mode 100644 src/commands/build/features/linter/index.ts create mode 100644 src/commands/build/features/redirects/index.ts create mode 100644 src/commands/build/features/search/config.ts create mode 100644 src/commands/build/features/search/index.spec.ts create mode 100644 src/commands/build/features/search/index.ts create mode 100644 src/commands/build/features/singlepage/config.ts create mode 100644 src/commands/build/features/singlepage/index.ts create mode 100644 src/commands/build/features/templating/config.ts create mode 100644 src/commands/build/features/templating/index.spec.ts create mode 100644 src/commands/build/features/templating/index.ts create mode 100644 src/commands/build/handler.ts create mode 100644 src/commands/build/index.spec.ts create mode 100644 src/commands/build/run.ts diff --git a/src/commands/build/__tests__/index.ts b/src/commands/build/__tests__/index.ts new file mode 100644 index 00000000..a117317e --- /dev/null +++ b/src/commands/build/__tests__/index.ts @@ -0,0 +1,120 @@ +import type {Run} from '../run'; +import type {BuildConfig, BuildRawConfig} from '..'; + +import {Mock, describe, expect, it, vi} from 'vitest'; +import {Build} from '..'; +import {handler as originalHandler} from '../handler'; +import {withConfigUtils} from '~/config'; + +export const handler = originalHandler as Mock; + +// eslint-disable-next-line no-var +var resolveConfig: Mock; + +vi.mock('shelljs'); +vi.mock('../handler'); +vi.mock('~/config', async (importOriginal) => { + resolveConfig = vi.fn((_path, {defaults, fallback}) => { + return defaults || fallback; + }); + + return { + ...((await importOriginal()) as {}), + resolveConfig, + }; +}); + +export async function runBuild(args: string) { + const build = new Build(); + + build.apply(); + + await build.parse(['node', 'index'].concat(args.split(' '))); +} + +export function testConfig(name: string, args: string, result: DeepPartial): void; +export function testConfig(name: string, args: string, error: Error): void; +export function testConfig( + name: string, + args: string, + config: DeepPartial, + result: DeepPartial, +): void; +export function testConfig( + name: string, + args: string, + config: DeepPartial, + error: Error, +): void; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function testConfig(name: string, args: string, config: any, result?: any): void { + it(name, async () => { + if (!result) { + result = config; + config = {}; + } + + resolveConfig.mockImplementation((path, {defaults}) => { + if (path.endsWith('.yfmlint')) { + return withConfigUtils(path, {}); + } + + if (path.endsWith('redirects.yaml')) { + return withConfigUtils(null, {}); + } + + return withConfigUtils(path, { + ...defaults, + ...config, + }); + }); + + handler.mockImplementation((run: Run) => { + expect(run.config).toMatchObject(result as Partial); + }); + + if (result instanceof Error) { + await expect(() => + runBuild('--input ./input --output ./output ' + args), + ).rejects.toThrow(result); + } else { + await runBuild('--input ./input --output ./output ' + args); + + expect(handler).toBeCalled(); + } + }); +} + +export function testBooleanFlag(name: string, arg: string, defaults: boolean) { + describe(name, () => { + testConfig('should handle default', '', { + [name]: defaults, + }); + + testConfig('should handle arg', arg, { + [name]: true, + }); + + testConfig( + 'should handle config enabled', + '', + { + [name]: true, + }, + { + [name]: true, + }, + ); + + testConfig( + 'should handle config disabled', + '', + { + [name]: false, + }, + { + [name]: false, + }, + ); + }); +} diff --git a/src/commands/build/config.ts b/src/commands/build/config.ts new file mode 100644 index 00000000..a09a1b48 --- /dev/null +++ b/src/commands/build/config.ts @@ -0,0 +1,148 @@ +import {bold, underline} from 'chalk'; +import {options as globalOptions} from '~/program/config'; +import {option} from '~/config'; +import {Stage} from '~/constants'; + +export enum OutputFormat { + md = 'md', + html = 'html', +} + +const outputFormat = option({ + flags: '-f, --output-format ', + defaultInfo: 'html', + choices: ['html', 'md'], + desc: ` + Format of output files. (html or md) + + If ${bold('html')} is selected, then renders md to static html files. + (See also ${underline('--static-content')} option) + + If ${bold('md')} is selected, then renders md to prepared md files + enriched by additional metadata. + (Useful for complex documentation servers with runtime rendering) + `, +}); + +const langs = option({ + flags: '--lang, --langs ', + desc: 'Allow loading custom resources into statically generated pages.', + // parser: toArray, +}); + +const vars = option({ + flags: '-v, --vars ', + desc: ` + Pass list of variables directly to build. + Variables should be passed in JSON format. + Passed variables overrides the same in presets.yaml + + Example: + {{PROGRAM}} -i ./ -o ./build -v '{"name":"test"}' + `, + parser: (value) => JSON.parse(value), +}); + +const varsPreset = option({ + flags: '--vars-preset ', + desc: ` + Select vars preset of documentation. + Selected preset will be merged with default section of presets.yaml + `, +}); + +const allowHtml = option({ + flags: '--allow-html', + desc: 'Allow to use HTML in Markdown files.', + defaultInfo: true, +}); + +const sanitizeHtml = option({ + flags: '--sanitize-html', + desc: 'Toggle transformed HTML sanitizing. (Slow but secure feature)', + defaultInfo: true, +}); + +const ignore = option({ + flags: '--ignore ', + desc: ` + Do not process paths matched by glob. + + Example: + {{PROGRAM}} -i ./input -o ./output --ignore *.bad.md + + or + + {{PROGRAM}} -i ./ -o ./build --ignore ./build + `, +}); + +// TODO: options below are not beautified. +const addMapFile = option({ + flags: '--add-map-file', + desc: 'Should add all paths of documentation into file.json.', +}); + +const removeHiddenTocItems = option({ + flags: '--remove-hidden-toc-items', + desc: 'Remove from Toc all items marked as hidden.', +}); + +const mergeIncludes = option({ + flags: '--merge-includes', + desc: 'Merge includes syntax during md to md processing.', +}); + +const resources = option({ + flags: '--resource, --resources ', + desc: 'Allow loading custom resources into statically generated pages.', + // parser: toArray, +}); + +const allowCustomResources = option({ + flags: '--allow-custom-resources', + desc: 'Allow loading custom resources into statically generated pages.', +}); + +const staticContent = option({ + flags: '--static-content', + desc: 'Allow loading custom resources into statically generated pages.', +}); + +const ignoreStage = option({ + flags: '--ignore-stage ', + defaultInfo: Stage.SKIP, + desc: 'Ignore tocs with stage.', +}); + +const addSystemMeta = option({ + flags: '--add-system-meta', + desc: 'Should add system section variables form presets into files meta data.', +}); + +const buildDisabled = option({ + flags: '--build-disabled', + desc: 'Disable building.', +}); + +export const options = { + input: globalOptions.input, + output: globalOptions.output, + config: globalOptions.config, + langs, + outputFormat, + varsPreset, + vars, + allowHtml, + sanitizeHtml, + addMapFile, + removeHiddenTocItems, + mergeIncludes, + resources, + allowCustomResources, + staticContent, + ignore, + ignoreStage, + addSystemMeta, + buildDisabled, +}; diff --git a/src/commands/build/features/changelogs/config.ts b/src/commands/build/features/changelogs/config.ts new file mode 100644 index 00000000..b880d1ff --- /dev/null +++ b/src/commands/build/features/changelogs/config.ts @@ -0,0 +1,10 @@ +import {option} from '~/config'; + +const changelogs = option({ + flags: '--changelogs', + desc: 'Beta functionality: Toggle processing of experimental changelogs syntax', +}); + +export const options = { + changelogs, +}; diff --git a/src/commands/build/features/changelogs/index.ts b/src/commands/build/features/changelogs/index.ts new file mode 100644 index 00000000..027abc62 --- /dev/null +++ b/src/commands/build/features/changelogs/index.ts @@ -0,0 +1,26 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; +import {defined} from '~/config'; +import {options} from './config'; + +export type ChangelogsArgs = { + changelogs: boolean | string; +}; + +export type ChangelogsConfig = { + changelogs: boolean | string; +}; + +export class Changelogs { + apply(program: Build) { + program.hooks.Command.tap('Changelogs', (command: Command) => { + command.addOption(options.changelogs); + }); + + program.hooks.Config.tap('Changelogs', (config, args) => { + config.changelogs = defined('changelogs', args, config) || false; + + return config; + }); + } +} diff --git a/src/commands/build/features/contributors/config.ts b/src/commands/build/features/contributors/config.ts new file mode 100644 index 00000000..5018f30d --- /dev/null +++ b/src/commands/build/features/contributors/config.ts @@ -0,0 +1,16 @@ +import {option} from '~/config'; + +const contributors = option({ + flags: '--contributors', + desc: 'Should attach contributors into files', +}); + +const ignoreAuthorPatterns = option({ + flags: '--ignore-author-patterns ', + desc: 'Ignore authors if they contain passed string', +}); + +export const options = { + contributors, + ignoreAuthorPatterns, +}; diff --git a/src/commands/build/features/contributors/index.ts b/src/commands/build/features/contributors/index.ts new file mode 100644 index 00000000..39e70793 --- /dev/null +++ b/src/commands/build/features/contributors/index.ts @@ -0,0 +1,55 @@ +import type {Build} from '../..'; +import type {Command} from '~/config'; +import type {VCSConnectorConfig} from '~/vcs-connector/connector-models'; + +import {defined} from '~/config'; +import {options} from './config'; + +interface VCSConfiguration { + /** + * Externally accessible base URI for a resource where a particular documentation + * source is hosted. + * + * This configuration parameter is used to directly control the Edit button behaviour + * in the Diplodoc documentation viewer(s). + * + * For example, if the following applies: + * - Repo with doc source is hosted on GitHub (say, https://github.com/foo-org/bar), + * - Within that particular repo, the directory that is being passed as an `--input` + * parameter to the CLI is located at `docs/`, + * - Whenever the Edit button is pressed, you wish to direct your readers to the + * respective document's source on `main` branch + * + * you should pass `https://github.com/foo-org/bar/tree/main/docs` as a value for this parameter. + */ + remoteBase?: string; + connector?: VCSConnectorConfig; +} + +export type ContributorsArgs = { + contributors?: boolean; + ignoreAuthorPatterns?: string[]; +}; + +export type ContributorsConfig = { + vcs: VCSConfiguration; + contributors: boolean; + ignoreAuthorPatterns: string[]; +}; + +export class Contributors { + apply(program: Build) { + program.hooks.Command.tap('Contributors', (command: Command) => { + command.addOption(options.contributors); + command.addOption(options.ignoreAuthorPatterns); + }); + + program.hooks.Config.tap('Contributors', (config, args) => { + config.vcs = defined('vcs', config) || {}; + config.contributors = defined('contributors', args, config) || false; + config.ignoreAuthorPatterns = defined('ignoreAuthorPatterns', args, config) || []; + + return config; + }); + } +} diff --git a/src/commands/build/features/legacy/config.ts b/src/commands/build/features/legacy/config.ts new file mode 100644 index 00000000..df014e36 --- /dev/null +++ b/src/commands/build/features/legacy/config.ts @@ -0,0 +1,68 @@ +import {bold} from 'chalk'; +import {option} from '~/config'; + +const disableLiquid = option({ + flags: '--disable-liquid', + desc: 'Disable template engine.', + defaultInfo: false, + deprecated: 'Use --no-template instead.', +}); + +const applyPresets = option({ + flags: '--apply-presets', + desc: 'Should apply presets.', + defaultInfo: true, + deprecated: 'Use --template-vars/--no-template-vars instead.', +}); + +const resolveConditions = option({ + flags: '--resolve-conditions', + desc: 'Should resolve conditions.', + defaultInfo: true, + deprecated: 'Use --template-conditions/--no-template-conditions instead.', +}); + +const conditionsInCode = option({ + flags: '--conditions-in-code', + desc: 'Meet conditions in code blocks.', + defaultInfo: false, + deprecated: 'Use --template=all or --template=code instead.', +}); + +const lintDisabled = option({ + flags: '--lint-disabled', + desc: 'Disable linting.', + hidden: true, + deprecated: `Use ${bold('--no-lint')} instead.`, +}); + +const allowHTML = option({ + flags: '--allowHTML', + desc: 'Allow to use HTML in Markdown files.', + defaultInfo: true, + deprecated: `Use ${bold('--allow-html')} for consistency.`, +}); + +const needToSanitizeHtml = option({ + flags: '--need-to-sanitize-html', + desc: 'Toggle transformed HTML sanitizing. (Slow but secure feature)', + defaultInfo: true, + deprecated: `Use ${bold('--sanitize-html')} instead.`, +}); + +const useLegacyConditions = option({ + flags: '--use-legacy-conditions', + desc: 'Temporal backward compatibility flag.', + defaultInfo: false, +}); + +export const options = { + disableLiquid, + applyPresets, + resolveConditions, + conditionsInCode, + lintDisabled, + allowHTML, + needToSanitizeHtml, + useLegacyConditions, +}; diff --git a/src/commands/build/features/legacy/index.spec.ts b/src/commands/build/features/legacy/index.spec.ts new file mode 100644 index 00000000..d6219379 --- /dev/null +++ b/src/commands/build/features/legacy/index.spec.ts @@ -0,0 +1,270 @@ +import {describe} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +describe('Build legacy feature', () => { + describe('config', () => { + describe('disableLiquid', () => { + test('should handle default', '', { + template: { + enabled: true, + }, + }); + + test('should handle arg', '--disable-liquid', { + template: { + enabled: false, + }, + }); + + test('should handle old arg with priority', '--disable-liquid --template all', { + template: { + enabled: false, + }, + }); + + test( + 'should handle config', + '', + { + disableLiquid: true, + }, + { + template: { + enabled: false, + }, + }, + ); + }); + + describe('applyPresets', () => { + test('should handle default', '', { + template: { + features: { + substitutions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--apply-presets', + { + applyPresets: false, + }, + { + template: { + features: { + substitutions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-apply-presets', { + template: { + features: { + substitutions: false, + }, + }, + }); + + test('should handle new arg', '--no-template-vars', { + template: { + features: { + substitutions: false, + }, + }, + }); + + test('should handle old arg with priority', '--no-apply-presets --template-vars', { + template: { + features: { + substitutions: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + applyPresets: false, + }, + { + template: { + features: { + substitutions: false, + }, + }, + }, + ); + }); + + describe('resolveConditions', () => { + test('should handle default', '', { + template: { + features: { + conditions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--resolve-conditions', + { + resolveConditions: false, + }, + { + template: { + features: { + conditions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-resolve-conditions', { + template: { + features: { + conditions: false, + }, + }, + }); + + test('should handle new arg', '--no-template-conditions', { + template: { + features: { + conditions: false, + }, + }, + }); + + test( + 'should handle old arg with priority', + '--no-resolve-conditions --template-conditions', + { + template: { + features: { + conditions: false, + }, + }, + }, + ); + + test( + 'should handle config', + '', + { + resolveConditions: false, + }, + { + template: { + features: { + conditions: false, + }, + }, + }, + ); + }); + + describe('conditionsInCode', () => { + test('should handle default', '', { + template: { + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle arg', '--conditions-in-code', { + template: { + scopes: { + text: true, + code: true, + }, + }, + }); + + test( + 'should handle negated arg with priority', + '--no-conditions-in-code', + { + conditionsInCode: true, + }, + { + template: { + scopes: { + text: true, + code: false, + }, + }, + }, + ); + + test('should handle old arg with priority', '--no-conditions-in-code --template all', { + template: { + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle negated new arg', '--no-template', { + template: { + enabled: false, + scopes: { + text: false, + code: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + conditionsInCode: true, + }, + { + template: { + scopes: { + text: true, + code: true, + }, + }, + }, + ); + }); + + describe('lintDisabled', () => { + test('should handle default', '', { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'disabled', + }, + }, + }, + }); + + test('should handle arg', '--lint-disabled', { + lint: {enabled: false}, + }); + + test( + 'should handle config', + '', + { + lintDisabled: true, + }, + { + lint: {enabled: false}, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/legacy/index.ts b/src/commands/build/features/legacy/index.ts new file mode 100644 index 00000000..c364335b --- /dev/null +++ b/src/commands/build/features/legacy/index.ts @@ -0,0 +1,115 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; +import type {VCSConnectorConfig} from '~/vcs-connector/connector-models'; + +import {defined, valuable} from '~/config'; +import {options} from './config'; + +export type LegacyArgs = { + disableLiquid?: boolean; + resolveConditions?: boolean; + conditionsInCode?: boolean; + applyPresets?: boolean; + + lintDisabled?: boolean; + allowHTML?: boolean; + needToSanitizeHtml?: boolean; + useLegacyConditions?: boolean; +}; + +export type LegacyRawConfig = { + disableLiquid: boolean; + resolveConditions: boolean; + conditionsInCode: boolean; + applyPresets: boolean; + + lintDisabled: boolean; + allowHTML: boolean; + needToSanitizeHtml: boolean; + useLegacyConditions: boolean; + + connector?: VCSConnectorConfig; +}; + +export type LegacyConfig = { + useLegacyConditions: boolean; +}; + +export class Legacy { + apply(program: Build) { + program.hooks.Command.tap('Legacy', (command: Command) => { + command + .addOption(options.disableLiquid) + .addOption(options.applyPresets) + .addOption(options.resolveConditions) + .addOption(options.conditionsInCode) + .addOption(options.lintDisabled) + .addOption(options.allowHTML) + .addOption(options.needToSanitizeHtml) + .addOption(options.useLegacyConditions); + }); + + program.hooks.Config.tap('Legacy', (config, args) => { + const disableLiquid = defined('disableLiquid', args, config); + const applyPresets = defined('applyPresets', args, config); + const resolveConditions = defined('resolveConditions', args, config); + const conditionsInCode = defined('conditionsInCode', args, config); + const lintDisabled = defined('lintDisabled', args, config); + const allowHTML = defined('allowHTML', args, config); + const needToSanitizeHtml = defined('needToSanitizeHtml', args, config); + const useLegacyConditions = defined('useLegacyConditions', args, config); + const vcsConnector = defined('connector', config); + + if (valuable(disableLiquid)) { + config.template.enabled = disableLiquid !== true; + } + + if (valuable(conditionsInCode)) { + config.template.scopes.code = conditionsInCode === true; + } + + if (valuable(applyPresets)) { + config.template.features.substitutions = applyPresets; + } + + if (valuable(resolveConditions)) { + config.template.features.conditions = resolveConditions; + } + + if (valuable(lintDisabled)) { + config.lint.enabled = lintDisabled !== true; + } + + if (valuable(allowHTML)) { + config.allowHtml = allowHTML; + config.lint.config['log-levels']['MD033'] = allowHTML ? 'disabled' : 'error'; + } + + if (valuable(needToSanitizeHtml)) { + config.sanitizeHtml = needToSanitizeHtml; + } + + if (valuable(vcsConnector)) { + config.vcs.connector = vcsConnector; + } + + config.useLegacyConditions = Boolean(useLegacyConditions); + + for (const prop of [ + 'disableLiquid', + 'applyPresets', + 'resolveConditions', + 'conditionsInCode', + 'lintDisabled', + 'allowHTML', + 'needToSanitizeHtml', + 'connector', + ]) { + // @ts-ignore + delete config[prop]; + } + + return config; + }); + } +} diff --git a/src/commands/build/features/linter/config.ts b/src/commands/build/features/linter/config.ts new file mode 100644 index 00000000..253f2518 --- /dev/null +++ b/src/commands/build/features/linter/config.ts @@ -0,0 +1,15 @@ +import {bold} from 'chalk'; +import {option} from '~/config'; + +const lint = option({ + flags: '--lint', + desc: ` + Toggle file linting. + + Enabled by default. Use ${bold('--no-lint')} to disable. + `, +}); + +export const options = { + lint, +}; diff --git a/src/commands/build/features/linter/index.spec.ts b/src/commands/build/features/linter/index.spec.ts new file mode 100644 index 00000000..473b6fb2 --- /dev/null +++ b/src/commands/build/features/linter/index.spec.ts @@ -0,0 +1,83 @@ +import {describe, vi} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +vi.mock('~/cmd/publish/upload'); + +describe('Build linter feature', () => { + describe('config', () => { + describe('lint', () => { + test('should handle default', '', { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'disabled', + }, + }, + }, + }); + + test('should handle arg', '--no-lint', { + lint: {enabled: false}, + }); + + test( + 'should handle config', + '', + { + lint: {enabled: false}, + }, + { + lint: {enabled: false}, + }, + ); + + test( + 'should handle simplified config', + '', + { + lint: false, + }, + { + lint: {enabled: false}, + }, + ); + + test( + 'should handle enabled allowHtml', + '', + { + allowHtml: true, + }, + { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'disabled', + }, + }, + }, + }, + ); + + test( + 'should handle disabled allowHtml', + '', + { + allowHtml: false, + }, + { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'error', + }, + }, + }, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/linter/index.ts b/src/commands/build/features/linter/index.ts new file mode 100644 index 00000000..6cb3a405 --- /dev/null +++ b/src/commands/build/features/linter/index.ts @@ -0,0 +1,97 @@ +import type {Build} from '../..'; +import type {Command} from '~/config'; + +import {resolve} from 'path'; +import shell from 'shelljs'; +import {LogLevels} from '@diplodoc/transform/lib/log'; + +import {configPath, resolveConfig, valuable} from '~/config'; +import {LINT_CONFIG_FILENAME} from '~/constants'; +import {options} from './config'; + +export type LintArgs = { + lint: boolean; +}; + +export type LintRawConfig = { + lint: + | boolean + | { + enabled: boolean; + config: string; + }; +}; + +export type LintConfig = { + lint: { + enabled: boolean; + config: LogLevelConfig; + }; +}; + +type LogLevelConfig = { + 'log-levels': Record; +}; + +// TODO(major): move to separated 'lint' command +export class Lint { + apply(program: Build) { + program.hooks.Command.tap('Lint', (command: Command) => { + command.addOption(options.lint); + }); + + let resolvedPath: AbsolutePath | null = null; + + program.hooks.Config.tapPromise('Lint', async (config, args) => { + let lint: LintConfig['lint'] | boolean = { + enabled: true, + config: {'log-levels': {}}, + }; + + if (valuable(config.lint)) { + lint = config.lint; + } + + if (typeof lint === 'boolean') { + lint = { + enabled: lint, + config: {'log-levels': {}}, + }; + } + + if (valuable(args.lint)) { + lint.enabled = Boolean(args.lint); + } + + config.lint = lint; + + if (config.lint.enabled) { + const configFilename = + typeof config.lint.config === 'string' + ? config.resolve(config.lint.config as string) + : resolve(args.input, LINT_CONFIG_FILENAME); + + const lintConfig = await resolveConfig>(configFilename, { + fallback: {'log-levels': {}}, + }); + + config.lint.config = lintConfig as LogLevelConfig; + resolvedPath = lintConfig[configPath]; + } + + config.lint.config = config.lint.config || {'log-levels': {}}; + config.lint.config['log-levels'] = config.lint.config['log-levels'] || {}; + config.lint.config['log-levels']['MD033'] = config.allowHtml + ? LogLevels.DISABLED + : LogLevels.ERROR; + + return config; + }); + + program.hooks.AfterRun.for('md').tap('Lint', async (run) => { + if (resolvedPath) { + shell.cp(resolvedPath, run.output); + } + }); + } +} diff --git a/src/commands/build/features/redirects/index.ts b/src/commands/build/features/redirects/index.ts new file mode 100644 index 00000000..7587d0ae --- /dev/null +++ b/src/commands/build/features/redirects/index.ts @@ -0,0 +1,71 @@ +import {ok} from 'node:assert'; +import {resolve} from 'node:path'; +import shell from 'shelljs'; +import {Build} from '../..'; + +import {REDIRECTS_FILENAME} from '~/constants'; +import {configPath, resolveConfig} from '~/config'; + +interface Redirect { + from: string; + to: string; +} + +interface RedirectsConfig { + common: Redirect[]; + [lang: string]: Redirect[]; +} + +export class Redirects { + apply(program: Build) { + let resolvedPath: string | null = null; + + program.hooks.BeforeRun.for('md').tap('Redirects', async (run) => { + try { + const redirects = await resolveConfig( + resolve(run.originalInput, REDIRECTS_FILENAME), + { + fallback: {common: []}, + }, + ); + + if (redirects[configPath]) { + validateRedirects(redirects, redirects[configPath]); + resolvedPath = redirects[configPath]; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + run.logger.error(error.message || error); + } + }); + + program.hooks.AfterRun.for('md').tap('Redirects', async (run) => { + if (resolvedPath) { + shell.cp(resolvedPath, run.output); + } + }); + } +} + +function validateRedirects(redirectsConfig: RedirectsConfig, pathToRedirects: string) { + const redirects: Redirect[] = Object.keys(redirectsConfig).reduce( + (res, redirectSectionName) => { + const sectionRedirects = redirectsConfig[redirectSectionName]; + res.push(...sectionRedirects); + return res; + }, + [] as Redirect[], + ); + + const getContext = (from: string, to: string) => ` [Context: \n- from: ${from}\n- to: ${to} ]`; + const formatMessage = (message: string, pathname: string, from: string, to: string) => + `${pathname}: ${message} ${getContext(from, to)}`; + + redirects.forEach(({from, to}) => { + ok( + from && to, + formatMessage('One of the two parameters is missing', pathToRedirects, from, to), + ); + ok(from !== to, formatMessage('Parameters must be different', pathToRedirects, from, to)); + }); +} diff --git a/src/commands/build/features/search/config.ts b/src/commands/build/features/search/config.ts new file mode 100644 index 00000000..51ba4501 --- /dev/null +++ b/src/commands/build/features/search/config.ts @@ -0,0 +1,15 @@ +import {option} from '~/config'; + +const search = option({ + flags: '--search', + desc: ` + Enable search functionality. + + From command args only local search can be enabled. + Use config to configure alternate search strategy. + `, +}); + +export const options = { + search, +}; diff --git a/src/commands/build/features/search/index.spec.ts b/src/commands/build/features/search/index.spec.ts new file mode 100644 index 00000000..0c3649de --- /dev/null +++ b/src/commands/build/features/search/index.spec.ts @@ -0,0 +1,72 @@ +import {describe} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +describe('Build search feature', () => { + describe('config', () => { + describe('search', () => { + test('should handle default', '', { + search: { + enabled: false, + provider: 'local', + }, + }); + + test('should handle arg', '--search', { + search: { + enabled: true, + provider: 'local', + }, + }); + + test( + 'should handle arg with priority', + '--search', + { + search: { + enabled: false, + provider: 'custom', + }, + }, + { + search: { + enabled: true, + provider: 'custom', + }, + }, + ); + + test( + 'should handle config', + '', + { + search: { + enabled: true, + provider: 'custom', + searchUrl: 'test/?q={{query}}', + }, + }, + { + search: { + enabled: true, + provider: 'custom', + searchUrl: 'test/?q={{query}}', + }, + }, + ); + + test( + 'should handle simplified config', + '', + { + search: true, + }, + { + search: { + enabled: true, + provider: 'local', + }, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/search/index.ts b/src/commands/build/features/search/index.ts new file mode 100644 index 00000000..60818549 --- /dev/null +++ b/src/commands/build/features/search/index.ts @@ -0,0 +1,58 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; + +import {valuable} from '~/config'; +import {options} from './config'; + +export type SearchArgs = { + search: boolean; +}; + +export type SearchRawConfig = { + search: boolean | Config; +}; + +export type SearchConfig = { + search: Config; +}; + +type Config = { + enabled: boolean; + provider: string; +} & { + [prop: string]: unknown; +}; + +export class Search { + apply(program: Build) { + program.hooks.Command.tap('Search', (command: Command) => { + command.addOption(options.search); + }); + + program.hooks.Config.tap('Search', (config, args) => { + let search: Config | boolean = { + enabled: false, + provider: 'local', + }; + + if (valuable(config.search)) { + search = config.search; + } + + if (typeof search === 'boolean') { + search = { + enabled: search, + provider: 'local', + }; + } + + if (valuable(args.search)) { + search.enabled = Boolean(args.search); + } + + config.search = search; + + return config; + }); + } +} diff --git a/src/commands/build/features/singlepage/config.ts b/src/commands/build/features/singlepage/config.ts new file mode 100644 index 00000000..a117e968 --- /dev/null +++ b/src/commands/build/features/singlepage/config.ts @@ -0,0 +1,10 @@ +import {option} from '~/config'; + +const singlePage = option({ + flags: '--single-page', + desc: 'Beta functionality: Build a single page in the output folder also.', +}); + +export const options = { + singlePage, +}; diff --git a/src/commands/build/features/singlepage/index.ts b/src/commands/build/features/singlepage/index.ts new file mode 100644 index 00000000..be266f63 --- /dev/null +++ b/src/commands/build/features/singlepage/index.ts @@ -0,0 +1,26 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; +import {defined} from '~/config'; +import {options} from './config'; + +export type SinglePageArgs = { + singlePage: boolean; +}; + +export type SinglePageConfig = { + singlePage: boolean; +}; + +export class SinglePage { + apply(program: Build) { + program.hooks.Command.tap('SinglePage', (command: Command) => { + command.addOption(options.singlePage); + }); + + program.hooks.Config.tap('SinglePage', (config, args) => { + config.singlePage = defined('singlePage', args, config) || false; + + return config; + }); + } +} diff --git a/src/commands/build/features/templating/config.ts b/src/commands/build/features/templating/config.ts new file mode 100644 index 00000000..f50f5ce5 --- /dev/null +++ b/src/commands/build/features/templating/config.ts @@ -0,0 +1,55 @@ +import {bold, cyan, green} from 'chalk'; +import {option} from '~/config'; + +const template = option({ + flags: '--template ', + desc: ` + Select liquid template engine mode. + By default liquid ignores code blocs. (${bold('text')} mode) + Use ${bold('all')} or ${bold('code')} mode to process code blocks. + Use ${bold('--no-template')} to completely disable template engine. + + Read more about templating ${cyan('https://diplodoc.com/docs/en/syntax/vars')} + `, + choices: ['all', 'text', 'code'], +}); + +const noTemplate = option({ + flags: '--no-template', + desc: 'Manual negation for --template', + hidden: true, + default: false, +}); + +const templateVars = option({ + flags: '--template-vars', + desc: ` + Toggle processing of terms decorated by double curly braces in Toc and Md files. (Enabled by default) + + Read more about substitutions ${cyan('https://diplodoc.com/docs/en/syntax/vars#subtitudes')} + + Example: + Some text ${green('{{some-variable}}')} end of text. + `, + defaultInfo: true, +}); + +const templateConditions = option({ + flags: '--template-conditions', + desc: ` + Toggle processing of conditions in Toc and Md files. (Enabled by default) + + Read more about conditions ${cyan('https://diplodoc.com/docs/en/syntax/vars#conditions')} + + Example: + Some text ${green('{% if var == "any" %}')} extra ${green('{% endif %}')} end of text. + `, + defaultInfo: true, +}); + +export const options = { + template, + noTemplate, + templateVars, + templateConditions, +}; diff --git a/src/commands/build/features/templating/index.spec.ts b/src/commands/build/features/templating/index.spec.ts new file mode 100644 index 00000000..3c1befc2 --- /dev/null +++ b/src/commands/build/features/templating/index.spec.ts @@ -0,0 +1,206 @@ +import {describe} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +describe('Build template feature', () => { + describe('config', () => { + describe('template', () => { + test('should handle default', '', { + template: { + enabled: true, + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle arg `all`', '--template all', { + template: { + enabled: true, + scopes: { + text: true, + code: true, + }, + }, + }); + + test('should handle arg `text`', '--template text', { + template: { + enabled: true, + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle arg `code`', '--template code', { + template: { + enabled: true, + scopes: { + text: false, + code: true, + }, + }, + }); + + test('should handle negated arg', '--no-template', { + template: { + enabled: false, + scopes: { + text: false, + code: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + template: { + enabled: false, + }, + }, + { + template: { + enabled: false, + scopes: { + text: true, + code: false, + }, + }, + }, + ); + + test( + 'should handle siplified config', + '', + { + template: false, + }, + { + template: { + enabled: false, + scopes: { + text: true, + code: false, + }, + }, + }, + ); + }); + + describe('templateVars', () => { + test('should handle default', '', { + template: { + features: { + substitutions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--template-vars', + { + template: { + features: { + substitutions: false, + }, + }, + }, + { + template: { + features: { + substitutions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-template-vars', { + template: { + features: { + substitutions: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + template: { + features: { + substitutions: false, + }, + }, + }, + { + template: { + features: { + substitutions: false, + }, + }, + }, + ); + }); + + describe('templateConditions', () => { + test('should handle default', '', { + template: { + features: { + conditions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--template-conditions', + { + template: { + features: { + conditions: false, + }, + }, + }, + { + template: { + features: { + conditions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-template-conditions', { + template: { + features: { + conditions: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + template: { + features: { + conditions: false, + }, + }, + }, + { + template: { + features: { + conditions: false, + }, + }, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/templating/index.ts b/src/commands/build/features/templating/index.ts new file mode 100644 index 00000000..6fab837f --- /dev/null +++ b/src/commands/build/features/templating/index.ts @@ -0,0 +1,94 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; +import {defined, valuable} from '~/config'; +import {options} from './config'; + +const merge = (acc: Hash, ...sources: Hash[]) => { + for (const source of sources) { + for (const [key, value] of Object.entries(source)) { + if (!acc[key] || !value) { + acc[key] = value; + } else if (typeof value === 'object') { + acc[key] = merge({}, acc[key], value); + } + } + } + + return acc; +}; + +export type TemplatingArgs = { + template?: boolean | 'all' | 'text' | 'code'; + templateVars?: boolean; + templateConditions?: boolean; +}; + +export type TemplatingConfig = { + template: { + enabled: boolean; + scopes: { + text: boolean; + code: boolean; + }; + features: { + substitutions: boolean; + conditions: boolean; + cycles: boolean; + }; + }; +}; + +export type TemplatingRawConfig = { + template: boolean | DeepPartial; +}; + +export class Templating { + apply(program: Build) { + program.hooks.Command.tap('Templating', (command: Command) => { + command + .addOption(options.template) + .addOption(options.noTemplate) + .addOption(options.templateVars) + .addOption(options.templateConditions); + }); + + program.hooks.Config.tap('Templating', (config, args) => { + const template = defined('template', args); + const templateVars = defined('templateVars', args); + const templateConditions = defined('templateConditions', args); + + config.template = merge( + { + enabled: (config as TemplatingRawConfig).template !== false, + scopes: { + text: true, + code: false, + }, + features: { + substitutions: true, + conditions: true, + cycles: true, + }, + }, + config.template || {}, + ) as TemplatingConfig['template']; + + if (valuable(template)) { + config.template.enabled = template !== false; + + config.template.scopes.text = ['all', 'text'].includes(template as string); + config.template.scopes.code = ['all', 'code'].includes(template as string); + } + + if (valuable(templateVars)) { + config.template.features.substitutions = templateVars; + } + + if (valuable(templateConditions)) { + config.template.features.conditions = templateConditions; + } + + return config; + }); + } +} diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts new file mode 100644 index 00000000..fbf995a2 --- /dev/null +++ b/src/commands/build/handler.ts @@ -0,0 +1,107 @@ +import type {Run} from './run'; + +import 'threads/register'; + +import glob from 'glob'; +import {join} from 'path'; +import shell from 'shelljs'; + +import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; + +import {BUNDLE_FOLDER} from '~/constants'; +import {ArgvService, Includers, SearchService} from '~/services'; +import { + initLinterWorkers, + processAssets, + processChangelogs, + processExcludedFiles, + processLinter, + processLogs, + processPages, + processServiceFiles, +} from '~/steps'; +import {prepareMapFile} from '~/steps/processMapFile'; +import {copyFiles} from '~/utils'; + +export async function handler(run: Run) { + const tmpInputFolder = run.input; + const tmpOutputFolder = run.output; + + if (typeof VERSION !== 'undefined') { + console.log(`Using v${VERSION} version`); + } + + try { + ArgvService.init(run.legacyConfig); + SearchService.init(); + // TODO: Remove duplicated types from openapi-extension + // @ts-ignore + Includers.init([OpenapiIncluder]); + + const { + output: outputFolderPath, + outputFormat, + lintDisabled, + buildDisabled, + addMapFile, + } = ArgvService.getConfig(); + + preparingTemporaryFolders(); + + await processServiceFiles(); + processExcludedFiles(); + + if (addMapFile) { + prepareMapFile(); + } + + const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + + if (!lintDisabled) { + /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ + await initLinterWorkers(); + } + + const processes = [ + !lintDisabled && processLinter(), + !buildDisabled && processPages(outputBundlePath), + ].filter(Boolean) as Promise[]; + + await Promise.all(processes); + + if (!buildDisabled) { + // process additional files + processAssets({ + run, + outputFormat, + outputBundlePath, + tmpOutputFolder, + }); + + await processChangelogs(); + + await SearchService.release(); + } + } catch (error) { + run.logger.error(error); + } finally { + processLogs(tmpInputFolder); + } +} + +function preparingTemporaryFolders() { + const args = ArgvService.getConfig(); + + copyFiles( + args.rootInput, + args.input, + glob.sync('**', { + cwd: args.rootInput, + nodir: true, + follow: true, + ignore: ['node_modules/**', '*/node_modules/**'], + }), + ); + + shell.chmod('-R', 'u+w', args.input); +} diff --git a/src/commands/build/index.spec.ts b/src/commands/build/index.spec.ts new file mode 100644 index 00000000..45b90235 --- /dev/null +++ b/src/commands/build/index.spec.ts @@ -0,0 +1,263 @@ +import {describe, expect, it} from 'vitest'; +import {handler, runBuild as run, testConfig as test, testBooleanFlag} from './__tests__'; + +describe('Build command', () => { + describe('config', () => { + it('should fail without required output prop', async () => { + await expect(() => run('--input ./input')).rejects.toThrow( + `error: required option '-o, --output ' not specified`, + ); + }); + + it('should handle required props in args', async () => { + await run('--input ./input --output ./output'); + + expect(handler).toBeCalled(); + }); + + describe('input', () => { + test('should be absolute', '--input ./input', { + input: expect.stringMatching(/^(\/|[A-Z]:\\).*?(\/|\\)input$/), + }); + }); + + describe('output', () => { + test('should be absolute', '--output ./output', { + output: expect.stringMatching(/^(\/|[A-Z]:\\).*?(\/|\\)output$/), + }); + }); + + describe('langs', () => { + test('should handle default', '', { + langs: ['ru'], + }); + + test('should handle arg', '--langs en', { + langs: ['en'], + }); + + test('should handle shorthand arg', '--lang en', { + langs: ['en'], + }); + + test('should handle multiple arg', '--lang en --lang ru', { + langs: ['en', 'ru'], + }); + + test('should handle multiple different arg', '--lang en --langs ru', { + langs: ['en', 'ru'], + }); + + test( + 'should handle config', + '', + { + langs: ['ru', 'en'], + }, + { + langs: ['ru', 'en'], + }, + ); + + test( + 'should handle empty config', + '', + { + langs: [], + }, + { + langs: ['ru'], + }, + ); + + test( + 'should fail on unlisted lang', + '', + { + // @ts-ignore + lang: 'fr', + langs: ['ru', 'en'], + }, + new Error(`Configured default lang 'fr' is not listed in langs (ru, en)`), + ); + }); + + describe('lang', () => { + test('should handle default', '', { + lang: 'ru', + }); + + test( + 'should handle config', + '', + { + lang: 'en', + }, + { + lang: 'en', + }, + ); + + test( + 'should handle first lang from langs', + '', + { + langs: ['en', 'ru'], + }, + { + lang: 'en', + }, + ); + }); + + describe('outputFormat', () => { + test('should handle default', '', { + outputFormat: 'html', + }); + + test('should handle arg', '--output-format md', { + outputFormat: 'md', + }); + + test('should handle shorthand arg', '-f md', { + outputFormat: 'md', + }); + + test( + 'should handle config', + '', + { + outputFormat: 'md', + }, + { + outputFormat: 'md', + }, + ); + + it('should fail on unknown format', async () => { + await expect(() => + run('--input ./input --output ./output --output-format other'), + ).rejects.toThrow( + `error: option '-f, --output-format ' argument 'other' is invalid. Allowed choices are html, md.`, + ); + }); + }); + + describe('varsPreset', () => { + test('should handle default', '', { + varsPreset: 'default', + }); + + test('should handle arg', '--vars-preset public', { + varsPreset: 'public', + }); + + test( + 'should handle config', + '', + { + varsPreset: 'public', + }, + { + varsPreset: 'public', + }, + ); + }); + + describe('vars', () => { + test('should handle default', '', { + vars: {}, + }); + + test('should handle arg', '--vars {"a":1}', { + vars: {a: 1}, + }); + + test('should handle shorthand arg', '-v {"a":1}', { + vars: {a: 1}, + }); + + test( + 'should handle config', + '', + { + vars: {a: 1}, + }, + { + vars: {a: 1}, + }, + ); + + // TODO: should merge args ang config + // test('should merge args ang config') + }); + + describe('ignoreStage', () => { + test('should handle default', '', { + ignoreStage: 'skip', + }); + + test('should handle arg', '--ignore-stage preview', { + ignoreStage: 'preview', + }); + + test( + 'should handle config', + '', + { + ignoreStage: 'preview', + }, + { + ignoreStage: 'preview', + }, + ); + }); + + describe('ignore', () => { + test('should handle default', '', { + ignore: [], + }); + + test('should handle arg', '--ignore **/*.md', { + ignore: ['**/*.md'], + }); + + test('should handle args', '--ignore **/*.md --ignore **/*.yaml', { + ignore: ['**/*.md', '**/*.yaml'], + }); + + test( + 'should handle config', + '', + { + ignore: ['**/*.md'], + }, + { + ignore: ['**/*.md'], + }, + ); + + // TODO: should merge args ang config + // test('should merge args ang config') + }); + + testBooleanFlag('addMapFile', '--add-map-file', false); + testBooleanFlag('removeHiddenTocItems', '--remove-hidden-toc-items', false); + testBooleanFlag('allowCustomResources', '--allow-custom-resources', false); + testBooleanFlag('staticContent', '--static-content', false); + testBooleanFlag('addSystemMeta', '--add-system-meta', false); + testBooleanFlag('buildDisabled', '--build-disabled', false); + testBooleanFlag('allowHtml', '--allow-html', true); + testBooleanFlag('sanitizeHtml', '--sanitize-html', true); + + // test('should handle required props in config', '', { + // input: './input', + // output: './output', + // }, { + // input: './input', + // output: './output', + // }); + }); + + // describe('apply', () => {}); +}); diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts new file mode 100644 index 00000000..06a909d7 --- /dev/null +++ b/src/commands/build/run.ts @@ -0,0 +1,108 @@ +import type {YfmArgv} from '~/models'; + +import {join, resolve} from 'path'; +import {configPath} from '~/config'; +import { + BUNDLE_FOLDER, + REDIRECTS_FILENAME, + TMP_INPUT_FOLDER, + TMP_OUTPUT_FOLDER, + YFM_CONFIG_FILENAME, +} from '~/constants'; +import {Logger} from '~/logger'; +import {BuildConfig} from '.'; + +/** + * This is transferable context for build command. + * Use this context to communicate with lower data processing levels. + */ +export class Run { + readonly originalInput: AbsolutePath; + + readonly originalOutput: AbsolutePath; + + readonly input: AbsolutePath; + + readonly output: AbsolutePath; + + readonly legacyConfig: YfmArgv; + + readonly logger: Logger; + + readonly config: BuildConfig; + + get bundlePath() { + return join(this.originalOutput, BUNDLE_FOLDER); + } + + get configPath() { + return this.config[configPath] || join(this.config.input, YFM_CONFIG_FILENAME); + } + + get redirectsPath() { + return join(this.originalInput, REDIRECTS_FILENAME); + } + + constructor(config: BuildConfig) { + this.config = config; + this.originalInput = config.input; + this.originalOutput = config.output; + + // TODO: use root instead + // We need to create system where we can safely work with original input. + this.input = resolve(config.output, TMP_INPUT_FOLDER); + this.output = resolve(config.output, TMP_OUTPUT_FOLDER); + + this.legacyConfig = { + rootInput: this.originalInput, + input: this.input, + output: this.output, + quiet: config.quiet, + addSystemMeta: config.addSystemMeta, + addMapFile: config.addMapFile, + staticContent: config.staticContent, + strict: config.strict, + langs: config.langs, + lang: config.lang, + ignoreStage: config.ignoreStage, + singlePage: config.singlePage, + removeHiddenTocItems: config.removeHiddenTocItems, + allowCustomResources: config.allowCustomResources, + resources: config.resources, + analytics: config.analytics, + varsPreset: config.varsPreset, + vars: config.vars, + outputFormat: config.outputFormat, + allowHTML: config.allowHtml, + needToSanitizeHtml: config.sanitizeHtml, + useLegacyConditions: config.useLegacyConditions, + + ignore: config.ignore, + + applyPresets: config.template.features.substitutions, + resolveConditions: config.template.features.conditions, + conditionsInCode: config.template.scopes.code, + disableLiquid: !config.template.enabled, + + buildDisabled: config.buildDisabled, + + lintDisabled: !config.lint.enabled, + // @ts-ignore + lintConfig: config.lint.config, + + vcs: config.vcs, + connector: config.vcs.connector, + contributors: config.contributors, + ignoreAuthorPatterns: config.ignoreAuthorPatterns, + + changelogs: config.changelogs, + search: config.search, + + included: config.mergeIncludes, + }; + + this.logger = new Logger(config, [ + (_level, message) => message.replace(new RegExp(this.input, 'ig'), ''), + ]); + } +} From 6d0b7e05fe998fba576938fd8ac4d20b398d7d1a Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 12 Nov 2024 12:43:18 +0300 Subject: [PATCH 3/9] fix: Attach new build system to main program --- src/cmd/build/index.ts | 277 ----------------------------- src/cmd/index.ts | 3 - src/commands/build/index.ts | 335 ++++++++++++++++++++++++++++++------ src/config/index.ts | 14 +- src/logger/index.ts | 20 ++- src/program/config.ts | 20 ++- src/program/types.ts | 6 +- src/resolvers/md2html.ts | 7 +- src/services/argv.ts | 9 +- src/services/search.ts | 7 +- src/steps/processAssets.ts | 47 ++--- src/validator.ts | 158 ----------------- 12 files changed, 345 insertions(+), 558 deletions(-) delete mode 100644 src/cmd/build/index.ts delete mode 100644 src/cmd/index.ts delete mode 100644 src/validator.ts diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts deleted file mode 100644 index 91790952..00000000 --- a/src/cmd/build/index.ts +++ /dev/null @@ -1,277 +0,0 @@ -import glob from 'glob'; -import {Arguments, Argv} from 'yargs'; -import {join, resolve} from 'path'; -import shell from 'shelljs'; - -import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; - -import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants'; -import {argvValidator} from '../../validator'; -import {ArgvService, Includers, SearchService} from '../../services'; -import { - initLinterWorkers, - processAssets, - processChangelogs, - processExcludedFiles, - processLinter, - processLogs, - processPages, - processServiceFiles, -} from '../../steps'; -import {prepareMapFile} from '../../steps/processMapFile'; -import {copyFiles, logger} from '../../utils'; - -export const build = { - command: ['build', '$0'], - description: 'Build documentation in target directory', - handler, - builder, -}; - -function builder(argv: Argv) { - return argv - .option('input', { - alias: 'i', - describe: 'Path to input folder with .md files', - type: 'string', - group: 'Build options:', - }) - .option('output', { - alias: 'o', - describe: 'Path to output folder', - type: 'string', - group: 'Build options:', - }) - .option('varsPreset', { - default: 'default', - describe: 'Target vars preset of documentation ', - group: 'Build options:', - }) - .option('output-format', { - default: 'html', - describe: 'Format of output file ', - group: 'Build options:', - }) - .option('vars', { - alias: 'v', - default: '{}', - describe: 'List of markdown variables', - group: 'Build options:', - }) - .option('apply-presets', { - default: true, - describe: 'Should apply presets. Only for --output-format=md', - type: 'boolean', - group: 'Build options:', - }) - .option('resolve-conditions', { - default: true, - describe: 'Should resolve conditions. Only for --output-format=md', - type: 'boolean', - group: 'Build options:', - }) - .option('conditions-in-code', { - default: false, - describe: 'Meet conditions in code blocks', - type: 'boolean', - group: 'Build options:', - }) - .option('disable-liquid', { - default: false, - describe: 'Disable template engine', - type: 'boolean', - group: 'Build options:', - }) - .option('allowHTML', { - default: false, - describe: 'Allow to use HTML in Markdown files', - type: 'boolean', - group: 'Build options:', - }) - .option('ignore-stage', { - default: Stage.SKIP, - describe: 'Ignore tocs with stage', - group: 'Build options:', - }) - .option('ignore-author-patterns', { - default: [] as string[], - describe: 'Ignore authors if they contain passed string', - group: 'Build options:', - type: 'array', - }) - .option('contributors', { - default: false, - describe: 'Should attach contributors into files', - type: 'boolean', - group: 'Build options:', - }) - .option('add-system-meta', { - default: false, - describe: 'Should add system section variables form presets into files meta data', - type: 'boolean', - group: 'Build options:', - }) - .option('add-map-file', { - default: false, - describe: 'Should add all paths of documentation into file.json', - type: 'boolean', - group: 'Build options:', - }) - .option('single-page', { - default: false, - describe: 'Beta functionality: Build a single page in the output folder also', - type: 'boolean', - group: 'Build options:', - }) - .option('publish', { - default: false, - describe: 'Should upload output files to S3 storage', - type: 'boolean', - group: 'Build options:', - }) - .option('remove-hidden-toc-items', { - default: false, - describe: 'Remove hidden toc items', - type: 'boolean', - group: 'Build options:', - }) - .option('lint-disabled', { - default: false, - describe: 'Disable linting', - type: 'boolean', - group: 'Build options:', - }) - .option('build-disabled', { - default: false, - describe: 'Disable building', - type: 'boolean', - group: 'Build options:', - }) - .option('allow-custom-resources', { - default: false, - describe: 'Allow loading custom resources', - type: 'boolean', - group: 'Build options:', - }) - .option('static-content', { - default: false, - describe: 'Include static content in the page', - type: 'boolean', - group: 'Build options:', - }) - .option('need-to-sanitize-html', { - default: true, - describe: 'Enable sanitize html', - type: 'boolean', - group: 'Build options:', - }) - .option('search', {}) - .check(argvValidator) - .example('yfm -i ./input -o ./output', '') - .demandOption( - ['input', 'output'], - 'Please provide input and output arguments to work with this tool', - ); -} - -async function handler(args: Arguments) { - const userOutputFolder = resolve(args.output); - const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER); - const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER); - - if (typeof VERSION !== 'undefined') { - console.log(`Using v${VERSION} version`); - } - - try { - ArgvService.init({ - ...args, - rootInput: args.input, - input: tmpInputFolder, - output: tmpOutputFolder, - }); - SearchService.init(); - Includers.init([OpenapiIncluder as any]); - - const { - output: outputFolderPath, - outputFormat, - lintDisabled, - buildDisabled, - addMapFile, - } = ArgvService.getConfig(); - - preparingTemporaryFolders(userOutputFolder); - - await processServiceFiles(); - processExcludedFiles(); - - if (addMapFile) { - prepareMapFile(); - } - - const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); - - if (!lintDisabled) { - /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ - await initLinterWorkers(); - } - - const processes = [ - !lintDisabled && processLinter(), - !buildDisabled && processPages(outputBundlePath), - ].filter(Boolean) as Promise[]; - - await Promise.all(processes); - - if (!buildDisabled) { - // process additional files - processAssets({ - args, - outputFormat, - outputBundlePath, - tmpOutputFolder, - userOutputFolder, - }); - - await processChangelogs(); - - await SearchService.release(); - - // Copy all generated files to user' output folder - shell.cp('-r', join(tmpOutputFolder, '*'), userOutputFolder); - if (glob.sync('.*', {cwd: tmpOutputFolder}).length) { - shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder); - } - } - } catch (err) { - logger.error('', err.message); - } finally { - processLogs(tmpInputFolder); - - shell.rm('-rf', tmpInputFolder, tmpOutputFolder); - } -} - -function preparingTemporaryFolders(userOutputFolder: string) { - const args = ArgvService.getConfig(); - - shell.mkdir('-p', userOutputFolder); - - // Create temporary input/output folders - shell.rm('-rf', args.input, args.output); - shell.mkdir(args.input, args.output); - - copyFiles( - args.rootInput, - args.input, - glob.sync('**', { - cwd: args.rootInput, - nodir: true, - follow: true, - ignore: ['node_modules/**', '*/node_modules/**'], - }), - ); - - shell.chmod('-R', 'u+w', args.input); -} diff --git a/src/cmd/index.ts b/src/cmd/index.ts deleted file mode 100644 index faea8f7e..00000000 --- a/src/cmd/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export {build} from './build'; -// export {publish} from './publish'; -// export {translate} from './translate'; diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index ea9f8c77..bf08189d 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -1,88 +1,309 @@ -import type {IProgram} from '~/program'; -import yargs from 'yargs'; -import {hideBin} from 'yargs/helpers'; -import {Help} from 'commander'; -import log from '@diplodoc/transform/lib/log'; +import type {IProgram, ProgramArgs, ProgramConfig} from '~/program'; +import type {DocAnalytics} from '@diplodoc/client'; + +import {ok} from 'node:assert'; +import {join} from 'node:path'; +import glob from 'glob'; +import {pick} from 'lodash'; +import {AsyncParallelHook, AsyncSeriesHook, HookMap} from 'tapable'; import {BaseProgram} from '~/program/base'; -import {Command} from '~/config'; -import {build} from '~/cmd'; +import {Lang, Stage, YFM_CONFIG_FILENAME} from '~/constants'; +import {Command, Config, configPath, defined, valuable} from '~/config'; +import {OutputFormat, options} from './config'; +import {Run} from './run'; + +import { + Templating, + TemplatingArgs, + TemplatingConfig, + TemplatingRawConfig, +} from './features/templating'; +import {Contributors, ContributorsArgs, ContributorsConfig} from './features/contributors'; +import {SinglePage, SinglePageArgs, SinglePageConfig} from './features/singlepage'; +import {Redirects} from './features/redirects'; +import {Lint, LintArgs, LintConfig, LintRawConfig} from './features/linter'; +import {Changelogs, ChangelogsArgs, ChangelogsConfig} from './features/changelogs'; +import {Search, SearchArgs, SearchConfig, SearchRawConfig} from './features/search'; +import {Legacy, LegacyArgs, LegacyConfig, LegacyRawConfig} from './features/legacy'; +import shell from 'shelljs'; + +export enum ResourceType { + style = 'style', + script = 'script', + csp = 'csp', +} + +// TODO: Move to isolated feature? +export type Resources = { + [key in ResourceType]?: string[]; +}; + +type BaseArgs = {output: AbsolutePath}; + +type BaseConfig = { + lang: `${Lang}`; + // TODO(patch): exetend langs list by newly supported langs or change type to string + langs: `${Lang}`[]; + outputFormat: `${OutputFormat}`; + varsPreset: string; + vars: Hash; + allowHtml: boolean; + sanitizeHtml: boolean; + // TODO(minor): string[] + ignoreStage: string; + ignore: string[]; + addSystemMeta: boolean; + // TODO(minor): we can generate this file all time + addMapFile: boolean; + // TODO(major): can this be solved by `when` prop in toc? + removeHiddenTocItems: boolean; + mergeIncludes: boolean; + // TODO(major): use as default behavior + staticContent: boolean; + // TODO(major): wtf? if we don't need to build, why we call build command? + buildDisabled: boolean; + allowCustomResources: boolean; + resources: Resources; + // TODO: explicitly handle + analytics: DocAnalytics; +}; + +export type {Run}; const command = 'Build'; -export type BuildArgs = {}; +const hooks = () => ({ + /** + * Async series hook which runs before start of any Run type.

+ * Args: + * - run - [Build.Run](./Run.ts) constructed context.
+ * Best place to subscribe on Run hooks. + */ + BeforeAnyRun: new AsyncSeriesHook(['run'], `${command}.BeforeAnyRun`), + /** + * Async series hook map which runs before start of target Run type.

+ * Args: + * - run - [Build.Run](./Run.ts) constructed context.
+ * Best place to subscribe on target Run hooks. + */ + BeforeRun: new HookMap( + (format: `${OutputFormat}`) => + new AsyncSeriesHook(['run'], `${command}.${format}.BeforeRun`), + ), + /** + * Async parallel hook which runs on start of any Run type.

+ * Args: + * - run - [Build.Run](./Run.ts) constructed context.
+ * Best place to do something in parallel with main build process. + */ + Run: new AsyncParallelHook(['run'], `${command}.Run`), + // TODO: decompose handler and describe this hook + AfterRun: new HookMap( + (format: `${OutputFormat}`) => + new AsyncSeriesHook(['run'], `${command}.${format}.AfterRun`), + ), + // TODO: decompose handler and describe this hook + AfterAnyRun: new AsyncSeriesHook(['run'], `${command}.AfterAnyRun`), +}); -export type BuildConfig = {}; +export type BuildArgs = ProgramArgs & + BaseArgs & + Partial< + TemplatingArgs & + ContributorsArgs & + SinglePageArgs & + LintArgs & + ChangelogsArgs & + SearchArgs & + LegacyArgs + >; -const parser = yargs() - .command(build) - .option('config', { - alias: 'c', - describe: 'YFM configuration file', - type: 'string', - }) - .option('strict', { - alias: 's', - default: false, - describe: 'Run in strict mode', - type: 'boolean', - }) - .option('quiet', { - alias: 'q', - default: false, - describe: "Run in quiet mode. Don't write logs to stdout", - type: 'boolean', - }) - .group(['config', 'strict', 'quiet', 'help', 'version'], 'Common options:') - .version(typeof VERSION === 'undefined' ? '' : VERSION) - .help(); +export type BuildRawConfig = BaseArgs & + ProgramConfig & + BaseConfig & + TemplatingRawConfig & + ContributorsConfig & + SinglePageConfig & + LintRawConfig & + ChangelogsConfig & + SearchRawConfig & + LegacyRawConfig; + +export type BuildConfig = Config< + BaseArgs & + ProgramConfig & + BaseConfig & + TemplatingConfig & + ContributorsConfig & + SinglePageConfig & + LintConfig & + ChangelogsConfig & + SearchConfig & + LegacyConfig +>; + +export type BuildHooks = ReturnType; export class Build // eslint-disable-next-line new-cap - extends BaseProgram(command, { + extends BaseProgram(command, { config: { - // scope: 'build', - defaults: () => ({}), + scope: 'build', + defaults: () => + ({ + langs: [], + outputFormat: OutputFormat.html, + varsPreset: 'default', + vars: {}, + ignore: [], + allowHtml: true, + sanitizeHtml: true, + addMapFile: false, + removeHiddenTocItems: false, + mergeIncludes: false, + resources: [], + allowCustomResources: false, + staticContent: false, + ignoreStage: Stage.SKIP, + addSystemMeta: false, + buildDisabled: false, + lint: {enabled: true, config: {'log-levels': {}}}, + }) as Partial, }, command: { isDefault: true, }, - hooks: () => {}, + hooks: hooks(), }) implements IProgram { - readonly command = new Command('build') - .allowUnknownOption(true) - .description('Build documentation in target directory'); + readonly templating = new Templating(); + + readonly contributors = new Contributors(); + + readonly singlepage = new SinglePage(); + + readonly redirects = new Redirects(); + + readonly linter = new Lint(); + + readonly changelogs = new Changelogs(); - protected options = []; + readonly search = new Search(); + + readonly legacy = new Legacy(); + + readonly command = new Command('build').description('Build documentation in target directory'); + + readonly options = [ + options.input('./'), + options.output({required: true}), + options.langs, + options.outputFormat, + options.varsPreset, + options.vars, + options.allowHtml, + options.sanitizeHtml, + options.addMapFile, + options.removeHiddenTocItems, + options.mergeIncludes, + options.resources, + options.allowCustomResources, + options.staticContent, + options.addSystemMeta, + options.ignore, + options.ignoreStage, + options.config(YFM_CONFIG_FILENAME), + options.buildDisabled, + ]; apply(program?: IProgram) { - super.apply(program); + this.hooks.Config.tap('Build', (config, args) => { + const langs = defined('langs', args, config) || []; + const lang = defined('lang', config); - this.command.createHelp = function () { - const help = new Help(); - help.formatHelp = () => parser.getHelp(); - return help; - }; - } + if (valuable(lang)) { + if (!langs.length) { + langs.push(lang); + } - async action() { - await parser.parse(hideBin(process.argv), {}, (err, {strict}, output) => { - if (err) { - console.error(err); - process.exit(1); + ok( + langs.includes(lang), + `Configured default lang '${lang}' is not listed in langs (${langs.join(', ')})`, + ); } - const {warn, error} = log.get(); - - if ((strict && warn.length) || error.length) { - process.exit(1); + if (!langs.length) { + langs.push(Lang.RU); } - console.log(output); + const options = [...this.options, ...(program?.options || [])].map((option) => + option.attributeName(), + ); + + Object.assign(config, pick(args, options)); + + config.langs = langs; + config.lang = lang || langs[0]; - process.exit(0); + return config; + }); + + this.hooks.AfterRun.for('md').tap('Build', async (run) => { + if (run.config[configPath]) { + shell.cp(run.config[configPath], run.output); + } }); + + this.templating.apply(this); + this.contributors.apply(this); + this.singlepage.apply(this); + this.redirects.apply(this); + this.linter.apply(this); + this.changelogs.apply(this); + this.search.apply(this); + this.legacy.apply(this); + + super.apply(program); + } + + async action() { + const run = new Run(this.config); + + run.logger.pipe(this.logger); + + // console.log(run.config); + + shell.mkdir('-p', run.originalOutput); + + // Create temporary input/output folders + shell.rm('-rf', run.input, run.output); + shell.mkdir('-p', run.input, run.output); + + await this.hooks.BeforeAnyRun.promise(run); + await this.hooks.BeforeRun.for(this.config.outputFormat).promise(run); + await Promise.all([this.handler(run), this.hooks.Run.promise(run)]); + await this.hooks.AfterRun.for(this.config.outputFormat).promise(run); + await this.hooks.AfterAnyRun.promise(run); + + // Copy all generated files to user' output folder + shell.cp('-r', join(run.output, '*'), run.originalOutput); + + if (glob.sync('.*', {cwd: run.output}).length) { + shell.cp('-r', join(run.output, '.*'), run.originalOutput); + } + + shell.rm('-rf', run.input, run.output); + } + + /** + * Loads handler in async mode to not initialise all deps on startup. + */ + private async handler(run: Run) { + // @ts-ignore + const {handler} = await import('./handler'); + + return handler(run); } } diff --git a/src/config/index.ts b/src/config/index.ts index 0eb966b2..ca43c219 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -146,13 +146,17 @@ type ConfigUtils = { export type Config = T & ConfigUtils; -export function withConfigUtils(path: string, config: T): Config { +export function withConfigUtils(path: string | null, config: T): Config { return { ...config, - resolve(subpath: string): AbsolutePath { + resolve: (subpath: string): AbsolutePath => { + if (path === null) { + return resolve(subpath) as AbsolutePath; + } + return resolve(dirname(path), subpath) as AbsolutePath; }, - [configPath]: resolve(path), + [configPath]: path === null ? path : resolve(path), }; } @@ -169,7 +173,7 @@ export async function resolveConfig( } = {}, ): Promise> { try { - const content = await readFile(path, 'utf8'); + const content = (await readFile(path, 'utf8')) || '{}'; const data = load(content) as Hash; return withConfigUtils(path, { @@ -184,7 +188,7 @@ export async function resolveConfig( case 'ScopeException': case 'ENOENT': if (fallback) { - return withConfigUtils(path, fallback); + return withConfigUtils(null, fallback); } else { throw error; } diff --git a/src/logger/index.ts b/src/logger/index.ts index 6da2e79e..2f8c31ab 100644 --- a/src/logger/index.ts +++ b/src/logger/index.ts @@ -123,7 +123,9 @@ export class Logger implements LogConsumer { const _writer = this[Symbol.for(level) as keyof LogConsumer]; const _color = color || colors[level]; - const topic = (...messages: string[]) => { + const topic = (...messages: unknown[]) => { + messages = messages.map(extractMessage); + const message = this.filters.reduce((message, filter) => { return filter(level, message); }, messages.join(' ')); @@ -179,3 +181,19 @@ export class Logger implements LogConsumer { } } } + +function extractMessage(error: unknown): string { + if (!error) { + return ''; + } + + if (typeof error === 'string') { + return error; + } + + if (typeof error === 'object' && 'message' in error) { + return String(error.message); + } + + return String(error); +} diff --git a/src/program/config.ts b/src/program/config.ts index 2a34f736..cd63fd93 100644 --- a/src/program/config.ts +++ b/src/program/config.ts @@ -1,6 +1,6 @@ import {resolve} from 'node:path'; import {bold} from 'chalk'; -import {option, toArray} from '~/config'; +import {OptionInfo, option, toArray} from '~/config'; export const NAME = 'yfm'; @@ -48,21 +48,31 @@ const extensions = option({ parser: toArray, }); -const input = (defaultPath?: string) => - option({ +const input = (defaults: string | Partial = {}) => { + const defaultPath = typeof defaults === 'string' ? defaults : defaults.default; + const overrides = typeof defaults === 'string' ? {} : defaults; + + return option({ + ...overrides, flags: '-i, --input ', desc: `Configure path to {{PROGRAM}} input directory.`, default: defaultPath ? absolute(defaultPath) : undefined, parser: absolute, }); +}; -const output = (defaultPath?: string) => - option({ +const output = (defaults: string | Partial = {}) => { + const defaultPath = typeof defaults === 'string' ? defaults : defaults.default; + const overrides = typeof defaults === 'string' ? {} : defaults; + + return option({ + ...overrides, flags: '-o, --output ', desc: `Configure path to {{PROGRAM}} output directory.`, default: defaultPath ? absolute(defaultPath) : undefined, parser: absolute, }); +}; const config = (defaultConfig: string) => option({ diff --git a/src/program/types.ts b/src/program/types.ts index 24b1558e..8992d072 100644 --- a/src/program/types.ts +++ b/src/program/types.ts @@ -1,5 +1,5 @@ import type {Hook, HookMap} from 'tapable'; -import type {Command} from '~/config'; +import type {Command, ExtendedOption} from '~/config'; import type {Logger} from '~/logger'; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -24,13 +24,15 @@ export interface ICallable { * 3. Program can be subprogram. This can be detected by non empty param passed to `apply` method. * But anyway program should be independent unit. * 4. Optional 'action' method - is a main place for hooks call. - * For compatibility with Commander.Command->action methos result shoul be void. + * For compatibility with Commander.Command->action method result should be void. * 5. Complex hook calls should be designed as external private methods named as 'hookMethodName' * (example: hookConfig) */ export interface IProgram extends ICallable { command: Command; + options: Readonly; + parent?: IParent; action?: (props: Args) => Promise | void; diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index cd2a1c18..0c9d572c 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -99,12 +99,7 @@ const getFileProps = async (options: ResolverOptions) => { }, lang, langs, - search: search - ? { - ...(search === true ? {provider: 'local'} : search), - ...SearchService.config(lang), - } - : undefined, + search: search.enabled ? SearchService.config(lang) : undefined, analytics, }; }; diff --git a/src/services/argv.ts b/src/services/argv.ts index 4941a615..8a6b6adb 100644 --- a/src/services/argv.ts +++ b/src/services/argv.ts @@ -10,14 +10,7 @@ function getConfig() { // eslint-disable-next-line @typescript-eslint/no-explicit-any function init(argv: any) { - _argv = { - ...argv, - ignore: Array.isArray(argv.ignore) ? argv.ignore : [], - } as YfmArgv; - - if (argv.vars) { - _argv.vars = JSON.parse(argv.vars); - } + _argv = argv as YfmArgv; try { const ignorefile = readFileSync(join(_argv.rootInput, '.yfmignore'), 'utf8'); diff --git a/src/services/search.ts b/src/services/search.ts index 5e478f1c..a53fd646 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -22,15 +22,13 @@ function init() { function isSearchEnabled() { const {search} = ArgvService.getConfig(); - return Boolean(search); + return Boolean(search.enabled); } function isLocalSearchEnabled() { const {search} = ArgvService.getConfig(); - return ( - isSearchEnabled() && (search === true || search!.provider === 'local' || !search!.provider) - ); + return isSearchEnabled() && search.provider === 'local'; } function add(path: string, info: DocInnerProps) { @@ -129,6 +127,7 @@ function config(lang: string) { const short = (link: string) => link.replace(output, '').replace(/^\/?/, ''); return { + provider: 'local', api: short(apiLink()), link: short(pageLink(lang)), resources: { diff --git a/src/steps/processAssets.ts b/src/steps/processAssets.ts index 5050adbc..90dada29 100644 --- a/src/steps/processAssets.ts +++ b/src/steps/processAssets.ts @@ -1,8 +1,9 @@ +import type {Run} from '~/commands/build'; + import walkSync from 'walk-sync'; import {load} from 'js-yaml'; import {readFileSync} from 'fs'; -import shell from 'shelljs'; -import {join, resolve, sep} from 'path'; +import {join, relative} from 'path'; import {ArgvService, TocService} from '../services'; import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; @@ -10,12 +11,7 @@ import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; import {DocLeadingPageData, LINK_KEYS} from '@diplodoc/client/ssr'; import {isLocalUrl} from '@diplodoc/transform/lib/utils'; -import { - ASSETS_FOLDER, - LINT_CONFIG_FILENAME, - REDIRECTS_FILENAME, - YFM_CONFIG_FILENAME, -} from '../constants'; +import {ASSETS_FOLDER} from '../constants'; import {Resources} from '../models'; import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; @@ -28,7 +24,7 @@ import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; */ type Props = { - args: string[]; + run: Run; outputBundlePath: string; outputFormat: string; tmpOutputFolder: string; @@ -36,13 +32,13 @@ type Props = { /* * Processes assets files (everything except .md files) */ -export function processAssets({args, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { +export function processAssets({run, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { switch (outputFormat) { case 'html': processAssetsHtmlRun({outputBundlePath}); break; case 'md': - processAssetsMdRun({args, tmpOutputFolder}); + processAssetsMdRun({run, tmpOutputFolder}); break; } } @@ -66,16 +62,8 @@ function processAssetsHtmlRun({outputBundlePath}) { copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); } -function processAssetsMdRun({args, tmpOutputFolder}) { - const {input: inputFolderPath, allowCustomResources, resources} = ArgvService.getConfig(); - - const pathToConfig = args.config || join(args.input, YFM_CONFIG_FILENAME); - const pathToRedirects = join(args.input, REDIRECTS_FILENAME); - const pathToLintConfig = join(args.input, LINT_CONFIG_FILENAME); - - shell.cp(resolve(pathToConfig), tmpOutputFolder); - shell.cp(resolve(pathToRedirects), tmpOutputFolder); - shell.cp(resolve(pathToLintConfig), tmpOutputFolder); +function processAssetsMdRun({run, tmpOutputFolder}: {run: Run; tmpOutputFolder: string}) { + const {allowCustomResources, resources} = run.config; if (resources && allowCustomResources) { const resourcePaths: string[] = []; @@ -90,17 +78,15 @@ function processAssetsMdRun({args, tmpOutputFolder}) { }); //copy resources - copyFiles(args.input, tmpOutputFolder, resourcePaths); + copyFiles(run.originalInput, tmpOutputFolder, resourcePaths); } const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { if (file.endsWith('.yaml')) { - const resolvedPathToFile = resolve(inputFolderPath, file); - - acc.push(resolvedPathToFile); + acc.push(join(run.input, file)); } return acc; - }, []); + }, [] as AbsolutePath[]); tocYamlFiles.forEach((yamlFile) => { const content = load(readFileSync(yamlFile, 'utf8')); @@ -118,16 +104,13 @@ function processAssetsMdRun({args, tmpOutputFolder}) { if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { const linkAbsolutePath = resolveRelativePath(yamlFile, link); - const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); + const linkRootPath = relative(run.input, linkAbsolutePath); acc.push(linkRootPath); } return acc; - }, - - [], - ); + }, [] as RelativePath[]); - copyFiles(args.input, tmpOutputFolder, localMediaLinks); + copyFiles(run.originalInput, tmpOutputFolder, localMediaLinks); }); } diff --git a/src/validator.ts b/src/validator.ts deleted file mode 100644 index f0289244..00000000 --- a/src/validator.ts +++ /dev/null @@ -1,158 +0,0 @@ -import {Arguments} from 'yargs'; -import {join, resolve} from 'path'; -import {readFileSync} from 'fs'; -import {load} from 'js-yaml'; -import merge from 'lodash/merge'; -import log from '@diplodoc/transform/lib/log'; -import {LINT_CONFIG_FILENAME, REDIRECTS_FILENAME, YFM_CONFIG_FILENAME} from './constants'; -import {ConnectorValidatorProps} from './vcs-connector/connector-models'; - -function notEmptyStringValidator(value: unknown): Boolean { - if (typeof value === 'string') { - return Boolean(value); - } - - return false; -} - -function requiredValueValidator(value: unknown): Boolean { - return Boolean(value); -} - -const validators: Record = { - storageEndpoint: { - errorMessage: 'Endpoint of S3 storage must be provided when publishes.', - validateFn: notEmptyStringValidator, - }, - storageBucket: { - errorMessage: 'Bucket name of S3 storage must be provided when publishes.', - validateFn: notEmptyStringValidator, - }, - storageKeyId: { - errorMessage: 'Key Id of S3 storage must be provided when publishes.', - validateFn: notEmptyStringValidator, - defaultValue: process.env.YFM_STORAGE_KEY_ID, - }, - storageSecretKey: { - errorMessage: 'Secret key of S3 storage must be provided when publishes.', - validateFn: notEmptyStringValidator, - defaultValue: process.env.YFM_STORAGE_SECRET_KEY, - }, - storageRegion: { - errorMessage: 'Region of S3 storage must be provided when publishes.', - validateFn: notEmptyStringValidator, - defaultValue: 'eu-central-1', - }, -}; - -interface Redirect { - from: string; - to: string; -} - -interface RedirectsConfig { - common: Redirect[]; - [lang: string]: Redirect[]; -} - -function validateRedirects(redirectsConfig: RedirectsConfig, pathToRedirects: string) { - const redirects: Redirect[] = Object.keys(redirectsConfig).reduce( - (res, redirectSectionName) => { - const sectionRedirects = redirectsConfig[redirectSectionName]; - res.push(...sectionRedirects); - return res; - }, - [] as Redirect[], - ); - - const getContext = (from: string, to: string) => ` [Context: \n- from: ${from}\n- to: ${to} ]`; - const formatMessage = (message: string, pathname: string, from: string, to: string) => - `${pathname}: ${message} ${getContext(from, to)}`; - - redirects.forEach((redirect) => { - const {from, to} = redirect; - - if (!from || !to) { - throw new Error( - formatMessage('One of the two parameters is missing', pathToRedirects, from, to), - ); - } - - if (from === to) { - throw new Error( - formatMessage('Parameters must be different', pathToRedirects, from, to), - ); - } - }); -} - -export function argvValidator(argv: Arguments): Boolean { - try { - // Combine passed argv and properties from configuration file. - const pathToConfig = argv.config - ? String(argv.config) - : join(String(argv.input), YFM_CONFIG_FILENAME); - const content = readFileSync(resolve(pathToConfig), 'utf8'); - Object.assign(argv, load(content) || {}); - } catch (error) { - if (error.name === 'YAMLException') { - log.error(`Error to parse ${YFM_CONFIG_FILENAME}: ${error.message}`); - } - } - - let lintConfig: unknown = {}; - try { - const pathToConfig = join(String(argv.input), LINT_CONFIG_FILENAME); - const content = readFileSync(resolve(pathToConfig), 'utf8'); - - lintConfig = load(content) || {}; - } catch (error) { - if (error.name === 'YAMLException') { - log.error(`Error to parse ${LINT_CONFIG_FILENAME}: ${error.message}`); - } - } finally { - const preparedLintConfig = merge(lintConfig, { - 'log-levels': { - MD033: argv.allowHTML ? 'disabled' : 'error', - }, - }); - - Object.assign(argv, {lintConfig: preparedLintConfig}); - } - - try { - const pathToRedirects = join(String(argv.input), REDIRECTS_FILENAME); - const redirectsContent = readFileSync(resolve(pathToRedirects), 'utf8'); - const redirects = load(redirectsContent); - - validateRedirects(redirects as RedirectsConfig, pathToRedirects); - } catch (error) { - if (error.name === 'YAMLException') { - log.error(`Error to parse ${REDIRECTS_FILENAME}: ${error.message}`); - } - - if (error.code !== 'ENOENT') { - throw error; - } - } - - if (argv.publish) { - for (const [field, validator] of Object.entries(validators)) { - const value = argv[field] ?? validator.defaultValue; - - if (!validator) { - continue; - } - - const validateFn = validator.validateFn ?? requiredValueValidator; - - if (!validateFn(value)) { - throw new Error(validator.errorMessage); - } - - argv[field] = value; - } - } - - return true; -} From 6a95af4bdf31fe1d9a156479dd53390f9d60e4b5 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 12 Nov 2024 12:44:01 +0300 Subject: [PATCH 4/9] fix: Enable new unit tests --- .github/workflows/tests.yml | 4 +- package-lock.json | 133 ++---------------------------------- package.json | 6 +- tsconfig.json | 3 +- vitest.config.mjs | 2 +- 5 files changed, 13 insertions(+), 135 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 480664ac..321d0bd5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,12 +26,14 @@ jobs: cache: 'npm' - name: Install packages for project run: npm ci + - name: Run unit tests + run: npm run test - run: npm run build - name: Install packages for tests run: | cd tests npm ci - - name: Run tests + - name: Run integration tests run: | cd tests npm run test diff --git a/package-lock.json b/package-lock.json index 0998c2b2..2bd768f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,8 +13,7 @@ "@diplodoc/translation": "^1.5.0", "katex": "^0.16.9", "shelljs": "0.8.5", - "threads": "1.7.0", - "yargs": "17.7.2" + "threads": "1.7.0" }, "bin": { "docs": "build/index.js", @@ -6108,49 +6107,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/cliui/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -8611,15 +8567,6 @@ "node": ">=6.9.0" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/get-east-asian-width": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz", @@ -12675,15 +12622,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -15584,6 +15522,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -15601,12 +15540,14 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, "license": "MIT" }, "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -15616,6 +15557,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -15678,15 +15620,6 @@ "node": ">= 16" } }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", @@ -15703,24 +15636,6 @@ "node": ">= 6" } }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/yargs-parser": { "version": "20.2.9", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", @@ -15730,44 +15645,6 @@ "node": ">=10" } }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/yargs/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 07dc665e..8f8ea03c 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ "src" ], "scripts": { - "build": "node scripts/build.cli.js", + "build": "npm run build:clean && node scripts/build.cli.js", + "build:clean": "rm -rf build assets coverage", "lint": "eslint \"src/**/*.{js,jsx,ts,tsx}\"", "lint:fix": "npm run lint -- --fix", "prepublishOnly": "npm run lint && npm run build", @@ -59,8 +60,7 @@ "@diplodoc/translation": "^1.5.0", "katex": "^0.16.9", "shelljs": "0.8.5", - "threads": "1.7.0", - "yargs": "17.7.2" + "threads": "1.7.0" }, "devDependencies": { "@aws-sdk/client-s3": "^3.525.0", diff --git a/tsconfig.json b/tsconfig.json index 77b9d088..3bc49dd1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,13 +4,12 @@ "lib": ["ES2019"], "target": "ES2019", "outDir": "build", - "module": "preserve", + "module": "es2022", "moduleResolution": "bundler", "paths": { "~/*": ["./src/*"] } }, "include": ["src"], - "exclude": ["node_modules"], "types": ["node"] } diff --git a/vitest.config.mjs b/vitest.config.mjs index 8176b495..0bc0ab79 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -11,7 +11,7 @@ export default defineConfig({ enabled: true, provider: 'v8', include: [ - 'src/cmd', + 'src/commands', 'src/program', 'src/config', 'src/logger', From 8bb712e3dddaadaf8f0187ea4775822bf5db123f Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 12 Nov 2024 15:53:32 +0300 Subject: [PATCH 5/9] fix: Fix post build logs processing --- src/commands/build/index.ts | 15 ++------------- src/index.ts | 2 +- src/program/base.ts | 11 ++++++++--- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index bf08189d..73f979a6 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -12,6 +12,7 @@ import {Lang, Stage, YFM_CONFIG_FILENAME} from '~/constants'; import {Command, Config, configPath, defined, valuable} from '~/config'; import {OutputFormat, options} from './config'; import {Run} from './run'; +import {handler} from './handler'; import { Templating, @@ -273,8 +274,6 @@ export class Build run.logger.pipe(this.logger); - // console.log(run.config); - shell.mkdir('-p', run.originalOutput); // Create temporary input/output folders @@ -283,7 +282,7 @@ export class Build await this.hooks.BeforeAnyRun.promise(run); await this.hooks.BeforeRun.for(this.config.outputFormat).promise(run); - await Promise.all([this.handler(run), this.hooks.Run.promise(run)]); + await Promise.all([handler(run), this.hooks.Run.promise(run)]); await this.hooks.AfterRun.for(this.config.outputFormat).promise(run); await this.hooks.AfterAnyRun.promise(run); @@ -296,14 +295,4 @@ export class Build shell.rm('-rf', run.input, run.output); } - - /** - * Loads handler in async mode to not initialise all deps on startup. - */ - private async handler(run: Run) { - // @ts-ignore - const {handler} = await import('./handler'); - - return handler(run); - } } diff --git a/src/index.ts b/src/index.ts index 8fbded19..12fed307 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,7 +25,7 @@ if (require.main === module) { if (message) { // eslint-disable-next-line no-console - console.error(error.stack || error.message || error); + console.error(error.message || error); } } diff --git a/src/program/base.ts b/src/program/base.ts index 90292d5a..29364bbd 100644 --- a/src/program/base.ts +++ b/src/program/base.ts @@ -4,6 +4,7 @@ import {AsyncSeriesWaterfallHook, Hook, HookMap, SyncHook} from 'tapable'; import {isAbsolute, resolve} from 'node:path'; import {once} from 'lodash'; import {Logger} from '~/logger'; +import log from '@diplodoc/transform/lib/log'; import { resolveConfig, scope as scopeConfig, @@ -151,9 +152,13 @@ export const BaseProgram = < } private async post() { - const {error, warn} = this.logger; - if (error.count || (this.config.strict && warn.count)) { - throw new HandledError('There is some errors.'); + if (this.logger.error.count || (this.config.strict && this.logger.warn.count)) { + throw new HandledError('There is some processing errors.'); + } + + const {error, warn} = log.get(); + if (error.length || (this.config.strict && warn.length)) { + throw new HandledError('There is some processing errors.'); } } From cbed2ad8ba97e6b7bf8b9470f4c75f64a22a0e19 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Thu, 14 Nov 2024 19:23:34 +0300 Subject: [PATCH 6/9] fix: Fix context config prototype to ballow serialization --- src/config/index.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/config/index.ts b/src/config/index.ts index ca43c219..6a6df7fc 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -147,17 +147,22 @@ type ConfigUtils = { export type Config = T & ConfigUtils; export function withConfigUtils(path: string | null, config: T): Config { - return { - ...config, - resolve: (subpath: string): AbsolutePath => { - if (path === null) { - return resolve(subpath) as AbsolutePath; - } - - return resolve(dirname(path), subpath) as AbsolutePath; + return Object.create(config, { + resolve: { + enumerable: false, + value: (subpath: string): AbsolutePath => { + if (path === null) { + return resolve(subpath) as AbsolutePath; + } + + return resolve(dirname(path), subpath) as AbsolutePath; + }, }, - [configPath]: path === null ? path : resolve(path), - }; + [configPath]: { + enumerable: false, + value: path === null ? path : resolve(path), + } + }); } export async function resolveConfig( From f252192de06059bf0e459a91460757326dcd66c9 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Thu, 14 Nov 2024 19:26:02 +0300 Subject: [PATCH 7/9] fix: Fix global log processing to handle errors --- src/logger/index.ts | 40 ++++++++++++++++++++++++++++++++++++---- src/program/base.ts | 3 ++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/logger/index.ts b/src/logger/index.ts index 2f8c31ab..fe1383e5 100644 --- a/src/logger/index.ts +++ b/src/logger/index.ts @@ -64,6 +64,18 @@ const colors = { [LogLevel.ERROR]: red, }; + +/** + * Logger has three logging channels: info, warning, and error. + * There are also many topics that use one of these channels. + * + * By default, the logger has three topics named after the logging channels. + * New topics should always use one of defined channel. + * + * Loggers are also pipeable. + * In this mode, only the channel is passed along. + * Topics processing from the parent logger are ignored. + */ export class Logger implements LogConsumer { [INFO] = writer(this, LogLevel.INFO); @@ -71,11 +83,11 @@ export class Logger implements LogConsumer { [ERROR] = writer(this, LogLevel.ERROR); - [LogLevel.INFO] = this.topic(LogLevel.INFO, 'INFO'); + info = this.topic(LogLevel.INFO, 'INFO'); - [LogLevel.WARN] = this.topic(LogLevel.WARN, 'WARN'); + warn = this.topic(LogLevel.WARN, 'WARN'); - [LogLevel.ERROR] = this.topic(LogLevel.ERROR, 'ERR'); + error = this.topic(LogLevel.ERROR, 'ERR'); private options: LoggerOptions = { colors: true, @@ -103,6 +115,13 @@ export class Logger implements LogConsumer { return this; } + /** + * Pipe local log channels to parent log channels. + * This doesn't pipe topics processing. + * So if child and parent has the same topic with name 'proc', + * only local topic will be applied to message. + * Message will be decorated by local topic and will be passed to parent as raw string. + */ pipe(consumer: LogConsumer) { if (this.consumer && this.consumer !== consumer) { throw new Error('This log already piped to another consumer.'); @@ -119,8 +138,13 @@ export class Logger implements LogConsumer { return this; } + /** + * Defines new write decorator to one of defined log channeld. + * Each decorator adds colored prefix to messages and apply preconfigured filters. + */ topic(level: LogLevels, prefix: string, color?: Color) { - const _writer = this[Symbol.for(level) as keyof LogConsumer]; + const channel = Symbol.for(level) as keyof LogConsumer; + const _writer = this[channel]; const _color = color || colors[level]; const topic = (...messages: unknown[]) => { @@ -169,6 +193,14 @@ export class Logger implements LogConsumer { return this; } + stat(): Record { + return { + [LogLevel.INFO]: this[INFO].count, + [LogLevel.WARN]: this[WARN].count, + [LogLevel.ERROR]: this[ERROR].count, + } + } + [Write](level: LogLevels, message: string) { if (this.options.quiet) { return; diff --git a/src/program/base.ts b/src/program/base.ts index 29364bbd..1d0ba523 100644 --- a/src/program/base.ts +++ b/src/program/base.ts @@ -152,7 +152,8 @@ export const BaseProgram = < } private async post() { - if (this.logger.error.count || (this.config.strict && this.logger.warn.count)) { + const stat = this.logger.stat(); + if (stat.error || (this.config.strict && stat.warn)) { throw new HandledError('There is some processing errors.'); } From 4ca82ade75cd7eebc52708101cf41c73d53eff26 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Thu, 14 Nov 2024 21:15:09 +0300 Subject: [PATCH 8/9] chore: Small impovments --- src/commands/build/config.ts | 3 +- src/commands/build/handler.ts | 38 +++++-------------- src/commands/build/index.ts | 3 +- src/commands/build/run.ts | 2 +- src/config/index.ts | 2 +- src/logger/index.ts | 3 +- src/steps/processAssets.ts | 69 +++++++++++++---------------------- 7 files changed, 41 insertions(+), 79 deletions(-) diff --git a/src/commands/build/config.ts b/src/commands/build/config.ts index a09a1b48..804bf9e5 100644 --- a/src/commands/build/config.ts +++ b/src/commands/build/config.ts @@ -26,8 +26,7 @@ const outputFormat = option({ const langs = option({ flags: '--lang, --langs ', - desc: 'Allow loading custom resources into statically generated pages.', - // parser: toArray, + desc: 'Configure langs supported by build', }); const vars = option({ diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index fbf995a2..55ac4b27 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -3,12 +3,10 @@ import type {Run} from './run'; import 'threads/register'; import glob from 'glob'; -import {join} from 'path'; import shell from 'shelljs'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {BUNDLE_FOLDER} from '~/constants'; import {ArgvService, Includers, SearchService} from '~/services'; import { initLinterWorkers, @@ -24,9 +22,6 @@ import {prepareMapFile} from '~/steps/processMapFile'; import {copyFiles} from '~/utils'; export async function handler(run: Run) { - const tmpInputFolder = run.input; - const tmpOutputFolder = run.output; - if (typeof VERSION !== 'undefined') { console.log(`Using v${VERSION} version`); } @@ -38,15 +33,9 @@ export async function handler(run: Run) { // @ts-ignore Includers.init([OpenapiIncluder]); - const { - output: outputFolderPath, - outputFormat, - lintDisabled, - buildDisabled, - addMapFile, - } = ArgvService.getConfig(); + const {lintDisabled, buildDisabled, addMapFile} = ArgvService.getConfig(); - preparingTemporaryFolders(); + preparingTemporaryFolders(run); await processServiceFiles(); processExcludedFiles(); @@ -55,7 +44,7 @@ export async function handler(run: Run) { prepareMapFile(); } - const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + const outputBundlePath = run.bundlePath; if (!lintDisabled) { /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ @@ -71,12 +60,7 @@ export async function handler(run: Run) { if (!buildDisabled) { // process additional files - processAssets({ - run, - outputFormat, - outputBundlePath, - tmpOutputFolder, - }); + processAssets(run); await processChangelogs(); @@ -85,23 +69,21 @@ export async function handler(run: Run) { } catch (error) { run.logger.error(error); } finally { - processLogs(tmpInputFolder); + processLogs(run.input); } } -function preparingTemporaryFolders() { - const args = ArgvService.getConfig(); - +function preparingTemporaryFolders(run: Run) { copyFiles( - args.rootInput, - args.input, + run.originalInput, + run.input, glob.sync('**', { - cwd: args.rootInput, + cwd: run.originalInput, nodir: true, follow: true, ignore: ['node_modules/**', '*/node_modules/**'], }), ); - shell.chmod('-R', 'u+w', args.input); + shell.chmod('-R', 'u+w', run.input); } diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index 73f979a6..b3923c75 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -274,8 +274,6 @@ export class Build run.logger.pipe(this.logger); - shell.mkdir('-p', run.originalOutput); - // Create temporary input/output folders shell.rm('-rf', run.input, run.output); shell.mkdir('-p', run.input, run.output); @@ -287,6 +285,7 @@ export class Build await this.hooks.AfterAnyRun.promise(run); // Copy all generated files to user' output folder + shell.mkdir('-p', run.originalOutput); shell.cp('-r', join(run.output, '*'), run.originalOutput); if (glob.sync('.*', {cwd: run.output}).length) { diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts index 06a909d7..327bfff5 100644 --- a/src/commands/build/run.ts +++ b/src/commands/build/run.ts @@ -32,7 +32,7 @@ export class Run { readonly config: BuildConfig; get bundlePath() { - return join(this.originalOutput, BUNDLE_FOLDER); + return join(this.output, BUNDLE_FOLDER); } get configPath() { diff --git a/src/config/index.ts b/src/config/index.ts index 6a6df7fc..987c65c1 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -161,7 +161,7 @@ export function withConfigUtils(path: string | null, conf [configPath]: { enumerable: false, value: path === null ? path : resolve(path), - } + }, }); } diff --git a/src/logger/index.ts b/src/logger/index.ts index fe1383e5..6547c1ba 100644 --- a/src/logger/index.ts +++ b/src/logger/index.ts @@ -64,7 +64,6 @@ const colors = { [LogLevel.ERROR]: red, }; - /** * Logger has three logging channels: info, warning, and error. * There are also many topics that use one of these channels. @@ -198,7 +197,7 @@ export class Logger implements LogConsumer { [LogLevel.INFO]: this[INFO].count, [LogLevel.WARN]: this[WARN].count, [LogLevel.ERROR]: this[ERROR].count, - } + }; } [Write](level: LogLevels, message: string) { diff --git a/src/steps/processAssets.ts b/src/steps/processAssets.ts index 90dada29..6cd699e9 100644 --- a/src/steps/processAssets.ts +++ b/src/steps/processAssets.ts @@ -5,7 +5,7 @@ import {load} from 'js-yaml'; import {readFileSync} from 'fs'; import {join, relative} from 'path'; -import {ArgvService, TocService} from '../services'; +import {TocService} from '../services'; import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; import {DocLeadingPageData, LINK_KEYS} from '@diplodoc/client/ssr'; @@ -15,54 +15,38 @@ import {ASSETS_FOLDER} from '../constants'; import {Resources} from '../models'; import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; -/** - * @param {Array} args - * @param {string} outputBundlePath - * @param {string} outputFormat - * @param {string} tmpOutputFolder - * @return {void} - */ - -type Props = { - run: Run; - outputBundlePath: string; - outputFormat: string; - tmpOutputFolder: string; -}; /* * Processes assets files (everything except .md files) */ -export function processAssets({run, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { - switch (outputFormat) { +export function processAssets(run: Run) { + switch (run.config.outputFormat) { case 'html': - processAssetsHtmlRun({outputBundlePath}); + processAssetsHtmlRun(run); break; case 'md': - processAssetsMdRun({run, tmpOutputFolder}); + processAssetsMdRun(run); break; } } -function processAssetsHtmlRun({outputBundlePath}) { - const {input: inputFolderPath, output: outputFolderPath} = ArgvService.getConfig(); - - const documentationAssetFilePath: string[] = walkSync(inputFolderPath, { +function processAssetsHtmlRun(run: Run) { + const documentationAssetFilePath: string[] = walkSync(run.input, { directories: false, includeBasePath: false, ignore: ['**/*.yaml', '**/*.md'], }); - copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath); + copyFiles(run.input, run.output, documentationAssetFilePath); const bundleAssetFilePath: string[] = walkSync(ASSETS_FOLDER, { directories: false, includeBasePath: false, }); - copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); + copyFiles(ASSETS_FOLDER, run.bundlePath, bundleAssetFilePath); } -function processAssetsMdRun({run, tmpOutputFolder}: {run: Run; tmpOutputFolder: string}) { +function processAssetsMdRun(run: Run) { const {allowCustomResources, resources} = run.config; if (resources && allowCustomResources) { @@ -78,7 +62,7 @@ function processAssetsMdRun({run, tmpOutputFolder}: {run: Run; tmpOutputFolder: }); //copy resources - copyFiles(run.originalInput, tmpOutputFolder, resourcePaths); + copyFiles(run.originalInput, run.output, resourcePaths); } const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { @@ -96,21 +80,20 @@ function processAssetsMdRun({run, tmpOutputFolder}: {run: Run; tmpOutputFolder: } const contentLinks = findAllValuesByKeys(content as DocLeadingPageData, LINK_KEYS); - const localMediaLinks = contentLinks.reduce( - (acc: string[], link: string) => { - const linkHasMediaExt = new RegExp( - /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, - ).test(link); - - if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { - const linkAbsolutePath = resolveRelativePath(yamlFile, link); - const linkRootPath = relative(run.input, linkAbsolutePath); - - acc.push(linkRootPath); - } - return acc; - }, [] as RelativePath[]); - - copyFiles(run.originalInput, tmpOutputFolder, localMediaLinks); + const localMediaLinks = contentLinks.reduce((acc: string[], link: string) => { + const linkHasMediaExt = new RegExp( + /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, + ).test(link); + + if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { + const linkAbsolutePath = resolveRelativePath(yamlFile, link); + const linkRootPath = relative(run.input, linkAbsolutePath); + + acc.push(linkRootPath); + } + return acc; + }, [] as RelativePath[]); + + copyFiles(run.originalInput, run.output, localMediaLinks); }); } From 29d01f76eca8cf3f7dfd18a47d19356f3a4be946 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Wed, 27 Nov 2024 13:41:18 +0300 Subject: [PATCH 9/9] fix: Fix leading page urls for static build --- src/resolvers/md2html.ts | 2 +- src/utils/toc.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 0c9d572c..b221750f 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -145,7 +145,7 @@ function getHref(root: string, path: string, href: string) { href = href.replace(/\.(md|ya?ml)$/gi, '.html'); } else if (!/.+\.\w+$/gi.test(href)) { // TODO: isFileExists index.md or index.yaml - href = href + '/index.html'; + href = href + (href.endsWith('/') ? '' : '/') + 'index.html'; } return href; diff --git a/src/utils/toc.ts b/src/utils/toc.ts index 2d33599e..9f903039 100644 --- a/src/utils/toc.ts +++ b/src/utils/toc.ts @@ -49,6 +49,10 @@ export function transformToc(toc: YfmToc, tocDir: string) { return href; } + if (href.endsWith('/')) { + href += 'index.yaml'; + } + const fileExtension: string = extname(href); const filename: string = basename(href, fileExtension) + '.html';