From 2a332ad540bf5488d1bdd142ef62b9313f36d091 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Mon, 22 Apr 2024 13:57:01 +0300 Subject: [PATCH] wip --- src/commands/build/__tests__/index.ts | 67 +++ src/commands/build/config.ts | 134 ++++++ .../build/features/contributors/config.ts | 16 + .../build/features/contributors/index.ts | 30 ++ src/commands/build/features/linter/config.ts | 18 + .../build/features/linter/index.spec.ts | 74 +++ src/commands/build/features/linter/index.ts | 77 +++ .../build/features/publishing/config.ts | 66 +++ .../build/features/publishing/index.spec.ts | 29 ++ .../build/features/publishing/index.ts | 84 ++++ .../build/features/redirects/index.ts | 71 +++ .../build/features/singlepage/config.ts | 10 + .../build/features/singlepage/index.ts | 26 + .../build/features/templating/config.ts | 88 ++++ .../build/features/templating/index.spec.ts | 454 ++++++++++++++++++ .../build/features/templating/index.ts | 133 +++++ src/commands/build/handler.ts | 78 +++ src/commands/build/index.spec.ts | 291 +++++++++++ src/commands/build/index.ts | 240 ++++++--- src/commands/build/run.ts | 54 +++ src/commands/publish/__tests__/index.ts | 84 ++++ src/models.ts | 8 +- src/resolvers/lintPage.ts | 2 +- src/resolvers/md2html.ts | 40 +- src/services/argv.ts | 4 - src/services/includers/index.ts | 9 +- src/steps/processAssets.ts | 123 ++--- src/steps/processExcludedFiles.ts | 15 +- src/steps/processLinter.ts | 5 +- src/steps/processPages.ts | 73 ++- 30 files changed, 2171 insertions(+), 232 deletions(-) create mode 100644 src/commands/build/__tests__/index.ts create mode 100644 src/commands/build/config.ts create mode 100644 src/commands/build/features/contributors/config.ts create mode 100644 src/commands/build/features/contributors/index.ts create mode 100644 src/commands/build/features/linter/config.ts create mode 100644 src/commands/build/features/linter/index.spec.ts create mode 100644 src/commands/build/features/linter/index.ts create mode 100644 src/commands/build/features/publishing/config.ts create mode 100644 src/commands/build/features/publishing/index.spec.ts create mode 100644 src/commands/build/features/publishing/index.ts create mode 100644 src/commands/build/features/redirects/index.ts create mode 100644 src/commands/build/features/singlepage/config.ts create mode 100644 src/commands/build/features/singlepage/index.ts create mode 100644 src/commands/build/features/templating/config.ts create mode 100644 src/commands/build/features/templating/index.spec.ts create mode 100644 src/commands/build/features/templating/index.ts create mode 100644 src/commands/build/handler.ts create mode 100644 src/commands/build/index.spec.ts create mode 100644 src/commands/build/run.ts create mode 100644 src/commands/publish/__tests__/index.ts diff --git a/src/commands/build/__tests__/index.ts b/src/commands/build/__tests__/index.ts new file mode 100644 index 00000000..fca726cc --- /dev/null +++ b/src/commands/build/__tests__/index.ts @@ -0,0 +1,67 @@ +import type {Run} from '../run'; +import type {Mock} from 'vitest'; +import type {BuildConfig} from '..'; + +import {expect, it, vi} from 'vitest'; +import {Build} from '..'; +import {handler as originalHandler} from '../handler'; + +export const handler = originalHandler as Mock; + +// eslint-disable-next-line no-var +var resolveConfig: Mock; + +vi.mock('../handler'); +vi.mock('~/config', async (importOriginal) => { + resolveConfig = vi.fn((_path, {defaults, fallback}) => { + return defaults || fallback; + }); + + return { + ...((await importOriginal()) as {}), + resolveConfig, + }; +}); + +export async function runBuild(args: string) { + const build = new Build(); + + build.apply(); + + await build.parse(['node', 'index'].concat(args.split(' '))); +} + +type DeepPartial = { + [P in keyof T]?: T[P] extends Record ? DeepPartial : T[P]; +}; + +export function testConfig(name: string, args: string, result: DeepPartial): void; +export function testConfig( + name: string, + args: string, + config: DeepPartial, + result: DeepPartial, +): void; +export function testConfig(name: string, args: string, config: any, result?: any): void { + it(name, async () => { + if (!result) { + result = config; + config = {}; + } + + resolveConfig.mockImplementation((_path, {defaults}) => { + return { + ...defaults, + ...config, + }; + }); + + handler.mockImplementation((run: Run) => { + expect(run.config).toMatchObject(result as Partial); + }); + + await runBuild('--input ./input --output ./output ' + args); + + expect(handler).toBeCalled(); + }); +} diff --git a/src/commands/build/config.ts b/src/commands/build/config.ts new file mode 100644 index 00000000..f0af4ab0 --- /dev/null +++ b/src/commands/build/config.ts @@ -0,0 +1,134 @@ +import {bold, underline} from 'chalk'; +import {options as globalOptions} from '~/program/config'; +import {option} from '~/config'; +import {Stage} from '~/constants'; + +// need-to-sanitize-html + +export enum OutputFormat { + md = 'md', + html = 'html', +} + +const outputFormat = option({ + flags: '-f, --output-format ', + defaultInfo: 'html', + choices: ['html', 'md'], + desc: ` + Format of output files. (html or md) + + If ${bold('html')} is selected, then renders md to static html files. + (See also ${underline('--static-content')} option) + + If ${bold('md')} is selected, then renders md to prepared md filed + enriched by additional metadata. + (Useful for complex documentation servers with runtime rendering) + `, +}); + +const vars = option({ + flags: '-v, --vars ', + desc: ` + Pass list of variables directly to build. + Variables should be passed in JSON format. + Passed variables overrides the same in presets.yaml + + Example: + {{PROGRAM}} -i ./ -o ./build -v '{"name":"test"}' + `, + parser: (value) => JSON.parse(value), +}); + +const varsPreset = option({ + flags: '--vars-preset ', + desc: ` + Select vars preset of documentation. + Selected preset will be merged with default section of presets.yaml + `, +}); + +const allowHTML = option({ + flags: '--allowHTML', + desc: 'Allow to use HTML in Markdown files.', + defaultInfo: true, + deprecated: 'Use --allow-html for consistency.', +}); + +const allowHtml = option({ + flags: '--allow-html', + desc: 'Allow to use HTML in Markdown files.', + defaultInfo: true, +}); + +const hidden = option({ + flags: '--hidden ', + desc: ` + Do not process paths matched by glob. + + Example: + {{PROGRAM}} -i ./input -o ./output --hidden *.bad.md + `, +}); + +// TODO: options below are not beautified. +const addMapFile = option({ + flags: '--add-map-file', + desc: 'Should add all paths of documentation into file.json.', +}); + +const removeHiddenTocItems = option({ + flags: '--remove-hidden-toc-items', + desc: 'Remove from Toc all items marked as hidden.', +}); + +const resources = option({ + flags: '--resource, --resources ', + desc: 'Allow loading custom resources into statically generated pages.', + // parser: toArray, +}); + +const allowCustomResources = option({ + flags: '--allow-custom-resources', + desc: 'Allow loading custom resources into statically generated pages.', +}); + +const staticContent = option({ + flags: '--static-content', + desc: 'Allow loading custom resources into statically generated pages.', +}); + +const ignoreStage = option({ + flags: '--ignore-stage ', + defaultInfo: Stage.SKIP, + desc: 'Ignore tocs with stage.', +}); + +const addSystemMeta = option({ + flags: '--add-system-meta', + desc: 'Should add system section variables form presets into files meta data.', +}); + +const buildDisabled = option({ + flags: '--build-disabled', + desc: 'Disable building.', +}); + +export const options = { + input: globalOptions.input, + output: globalOptions.output, + config: globalOptions.config, + outputFormat, + varsPreset, + vars, + allowHTML, + allowHtml, + addMapFile, + removeHiddenTocItems, + resources, + allowCustomResources, + staticContent, + hidden, + ignoreStage, + addSystemMeta, + buildDisabled, +}; diff --git a/src/commands/build/features/contributors/config.ts b/src/commands/build/features/contributors/config.ts new file mode 100644 index 00000000..5018f30d --- /dev/null +++ b/src/commands/build/features/contributors/config.ts @@ -0,0 +1,16 @@ +import {option} from '~/config'; + +const contributors = option({ + flags: '--contributors', + desc: 'Should attach contributors into files', +}); + +const ignoreAuthorPatterns = option({ + flags: '--ignore-author-patterns ', + desc: 'Ignore authors if they contain passed string', +}); + +export const options = { + contributors, + ignoreAuthorPatterns, +}; diff --git a/src/commands/build/features/contributors/index.ts b/src/commands/build/features/contributors/index.ts new file mode 100644 index 00000000..c3468a4c --- /dev/null +++ b/src/commands/build/features/contributors/index.ts @@ -0,0 +1,30 @@ +import type {Build} from '../..'; +import type {Command} from '~/config'; +import {defined} from '~/config'; +import {options} from './config'; + +export type ContributorsArgs = { + contributors?: boolean; + ignoreAuthorPatterns?: string[]; +}; + +export type ContributorsConfig = { + contributors: boolean; + ignoreAuthorPatterns: string[]; +}; + +export class Contributors { + apply(program: Build) { + program.hooks.Command.tap('Contributors', (command: Command) => { + command.addOption(options.contributors); + command.addOption(options.ignoreAuthorPatterns); + }); + + program.hooks.Config.tap('Contributors', (config, args) => { + config.contributors = defined('contributors', args, config) || false; + config.ignoreAuthorPatterns = defined('ignoreAuthorPatterns', args, config) || []; + + return config; + }); + } +} diff --git a/src/commands/build/features/linter/config.ts b/src/commands/build/features/linter/config.ts new file mode 100644 index 00000000..2c2bc34e --- /dev/null +++ b/src/commands/build/features/linter/config.ts @@ -0,0 +1,18 @@ +import {option} from '~/config'; + +const lint = option({ + flags: '--lint', + desc: 'Toggle file linting.', +}); + +const lintDisabled = option({ + flags: '--lint-disabled', + desc: 'Disable linting.', + hidden: true, + deprecated: 'Use --no-lint instead', +}); + +export const options = { + lint, + lintDisabled, +}; diff --git a/src/commands/build/features/linter/index.spec.ts b/src/commands/build/features/linter/index.spec.ts new file mode 100644 index 00000000..be554eb0 --- /dev/null +++ b/src/commands/build/features/linter/index.spec.ts @@ -0,0 +1,74 @@ +import {describe, vi} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +vi.mock('~/cmd/publish/upload'); + +describe('Build linter feature', () => { + describe('config', () => { + describe('lintDisabled', () => { + test('should handle default', '', { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'disabled', + }, + } + }, + }); + + test('should handle arg', '--lint-disabled', { + lint: {enabled: false} + }); + + test( + 'should handle config', + '', + { + lintDisabled: true, + }, + { + lint: {enabled: false} + }, + ); + + test( + 'should handle enabled allowHtml', + '', + { + lintDisabled: false, + allowHtml: true, + }, + { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'disabled', + }, + } + }, + }, + ); + + test( + 'should handle disabled allowHtml', + '', + { + lintDisabled: false, + allowHtml: false, + }, + { + lint: { + enabled: true, + config: { + 'log-levels': { + MD033: 'error', + }, + } + }, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/linter/index.ts b/src/commands/build/features/linter/index.ts new file mode 100644 index 00000000..93c61989 --- /dev/null +++ b/src/commands/build/features/linter/index.ts @@ -0,0 +1,77 @@ +import type {Build} from '../..'; +import type {Command} from '~/config'; +import { join, resolve } from 'node:path'; +import shell from 'shelljs'; +import {defined, resolveConfig} from '~/config'; +import {LINT_CONFIG_FILENAME} from '~/constants'; +import {initLinterWorkers, processLinter} from '~/steps'; +import {options} from './config'; + +export type LintArgs = { + lint: boolean; + lintDisabled: boolean; +}; + +export type LintConfig = { + lintDisabled?: boolean; + lint: { + enabled: boolean; + config: { + 'log-levels': Record; + }; + }; +}; + +// TODO(major): move to separated 'lint' command +export class Lint { + apply(program: Build) { + program.hooks.Command.tap('Lint', (command: Command) => { + command.addOption(options.lint); + command.addOption(options.lintDisabled); + }); + + program.hooks.Config.tapPromise('Lint', async (config, args) => { + const lintDisabled = defined('lintDisabled', args, config) || false; + const lintEnabled = args.lint !== false || config.lint && config.lint.enabled !== false; + if (lintDisabled || !lintEnabled) { + config.lint = {enabled: false, config: {}}; + return config; + } + + config.lint = config.lint || {}; + config.lint.enabled = true; + + const lintConfig = await resolveConfig>( + resolve(args.input, LINT_CONFIG_FILENAME), + { + fallback: {}, + }, + ); + + lintConfig['log-levels'] = lintConfig['log-levels'] || {}; + lintConfig['log-levels']['MD033'] = config.allowHtml ? 'disabled' : 'error'; + + config.lint.config = lintConfig as LintConfig['lint']['config']; + + return config; + }); + + program.hooks.BeforeAnyRun.tapPromise('Lint', async (run) => { + if (!run.config.lint.enabled) { + return; + } + + program.hooks.Run.tapPromise('Lint', async (run) => { + await processLinter(run); + }); + + program.hooks.AfterRun.for('md').tapPromise('Lint', async (run) => { + const lintPath = join(run.originalInput, LINT_CONFIG_FILENAME); + + shell.cp(lintPath, run.output); + }); + + await initLinterWorkers(run); + }); + } +} diff --git a/src/commands/build/features/publishing/config.ts b/src/commands/build/features/publishing/config.ts new file mode 100644 index 00000000..b399314a --- /dev/null +++ b/src/commands/build/features/publishing/config.ts @@ -0,0 +1,66 @@ +import {option} from '~/config'; + +const publish = option({ + flags: '--publish', + desc: 'Should upload output files to S3 storage.', + deprecated: 'Use separated publish command instead.', +}); + +const storageEndpoint = option({ + flags: '--storage-endpoint ', + desc: 'Endpoint of S3 storage.', + deprecated: 'Use separated publish command instead.', + hidden: true, +}); + +const storageBucket = option({ + flags: '--storage-bucket ', + desc: 'Bucket name of S3 storage.', + deprecated: 'Use separated publish command instead.', + hidden: true, +}); + +const storagePrefix = option({ + flags: '--storage-prefix ', + desc: 'Bucket internal scope of S3 storage.', + env: 'YFM_STORAGE_PREFIX', + // defaultInfo: process.env.YFM_STORAGE_PREFIX, + deprecated: 'Use separated publish command instead.', + hidden: true, +}); + +const storageKeyId = option({ + flags: '--storage-key-id ', + desc: 'Key Id of S3 storage.', + env: 'YFM_STORAGE_KEY_ID', + // defaultInfo: process.env.YFM_STORAGE_KEY_ID, + deprecated: 'Use separated publish command instead.', + hidden: true, +}); + +const storageSecretKey = option({ + flags: '--storage-secret-key ', + desc: 'Secret key of S3 storage.', + env: 'YFM_STORAGE_SECRET_KEY', + // defaultInfo: process.env.YFM_STORAGE_SECRET_KEY, + deprecated: 'Use separated publish command instead.', + hidden: true, +}); + +const storageRegion = option({ + flags: '--storage-region ', + desc: 'Region of S3 storage.', + defaultInfo: 'eu-central-1', + deprecated: 'Use separated publish command instead.', + hidden: true, +}); + +export const options = { + publish, + storageEndpoint, + storageBucket, + storagePrefix, + storageKeyId, + storageSecretKey, + storageRegion, +}; diff --git a/src/commands/build/features/publishing/index.spec.ts b/src/commands/build/features/publishing/index.spec.ts new file mode 100644 index 00000000..e486ed89 --- /dev/null +++ b/src/commands/build/features/publishing/index.spec.ts @@ -0,0 +1,29 @@ +import {describe, vi} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +vi.mock('~/cmd/publish/upload'); + +describe('Build publish feature', () => { + describe('config', () => { + describe('publish', () => { + test('should handle default', '', { + publish: false, + }); + + test('should handle arg', '--publish', { + publish: true, + }); + + test( + 'should handle config', + '', + { + publish: true, + }, + { + publish: true, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/publishing/index.ts b/src/commands/build/features/publishing/index.ts new file mode 100644 index 00000000..a6dc3af5 --- /dev/null +++ b/src/commands/build/features/publishing/index.ts @@ -0,0 +1,84 @@ +import type {Build, Run} from '../..'; +import type {Command} from '~/config'; +import {defined} from '~/config'; +import {Run as UploadRun, upload} from '~/commands/publish'; +import {options} from './config'; + +export type PublishingArgs = { + publish: boolean; + storageEndpoint: string; + storageRegion: string; + storageBucket: string; + storagePrefix: string; + storageKeyId: string; + storageSecretKey: string; +}; + +export type PublishingConfig = { + publish: boolean; +} & StorageInfo; + +type StorageInfo = { + endpoint: string; + region: string; + bucket: string; + prefix: string; + accessKeyId: string; + secretAccessKey: string; +}; + +/** + * This is deprecated build feature. + * We need to migrate users to separate publish command. + */ +export class Publishing { + apply(program: Build) { + let props: StorageInfo | null = null; + + program.hooks.Command.tap('Publishing', (command: Command) => { + command + .addOption(options.publish) + .addOption(options.storageEndpoint) + .addOption(options.storageRegion) + .addOption(options.storageBucket) + .addOption(options.storagePrefix) + .addOption(options.storageKeyId) + .addOption(options.storageSecretKey); + }); + + program.hooks.Config.tap('Publishing', (config, args) => { + if (args.storageSecretKey || args.storageKeyId) { + throw new Error('Storage secret key should not be stored in config.'); + } + + config.publish = defined('publish', args, config) || false; + + if (config.publish) { + props = { + endpoint: defined('storageEndpoint', args, config) || '', + region: + defined('storageRegion', args, config) || options.storageRegion.defaultInfo, + bucket: defined('storageBucket', args, config) || '', + prefix: defined('storagePrefix', args, config) || '', + accessKeyId: defined('storageKeyId', args) || '', + secretAccessKey: defined('storageSecretKey', args) || '', + }; + } + + return config; + }); + + program.hooks.AfterAnyRun.tapPromise('Publishing', async (run: Run) => { + if (props) { + await upload( + new UploadRun({ + input: run.output, + hidden: run.config.hidden, + quiet: run.config.quiet, + ...props, + }), + ); + } + }); + } +} diff --git a/src/commands/build/features/redirects/index.ts b/src/commands/build/features/redirects/index.ts new file mode 100644 index 00000000..d3e52fd8 --- /dev/null +++ b/src/commands/build/features/redirects/index.ts @@ -0,0 +1,71 @@ +import {ok} from 'node:assert'; +import {resolve} from 'node:path'; +import shell from 'shelljs'; +import {Build} from '../..'; + +import {REDIRECTS_FILENAME} from '~/constants'; +import {configPath, resolveConfig} from '~/config'; + +interface Redirect { + from: string; + to: string; +} + +interface RedirectsConfig { + common: Redirect[]; + [lang: string]: Redirect[]; +} + +export class Redirects { + apply(program: Build) { + let resolvedPath: string | null = null; + + program.hooks.BeforeRun.for('md').tap('Redirects', async (run) => { + try { + const redirects = await resolveConfig( + resolve(run.originalInput, REDIRECTS_FILENAME), + { + fallback: {common: []}, + }, + ); + + if (redirects[configPath]) { + resolvedPath = redirects[configPath]; + validateRedirects(redirects, resolvedPath); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + run.logger.error(error.message || error); + } + }); + + program.hooks.AfterRun.for('md').tap('Redirects', async (run) => { + if (resolvedPath) { + shell.cp(resolvedPath, run.output); + } + }); + } +} + +function validateRedirects(redirectsConfig: RedirectsConfig, pathToRedirects: string) { + const redirects: Redirect[] = Object.keys(redirectsConfig).reduce( + (res, redirectSectionName) => { + const sectionRedirects = redirectsConfig[redirectSectionName]; + res.push(...sectionRedirects); + return res; + }, + [] as Redirect[], + ); + + const getContext = (from: string, to: string) => ` [Context: \n- from: ${from}\n- to: ${to} ]`; + const formatMessage = (message: string, pathname: string, from: string, to: string) => + `${pathname}: ${message} ${getContext(from, to)}`; + + redirects.forEach(({from, to}) => { + ok( + from && to, + formatMessage('One of the two parameters is missing', pathToRedirects, from, to), + ); + ok(from !== to, formatMessage('Parameters must be different', pathToRedirects, from, to)); + }); +} diff --git a/src/commands/build/features/singlepage/config.ts b/src/commands/build/features/singlepage/config.ts new file mode 100644 index 00000000..a117e968 --- /dev/null +++ b/src/commands/build/features/singlepage/config.ts @@ -0,0 +1,10 @@ +import {option} from '~/config'; + +const singlePage = option({ + flags: '--single-page', + desc: 'Beta functionality: Build a single page in the output folder also.', +}); + +export const options = { + singlePage, +}; diff --git a/src/commands/build/features/singlepage/index.ts b/src/commands/build/features/singlepage/index.ts new file mode 100644 index 00000000..be266f63 --- /dev/null +++ b/src/commands/build/features/singlepage/index.ts @@ -0,0 +1,26 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; +import {defined} from '~/config'; +import {options} from './config'; + +export type SinglePageArgs = { + singlePage: boolean; +}; + +export type SinglePageConfig = { + singlePage: boolean; +}; + +export class SinglePage { + apply(program: Build) { + program.hooks.Command.tap('SinglePage', (command: Command) => { + command.addOption(options.singlePage); + }); + + program.hooks.Config.tap('SinglePage', (config, args) => { + config.singlePage = defined('singlePage', args, config) || false; + + return config; + }); + } +} diff --git a/src/commands/build/features/templating/config.ts b/src/commands/build/features/templating/config.ts new file mode 100644 index 00000000..8ff4ee61 --- /dev/null +++ b/src/commands/build/features/templating/config.ts @@ -0,0 +1,88 @@ +import {bold, cyan, green} from 'chalk'; +import {option} from '~/config'; + +const disableLiquid = option({ + flags: '--disable-liquid', + desc: 'Disable template engine.', + defaultInfo: false, + deprecated: 'Use --no-template instead.', +}); + +const applyPresets = option({ + flags: '--apply-presets', + desc: 'Should apply presets.', + defaultInfo: true, + deprecated: 'Use --template-vars/--no-template-vars instead.', +}); + +const resolveConditions = option({ + flags: '--resolve-conditions', + desc: 'Should resolve conditions.', + defaultInfo: true, + deprecated: 'Use --template-conditions/--no-template-conditions instead.', +}); + +const conditionsInCode = option({ + flags: '--conditions-in-code', + desc: 'Meet conditions in code blocks.', + defaultInfo: false, + deprecated: 'Use --template=all or --template=code instead.', +}); + +const template = option({ + flags: '--template ', + desc: ` + Select liquid template engine mode. + By default liquid ignores code blocs. (${bold('text')} mode) + Use ${bold('all')} or ${bold('code')} mode to process code blocks. + Use ${bold('--no-template')} to completely disable template engine. + + Read more about templating ${cyan('https://diplodoc.com/docs/en/syntax/vars')} + `, + choices: ['all', 'text', 'code'], +}); + +const noTemplate = option({ + flags: '--no-template', + desc: 'Manual negation for --template', + hidden: true, + default: false, +}); + +const templateVars = option({ + flags: '--template-vars', + desc: ` + Toggle processing of terms decorated by double curly braces in Toc and Md files. (Enabled by default) + + Read more about substitutions ${cyan('https://diplodoc.com/docs/en/syntax/vars#subtitudes')} + + Example: + Some text ${green('{{some-variable}}')} end of text. + `, + defaultInfo: true, +}); + +const templateConditions = option({ + flags: '--template-conditions', + desc: ` + Toggle processing of conditions in Toc and Md files. (Enabled by default) + + Read more about conditions ${cyan('https://diplodoc.com/docs/en/syntax/vars#conditions')} + + Example: + Some text ${green('{% if var == "any" %}')} extra ${green('{% endif %}')} end of text. + `, + defaultInfo: true, +}); + +export const options = { + template, + noTemplate, + templateVars, + templateConditions, + + disableLiquid, + applyPresets, + resolveConditions, + conditionsInCode, +}; diff --git a/src/commands/build/features/templating/index.spec.ts b/src/commands/build/features/templating/index.spec.ts new file mode 100644 index 00000000..2dfa8f25 --- /dev/null +++ b/src/commands/build/features/templating/index.spec.ts @@ -0,0 +1,454 @@ +import {describe} from 'vitest'; +import {testConfig as test} from '../../__tests__'; + +describe('Build template feature', () => { + describe('config', () => { + describe('disableLiquid', () => { + test('should handle default', '', { + disableLiquid: false, + template: { + enabled: true, + }, + }); + + test('should handle arg', '--disable-liquid', { + disableLiquid: true, + template: { + enabled: false, + }, + }); + + test('should handle new arg', '--no-template', { + disableLiquid: true, + template: { + enabled: false, + }, + }); + + test('should handle new arg with priority', '--disable-liquid --template all', { + disableLiquid: false, + template: { + enabled: true, + }, + }); + + test( + 'should handle config', + '', + { + disableLiquid: true, + }, + { + disableLiquid: true, + template: { + enabled: false, + }, + }, + ); + }); + + describe('applyPresets', () => { + test('should handle default', '', { + applyPresets: true, + template: { + features: { + substitutions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--apply-presets', + { + applyPresets: false, + }, + { + applyPresets: true, + template: { + features: { + substitutions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-apply-presets', { + applyPresets: false, + template: { + features: { + substitutions: false, + }, + }, + }); + + test('should handle new arg', '--no-template-vars', { + applyPresets: false, + template: { + features: { + substitutions: false, + }, + }, + }); + + test('should handle new arg with priority', '--no-apply-presets --template-vars', { + applyPresets: true, + template: { + features: { + substitutions: true, + }, + }, + }); + + test( + 'should handle config', + '', + { + applyPresets: false, + }, + { + applyPresets: false, + template: { + features: { + substitutions: false, + }, + }, + }, + ); + }); + + describe('resolveConditions', () => { + test('should handle default', '', { + resolveConditions: true, + template: { + features: { + conditions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--resolve-conditions', + { + resolveConditions: false, + }, + { + resolveConditions: true, + template: { + features: { + conditions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-resolve-conditions', { + resolveConditions: false, + template: { + features: { + conditions: false, + }, + }, + }); + + test('should handle new arg', '--no-template-conditions', { + resolveConditions: false, + template: { + features: { + conditions: false, + }, + }, + }); + + test( + 'should handle new arg with priority', + '--no-resolve-conditions --template-conditions', + { + resolveConditions: true, + template: { + features: { + conditions: true, + }, + }, + }, + ); + + test( + 'should handle config', + '', + { + resolveConditions: false, + }, + { + resolveConditions: false, + template: { + features: { + conditions: false, + }, + }, + }, + ); + }); + + describe('conditionsInCode', () => { + test('should handle default', '', { + conditionsInCode: false, + template: { + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle arg', '--conditions-in-code', { + conditionsInCode: true, + template: { + scopes: { + text: true, + code: true, + }, + }, + }); + + test( + 'should handle negated arg with priority', + '--no-conditions-in-code', + { + conditionsInCode: true, + }, + { + conditionsInCode: false, + template: { + scopes: { + text: true, + code: false, + }, + }, + }, + ); + + test('should handle new arg with priority', '--no-conditions-in-code --template all', { + conditionsInCode: true, + template: { + scopes: { + text: true, + code: true, + }, + }, + }); + + test('should handle negated new arg', '--no-template', { + conditionsInCode: false, + template: { + enabled: false, + scopes: { + text: false, + code: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + conditionsInCode: true, + }, + { + conditionsInCode: true, + template: { + scopes: { + text: true, + code: true, + }, + }, + }, + ); + }); + + describe('template', () => { + test('should handle default', '', { + template: { + enabled: true, + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle arg `all`', '--template all', { + template: { + enabled: true, + scopes: { + text: true, + code: true, + }, + }, + }); + + test('should handle arg `text`', '--template text', { + template: { + enabled: true, + scopes: { + text: true, + code: false, + }, + }, + }); + + test('should handle arg `text`', '--template code', { + template: { + enabled: true, + scopes: { + text: false, + code: true, + }, + }, + }); + + test('should handle negated arg', '--no-template', { + template: { + enabled: false, + scopes: { + text: false, + code: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + template: { + enabled: false, + }, + }, + { + template: { + enabled: false, + scopes: { + text: true, + code: false, + }, + }, + }, + ); + }); + + describe('templateVars', () => { + test('should handle default', '', { + template: { + features: { + substitutions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--template-vars', + { + template: { + features: { + substitutions: false, + }, + }, + }, + { + template: { + features: { + substitutions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-template-vars', { + template: { + features: { + substitutions: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + template: { + features: { + substitutions: false, + }, + }, + }, + { + template: { + features: { + substitutions: false, + }, + }, + }, + ); + }); + + describe('templateConditions', () => { + test('should handle default', '', { + template: { + features: { + conditions: true, + }, + }, + }); + + test( + 'should handle arg with priority', + '--template-conditions', + { + template: { + features: { + conditions: false, + }, + }, + }, + { + template: { + features: { + conditions: true, + }, + }, + }, + ); + + test('should handle negated arg', '--no-template-conditions', { + template: { + features: { + conditions: false, + }, + }, + }); + + test( + 'should handle config', + '', + { + template: { + features: { + conditions: false, + }, + }, + }, + { + template: { + features: { + conditions: false, + }, + }, + }, + ); + }); + }); +}); diff --git a/src/commands/build/features/templating/index.ts b/src/commands/build/features/templating/index.ts new file mode 100644 index 00000000..cc510e1c --- /dev/null +++ b/src/commands/build/features/templating/index.ts @@ -0,0 +1,133 @@ +import type {Build} from '~/commands'; +import type {Command} from '~/config'; +import get from 'lodash/get'; +import {defined, deprecated, valuable} from '~/config'; +import {options} from './config'; + +const merge = (acc: Hash, ...sources: Hash[]) => { + for (const source of sources) { + for (const [key, value] of Object.entries(source)) { + if (!acc[key] || !value) { + acc[key] = value; + } else if (typeof value === 'object') { + acc[key] = merge({}, acc[key], value); + } + } + } + + return acc; +}; + +export type TemplatingArgs = { + template?: boolean | 'all' | 'text' | 'code'; + templateVars?: boolean; + templateConditions?: boolean; + disableLiquid?: boolean; + resolveConditions?: boolean; + conditionsInCode?: boolean; + applyPresets?: boolean; +}; + +export type TemplatingConfig = { + template: { + enabled: boolean; + scopes: { + text: boolean; + code: boolean; + }; + features: { + substitutions: boolean; + conditions: boolean; + cycles: boolean; + }; + }; + + disableLiquid: boolean; + resolveConditions: boolean; + conditionsInCode: boolean; + applyPresets: boolean; +}; + +export class Templating { + apply(program: Build) { + program.hooks.Command.tap('Templating', (command: Command) => { + command + .addOption(options.template) + .addOption(options.noTemplate) + .addOption(options.templateVars) + .addOption(options.templateConditions) + .addOption(options.disableLiquid) + .addOption(options.applyPresets) + .addOption(options.resolveConditions) + .addOption(options.conditionsInCode); + }); + + program.hooks.Config.tap('Templating', (config, args) => { + const disableLiquid = defined('disableLiquid', args, config); + const applyPresets = defined('applyPresets', args, config); + const resolveConditions = defined('resolveConditions', args, config); + const conditionsInCode = defined('conditionsInCode', args, config); + const template = defined('template', args); + const templateVars = defined('templateVars', args); + const templateConditions = defined('templateConditions', args); + + config.template = merge( + { + enabled: true, + scopes: { + text: true, + code: false, + }, + features: { + substitutions: true, + conditions: true, + cycles: true, + }, + }, + config.template || {}, + ) as TemplatingConfig['template']; + + if (valuable(disableLiquid)) { + config.template.enabled = disableLiquid !== true; + } + + if (valuable(conditionsInCode)) { + config.template.scopes.code = conditionsInCode === true; + } + + if (valuable(applyPresets)) { + config.template.features.substitutions = applyPresets; + } + + if (valuable(resolveConditions)) { + config.template.features.conditions = resolveConditions; + } + + if (valuable(template)) { + config.template.enabled = template !== false; + + config.template.scopes.text = ['all', 'text'].includes(template as string); + config.template.scopes.code = ['all', 'code'].includes(template as string); + } + + if (valuable(templateVars)) { + config.template.features.substitutions = templateVars; + } + + if (valuable(templateConditions)) { + config.template.features.conditions = templateConditions; + } + + deprecated(config, 'disableLiquid', () => !get(config, 'template.enabled')); + deprecated(config, 'applyPresets', () => + get(config, 'template.features.substitutions'), + ); + deprecated(config, 'resolveConditions', () => + get(config, 'template.features.conditions'), + ); + deprecated(config, 'conditionsInCode', () => get(config, 'template.scopes.code')); + + return config; + }); + } +} diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts new file mode 100644 index 00000000..1be7fba5 --- /dev/null +++ b/src/commands/build/handler.ts @@ -0,0 +1,78 @@ +import type {Run} from './run'; + +import 'threads/register'; + +import {join} from 'node:path'; +import glob from 'glob'; +import shell from 'shelljs'; +import {ArgvService, Includers} from '~/services'; +import { + processAssets, + processExcludedFiles, + processLogs, + processPages, + processServiceFiles, +} from '~/steps'; +import {prepareMapFile} from '~/steps/processMapFile'; +import {copyFiles} from '~/utils'; + +export async function handler(run: Run) { + try { + ArgvService.init({ + ...run.config, + rootInput: run.originalInput, + input: run.input, + output: run.output, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Includers.init(); + + const {buildDisabled, addMapFile} = run.config; + + preparingTemporaryFolders(run); + + await processServiceFiles(); + await processExcludedFiles(run); + + if (addMapFile) { + prepareMapFile(); + } + + if (!buildDisabled) { + await processPages(run); + processAssets(run); + + // Copy all generated files to user' output folder + shell.cp('-r', [join(run.output, '*'), join(run.output, '.*')], run.originalOutput); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + console.log(error); + run.logger.error(error.message); + } finally { + processLogs(run.input); + + shell.rm('-rf', run.input, run.output); + } +} + +function preparingTemporaryFolders(run: Run) { + shell.mkdir('-p', run.originalOutput); + + // Create temporary input/output folders + shell.rm('-rf', run.input, run.output); + shell.mkdir(run.input, run.output); + + copyFiles( + run.originalInput, + run.input, + glob.sync('**', { + cwd: run.originalInput, + nodir: true, + follow: true, + ignore: ['node_modules/**', '*/node_modules/**'], + }), + ); + + shell.chmod('-R', 'u+w', run.input); +} diff --git a/src/commands/build/index.spec.ts b/src/commands/build/index.spec.ts new file mode 100644 index 00000000..d92a0f53 --- /dev/null +++ b/src/commands/build/index.spec.ts @@ -0,0 +1,291 @@ +import {describe, expect, it} from 'vitest'; +import {handler, runBuild as run, testConfig as test} from './__tests__'; + +describe('Build command', () => { + describe('config', () => { + it('should fail without required input prop', async () => { + expect(() => run('')).rejects.toThrow( + `error: required option '-i, --input ' not specified`, + ); + }); + + it('should fail without required output prop', async () => { + expect(() => run('--input ./input')).rejects.toThrow( + `error: required option '-o, --output ' not specified`, + ); + }); + + it('should handle required props in args', async () => { + await run('--input ./input --output ./output'); + + expect(handler).toBeCalled(); + }); + + describe('outputFormat', () => { + test('should handle default', '', { + outputFormat: 'html', + }); + + test('should handle arg', '--output-format md', { + outputFormat: 'md', + }); + + test('should handle shorthand arg', '-f md', { + outputFormat: 'md', + }); + + test( + 'should handle config', + '', + { + outputFormat: 'md', + }, + { + outputFormat: 'md', + }, + ); + + it('should fail on unknown format', async () => { + expect(() => + run('--input ./input --output ./output --output-format other'), + ).rejects.toThrow( + `error: option '-f, --output-format ' argument 'other' is invalid. Allowed choices are html, md.`, + ); + }); + }); + + describe('varsPreset', () => { + test('should handle default', '', { + varsPreset: 'default', + }); + + test('should handle arg', '--vars-preset public', { + varsPreset: 'public', + }); + + test( + 'should handle config', + '', + { + varsPreset: 'public', + }, + { + varsPreset: 'public', + }, + ); + }); + + describe('vars', () => { + test('should handle default', '', { + vars: {}, + }); + + test('should handle arg', '--vars {"a":1}', { + vars: {a: 1}, + }); + + test('should handle shorthand arg', '-v {"a":1}', { + vars: {a: 1}, + }); + + test( + 'should handle config', + '', + { + vars: {a: 1}, + }, + { + vars: {a: 1}, + }, + ); + + // TODO: should merge args ang config + // test('should merge args ang config') + }); + + describe('addMapFile', () => { + test('should handle default', '', { + addMapFile: false, + }); + + test('should handle arg', '--add-map-file', { + addMapFile: true, + }); + + test( + 'should handle config', + '', + { + addMapFile: true, + }, + { + addMapFile: true, + }, + ); + }); + + describe('removeHiddenTocItems', () => { + test('should handle default', '', { + removeHiddenTocItems: false, + }); + + test('should handle arg', '--remove-hidden-toc-items', { + removeHiddenTocItems: true, + }); + + test( + 'should handle config', + '', + { + removeHiddenTocItems: true, + }, + { + removeHiddenTocItems: true, + }, + ); + }); + + describe('allowCustomResources', () => { + test('should handle default', '', { + allowCustomResources: false, + }); + + test('should handle arg', '--allow-custom-resources', { + allowCustomResources: true, + }); + + test( + 'should handle config', + '', + { + allowCustomResources: true, + }, + { + allowCustomResources: true, + }, + ); + }); + + describe('staticContent', () => { + test('should handle default', '', { + staticContent: false, + }); + + test('should handle arg', '--static-content', { + staticContent: true, + }); + + test( + 'should handle config', + '', + { + staticContent: true, + }, + { + staticContent: true, + }, + ); + }); + + describe('addSystemMeta', () => { + test('should handle default', '', { + addSystemMeta: false, + }); + + test('should handle arg', '--add-system-meta', { + addSystemMeta: true, + }); + + test( + 'should handle config', + '', + { + addSystemMeta: true, + }, + { + addSystemMeta: true, + }, + ); + }); + + describe('ignoreStage', () => { + test('should handle default', '', { + ignoreStage: 'skip', + }); + + test('should handle arg', '--ignore-stage preview', { + ignoreStage: 'preview', + }); + + test( + 'should handle config', + '', + { + ignoreStage: 'preview', + }, + { + ignoreStage: 'preview', + }, + ); + }); + + describe('hidden', () => { + test('should handle default', '', { + hidden: [], + }); + + test('should handle arg', '--hidden **/*.md', { + hidden: ['**/*.md'], + }); + + test('should handle args', '--hidden **/*.md --hidden **/*.yaml', { + hidden: ['**/*.md', '**/*.yaml'], + }); + + test( + 'should handle config', + '', + { + hidden: ['**/*.md'], + }, + { + hidden: ['**/*.md'], + }, + ); + + // TODO: should merge args ang config + // test('should merge args ang config') + }); + + describe('buildDisabled', () => { + test('should handle default', '', { + buildDisabled: false, + }); + + test('should handle arg', '--build-disabled', { + buildDisabled: true, + }); + + test( + 'should handle config', + '', + { + buildDisabled: true, + }, + { + buildDisabled: true, + }, + ); + }); + + // test('should handle required props in config', '', { + // input: './input', + // output: './output', + // }, { + // input: './input', + // output: './output', + // }); + }); + + // describe('apply', () => {}); +}); diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index da979e4f..3e2faf65 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -1,88 +1,212 @@ -import type {IProgram} from '~/program'; -import yargs from 'yargs'; -import {hideBin} from 'yargs/helpers'; -import {Help} from 'commander'; -import log from '@diplodoc/transform/lib/log'; - +import type {IProgram, Program, ProgramArgs, ProgramConfig} from '~/program'; +import type {Config} from '~/config'; +import {ok} from 'node:assert'; +import {pick} from 'lodash'; +import {AsyncParallelHook, AsyncSeriesHook, HookMap} from 'tapable'; import {BaseProgram} from '~/program/base'; -import {Command} from '~/config'; -import {build} from '~/cmd'; +import {Stage, YFM_CONFIG_FILENAME} from '~/constants'; +import {Command, defined, deprecated} from '~/config'; +import {OutputFormat, options} from './config'; +import {Run} from './run'; + +import {Templating, TemplatingArgs, TemplatingConfig} from './features/templating'; +import {Publishing, PublishingArgs, PublishingConfig} from './features/publishing'; +import {Contributors, ContributorsArgs, ContributorsConfig} from './features/contributors'; +import {SinglePage, SinglePageArgs, SinglePageConfig} from './features/singlepage'; +import {Redirects} from './features/redirects'; +import {Lint, LintArgs, LintConfig} from './features/linter'; + +type BaseArgs = {output: string}; + +type BaseConfig = { + outputFormat: `${OutputFormat}`; + varsPreset: string; + vars: Hash; + allowHtml: boolean; + // TODO(minor): string[] + ignoreStage: string; + hidden: string[]; + addSystemMeta: boolean; + addMapFile: boolean; + removeHiddenTocItems: boolean; + // TODO(major): wtf? if we don't need to build, why we call build command? + buildDisabled: boolean; + resources: string[]; + allowCustomResources: boolean; + // TODO(major): use as default behavior + staticContent: boolean; +}; + +export type {Run}; const command = 'Build'; -export type BuildArgs = {}; +const hooks = () => ({ + /** + * Async series hook which runs before start of any Run type.

+ * Args: + * - run - [Build.Run](./Run.ts) constructed context.
+ * Best place to subscribe on Run hooks. + */ + BeforeAnyRun: new AsyncSeriesHook(['run'], `${command}.BeforeAnyRun`), + /** + * Async series hook map which runs before start of target Run type.

+ * Args: + * - run - [Build.Run](./Run.ts) constructed context.
+ * Best place to subscribe on target Run hooks. + */ + BeforeRun: new HookMap( + (format: `${OutputFormat}`) => + new AsyncSeriesHook(['run'], `${command}.${format}.BeforeRun`), + ), + /** + * Async parallel hook which runs on start of any Run type.

+ * Args: + * - run - [Build.Run](./Run.ts) constructed context.
+ * Best place to do something in parallel with main build process. + */ + Run: new AsyncParallelHook(['run'], `${command}.Run`), + // TODO: decompose handler and describe this hook + AfterRun: new HookMap( + (format: `${OutputFormat}`) => + new AsyncSeriesHook(['run'], `${command}.${format}.AfterRun`), + ), + // TODO: decompose handler and describe this hook + AfterAnyRun: new AsyncSeriesHook(['run'], `${command}.AfterAnyRun`), +}); -export type BuildConfig = {}; +export type BuildArgs = ProgramArgs & + BaseArgs & + Partial; -const parser = yargs - .command(build) - .option('config', { - alias: 'c', - describe: 'YFM configuration file', - type: 'string', - }) - .option('strict', { - alias: 's', - default: false, - describe: 'Run in strict mode', - type: 'boolean', - }) - .option('quiet', { - alias: 'q', - default: false, - describe: "Run in quiet mode. Don't write logs to stdout", - type: 'boolean', - }) - .group(['config', 'strict', 'quiet', 'help', 'version'], 'Common options:') - .version(typeof VERSION !== 'undefined' ? VERSION : '') - .help(); +export type BuildConfig = Config< + BaseArgs & + ProgramConfig & + BaseConfig & + TemplatingConfig & + PublishingConfig & + ContributorsConfig & + SinglePageConfig & + LintConfig +>; + +export type BuildHooks = ReturnType; export class Build // eslint-disable-next-line new-cap - extends BaseProgram(command, { + extends BaseProgram(command, { config: { - // scope: 'build', - defaults: () => ({}), + scope: 'build', + defaults: () => ({ + outputFormat: OutputFormat.html, + varsPreset: 'default', + vars: {}, + hidden: [], + allowHtml: true, + addMapFile: false, + removeHiddenTocItems: false, + resources: [], + allowCustomResources: false, + staticContent: false, + ignoreStage: Stage.SKIP, + addSystemMeta: false, + buildDisabled: false, + lint: {enabled: true, config: {'log-levels': {}}}, + }), }, command: { isDefault: true, }, - hooks: () => {}, + hooks: hooks(), }) implements IProgram { - readonly command = new Command('build') - .allowUnknownOption(true) - .description('Build documentation in target directory'); + readonly templating = new Templating(); + + readonly publishing = new Publishing(); + + readonly contributors = new Contributors(); + + readonly singlepage = new SinglePage(); + + readonly redirects = new Redirects(); - protected options = []; + readonly linter = new Lint(); + + readonly command = new Command('build').description('Build documentation in target directory'); + + protected options = [ + options.input(), + options.output(), + options.outputFormat, + options.varsPreset, + options.vars, + options.allowHtml, + options.allowHTML, + options.addMapFile, + options.removeHiddenTocItems, + options.resources, + options.allowCustomResources, + options.staticContent, + options.addSystemMeta, + options.hidden, + options.ignoreStage, + options.config(YFM_CONFIG_FILENAME), + options.buildDisabled, + ]; + + apply(program?: Program) { + this.templating.apply(this); + this.publishing.apply(this); + this.contributors.apply(this); + this.singlepage.apply(this); + this.redirects.apply(this); + this.linter.apply(this); - apply(program?: IProgram) { super.apply(program); - this.command.createHelp = function () { - const help = new Help(); - help.formatHelp = () => parser.getHelp(); - return help; - }; + this.hooks.Config.tap('Build', (config, args) => { + const options = this.options.map((option) => option.attributeName()); + + const allowHtml = defined('allowHtml', args, config); + const allowHTML = defined('allowHTML', args, config); + + ok( + (allowHtml !== null && allowHTML !== null && allowHtml === allowHTML) || + allowHtml === null || + allowHTML === null, + 'Options conflict: both allowHtml and allowHTML are configured', + ); + + Object.assign(config, pick(args, options)); + + deprecated(config, 'allowHTML', () => config.allowHtml); + + return config; + }); } async action() { - await parser.parse(hideBin(process.argv), {}, (err, {strict}, output) => { - if (err) { - console.error(err); - process.exit(1); - } + const run = new Run(this.config); - const {warn, error} = log.get(); + console.log(this.config); - if ((strict && warn.length) || error.length) { - process.exit(1); - } + run.logger.pipe(this.logger); - console.log(output); + await this.hooks.BeforeAnyRun.promise(run); + await this.hooks.BeforeRun.for(this.config.outputFormat).promise(run); + await Promise.all([this.handler(run), this.hooks.Run.promise(run)]); + await this.hooks.AfterRun.for(this.config.outputFormat).promise(run); + await this.hooks.AfterAnyRun.promise(run); + } - process.exit(0); - }); + /** + * Loads handler in async mode to not initialise all deps on startup. + */ + private async handler(run: Run) { + // @ts-ignore + const {handler} = await import('./handler'); + + return handler(run); } } diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts new file mode 100644 index 00000000..bd1c7c59 --- /dev/null +++ b/src/commands/build/run.ts @@ -0,0 +1,54 @@ +import { join, resolve } from 'node:path'; +import { configPath, deprecated } from '~/config'; +import { + BUNDLE_FOLDER, + REDIRECTS_FILENAME, + TMP_INPUT_FOLDER, + TMP_OUTPUT_FOLDER, + YFM_CONFIG_FILENAME +} from '~/constants'; +import {Logger} from '~/logger'; +import {BuildConfig} from '.'; + +/** + * This is transferable context for build command. + * Use this context to communicate with lower data processing levels. + */ +export class Run { + readonly originalInput: string; + + readonly originalOutput: string; + + readonly input: string; + + readonly output: string; + + readonly logger: Logger; + + get bundlePath() { + return join(this.originalOutput, BUNDLE_FOLDER); + } + + get configPath() { + return this.config[configPath] || join(this.config.input, YFM_CONFIG_FILENAME); + } + + get redirectsPath() { + return join(this.originalInput, REDIRECTS_FILENAME); + } + + constructor(readonly config: BuildConfig) { + this.originalInput = config.input; + this.originalOutput = config.output; + deprecated(this, 'rootInput', () => config.input); + + // TODO: use root instead + // We need to create system where we can safely work with original input. + this.input = resolve(config.output, TMP_INPUT_FOLDER); + this.output = resolve(config.output, TMP_OUTPUT_FOLDER); + + this.logger = new Logger(config, [ + (message) => message.replace(new RegExp(this.input, 'ig'), ''), + ]); + } +} diff --git a/src/commands/publish/__tests__/index.ts b/src/commands/publish/__tests__/index.ts new file mode 100644 index 00000000..439d19e1 --- /dev/null +++ b/src/commands/publish/__tests__/index.ts @@ -0,0 +1,84 @@ +import type {Run} from '../run'; +import type {Mock} from 'vitest'; +import type {PublishConfig} from '..'; + +import {expect, it, vi} from 'vitest'; +import {Publish} from '..'; +import {upload as originalUpload} from '../upload'; + +export const upload = originalUpload as Mock; + +// eslint-disable-next-line no-var +var resolveConfig: Mock; + +vi.mock('../upload'); +vi.mock('~/config', async (importOriginal) => { + resolveConfig = vi.fn((_path, {defaults, fallback}) => { + return defaults || fallback; + }); + + return { + ...((await importOriginal()) as {}), + resolveConfig, + }; +}); + +export async function runPublish(args: string) { + const publish = new Publish(); + + publish.apply(); + + await publish.parse(['node', 'index'].concat(args.split(' '))); +} + +type DeepPartial = { + [P in keyof T]?: T[P] extends Record ? DeepPartial : T[P]; +}; + +export function testConfig(name: string, args: string, result: DeepPartial): void; +export function testConfig(name: string, args: string, result: Error | string): void; +export function testConfig( + name: string, + args: string, + config: DeepPartial, + result: DeepPartial, +): void; +export function testConfig( + name: string, + args: string, + config: DeepPartial, + result: Error | string, +): void; +export function testConfig(name: string, args: string, config: any, result?: any): void { + it(name, async () => { + if (!result) { + result = config; + config = {}; + } + + resolveConfig.mockImplementation((_path, {defaults}) => { + return { + ...defaults, + ...config, + }; + }); + + upload.mockImplementation((run: Run) => { + expect(run.config).toMatchObject(result as Partial); + }); + + try { + await runPublish('--input ./input --access-key-id 1 --secret-access-key 1 ' + args); + expect(upload).toBeCalled(); + } catch (error: any) { + const message = error.message || error; + if (result instanceof Error) { + expect(message).toEqual(result.message); + } else if (typeof result === 'string') { + expect(message).toEqual(result); + } else { + throw error; + } + } + }); +} diff --git a/src/models.ts b/src/models.ts index 5a8020ad..e91493ec 100644 --- a/src/models.ts +++ b/src/models.ts @@ -114,18 +114,14 @@ export interface YfmTocInclude { export type YfmTocIncluders = YfmTocIncluder[]; export type YfmTocIncluder = { - name: YfmTocIncluderName; + name: string; // arbitrary includer parameters // eslint-disable-next-line @typescript-eslint/no-explicit-any } & Record; -export const includersNames = ['sourcedocs', 'openapi', 'generic', 'unarchive'] as const; - -export type YfmTocIncluderName = (typeof includersNames)[number]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any export type Includer = { - name: YfmTocIncluderName; + name: string; includerFunction: IncluderFunction; }; diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index a98ed0b7..8e659479 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -95,7 +95,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins(); const vars = getVarsPerFile(filePath); const root = resolve(input); - const path: string = resolve(input, filePath); + const path = resolve(input, filePath); let preparedContent = content; /* Relative path from folder of .md file to root of user' output folder */ diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 19d229ac..308f1f6b 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -1,9 +1,10 @@ +import type {DocInnerProps} from '@diplodoc/client'; +import type {Run} from '~/commands/build'; import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, join, resolve, sep} from 'path'; import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import {isString} from 'lodash'; -import type {DocInnerProps} from '@diplodoc/client'; import transform, {Output} from '@diplodoc/transform'; import liquid from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; @@ -46,7 +47,7 @@ const getFileMeta = async ({fileExtension, metadata, inputPath}: ResolverOptions const content: string = readFileSync(resolvedPath, 'utf8'); const transformFn: Function = FileTransformer[fileExtension]; - const {result} = transformFn(content, {path: inputPath}); + const {result} = transformFn(inputPath, content, run); const vars = getVarsPerFile(inputPath); const updatedMetadata = metadata?.isContributorsEnabled @@ -163,34 +164,33 @@ function YamlFileTransformer(content: string, transformOptions: FileTransformOpt }; } -export function liquidMd2Html(input: string, vars: Record, path: string) { - const {conditionsInCode} = ArgvService.getConfig(); +export function liquidMd2Html(path: string, content: string) { + const vars = getVarsPerFile(path); - return liquid(input, vars, path, { - conditionsInCode, + return liquid(content, vars, path, { + conditionsInCode: run.config.template.scopes.code, withSourceMap: true, }); } -function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output { - const {input, ...options} = ArgvService.getConfig(); - const {path: filePath} = transformOptions; - +function MdFileTransformer(path: string, content: string, run: Run): Output { const plugins = PluginService.getPlugins(); - const vars = getVarsPerFile(filePath); - const root = resolve(input); - const path: string = resolve(input, filePath); + const vars = getVarsPerFile(path); return transform(content, { - ...options, - plugins: plugins as MarkdownItPluginCb[], vars, - root, - path, - assetsPublicPath: getAssetsPublicPath(filePath), - rootPublicPath: getAssetsRootPath(filePath), - getVarsPerFile: getVarsPerRelativeFile, + path: resolve(run.input, path), + rootPublicPath: getAssetsRootPath(path), getPublicPath, extractTitle: true, + allowHTML: run.config.allowHtml, + conditionsInCode: run.config.template.scopes.code, + disableLiquid: !run.config.template.enabled, + needToSanitizeHtml: run.config.sanitizeHtml, + plugins: plugins as MarkdownItPluginCb[], + root: run.input, + extractChangelogs: run.config.changelogs.enabled, + assetsPublicPath: getAssetsPublicPath(path), + getVarsPerFile: getVarsPerRelativeFile, }); } diff --git a/src/services/argv.ts b/src/services/argv.ts index 4941a615..f129f1a5 100644 --- a/src/services/argv.ts +++ b/src/services/argv.ts @@ -15,10 +15,6 @@ function init(argv: any) { ignore: Array.isArray(argv.ignore) ? argv.ignore : [], } as YfmArgv; - if (argv.vars) { - _argv.vars = JSON.parse(argv.vars); - } - try { const ignorefile = readFileSync(join(_argv.rootInput, '.yfmignore'), 'utf8'); const ignore = ignorefile.split('\n'); diff --git a/src/services/includers/index.ts b/src/services/includers/index.ts index 402c3f22..15f3253e 100644 --- a/src/services/includers/index.ts +++ b/src/services/includers/index.ts @@ -5,6 +5,7 @@ import {isObject} from 'lodash'; import {ArgvService} from '../index'; import {IncludeMode} from '../../constants'; import {generic, sourcedocs, unarchive} from './batteries'; +import openapi from '@diplodoc/openapi-extension/includer'; import type { Includer, @@ -39,16 +40,12 @@ class IncludersError extends Error { } } -function init(custom: Includer[] = []) { +function init() { if (includersMap) { return; } - includersMap = {generic, sourcedocs, unarchive}; - - for (const includer of custom) { - includersMap[includer.name] = includer; - } + includersMap = {generic, sourcedocs, unarchive, openapi}; } async function applyIncluders(path: string, item: YfmToc, vars: YfmPreset) { diff --git a/src/steps/processAssets.ts b/src/steps/processAssets.ts index 435b0906..505e6418 100644 --- a/src/steps/processAssets.ts +++ b/src/steps/processAssets.ts @@ -2,139 +2,96 @@ import walkSync from 'walk-sync'; import {load} from 'js-yaml'; import {readFileSync} from 'fs'; import shell from 'shelljs'; -import {join, resolve, sep} from 'path'; - -import {ArgvService, TocService} from '../services'; -import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; +import {resolve} from 'path'; +import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {TocService} from '../services'; +import {copyFiles, findAllValuesByKeys} from '../utils'; import {LINK_KEYS} from '@diplodoc/client/ssr'; import {isLocalUrl} from '@diplodoc/transform/lib/utils'; -import { - ASSETS_FOLDER, - LINT_CONFIG_FILENAME, - REDIRECTS_FILENAME, - RTL_LANGS, - YFM_CONFIG_FILENAME, -} from '../constants'; +import {ASSETS_FOLDER} from '../constants'; import {Resources} from '../models'; -import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {Run} from '~/commands/build'; /** - * @param {Array} args - * @param {string} outputBundlePath - * @param {string} outputFormat - * @param {string} tmpOutputFolder - * @return {void} - */ - -type Props = { - args: string[]; - outputBundlePath: string; - outputFormat: string; - tmpOutputFolder: string; -}; -/* * Processes assets files (everything except .md files) */ -export function processAssets({args, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { - switch (outputFormat) { +export function processAssets(run: Run) { + switch (run.config.outputFormat) { case 'html': - processAssetsHtmlRun({outputBundlePath}); + processAssetsHtmlRun(run); break; case 'md': - processAssetsMdRun({args, tmpOutputFolder}); + processAssetsMdRun(run); break; } } -function processAssetsHtmlRun({outputBundlePath}) { - const {input: inputFolderPath, output: outputFolderPath, langs} = ArgvService.getConfig(); - - const documentationAssetFilePath: string[] = walkSync(inputFolderPath, { +function processAssetsHtmlRun(run: Run) { + const documentationAssetFilePath: string[] = walkSync(run.input, { directories: false, includeBasePath: false, ignore: ['**/*.yaml', '**/*.md'], }); - copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath); - - const hasRTLlang = hasIntersection(langs, RTL_LANGS); const bundleAssetFilePath: string[] = walkSync(ASSETS_FOLDER, { directories: false, includeBasePath: false, - ignore: !hasRTLlang && ['**/*.rtl.css'], }); - copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); + copyFiles(run.input, run.output, documentationAssetFilePath); + copyFiles(ASSETS_FOLDER, run.bundlePath, bundleAssetFilePath); } -function processAssetsMdRun({args, tmpOutputFolder}) { - const {input: inputFolderPath, allowCustomResources, resources} = ArgvService.getConfig(); - - const pathToConfig = args.config || join(args.input, YFM_CONFIG_FILENAME); - const pathToRedirects = join(args.input, REDIRECTS_FILENAME); - const pathToLintConfig = join(args.input, LINT_CONFIG_FILENAME); +function processAssetsMdRun(run: Run) { + const {allowCustomResources, resources} = run.config; - shell.cp(resolve(pathToConfig), tmpOutputFolder); - shell.cp(resolve(pathToRedirects), tmpOutputFolder); - shell.cp(resolve(pathToLintConfig), tmpOutputFolder); + shell.cp(run.configPath, run.output); + shell.cp(run.redirectsPath, run.output); if (resources && allowCustomResources) { const resourcePaths: string[] = []; // collect paths of all resources - Object.keys(resources).forEach( - (type) => - resources[type as keyof Resources]?.forEach((path: string) => - resourcePaths.push(path), - ), + Object.keys(resources).forEach((type) => + resources[type as keyof Resources]?.forEach((path: string) => resourcePaths.push(path)), ); //copy resources - copyFiles(args.input, tmpOutputFolder, resourcePaths); + copyFiles(run.originalInput, run.output, resourcePaths); } - const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { + const yamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { if (file.endsWith('.yaml')) { - const resolvedPathToFile = resolve(inputFolderPath, file); + const resolvedPathToFile = resolve(run.input, file); acc.push(resolvedPathToFile); } + return acc; - }, []); + }, [] as string[]); - tocYamlFiles.forEach((yamlFile) => { + const contentLinks = yamlFiles.reduce((acc, yamlFile) => { const content = load(readFileSync(yamlFile, 'utf8')); if (!Object.prototype.hasOwnProperty.call(content, 'blocks')) { - return; + return acc; } - const contentLinks = findAllValuesByKeys(content, LINK_KEYS); - const localMediaLinks = contentLinks.reduce( - (acc, link) => { - const linkHasMediaExt = new RegExp( - /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, - ).test(link); - - if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { - const linkAbsolutePath = resolveRelativePath(yamlFile, link); - const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); - - acc.push(linkRootPath); - } - return acc; - }, - - [], - ); - - copyFiles(args.input, tmpOutputFolder, localMediaLinks); - }); + return acc.concat((findAllValuesByKeys(content, LINK_KEYS) as string[]) + .filter((link) => isMediaLink(link) && isLocalUrl(link)) + .map((link) => resolveRelativePath(yamlFile, link)) + .filter((link) => isFileExists(link)) + .map((link) => link + .replace(`${run.originalInput}`, '') + .replace(/^[/\\]/, '') + )); + }, [] as string[]); + + copyFiles(run.input, run.output, [...new Set(contentLinks)]); } -function hasIntersection(array1, array2) { - const set1 = new Set(array1); - return array2.some((element) => set1.has(element)); +function isMediaLink(link: string) { + return /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/.test(link); } diff --git a/src/steps/processExcludedFiles.ts b/src/steps/processExcludedFiles.ts index 7dfbc683..fd0ceb01 100644 --- a/src/steps/processExcludedFiles.ts +++ b/src/steps/processExcludedFiles.ts @@ -1,4 +1,5 @@ -import {relative, resolve} from 'path'; +import type {Run} from '~/commands/build'; +import {relative, resolve} from 'node:path'; import walkSync from 'walk-sync'; import shell from 'shelljs'; @@ -9,10 +10,10 @@ import {convertBackSlashToSlash} from '../utils'; * Removes all content files that unspecified in toc files or ignored. * @return {void} */ -export function processExcludedFiles() { - const {input: inputFolderPath, output: outputFolderPath, ignore} = ArgvService.getConfig(); +export async function processExcludedFiles(run: Run) { + const {ignore} = ArgvService.getConfig(); - const allContentFiles: string[] = walkSync(inputFolderPath, { + const allContentFiles: string[] = walkSync(run.input, { directories: false, includeBasePath: true, globs: ['**/*.md', '**/index.yaml', ...ignore], @@ -20,7 +21,7 @@ export function processExcludedFiles() { ignore: ['**/_*/**/*'], }); const navigationPaths = TocService.getNavigationPaths().map((filePath) => - convertBackSlashToSlash(resolve(inputFolderPath, filePath)), + convertBackSlashToSlash(resolve(run.input, filePath)), ); const tocSpecifiedFiles = new Set(navigationPaths); const excludedFiles = allContentFiles.filter((filePath) => !tocSpecifiedFiles.has(filePath)); @@ -28,8 +29,8 @@ export function processExcludedFiles() { shell.rm('-f', excludedFiles); const includedTocPaths = TocService.getIncludedTocPaths().map((filePath) => { - const relativeTocPath = relative(inputFolderPath, filePath); - const destTocPath = resolve(outputFolderPath, relativeTocPath); + const relativeTocPath = relative(run.input, filePath); + const destTocPath = resolve(run.output, relativeTocPath); return convertBackSlashToSlash(destTocPath); }); diff --git a/src/steps/processLinter.ts b/src/steps/processLinter.ts index 13c3c050..0bb85842 100644 --- a/src/steps/processLinter.ts +++ b/src/steps/processLinter.ts @@ -8,11 +8,12 @@ import {logger} from '../utils'; import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '../constants'; import {lintPage} from '../resolvers'; import {splitOnChunks} from '../utils/worker'; +import {Run} from '~/commands/build'; let processLinterWorkers: (ProcessLinterWorker & Thread)[]; let navigationPathsChunks: string[][]; -export async function processLinter(): Promise { +export async function processLinter(run: Run): Promise { const argvConfig = ArgvService.getConfig(); const navigationPaths = TocService.getNavigationPaths(); @@ -62,7 +63,7 @@ export async function processLinter(): Promise { ); } -export async function initLinterWorkers() { +export async function initLinterWorkers(run: Run) { const navigationPaths = TocService.getNavigationPaths(); const chunkSize = getChunkSize(navigationPaths); diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index c7fd4bd9..2aeb3f6f 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -23,7 +23,7 @@ import { YfmToc, } from '../models'; import {resolveMd2HTML, resolveMd2Md} from '../resolvers'; -import {ArgvService, LeadingService, PluginService, TocService} from '../services'; +import {LeadingService, PluginService, TocService} from '../services'; import { generateStaticMarkup, joinSinglePageResults, @@ -33,20 +33,14 @@ import { import {getVCSConnector} from '../vcs-connector'; import {VCSConnector} from '../vcs-connector/connector-models'; import {generateStaticRedirect} from '../utils/redirect'; +import {Run} from '~/commands/build'; const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; // Processes files of documentation (like index.yaml, *.md) -export async function processPages(outputBundlePath: string): Promise { - const { - input: inputFolderPath, - output: outputFolderPath, - outputFormat, - singlePage, - resolveConditions, - } = ArgvService.getConfig(); - +export async function processPages(run: Run): Promise { + const {outputFormat, singlePage} = run.config; const vcsConnector = await getVCSConnector(); PluginService.setPlugins(); @@ -59,35 +53,34 @@ export async function processPages(outputBundlePath: string): Promise { asyncify(async (pathToFile: string) => { const pathData = getPathData( pathToFile, - inputFolderPath, - outputFolderPath, + run.input, + run.output, outputFormat, - outputBundlePath, + run.bundlePath, ); logger.proc(pathToFile); const metaDataOptions = getMetaDataOptions( + run, pathData, - inputFolderPath.length, vcsConnector, ); await preparingPagesByOutputFormat( + run, pathData, - metaDataOptions, - resolveConditions, - singlePage, + metaDataOptions ); }), ); if (singlePage) { - await saveSinglePages(); + await saveSinglePages(run); } if (outputFormat === 'html') { - saveRedirectPage(outputFolderPath); + saveRedirectPage(run); } } @@ -126,13 +119,8 @@ function getPathData( return pathData; } -async function saveSinglePages() { - const { - input: inputFolderPath, - lang: configLang, - langs: configLangs, - resources, - } = ArgvService.getConfig(); +async function saveSinglePages(run: Run) { + const {lang, langs, resources} = run.config; try { await Promise.all( @@ -143,19 +131,16 @@ async function saveSinglePages() { const singlePageBody = joinSinglePageResults( singlePageResults[tocDir], - inputFolderPath, + run.input, tocDir, ); - const tocPath = join(relative(inputFolderPath, tocDir), 'toc.yaml'); + const tocPath = join(relative(run.input, tocDir), 'toc.yaml'); const toc: YfmToc | null = TocService.getForPath(tocPath) || null; const preparedToc = transformTocForSinglePage(toc, { - root: inputFolderPath, + root: run.input, currentPath: join(tocDir, SINGLE_PAGE_FILENAME), }) as YfmToc; - const lang = configLang ?? Lang.RU; - const langs = configLangs?.length ? configLangs : [lang]; - const pageData = { data: { leading: false as const, @@ -188,14 +173,12 @@ async function saveSinglePages() { } } -function saveRedirectPage(outputDir: string): void { - const {lang, langs} = ArgvService.getConfig(); - +function saveRedirectPage(run: Run): void { + const {lang, langs} = run.config; const redirectLang = lang || langs?.[0] || Lang.RU; const redirectLangRelativePath = `./${redirectLang}/index.html`; - - const redirectPagePath = join(outputDir, 'index.html'); - const redirectLangPath = join(outputDir, redirectLangRelativePath); + const redirectPagePath = join(run.output, 'index.html'); + const redirectLangPath = join(run.output, redirectLangRelativePath); if (!existsSync(redirectPagePath) && existsSync(redirectLangPath)) { const content = generateStaticRedirect(redirectLang, redirectLangRelativePath); @@ -227,11 +210,12 @@ function savePageResultForSinglePage(pageProps: DocInnerProps, pathData: PathDat } function getMetaDataOptions( + run: Run, pathData: PathData, - inputFolderPathLength: number, vcsConnector?: VCSConnector, ): MetaDataOptions { - const {contributors, addSystemMeta, resources, allowCustomResources} = ArgvService.getConfig(); + const inputFolderPathLength = run.input.length; + const {contributors, addSystemMeta, resources, allowCustomResources} = run.config; const metaDataOptions: MetaDataOptions = { vcsConnector, @@ -247,7 +231,7 @@ function getMetaDataOptions( if (allowCustomResources && resources) { const allowedResources = Object.entries(resources).reduce((acc: Resources, [key, val]) => { if (Object.keys(ResourceType).includes(key)) { - acc[key as keyof typeof ResourceType] = val; + acc[key as keyof ResourceType] = val; } return acc; }, {}); @@ -259,10 +243,9 @@ function getMetaDataOptions( } async function preparingPagesByOutputFormat( + run: Run, path: PathData, metaDataOptions: MetaDataOptions, - resolveConditions: boolean, - singlePage: boolean, ): Promise { const { filename, @@ -273,14 +256,14 @@ async function preparingPagesByOutputFormat( outputFormat, pathToFile, } = path; - const {allowCustomResources} = ArgvService.getConfig(); + const {allowCustomResources, singlePage, template} = run.config; try { shell.mkdir('-p', outputDir); const isYamlFileExtension = fileExtension === '.yaml'; - if (resolveConditions && fileBaseName === 'index' && isYamlFileExtension) { + if (template.features.conditions && fileBaseName === 'index' && isYamlFileExtension) { LeadingService.filterFile(pathToFile); }