From 991e65bdb304bd3c57e144a651632d6948cd360f Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 3 Dec 2024 13:16:11 +0300 Subject: [PATCH 1/3] feat: Add build run fs utils --- src/commands/build/__tests__/index.ts | 10 ++++ src/commands/build/handler.ts | 25 --------- src/commands/build/index.ts | 17 +++--- src/commands/build/run.ts | 78 ++++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 34 deletions(-) diff --git a/src/commands/build/__tests__/index.ts b/src/commands/build/__tests__/index.ts index a117317e..70d9841a 100644 --- a/src/commands/build/__tests__/index.ts +++ b/src/commands/build/__tests__/index.ts @@ -13,6 +13,12 @@ var resolveConfig: Mock; vi.mock('shelljs'); vi.mock('../handler'); +vi.mock('../run', async (importOriginal) => { + return { + ...((await importOriginal()) as {}), + copy: vi.fn(), + }; +}); vi.mock('~/config', async (importOriginal) => { resolveConfig = vi.fn((_path, {defaults, fallback}) => { return defaults || fallback; @@ -28,6 +34,10 @@ export async function runBuild(args: string) { const build = new Build(); build.apply(); + build.hooks.BeforeAnyRun.tap('Tests', (run) => { + run.copy = vi.fn(); + run.write = vi.fn(); + }); await build.parse(['node', 'index'].concat(args.split(' '))); } diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index 55ac4b27..e37859fa 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -2,9 +2,6 @@ import type {Run} from './run'; import 'threads/register'; -import glob from 'glob'; -import shell from 'shelljs'; - import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; import {ArgvService, Includers, SearchService} from '~/services'; @@ -19,13 +16,8 @@ import { processServiceFiles, } from '~/steps'; import {prepareMapFile} from '~/steps/processMapFile'; -import {copyFiles} from '~/utils'; export async function handler(run: Run) { - if (typeof VERSION !== 'undefined') { - console.log(`Using v${VERSION} version`); - } - try { ArgvService.init(run.legacyConfig); SearchService.init(); @@ -35,8 +27,6 @@ export async function handler(run: Run) { const {lintDisabled, buildDisabled, addMapFile} = ArgvService.getConfig(); - preparingTemporaryFolders(run); - await processServiceFiles(); processExcludedFiles(); @@ -72,18 +62,3 @@ export async function handler(run: Run) { processLogs(run.input); } } - -function preparingTemporaryFolders(run: Run) { - copyFiles( - run.originalInput, - run.input, - glob.sync('**', { - cwd: run.originalInput, - nodir: true, - follow: true, - ignore: ['node_modules/**', '*/node_modules/**'], - }), - ); - - shell.chmod('-R', 'u+w', run.input); -} diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index b3923c75..daa26ba5 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -3,7 +3,6 @@ import type {DocAnalytics} from '@diplodoc/client'; import {ok} from 'node:assert'; import {join} from 'node:path'; -import glob from 'glob'; import {pick} from 'lodash'; import {AsyncParallelHook, AsyncSeriesHook, HookMap} from 'tapable'; @@ -270,6 +269,10 @@ export class Build } async action() { + if (typeof VERSION !== 'undefined') { + console.log(`Using v${VERSION} version`); + } + const run = new Run(this.config); run.logger.pipe(this.logger); @@ -280,17 +283,15 @@ export class Build await this.hooks.BeforeAnyRun.promise(run); await this.hooks.BeforeRun.for(this.config.outputFormat).promise(run); + + await run.copy(run.originalInput, run.input, ['node_modules/**', '*/node_modules/**']); + await Promise.all([handler(run), this.hooks.Run.promise(run)]); + await this.hooks.AfterRun.for(this.config.outputFormat).promise(run); await this.hooks.AfterAnyRun.promise(run); - // Copy all generated files to user' output folder - shell.mkdir('-p', run.originalOutput); - shell.cp('-r', join(run.output, '*'), run.originalOutput); - - if (glob.sync('.*', {cwd: run.output}).length) { - shell.cp('-r', join(run.output, '.*'), run.originalOutput); - } + await run.copy(run.output, run.originalOutput); shell.rm('-rf', run.input, run.output); } diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts index 327bfff5..cf071026 100644 --- a/src/commands/build/run.ts +++ b/src/commands/build/run.ts @@ -1,6 +1,10 @@ import type {YfmArgv} from '~/models'; -import {join, resolve} from 'path'; +// import {ok} from 'node:assert'; +import {dirname, join, resolve} from 'node:path'; +import {access, link, mkdir, readFile, stat, unlink, writeFile} from 'node:fs/promises'; +import {glob} from 'glob'; + import {configPath} from '~/config'; import { BUNDLE_FOLDER, @@ -11,6 +15,17 @@ import { } from '~/constants'; import {Logger} from '~/logger'; import {BuildConfig} from '.'; +// import {InsecureAccessError} from './errors'; + +type FileSystem = { + access: typeof access; + stat: typeof stat; + link: typeof link; + unlink: typeof unlink; + mkdir: typeof mkdir; + readFile: typeof readFile; + writeFile: typeof writeFile; +}; /** * This is transferable context for build command. @@ -31,6 +46,8 @@ export class Run { readonly config: BuildConfig; + readonly fs: FileSystem = {access, stat, link, unlink, mkdir, readFile, writeFile}; + get bundlePath() { return join(this.output, BUNDLE_FOLDER); } @@ -43,6 +60,8 @@ export class Run { return join(this.originalInput, REDIRECTS_FILENAME); } + private _copyMap: Record = {}; + constructor(config: BuildConfig) { this.config = config; this.originalInput = config.input; @@ -105,4 +124,61 @@ export class Run { (_level, message) => message.replace(new RegExp(this.input, 'ig'), ''), ]); } + + write = async (path: AbsolutePath, content: string | Buffer) => { + await this.fs.mkdir(dirname(path), {recursive: true}); + await this.fs.writeFile(path, content, 'utf8'); + }; + + copy = async (from: AbsolutePath, to: AbsolutePath, ignore?: string[]) => { + const isFile = (await this.fs.stat(from)).isFile(); + const hardlink = async (from: AbsolutePath, to: AbsolutePath) => { + // const realpath = this.realpath(from); + // + // ok( + // realpath[0].startsWith(this.originalInput), + // new InsecureAccessError(realpath[0], realpath), + // ); + + await this.fs.unlink(to).catch(() => {}); + await this.fs.link(from, to); + this._copyMap[to] = from; + }; + + if (isFile) { + await this.fs.mkdir(dirname(to), {recursive: true}); + await hardlink(from, to); + + return; + } + + const dirs = new Set(); + // TODO: check dotfiles copy + const files = (await glob('*/**', { + cwd: from, + nodir: true, + follow: true, + ignore, + })) as RelativePath[]; + + for (const file of files) { + const dir = join(to, dirname(file)); + if (!dirs.has(dir)) { + await this.fs.mkdir(dir, {recursive: true}); + dirs.add(dir); + } + + await hardlink(join(from, file), join(to, file)); + } + }; + + realpath = (path: AbsolutePath): AbsolutePath[] => { + const stack = [path]; + while (this._copyMap[path]) { + path = this._copyMap[path]; + stack.unshift(path); + } + + return stack; + }; } From 513fedf08b30df56feb7cd39bfa5b07fcf6e8eef Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Tue, 3 Dec 2024 13:19:11 +0300 Subject: [PATCH 2/3] fix: Update glob --- package-lock.json | 343 ++++++++++++++++-- package.json | 4 +- src/commands/build/run.ts | 4 +- src/commands/translate/utils/config.ts | 4 +- src/services/includers/batteries/generic.ts | 13 +- src/services/leading.ts | 3 +- src/steps/processChangelogs.ts | 6 +- src/utils/glob.ts | 12 - src/utils/index.ts | 1 - .../load-custom-resources.spec.ts.snap | 20 + tests/e2e/__snapshots__/rtl.spec.ts.snap | 6 + 11 files changed, 346 insertions(+), 70 deletions(-) delete mode 100644 src/utils/glob.ts diff --git a/package-lock.json b/package-lock.json index 6dcff74a..ec330b54 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,6 @@ "@octokit/core": "4.2.4", "@types/async": "^3.2.15", "@types/chalk": "2.2.0", - "@types/glob": "^8.1.0", "@types/html-escaper": "^3.0.0", "@types/js-yaml": "4.0.9", "@types/json-stringify-safe": "^5.0.3", @@ -52,7 +51,7 @@ "commander": "^12.0.0", "csp-header": "^5.2.1", "esbuild": "^0.23.1", - "glob": "^8.0.3", + "glob": "^10.4.5", "html-escaper": "^3.0.3", "husky": "8.0.3", "js-yaml": "4.1.0", @@ -70,6 +69,7 @@ "typescript": "^5.4.5", "vite-tsconfig-paths": "^4.2.3", "vitest": "^1.1.3", + "vitest-when": "^0.5.0", "walk-sync": "^3.0.0" }, "engines": { @@ -3271,6 +3271,84 @@ "deprecated": "Use @eslint/object-schema instead", "license": "BSD-3-Clause" }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", @@ -3528,6 +3606,17 @@ "@octokit/openapi-types": "^18.0.0" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@pkgr/core": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", @@ -4718,17 +4807,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/minimatch": "^5.1.2", - "@types/node": "*" - } - }, "node_modules/@types/hoist-non-react-statics": { "version": "3.3.5", "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.5.tgz", @@ -8488,6 +8566,36 @@ "is-callable": "^1.1.3" } }, + "node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", @@ -8672,21 +8780,21 @@ "license": "ISC" }, "node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "deprecated": "Glob versions prior to v9 are no longer supported", + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", "dev": true, "license": "ISC", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" }, - "engines": { - "node": ">=12" + "bin": { + "glob": "dist/esm/bin.mjs" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -8704,19 +8812,6 @@ "node": ">=10.13.0" } }, - "node_modules/glob/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/global-modules": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-2.0.0.tgz", @@ -9735,6 +9830,22 @@ "node": ">= 0.4" } }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -11266,6 +11377,16 @@ "node": ">= 6" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mlly": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.7.3.tgz", @@ -11651,6 +11772,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -11757,6 +11885,30 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "license": "MIT" }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -13365,6 +13517,39 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/string-width/node_modules/ansi-regex": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", @@ -13505,6 +13690,20 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", @@ -15151,6 +15350,25 @@ } } }, + "node_modules/vitest-when": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/vitest-when/-/vitest-when-0.5.0.tgz", + "integrity": "sha512-BYDfzSawgKsV5GX3bU9ZbURuljjBCqi5KPtE2hBn/DsCRThU0z4qH0PAhJGemyKNnR01ADObXkmm1UPDHGzVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pretty-format": "^29.7.0" + }, + "peerDependencies": { + "@vitest/expect": ">=0.31.0 <3.0.0", + "vitest": ">=0.31.0 <3.0.0" + }, + "peerDependenciesMeta": { + "@vitest/expect": { + "optional": true + } + } + }, "node_modules/vitest/node_modules/execa": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", @@ -15547,6 +15765,57 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", diff --git a/package.json b/package.json index f1f86472..092bb027 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "@octokit/core": "4.2.4", "@types/async": "^3.2.15", "@types/chalk": "2.2.0", - "@types/glob": "^8.1.0", "@types/html-escaper": "^3.0.0", "@types/js-yaml": "4.0.9", "@types/json-stringify-safe": "^5.0.3", @@ -95,7 +94,7 @@ "commander": "^12.0.0", "csp-header": "^5.2.1", "esbuild": "^0.23.1", - "glob": "^8.0.3", + "glob": "^10.4.5", "html-escaper": "^3.0.3", "husky": "8.0.3", "js-yaml": "4.1.0", @@ -113,6 +112,7 @@ "typescript": "^5.4.5", "vite-tsconfig-paths": "^4.2.3", "vitest": "^1.1.3", + "vitest-when": "^0.5.0", "walk-sync": "^3.0.0" }, "engines": { diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts index cf071026..7359aff9 100644 --- a/src/commands/build/run.ts +++ b/src/commands/build/run.ts @@ -127,6 +127,7 @@ export class Run { write = async (path: AbsolutePath, content: string | Buffer) => { await this.fs.mkdir(dirname(path), {recursive: true}); + await this.fs.unlink(path).catch(() => {}); await this.fs.writeFile(path, content, 'utf8'); }; @@ -154,8 +155,9 @@ export class Run { const dirs = new Set(); // TODO: check dotfiles copy - const files = (await glob('*/**', { + const files = (await glob('**', { cwd: from, + dot: true, nodir: true, follow: true, ignore, diff --git a/src/commands/translate/utils/config.ts b/src/commands/translate/utils/config.ts index 6073d663..ebb04852 100644 --- a/src/commands/translate/utils/config.ts +++ b/src/commands/translate/utils/config.ts @@ -1,7 +1,7 @@ import {ok} from 'node:assert'; import {dirname, isAbsolute, relative, resolve} from 'node:path'; import {readFileSync} from 'node:fs'; -import glob from 'glob'; +import {globSync} from 'glob'; import {merge} from 'lodash'; import {filter} from 'minimatch'; import {defined} from '~/config'; @@ -117,7 +117,7 @@ export function resolveFiles( return acc.concat(path); }, [] as string[]); } else { - result = glob.sync(extmatch, { + result = globSync(extmatch, { cwd: input, nodir: true, ignore: ['node_modules/**', '*/node_modules/**'], diff --git a/src/services/includers/batteries/generic.ts b/src/services/includers/batteries/generic.ts index 84fae9c6..e6f12226 100644 --- a/src/services/includers/batteries/generic.ts +++ b/src/services/includers/batteries/generic.ts @@ -6,7 +6,7 @@ import {dump} from 'js-yaml'; import {getRealPath} from '@diplodoc/transform/lib/utilsFS'; -import {glob} from '../../../utils/glob'; +import {glob} from 'glob'; import {IncluderFunctionParams} from '../../../models'; @@ -56,17 +56,10 @@ async function includerFunction(params: IncluderFunctionParams) { ? join(writeBasePath, tocDirPath, input) : join(readBasePath, tocDirPath, input); - let cache = {}; - let found = []; - - ({ - state: {found, cache}, - } = await glob(MD_GLOB, { + const found = await glob(MD_GLOB, { cwd: contentPath, - nosort: true, nocase: true, - cache, - })); + }); const writePath = getRealPath(join(writeBasePath, tocDirPath, item.include.path)); diff --git a/src/services/leading.ts b/src/services/leading.ts index d785a996..1a3b2e35 100644 --- a/src/services/leading.ts +++ b/src/services/leading.ts @@ -1,5 +1,5 @@ import {dirname, resolve} from 'path'; -import {readFileSync, writeFileSync} from 'fs'; +import {readFileSync, unlinkSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; @@ -74,6 +74,7 @@ function filterFile(path: string) { } }); + unlinkSync(filePath); writeFileSync(filePath, dump(parsedIndex)); } catch (error) { log.error(`Error while filtering index file: ${path}. Error message: ${error}`); diff --git a/src/steps/processChangelogs.ts b/src/steps/processChangelogs.ts index 084b1b09..0c8c4204 100644 --- a/src/steps/processChangelogs.ts +++ b/src/steps/processChangelogs.ts @@ -1,4 +1,4 @@ -import {glob} from '../utils/glob'; +import {glob} from 'glob'; import {dirname, join, normalize, resolve} from 'node:path'; import {ArgvService} from '../services'; import {copyFile, mkdir, readFile, writeFile} from 'node:fs/promises'; @@ -37,12 +37,10 @@ export async function processChangelogs() { return; } - const result = await glob('**/**/__changes-*.json', { + const files = await glob('**/**/__changes-*.json', { cwd: outputFolderPath, }); - const files = result.state.found; - if (!files.length) { return; } diff --git a/src/utils/glob.ts b/src/utils/glob.ts deleted file mode 100644 index 41a5b188..00000000 --- a/src/utils/glob.ts +++ /dev/null @@ -1,12 +0,0 @@ -import libglob, {IGlob, IOptions} from 'glob'; - -export type Glob = {state: IGlob}; - -const glob = async (pattern: string, options: IOptions): Promise => - new Promise((res, rej) => { - const state: IGlob = libglob(pattern, options, (err) => (err ? rej(err) : res({state}))); - }); - -export {glob}; - -export default {glob}; diff --git a/src/utils/index.ts b/src/utils/index.ts index 70f1c8bc..a495e0b1 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -5,5 +5,4 @@ export * from './url'; export * from './path'; export * from './toc'; export * from './presets'; -export * from './glob'; export * from './file'; diff --git a/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap b/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap index 3d21c832..14efbcae 100644 --- a/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap +++ b/tests/e2e/__snapshots__/load-custom-resources.spec.ts.snap @@ -1,7 +1,17 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Allow load custom resources md2html single page with custom resources: .yfm 1`] = ` +"resources: + style: + - _assets/style/test.css + script: + - _assets/script/test1.js +" +`; + exports[`Allow load custom resources md2html single page with custom resources: filelist 1`] = ` "[ + ".yfm", "_assets/script/test1.js", "_assets/style/test.css", "_bundle/search-async-0", @@ -326,8 +336,18 @@ exports[`Allow load custom resources md2html single page with custom resources: exports[`Allow load custom resources md2html single page with custom resources: toc.js 1`] = `"window.__DATA__.data.toc = {"title":"Documentation","href":"index.html","items":[{"name":"Documentation","href":"page.html","id":"Documentation-RANDOM"},{"name":"Config","href":"project/config.html","id":"Config-RANDOM"}]};"`; +exports[`Allow load custom resources md2html with custom resources: .yfm 1`] = ` +"resources: + style: + - _assets/style/test.css + script: + - _assets/script/test1.js +" +`; + exports[`Allow load custom resources md2html with custom resources: filelist 1`] = ` "[ + ".yfm", "_assets/script/test1.js", "_assets/style/test.css", "_bundle/search-async-0", diff --git a/tests/e2e/__snapshots__/rtl.spec.ts.snap b/tests/e2e/__snapshots__/rtl.spec.ts.snap index ed234ccf..815b870f 100644 --- a/tests/e2e/__snapshots__/rtl.spec.ts.snap +++ b/tests/e2e/__snapshots__/rtl.spec.ts.snap @@ -1,7 +1,10 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with only one rtl lang: .yfm 1`] = `"langs: ['he']"`; + exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with only one rtl lang: filelist 1`] = ` "[ + ".yfm", "_bundle/search-async-0", "_bundle/app-css-1", "_bundle/app-js-1", @@ -162,6 +165,8 @@ exports[`Generate html document with correct lang and dir attributes. Load corre exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with only one rtl lang: toc.js 1`] = `"window.__DATA__.data.toc = {"title":"Documentation","href":"index.html","items":[{"name":"Documentation","href":"page.html","id":"Documentation-RANDOM"}]};"`; +exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with rtl and ltr langs: .yfm 1`] = `"langs: ['ar', 'en']"`; + exports[`Generate html document with correct lang and dir attributes. Load correct bundles. documentation with rtl and ltr langs: ar/index.html 1`] = ` Date: Tue, 3 Dec 2024 21:45:02 +0300 Subject: [PATCH 3/3] chore: Rewrite presets on new architecture --- src/commands/build/__tests__/index.ts | 42 ++- src/commands/build/core/vars/VarsService.ts | 105 +++++++ .../vars/__snapshots__/index.spec.ts.snap | 57 ++++ src/commands/build/core/vars/index.spec.ts | 294 ++++++++++++++++++ src/commands/build/core/vars/index.ts | 3 + src/commands/build/core/vars/types.ts | 10 + .../build/features/templating/index.spec.ts | 82 ++++- .../build/features/templating/index.ts | 51 ++- src/commands/build/handler.ts | 7 +- src/commands/build/index.ts | 9 +- src/commands/build/run.ts | 36 ++- src/services/preset.ts | 45 +-- src/steps/processServiceFiles.ts | 67 +--- src/utils/common.ts | 25 ++ .../md2html-with-metadata/input/presets.yaml | 11 +- .../md2md-with-metadata/input/presets.yaml | 11 +- 16 files changed, 719 insertions(+), 136 deletions(-) create mode 100644 src/commands/build/core/vars/VarsService.ts create mode 100644 src/commands/build/core/vars/__snapshots__/index.spec.ts.snap create mode 100644 src/commands/build/core/vars/index.spec.ts create mode 100644 src/commands/build/core/vars/index.ts create mode 100644 src/commands/build/core/vars/types.ts diff --git a/src/commands/build/__tests__/index.ts b/src/commands/build/__tests__/index.ts index 70d9841a..de093631 100644 --- a/src/commands/build/__tests__/index.ts +++ b/src/commands/build/__tests__/index.ts @@ -1,7 +1,9 @@ import type {Run} from '../run'; import type {BuildConfig, BuildRawConfig} from '..'; +import {join} from 'node:path'; import {Mock, describe, expect, it, vi} from 'vitest'; +import {when} from 'vitest-when'; import {Build} from '..'; import {handler as originalHandler} from '../handler'; import {withConfigUtils} from '~/config'; @@ -30,15 +32,53 @@ vi.mock('~/config', async (importOriginal) => { }; }); -export async function runBuild(args: string) { +type BuildState = { + globs?: Hash; + files?: Hash; +}; +export function setupBuild(state: BuildState = {}): Build & {run: Run} { const build = new Build(); build.apply(); build.hooks.BeforeAnyRun.tap('Tests', (run) => { + (build as Build & {run: Run}).run = run; + + // @ts-ignore + run.glob = vi.fn(() => []); run.copy = vi.fn(); run.write = vi.fn(); + run.fs.writeFile = vi.fn(); + // @ts-ignore + run.fs.readFile = vi.fn(); + // @ts-ignore + run.logger.proc = vi.fn(); + // @ts-ignore + run.logger.info = vi.fn(); + // @ts-ignore + run.logger.warn = vi.fn(); + // @ts-ignore + run.logger.error = vi.fn(); + + if (state.globs) { + for (const [pattern, files] of Object.entries(state.globs)) { + when(run.glob).calledWith(pattern, expect.anything()).thenResolve(files); + } + } + + if (state.files) { + for (const [file, content] of Object.entries(state.files)) { + when(run.fs.readFile) + .calledWith(join(run.input, file), expect.anything()) + .thenResolve(content); + } + } }); + return build as Build & {run: Run}; +} + +export async function runBuild(args: string, build?: Build) { + build = build || setupBuild(); await build.parse(['node', 'index'].concat(args.split(' '))); } diff --git a/src/commands/build/core/vars/VarsService.ts b/src/commands/build/core/vars/VarsService.ts new file mode 100644 index 00000000..4fee897d --- /dev/null +++ b/src/commands/build/core/vars/VarsService.ts @@ -0,0 +1,105 @@ +import type {Preset, Presets} from './types'; + +import {dirname, join} from 'node:path'; +import {merge} from 'lodash'; +import {dump, load} from 'js-yaml'; + +import {Run} from '~/commands/build'; +import {freeze, own} from '~/utils'; +import {AsyncParallelHook, AsyncSeriesWaterfallHook} from 'tapable'; + +export type VarsServiceConfig = { + varsPreset: string; + vars: Hash; +}; + +type VarsServiceHooks = { + /** + * Async waterfall hook. + * Called after any presets.yaml was loaded. + */ + PresetsLoaded: AsyncSeriesWaterfallHook<[Presets, RelativePath]>; + /** + * Async parallel hook. + * Called after vars was resolved on any level. + * Vars data is sealed here. + */ + Resolved: AsyncParallelHook<[Preset, RelativePath]>; +}; + +export class VarsService { + hooks: VarsServiceHooks; + + private run: Run; + + private fs: Run['fs']; + + private logger: Run['logger']; + + private config: VarsServiceConfig; + + private cache: Record = {}; + + constructor(run: Run) { + this.run = run; + this.fs = run.fs; + this.logger = run.logger; + this.config = run.config; + this.hooks = { + PresetsLoaded: new AsyncSeriesWaterfallHook(['presets', 'path']), + Resolved: new AsyncParallelHook(['vars', 'path']), + }; + } + + async load(path: RelativePath) { + const varsPreset = this.config.varsPreset || 'default'; + const file = join(dirname(path), 'presets.yaml'); + + if (this.cache[file]) { + return this.cache[file]; + } + + this.logger.proc(path); + + const scopes = []; + + if (dirname(path) !== '.') { + scopes.push(await this.load(dirname(path))); + } + + try { + const presets = await this.hooks.PresetsLoaded.promise( + load(await this.fs.readFile(join(this.run.input, file), 'utf8')) as Presets, + file, + ); + + scopes.push(presets['default']); + + if (varsPreset && varsPreset !== 'default') { + scopes.push(presets[varsPreset] || {}); + } + } catch (error) { + if (!own(error, 'code') || error.code !== 'ENOENT') { + throw error; + } + } + + scopes.push(this.config.vars); + + this.cache[file] = freeze(merge({}, ...scopes)); + + await this.hooks.Resolved.promise(this.cache[file], file); + + return this.cache[file]; + } + + dump(presets: Hash): string { + return dump(presets, { + lineWidth: 120, + }); + } + + entries() { + return Object.entries(this.cache); + } +} diff --git a/src/commands/build/core/vars/__snapshots__/index.spec.ts.snap b/src/commands/build/core/vars/__snapshots__/index.spec.ts.snap new file mode 100644 index 00000000..115d72ec --- /dev/null +++ b/src/commands/build/core/vars/__snapshots__/index.spec.ts.snap @@ -0,0 +1,57 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`vars > service > load > should allow content extending in PresetsLoaded hook 1`] = ` +"field1: value1 +field2: value2 +" +`; + +exports[`vars > service > load > should allow content updating in PresetsLoaded hook 1`] = ` +"field1: value2 +" +`; + +exports[`vars > service > load > should load presets file default scope 1`] = ` +"field1: value1 +field2: value2 +" +`; + +exports[`vars > service > load > should load presets file target scope 1`] = ` +"field1: value3 +field2: value2 +" +`; + +exports[`vars > service > load > should load super layers 1`] = ` +"field1: value1 +override1: value1 +override2: value1 +override3: value1 +override4: value1 +field2: value1 +sub1: value1 +sub2: value1 +override5: value1 +override6: value1 +subsub1: value1 +subsub2: value1 +" +`; + +exports[`vars > service > load > should override default presets with vars 1`] = ` +"field1: value6 +field2: value2 +" +`; + +exports[`vars > service > load > should override target presets with vars 1`] = ` +"field1: value6 +field2: value2 +" +`; + +exports[`vars > service > load > should use vars if presets not found 1`] = ` +"field1: value6 +" +`; diff --git a/src/commands/build/core/vars/index.spec.ts b/src/commands/build/core/vars/index.spec.ts new file mode 100644 index 00000000..c12a68c2 --- /dev/null +++ b/src/commands/build/core/vars/index.spec.ts @@ -0,0 +1,294 @@ +import type {Run} from '~/commands/build'; +import type {VarsServiceConfig} from './VarsService'; + +import {join} from 'node:path'; +import {describe, expect, it, vi} from 'vitest'; +import {when} from 'vitest-when'; +import {dedent} from 'ts-dedent'; +import {YAMLException} from 'js-yaml'; + +import {VarsService} from './VarsService'; + +const ENOENT = Object.assign(new Error('ENOENT: no such file or directory'), { + code: 'ENOENT', +}); + +type Options = Partial; + +function prepare(content: string | Hash | Error, options: Options = {}) { + const input = '/dev/null/input' as AbsolutePath; + const output = '/dev/null/output' as AbsolutePath; + const run = { + input, + output, + config: { + varsPreset: options.varsPreset, + vars: options.vars || {}, + }, + logger: { + proc: vi.fn(), + }, + fs: { + readFile: vi.fn(), + }, + } as unknown as Run; + const service = new VarsService(run); + + if (content instanceof Error) { + when(run.fs.readFile) + .calledWith(join(input, './presets.yaml'), expect.anything()) + .thenReject(content); + } else { + if (typeof content === 'string') { + content = {'./presets.yaml': content}; + } + + for (const [file, data] of Object.entries(content)) { + when(run.fs.readFile) + .calledWith(join(input, file), expect.anything()) + .thenResolve(data); + } + } + + return service; +} + +async function call(content: string | Error, options: Options = {}) { + const service = prepare(content, options); + const result = await service.load('./presets.yaml' as RelativePath); + + expect(service.dump(result)).toMatchSnapshot(); +} + +function test(name: string, content: string | Error, options: Options = {}) { + it(name, async () => call(content, options)); +} + +describe('vars', () => { + describe('service', () => { + describe('load', () => { + test( + 'should load presets file default scope', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + ); + + test( + 'should load presets file target scope', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + {varsPreset: 'internal'}, + ); + + test( + 'should override default presets with vars', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + {vars: {field1: 'value6'}}, + ); + + test( + 'should override target presets with vars', + dedent` + default: + field1: value1 + field2: value2 + internal: + field1: value3 + external: + field1: value4 + `, + {varsPreset: 'internal', vars: {field1: 'value6'}}, + ); + + test('should use vars if presets not found', ENOENT, {vars: {field1: 'value6'}}); + + it('should throw parse error', async () => { + await expect(() => call('!@#', {vars: {field1: 'value6'}})).rejects.toThrow( + YAMLException, + ); + }); + + it('should load super layers', async () => { + const service = prepare( + { + './presets.yaml': dedent` + default: + field1: value1 + override1: value2 + override2: value2 + override3: value2 + override4: value2 + internal: + field2: value1 + override1: value1 + `, + './subfolder/presets.yaml': dedent` + default: + sub1: value1 + sub2: value2 + override2: value1 + override5: value2 + internal: + sub2: value1 + override3: value1 + override6: value2 + `, + './subfolder/subfolder/subfolder/presets.yaml': dedent` + default: + subsub1: value2 + override4: value2 + override5: value1 + internal: + subsub1: value1 + subsub2: value1 + override4: value1 + override6: value1 + `, + }, + {varsPreset: 'internal'}, + ); + + const result = await service.load( + './subfolder/subfolder/subfolder/presets.yaml' as RelativePath, + ); + + expect(service.dump(result)).toMatchSnapshot(); + }); + + it('should call PresetsLoaded hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + const spy = vi.fn(); + + service.hooks.PresetsLoaded.tap('Test', spy); + + await service.load('./presets.yaml' as RelativePath); + + expect(spy).toHaveBeenCalledWith({default: {field1: 'value1'}}, 'presets.yaml'); + }); + + it('should call Resolved hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + const spy = vi.fn(); + + service.hooks.Resolved.tap('Test', spy); + + await service.load('./presets.yaml' as RelativePath); + + expect(spy).toHaveBeenCalledWith({field1: 'value1'}, 'presets.yaml'); + }); + + it('should allow content updating in PresetsLoaded hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.PresetsLoaded.tap('Test', (presets) => { + presets.default.field1 = 'value2'; + + return presets; + }); + + const result = await service.load('./presets.yaml' as RelativePath); + + expect(service.dump(result)).toMatchSnapshot(); + }); + + it('should allow content extending in PresetsLoaded hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.PresetsLoaded.tap('Test', (presets) => { + presets.default.field2 = 'value2'; + + return presets; + }); + + const result = await service.load('./presets.yaml' as RelativePath); + + expect(service.dump(result)).toMatchSnapshot(); + }); + + it('should reject content updating in Resolved hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.Resolved.tap('Test', (vars) => { + vars.field1 = 'value2'; + }); + + await expect(() => + service.load('./presets.yaml' as RelativePath), + ).rejects.toThrow(); + }); + + it('should reject content extending in Resolved hook', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + service.hooks.Resolved.tap('Test', (vars) => { + vars.field2 = 'value2'; + }); + + await expect(() => + service.load('./presets.yaml' as RelativePath), + ).rejects.toThrow(); + }); + + it('should load content only once', async () => { + const service = prepare(dedent` + default: + field1: value1 + `); + + const spy1 = vi.fn(); + const spy2 = vi.fn(); + + service.hooks.PresetsLoaded.tap('Test', spy1); + service.hooks.Resolved.tap('Test', spy2); + + await service.load('./presets.yaml' as RelativePath); + await service.load('./presets.yaml' as RelativePath); + + expect(spy1).toHaveBeenCalledOnce(); + expect(spy2).toHaveBeenCalledOnce(); + }); + }); + }); +}); diff --git a/src/commands/build/core/vars/index.ts b/src/commands/build/core/vars/index.ts new file mode 100644 index 00000000..7f666f0d --- /dev/null +++ b/src/commands/build/core/vars/index.ts @@ -0,0 +1,3 @@ +export type {Preset, Presets} from './types'; + +export {VarsService} from './VarsService'; diff --git a/src/commands/build/core/vars/types.ts b/src/commands/build/core/vars/types.ts new file mode 100644 index 00000000..2dbb23d8 --- /dev/null +++ b/src/commands/build/core/vars/types.ts @@ -0,0 +1,10 @@ +export type Presets = { + default: Preset; +} & { + [prop: string]: Preset; +}; + +export type Preset = { + __system?: Hash; + __metadata?: Hash; +} & Hash; diff --git a/src/commands/build/features/templating/index.spec.ts b/src/commands/build/features/templating/index.spec.ts index 3c1befc2..ff220f81 100644 --- a/src/commands/build/features/templating/index.spec.ts +++ b/src/commands/build/features/templating/index.spec.ts @@ -1,5 +1,7 @@ -import {describe} from 'vitest'; -import {testConfig as test} from '../../__tests__'; +import {describe, expect, it} from 'vitest'; +import {runBuild, setupBuild, testConfig as test} from '../../__tests__'; +import {resolve} from 'node:path'; +import {dedent} from 'ts-dedent'; describe('Build template feature', () => { describe('config', () => { @@ -51,6 +53,10 @@ describe('Build template feature', () => { text: false, code: false, }, + features: { + conditions: false, + substitutions: false, + }, }, }); @@ -203,4 +209,76 @@ describe('Build template feature', () => { ); }); }); + + describe('run', () => { + const args = (...args: string[]) => + '-i /dev/null/input -o /dev/null/output ' + args.join(' '); + + it('should not save presets.yaml for html build', async () => { + const build = setupBuild({ + globs: { + '**/presets.yaml': ['./presets.yaml'], + }, + files: { + './presets.yaml': dedent` + default: + field: value + `, + }, + }); + + await runBuild(args('-f', 'html', '--no-template'), build); + + expect(build.run.write).not.toHaveBeenCalledWith( + resolve('/dev/null/output/.tmp_output/presets.yaml'), + `default:\n field: value\n`, + ); + }); + + it('should save presets.yaml for md build with disabled templating', async () => { + const build = setupBuild({ + globs: { + '**/presets.yaml': ['./presets.yaml'], + }, + files: { + './presets.yaml': dedent` + default: + field: value + `, + }, + }); + + await runBuild(args('-f', 'md', '--no-template'), build); + + expect(build.run.write).toHaveBeenCalledWith( + resolve('/dev/null/output/.tmp_output/presets.yaml'), + `default:\n field: value\n`, + ); + }); + + it('should filter presets.yaml for md build with disabled templating', async () => { + const build = setupBuild({ + globs: { + '**/presets.yaml': ['./presets.yaml'], + }, + files: { + './presets.yaml': dedent` + default: + field: value + internal: + field: value + external: + field: value + `, + }, + }); + + await runBuild(args('-f', 'md', '--no-template', '--vars-preset', 'internal'), build); + + expect(build.run.write).toHaveBeenCalledWith( + resolve('/dev/null/output/.tmp_output/presets.yaml'), + `default:\n field: value\ninternal:\n field: value\n`, + ); + }); + }); }); diff --git a/src/commands/build/features/templating/index.ts b/src/commands/build/features/templating/index.ts index 6fab837f..e159a726 100644 --- a/src/commands/build/features/templating/index.ts +++ b/src/commands/build/features/templating/index.ts @@ -1,21 +1,13 @@ import type {Build} from '~/commands'; import type {Command} from '~/config'; -import {defined, valuable} from '~/config'; -import {options} from './config'; +import type {Preset} from '~/commands/build/core/vars'; -const merge = (acc: Hash, ...sources: Hash[]) => { - for (const source of sources) { - for (const [key, value] of Object.entries(source)) { - if (!acc[key] || !value) { - acc[key] = value; - } else if (typeof value === 'object') { - acc[key] = merge({}, acc[key], value); - } - } - } +import {join} from 'node:path'; +import {dump} from 'js-yaml'; +import {merge} from 'lodash'; - return acc; -}; +import {defined, valuable} from '~/config'; +import {options} from './config'; export type TemplatingArgs = { template?: boolean | 'all' | 'text' | 'code'; @@ -88,7 +80,38 @@ export class Templating { config.template.features.conditions = templateConditions; } + if (!config.template.enabled) { + config.template.features.substitutions = false; + config.template.features.conditions = false; + } + return config; }); + + program.hooks.BeforeRun.for('md').tap('Build', (run) => { + const {varsPreset, template} = run.config; + const {substitutions, conditions} = template.features; + + // For case when we need to copy project from private to public repo and filter private presets. + if (!substitutions || !conditions) { + run.vars.hooks.PresetsLoaded.tapPromise('Build', async (presets, path) => { + const scopes = [ + {default: presets.default}, + varsPreset !== 'default' && + presets[varsPreset] && {[varsPreset]: presets[varsPreset]}, + ].filter(Boolean) as Preset[]; + const result = merge({}, ...scopes); + + await run.write( + join(run.output, path), + dump(result, { + lineWidth: 120, + }), + ); + + return presets; + }); + } + }); } } diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index e37859fa..abe17161 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -4,16 +4,16 @@ import 'threads/register'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {ArgvService, Includers, SearchService} from '~/services'; +import {ArgvService, Includers, PresetService, SearchService} from '~/services'; import { initLinterWorkers, + preparingTocFiles, processAssets, processChangelogs, processExcludedFiles, processLinter, processLogs, processPages, - processServiceFiles, } from '~/steps'; import {prepareMapFile} from '~/steps/processMapFile'; @@ -27,7 +27,8 @@ export async function handler(run: Run) { const {lintDisabled, buildDisabled, addMapFile} = ArgvService.getConfig(); - await processServiceFiles(); + PresetService.init(run.vars); + await preparingTocFiles(run); processExcludedFiles(); if (addMapFile) { diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index daa26ba5..acfe3472 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -2,7 +2,6 @@ import type {IProgram, ProgramArgs, ProgramConfig} from '~/program'; import type {DocAnalytics} from '@diplodoc/client'; import {ok} from 'node:assert'; -import {join} from 'node:path'; import {pick} from 'lodash'; import {AsyncParallelHook, AsyncSeriesHook, HookMap} from 'tapable'; @@ -286,6 +285,14 @@ export class Build await run.copy(run.originalInput, run.input, ['node_modules/**', '*/node_modules/**']); + const presets = (await run.glob('**/presets.yaml', { + cwd: run.input, + ignore: run.config.ignore, + })) as RelativePath[]; + for (const preset of presets) { + await run.vars.load(preset); + } + await Promise.all([handler(run), this.hooks.Run.promise(run)]); await this.hooks.AfterRun.for(this.config.outputFormat).promise(run); diff --git a/src/commands/build/run.ts b/src/commands/build/run.ts index 7359aff9..f563df66 100644 --- a/src/commands/build/run.ts +++ b/src/commands/build/run.ts @@ -1,4 +1,5 @@ import type {YfmArgv} from '~/models'; +import type {GlobOptions} from 'glob'; // import {ok} from 'node:assert'; import {dirname, join, resolve} from 'node:path'; @@ -13,9 +14,10 @@ import { TMP_OUTPUT_FOLDER, YFM_CONFIG_FILENAME, } from '~/constants'; -import {Logger} from '~/logger'; +import {LogLevel, Logger} from '~/logger'; import {BuildConfig} from '.'; // import {InsecureAccessError} from './errors'; +import {VarsService} from './core/vars'; type FileSystem = { access: typeof access; @@ -27,6 +29,10 @@ type FileSystem = { writeFile: typeof writeFile; }; +class RunLogger extends Logger { + proc = this.topic(LogLevel.INFO, 'PROC'); +} + /** * This is transferable context for build command. * Use this context to communicate with lower data processing levels. @@ -42,12 +48,14 @@ export class Run { readonly legacyConfig: YfmArgv; - readonly logger: Logger; + readonly logger: RunLogger; readonly config: BuildConfig; readonly fs: FileSystem = {access, stat, link, unlink, mkdir, readFile, writeFile}; + readonly vars: VarsService; + get bundlePath() { return join(this.output, BUNDLE_FOLDER); } @@ -72,6 +80,11 @@ export class Run { this.input = resolve(config.output, TMP_INPUT_FOLDER); this.output = resolve(config.output, TMP_OUTPUT_FOLDER); + this.logger = new RunLogger(config, [ + (_level, message) => message.replace(new RegExp(this.input, 'ig'), ''), + ]); + + this.vars = new VarsService(this); this.legacyConfig = { rootInput: this.originalInput, input: this.input, @@ -119,10 +132,6 @@ export class Run { included: config.mergeIncludes, }; - - this.logger = new Logger(config, [ - (_level, message) => message.replace(new RegExp(this.input, 'ig'), ''), - ]); } write = async (path: AbsolutePath, content: string | Buffer) => { @@ -131,6 +140,15 @@ export class Run { await this.fs.writeFile(path, content, 'utf8'); }; + glob = async (pattern: string | string[], options: GlobOptions) => { + return glob(pattern, { + dot: true, + nodir: true, + follow: true, + ...options, + }); + }; + copy = async (from: AbsolutePath, to: AbsolutePath, ignore?: string[]) => { const isFile = (await this.fs.stat(from)).isFile(); const hardlink = async (from: AbsolutePath, to: AbsolutePath) => { @@ -154,12 +172,8 @@ export class Run { } const dirs = new Set(); - // TODO: check dotfiles copy - const files = (await glob('**', { + const files = (await this.glob('**', { cwd: from, - dot: true, - nodir: true, - follow: true, ignore, })) as RelativePath[]; diff --git a/src/services/preset.ts b/src/services/preset.ts index c24533ba..13694c85 100644 --- a/src/services/preset.ts +++ b/src/services/preset.ts @@ -1,43 +1,30 @@ import {dirname, normalize} from 'path'; -import {DocPreset, YfmPreset} from '../models'; +import {YfmPreset} from '../models'; +import {VarsService} from '~/commands/build/core/vars'; export type PresetStorage = Map; let presetStorage: PresetStorage = new Map(); -function add(parsedPreset: DocPreset, path: string, varsPreset: string) { - const combinedValues = { - ...(parsedPreset.default || {}), - ...(parsedPreset[varsPreset] || {}), - __metadata: parsedPreset.__metadata, - } as YfmPreset; - - const key = dirname(normalize(path)); - presetStorage.set(key, combinedValues); +function init(vars: VarsService) { + for (const [path, values] of vars.entries()) { + presetStorage.set(dirname(path), values); + } } function get(path: string): YfmPreset { - let combinedValues: YfmPreset = {}; - let localPath = normalize(path); - - while (localPath !== '.') { - const presetValues: YfmPreset = presetStorage.get(localPath) || {}; - localPath = dirname(localPath); - - combinedValues = { - ...presetValues, - ...combinedValues, - }; + let vars = presetStorage.get(normalize(path)); + while (!vars) { + path = dirname(path); + vars = presetStorage.get(normalize(path)); + + if (path === '.') { + break; + } } - // Add root' presets - combinedValues = { - ...presetStorage.get('.'), - ...combinedValues, - }; - - return combinedValues; + return vars || {}; } function getPresetStorage(): Map { @@ -49,7 +36,7 @@ function setPresetStorage(preset: Map): void { } export default { - add, + init, get, getPresetStorage, setPresetStorage, diff --git a/src/steps/processServiceFiles.ts b/src/steps/processServiceFiles.ts index 7c315264..9c1fafe1 100644 --- a/src/steps/processServiceFiles.ts +++ b/src/steps/processServiceFiles.ts @@ -1,13 +1,7 @@ -import {dirname, resolve} from 'path'; import walkSync from 'walk-sync'; -import {readFileSync, writeFileSync} from 'fs'; -import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; -import {ArgvService, PresetService, TocService} from '../services'; -import {logger} from '../utils'; -import {DocPreset} from '../models'; -import shell from 'shelljs'; +import {ArgvService, TocService} from '../services'; const getFilePathsByGlobals = (globs: string[]): string[] => { const {input, ignore = []} = ArgvService.getConfig(); @@ -20,64 +14,7 @@ const getFilePathsByGlobals = (globs: string[]): string[] => { }); }; -export async function processServiceFiles(): Promise { - await preparingPresetFiles(); - await preparingTocFiles(); -} - -async function preparingPresetFiles() { - const { - input: inputFolderPath, - varsPreset = '', - outputFormat, - applyPresets, - resolveConditions, - } = ArgvService.getConfig(); - - try { - const presetsFilePaths = getFilePathsByGlobals(['**/presets.yaml']); - - for (const path of presetsFilePaths) { - logger.proc(path); - - const pathToPresetFile = resolve(inputFolderPath, path); - const content = readFileSync(pathToPresetFile, 'utf8'); - const parsedPreset = load(content) as DocPreset; - - PresetService.add(parsedPreset, path, varsPreset); - - if (outputFormat === 'md' && (!applyPresets || !resolveConditions)) { - // Should save filtered presets.yaml only when --apply-presets=false or --resolve-conditions=false - saveFilteredPresets(path, parsedPreset); - } - } - } catch (error) { - log.error(`Preparing presets.yaml files failed. Error: ${error}`); - throw error; - } -} - -function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { - const {output: outputFolderPath, varsPreset = ''} = ArgvService.getConfig(); - - const outputPath = resolve(outputFolderPath, path); - const filteredPreset: Record = { - default: parsedPreset.default, - }; - - if (parsedPreset[varsPreset]) { - filteredPreset[varsPreset] = parsedPreset[varsPreset]; - } - - const outputPreset = dump(filteredPreset, { - lineWidth: 120, - }); - - shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputPreset); -} - -async function preparingTocFiles(): Promise { +export async function preparingTocFiles(): Promise { try { const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']); await TocService.init(tocFilePaths); diff --git a/src/utils/common.ts b/src/utils/common.ts index 569226ce..bae1fff7 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -50,3 +50,28 @@ export function checkPathExists(path: string, parentFilePath: string) { return isFileExists(includePath); } + +export function own(box: unknown, field: T): box is {[p in T]: unknown} { + return ( + Boolean(box && typeof box === 'object') && Object.prototype.hasOwnProperty.call(box, field) + ); +} + +export function freeze(target: T, visited = new Set()): T { + if (!visited.has(target)) { + visited.add(target); + + if (Array.isArray(target)) { + target.forEach((item) => freeze(item, visited)); + } + + if (isObject(target) && !Object.isSealed(target)) { + Object.freeze(target); + Object.keys(target).forEach((key) => + freeze(target[key as keyof typeof target], visited), + ); + } + } + + return target; +} diff --git a/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml b/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml index a5dec198..d02a09a9 100644 --- a/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml +++ b/tests/mocks/metadata/md2html-with-metadata/input/presets.yaml @@ -1,5 +1,6 @@ -__metadata: - - name: test-yfm - content: inline test - - name: yfm-config - content: config test +default: + __metadata: + - name: test-yfm + content: inline test + - name: yfm-config + content: config test diff --git a/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml b/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml index a5dec198..d02a09a9 100644 --- a/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml +++ b/tests/mocks/metadata/md2md-with-metadata/input/presets.yaml @@ -1,5 +1,6 @@ -__metadata: - - name: test-yfm - content: inline test - - name: yfm-config - content: config test +default: + __metadata: + - name: test-yfm + content: inline test + - name: yfm-config + content: config test