From 6c822874d0ce602d91dca9022e26d7f89ad6dec3 Mon Sep 17 00:00:00 2001 From: 3y3 <3y3@ya.ru> Date: Fri, 29 Mar 2024 17:24:13 +0300 Subject: [PATCH] wip --- package-lock.json | 22 +- package.json | 2 + src/cmd/build/features/publishing/config.ts | 2 +- src/cmd/build/index.ts | 2 +- src/cmd/translation/__tests__/index.ts | 19 +- src/cmd/translation/config.ts | 64 ++- src/cmd/translation/handler.ts | 0 src/cmd/translation/index.spec.ts | 281 +++++++++- src/cmd/translation/index.ts | 53 +- src/cmd/translation/providers/yandex/auth.ts | 36 ++ .../translation/providers/yandex/config.ts | 13 +- src/cmd/translation/providers/yandex/index.ts | 30 +- src/cmd/translation/providers/yandex/oauth.ts | 31 -- .../translation/providers/yandex/provider.ts | 486 ++++++++++++------ .../providers/yandex/utils/errors.ts | 100 ++++ .../translation/providers/yandex/utils/fs.ts | 96 ++++ .../providers/yandex/utils/index.ts | 158 ++++++ .../providers/yandex/utils/translate.ts | 26 + src/cmd/translation/utils.ts | 126 +++++ src/config/index.ts | 23 +- src/program/config.ts | 12 +- src/program/index.ts | 2 +- 22 files changed, 1289 insertions(+), 295 deletions(-) create mode 100644 src/cmd/translation/handler.ts create mode 100644 src/cmd/translation/providers/yandex/auth.ts delete mode 100644 src/cmd/translation/providers/yandex/oauth.ts create mode 100644 src/cmd/translation/providers/yandex/utils/errors.ts create mode 100644 src/cmd/translation/providers/yandex/utils/fs.ts create mode 100644 src/cmd/translation/providers/yandex/utils/index.ts create mode 100644 src/cmd/translation/providers/yandex/utils/translate.ts create mode 100644 src/cmd/translation/utils.ts diff --git a/package-lock.json b/package-lock.json index d56b2d897..996914387 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,6 +48,7 @@ "async": "^3.2.4", "axios": "^1.6.7", "chalk": "^4.1.2", + "commander": "^12.0.0", "esbuild": "^0.20.0", "glob": "^8.0.3", "html-escaper": "^3.0.3", @@ -60,6 +61,7 @@ "node-html-parser": "^6.1.5", "simple-git": "3.22.0", "slugify": "^1.6.5", + "tapable": "^2.2.1", "tar-stream": "^3.1.4", "typescript": "^5.3.3", "vite-tsconfig-paths": "^4.2.3", @@ -4479,12 +4481,12 @@ } }, "node_modules/commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.0.0.tgz", + "integrity": "sha512-MwVNWlYjDTtOjX5PiD7o5pK0UrFU/OYgcJfjjK4RaHZETNtjJqrZa9Y9ds88+A+f+d5lv+561eZ+yCKoS3gbAA==", "dev": true, "engines": { - "node": "^12.20.0 || >=14" + "node": ">=18" } }, "node_modules/component-emitter": { @@ -7354,6 +7356,15 @@ "url": "https://opencollective.com/lint-staged" } }, + "node_modules/lint-staged/node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "dev": true, + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/lint-staged/node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -9572,8 +9583,9 @@ }, "node_modules/tapable": { "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", "dev": true, - "license": "MIT", "engines": { "node": ">=6" } diff --git a/package.json b/package.json index 698124844..b58d4b516 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "async": "^3.2.4", "axios": "^1.6.7", "chalk": "^4.1.2", + "commander": "^12.0.0", "esbuild": "^0.20.0", "glob": "^8.0.3", "html-escaper": "^3.0.3", @@ -82,6 +83,7 @@ "node-html-parser": "^6.1.5", "simple-git": "3.22.0", "slugify": "^1.6.5", + "tapable": "^2.2.1", "tar-stream": "^3.1.4", "typescript": "^5.3.3", "vite-tsconfig-paths": "^4.2.3", diff --git a/src/cmd/build/features/publishing/config.ts b/src/cmd/build/features/publishing/config.ts index 7edc045c2..b399314a3 100644 --- a/src/cmd/build/features/publishing/config.ts +++ b/src/cmd/build/features/publishing/config.ts @@ -48,7 +48,7 @@ const storageSecretKey = option({ }); const storageRegion = option({ - flags: '--storage-secret-key ', + flags: '--storage-region ', desc: 'Region of S3 storage.', defaultInfo: 'eu-central-1', deprecated: 'Use separated publish command instead.', diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index 54a19faca..018210411 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -131,7 +131,7 @@ export class Build protected options = [ options.input(), - options.output, + options.output(), options.outputFormat, options.varsPreset, options.vars, diff --git a/src/cmd/translation/__tests__/index.ts b/src/cmd/translation/__tests__/index.ts index 21e498b74..b9dd5f5af 100644 --- a/src/cmd/translation/__tests__/index.ts +++ b/src/cmd/translation/__tests__/index.ts @@ -62,20 +62,13 @@ export function testConfig(defaultArgs: string) { }; }); - try { + if (result instanceof Error || typeof result === 'string') { + const message = result.message || result; + await expect(async () => runTranslate(defaultArgs + ' ' + args)).rejects.toThrow(message); + } else { const instance = await runTranslate(defaultArgs + ' ' + args); - expect(instance.provider?.translate).toBeCalledWith( - expect.objectContaining(result), - ); - } catch (error: any) { - const message = error.message || error; - if (result instanceof Error) { - expect(message).toEqual(result.message); - } else if (typeof result === 'string') { - expect(message).toEqual(result); - } else { - throw error; - } + + expect(instance.provider?.translate).toBeCalledWith(expect.anything(), expect.objectContaining(result)); } }); } diff --git a/src/cmd/translation/config.ts b/src/cmd/translation/config.ts index 0604399fb..925b24928 100644 --- a/src/cmd/translation/config.ts +++ b/src/cmd/translation/config.ts @@ -1,6 +1,7 @@ -import {green} from 'chalk'; -import {option} from '~/config'; +import {cyan, green, underline} from 'chalk'; +import {option, toArray} from '~/config'; import {options as globalOptions} from '~/program'; +import {withListResolver} from './utils'; export const NAME = 'translate'; @@ -16,20 +17,60 @@ const provider = option({ desc: 'Configure translation service provider.', }); -const sourceLanguage = option({ - flags: '-sl, --source-language ', +const source = option({ + flags: '-sl, --source ', desc: ` The text language to translate from. - Specified in ISO 639-1 format (for example, ru). + Specified in ISO 639-1 format (for example, ru or ru-RU). `, }); -const targetLanguage = option({ - flags: '-tl, --target-language ', +const target = option({ + flags: '-tl, --target ', desc: ` The target language to translate the text. - Specified in ISO 639-1 format (for example, en). + Specified in ISO 639-1 format (for example, en or en-US). `, + parser: toArray, +}); + +const include = option({ + flags: '--include ', + desc: ` + Relative to input filtering rule for files need to be translated. + Can be direct file path, glob filter, file ${underline('filter list')}. + + Usage of include flag will reset default include rules. + If you need to apply also default rules use special ${cyan('--include ...')} + + Read more about ${underline('filter list')} format in documentation ${cyan('docs')}. + + Example: + {{PROGRAM}} --include some/direct/path.md + {{PROGRAM}} --include subpath/glob/**/*.md + {{PROGRAM}} --include filter.list + {{PROGRAM}} --include filter.list --include ... + `, + parser: withListResolver(toArray), +}); + +const exclude = option({ + flags: '--exclude ', + desc: ` + Relative to input filtering rule for files need to be translated. + Can be direct file path, glob filter, file ${underline('filter list')}. + + Read more about ${underline('filter list')} format in documentation ${cyan('docs')}. + + Example: + {{PROGRAM}} --exclude subpath/glob/**/*.md + `, + parser: withListResolver(toArray), +}); + +const dryRun = option({ + flags: '--dry-run', + desc: 'Do not execute target translation provider, but only calculate required quota.', }); export const options = { @@ -37,6 +78,9 @@ export const options = { output: globalOptions.output, config: globalOptions.config, provider, - sourceLanguage, - targetLanguage, + source, + target, + include, + exclude, + dryRun, }; diff --git a/src/cmd/translation/handler.ts b/src/cmd/translation/handler.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/cmd/translation/index.spec.ts b/src/cmd/translation/index.spec.ts index 0c274f3b6..e6e9b007f 100644 --- a/src/cmd/translation/index.spec.ts +++ b/src/cmd/translation/index.spec.ts @@ -5,48 +5,303 @@ import {runTranslate as run, testConfig} from './__tests__'; describe('Translate command', () => { describe('config', () => { describe('provider', () => { - const test = testConfig('-i input -o output --folder-id 1 --oauth-token token'); + const test = testConfig('--source ru --target en --folder 1 --auth t1.a'); test( 'should fail on unknown provider', - '--provider unknown --folder-id 1', + '--provider unknown --folder 1', `error: option '--provider ' argument 'unknown' is invalid. Allowed choices are yandex.`, ); - test('should handle default', '--folder-id 1', { + test('should handle default', '--folder 1', { provider: 'yandex', }); }); + describe('source', () => { + const test = testConfig('--target ru --folder 1 --auth t1.a'); + + test( + 'should handle partial arg', + '--source ru', + { + source: { + language: 'ru', + locale: '' + } + }, + ); + + test( + 'should handle full arg', + '--source ru-RU', + { + source: { + language: 'ru', + locale: 'RU' + } + }, + ); + + test( + 'should handle partial string config', + '', + { + // @ts-ignore + source: 'ru' + }, + { + source: { + language: 'ru', + locale: '' + } + }, + ); + + test( + 'should handle full string config', + '', + { + // @ts-ignore + source: 'ru-RU' + }, + { + source: { + language: 'ru', + locale: 'RU' + } + }, + ); + + test( + 'should handle partial object config', + '', + { + source: { + language: 'ru' + } + }, + { + source: { + language: 'ru', + locale: '' + } + }, + ); + + test( + 'should handle full object config', + '', + { + source: { + language: 'ru', + locale: 'RU' + } + }, + { + source: { + language: 'ru', + locale: 'RU' + } + }, + ); + + test( + 'should handle args with priority', + '--source ru', + { + // @ts-ignore + source: 'en-US' + }, + { + source: { + language: 'ru', + locale: '' + } + }, + ); + + test( + 'should fail on wrong type', + '', + { + // @ts-ignore + source: [{ + language: 'ru', + locale: 'RU' + }] + }, + `Field 'source' should be string or locale.`, + ); + }); + + describe('target', () => { + const test = testConfig('--source ru --folder 1 --auth t1.a'); + + test( + 'should handle partial arg', + '--target ru', + { + target: [{ + language: 'ru', + locale: '' + }] + }, + ); + + test( + 'should handle full arg', + '--target ru-RU', + { + target: [{ + language: 'ru', + locale: 'RU' + }] + }, + ); + + test( + 'should handle multi arg', + '--target ru-RU --target en-US', + { + target: [{ + language: 'ru', + locale: 'RU' + }, { + language: 'en', + locale: 'US' + }] + }, + ); + + test( + 'should handle partial string config', + '', + { + // @ts-ignore + target: 'ru' + }, + { + target: [{ + language: 'ru', + locale: '' + }] + }, + ); + + test( + 'should handle full string config', + '', + { + // @ts-ignore + target: 'ru-RU' + }, + { + target: [{ + language: 'ru', + locale: 'RU' + }] + }, + ); + + test( + 'should handle partial object config', + '', + { + target: { + // @ts-ignore + language: 'ru' + } + }, + { + target: [{ + language: 'ru', + locale: '' + }] + }, + ); + + test( + 'should handle full object config', + '', + { + target: { + // @ts-ignore + language: 'ru', + locale: 'RU' + } + }, + { + target: [{ + language: 'ru', + locale: 'RU' + }] + }, + ); + + test( + 'should handle multi object config', + '', + { + target: [{ + language: 'ru', + locale: 'RU' + }, { + language: 'en', + locale: 'US' + }] + }, + { + target: [{ + language: 'ru', + locale: 'RU' + }, { + language: 'en', + locale: 'US' + }] + }, + ); + + test( + 'should fail on wrong type', + '', + { + // @ts-ignore + target: 1 + }, + `Field 'target' should be string, locale or array.`, + ); + }); + describe('yandex provider', () => { - describe('folderId', () => { + describe('folder', () => { const test = testConfig( - '-i input -o output --oauth-token token', + '--source ru --target en --auth t1.a', ); - test('should handle arg', '--folder-id 1', { - folderId: '1', + test('should handle arg', '--folder 1', { + folder: '1', }); test( 'should handle config', '', { - folderId: '1', + folder: '1', }, { - folderId: '1', + folder: '1', }, ); test( 'should handle arg with priority', - '--folder-id 1', + '--folder 1', { - folderId: '2', + folder: '2', }, { - folderId: '1', + folder: '1', }, ); }); @@ -54,7 +309,7 @@ describe('Translate command', () => { }); it('should call provider translate with config', async () => { - const instance = await run('-o output --folder-id 1'); + const instance = await run('-o output --folder 1'); expect(instance.provider?.translate).toBeCalledWith(expect.objectContaining({})); }); diff --git a/src/cmd/translation/index.ts b/src/cmd/translation/index.ts index 36e364e24..70fc7fe89 100644 --- a/src/cmd/translation/index.ts +++ b/src/cmd/translation/index.ts @@ -1,15 +1,18 @@ import type {IProgram, ProgramArgs, ProgramConfig} from '~/program'; import type {BaseHooks} from '~/program/base'; +import type {Locale} from './utils'; import {ok} from 'assert'; import {pick} from 'lodash'; import {AsyncSeriesWaterfallHook, HookMap} from 'tapable'; import {BaseProgram} from '~/program/base'; -import {Command, args} from '~/config'; +import {Command, args, defined} from '~/config'; import {YFM_CONFIG_FILENAME} from '~/constants'; import {DESCRIPTION, NAME, options} from './config'; import {Extension as YandexTranslation} from './providers/yandex'; +import {resolveFiles, resolveSource, resolveTargets} from './utils'; + type Parent = IProgram & { translate: Translate; }; @@ -27,21 +30,27 @@ const hooks = () => ({ }); export interface IProvider { - translate(config: TranslateConfig): Promise; + translate(files: string[], config: TranslateConfig): Promise; } export type TranslateArgs = ProgramArgs & { output: string; provider: string; - sourceLanguage: string; - targetLanguage: string; + source?: string; + target?: string | string[]; + include?: string[]; + exclude?: string[]; }; export type TranslateConfig = Pick & { output: string; provider: string; - sourceLanguage: string; - targetLanguage: string; + source: Locale; + target: Locale[]; + include: string[]; + exclude: string[]; + files: string[]; + dryRun: boolean; }; export type TranslateHooks = ReturnType; @@ -51,7 +60,7 @@ export class Translate extends BaseProgram('Translate', { config: { defaults: () => ({}), - strictScope: 'translate', + strictScope: NAME, }, hooks: hooks(), }) @@ -83,10 +92,13 @@ export class Translate readonly options = [ options.input('./'), - options.output, + options.output('./'), options.provider, - options.sourceLanguage, - options.targetLanguage, + options.source, + options.target, + options.include, + options.exclude, + options.dryRun, options.config(YFM_CONFIG_FILENAME), ]; @@ -98,11 +110,16 @@ export class Translate super.apply(program); this.hooks.Config.tap('Translate', (config, args) => { - const options = this.options.map((option) => option.attributeName()); - - Object.assign(config, pick(args, options)); - - return config; + return Object.assign(config, { + ...pick(args, ['input', 'quiet', 'silent']) as ProgramArgs, + provider: defined('provider', args, config), + output: defined('output', args, config), + source: resolveSource(config, args), + target: resolveTargets(config, args), + include: defined('include', args, config) || [], + exclude: defined('exclude', args, config) || [], + dryRun: defined('dryRun', args, config) || false, + }); }); } @@ -125,7 +142,11 @@ export class Translate await this.parse(args(this.command)); } else { - await this.provider.translate(this.config); + const {input, include, exclude, source} = this.config; + const files = resolveFiles(input, include, exclude, source.language, ['.md', '.yaml']); + + await this.provider.translate(files, this.config); } } } + diff --git a/src/cmd/translation/providers/yandex/auth.ts b/src/cmd/translation/providers/yandex/auth.ts new file mode 100644 index 000000000..9f9786727 --- /dev/null +++ b/src/cmd/translation/providers/yandex/auth.ts @@ -0,0 +1,36 @@ +import {readFileSync} from 'fs'; + +const resolveKey = (data: string) => { + data = data.trim(); + + switch (true) { + case data.startsWith('y0_'): + return 'Bearer ' + data; + case data.startsWith('t1.'): + return 'Bearer ' + data; + case data.startsWith('AQVN'): + return 'Api-Key ' + data; + default: + return null; + } +}; + +export function getYandexAuth(path: string) { + if (path === null) { + throw new Error('No Auth'); + } + + let auth = resolveKey(path); + + if (auth !== null) { + return auth; + } + + auth = resolveKey(readFileSync(path, 'utf8')); + + if (auth === null) { + throw new Error('No Auth'); + } + + return auth; +} diff --git a/src/cmd/translation/providers/yandex/config.ts b/src/cmd/translation/providers/yandex/config.ts index 48847fd8e..ee8336843 100644 --- a/src/cmd/translation/providers/yandex/config.ts +++ b/src/cmd/translation/providers/yandex/config.ts @@ -1,12 +1,11 @@ import {cyan, gray} from 'chalk'; import {option, trim} from '~/config'; -const folderId = option({ - flags: '--folder-id ', +const folder = option({ + flags: '--folder ', desc: ` ID of the folder to which you have access. Required for authorization with a user account (see https://cloud.yandex.ru/ru/docs/iam/api-ref/UserAccount/#representation). - Don't specify this field if you make the request on behalf of a service account. `, }); @@ -29,8 +28,8 @@ const glossary = option({ ${configExample}`, }); -const oauthToken = option({ - flags: '--oauth-token ', +const auth = option({ + flags: '--auth ', desc: ` Authorization token for Translation API. (See more ${cyan('https://cloud.yandex.ru/en-ru/docs/translate/api-ref/authentication')} @@ -38,7 +37,7 @@ const oauthToken = option({ }); export const options = { - folderId, + folder, glossary, - oauthToken, + auth, }; diff --git a/src/cmd/translation/providers/yandex/index.ts b/src/cmd/translation/providers/yandex/index.ts index 351a329bf..462af9482 100644 --- a/src/cmd/translation/providers/yandex/index.ts +++ b/src/cmd/translation/providers/yandex/index.ts @@ -1,23 +1,23 @@ import type {IProgram} from '~/program'; -import type {TranslateArgs, TranslateConfig} from '~/cmd/translate'; +import type {TranslateArgs, TranslateConfig} from '~/cmd/translation'; import {ok} from 'assert'; -import {Translate} from '~/cmd/translate'; +import {Translate} from '~/cmd/translation'; import {defined, resolveConfig} from '~/config'; import {Provider} from './provider'; import {options} from './config'; -import {getYandexOAuthToken} from './oauth'; +import {getYandexAuth} from './auth'; const ExtensionName = 'YandexTranslation'; export type YandexTranslationArgs = TranslateArgs & { - folderId: string; - oauthToken: string; + folder: string; + auth: string; glossary: string; }; export type YandexTranslationConfig = TranslateConfig & { - folderId: string; - oauthToken: string; + folder: string; + auth: string; glossary: string; glossaryPairs: { sourceText: string; @@ -53,18 +53,22 @@ export class Extension { hooks.Provider.for('yandex').tap(ExtensionName, (_provider, config) => { hooks.Command.tap(ExtensionName, (command) => { command - .addOption(options.oauthToken) - .addOption(options.folderId) + .addOption(options.auth) + .addOption(options.folder) .addOption(options.glossary); }); hooks.Config.tapPromise(ExtensionName, async (config, args) => { - ok(!config.oauthToken, 'Do not store `oauthToken` in public config'); + ok(!config.auth, 'Do not store `authToken` in public config'); - config.folderId = defined('folderId', args, config); - config.oauthToken = defined('oauthToken', args) || (await getYandexOAuthToken()); + config.auth = getYandexAuth(args.auth); + config.folder = defined('folder', args, config); - if (config.glossary) { + ok(config.auth, 'Required param auth is not configured'); + ok(config.folder, 'Required param folder is not configured'); + + const glossary = defined('glossary', args, config); + if (glossary) { const glossaryConfig = await resolveConfig(config.glossary, { defaults: {glossaryPairs: []}, }); diff --git a/src/cmd/translation/providers/yandex/oauth.ts b/src/cmd/translation/providers/yandex/oauth.ts deleted file mode 100644 index b06daaff0..000000000 --- a/src/cmd/translation/providers/yandex/oauth.ts +++ /dev/null @@ -1,31 +0,0 @@ -import {readFile} from 'fs/promises'; -import {existsSync} from 'fs'; -import {env} from 'process'; -import {homedir} from 'os'; -import {join} from 'path'; - -const YANDEX_OAUTH_TOKEN_FILENAME = '.ya_oauth_token'; - -export async function getYandexOAuthToken() { - const {YANDEX_OAUTH_TOKEN} = env; - - return YANDEX_OAUTH_TOKEN ?? (await getYandexOAuthTokenFromHomeDir()); -} - -async function getYandexOAuthTokenFromHomeDir() { - const path = join(homedir(), YANDEX_OAUTH_TOKEN_FILENAME); - - const isFileExists = existsSync(path); - - if (!isFileExists) { - throw new Error(`OAuth token file ${path} not found`); - } - - const token = (await readFile(path, {encoding: 'utf8'})).trim(); - - if (!token?.length) { - throw new Error(`OAuth token content in ${path} is empty`); - } - - return token; -} diff --git a/src/cmd/translation/providers/yandex/provider.ts b/src/cmd/translation/providers/yandex/provider.ts index 3f2725962..6f64f9510 100644 --- a/src/cmd/translation/providers/yandex/provider.ts +++ b/src/cmd/translation/providers/yandex/provider.ts @@ -1,24 +1,29 @@ -import type {TranslateConfig} from '~/cmd/translate'; +import type {TranslateConfig} from '~/cmd/translation'; import type {YandexTranslationConfig} from '.'; -import {mkdir, readFile, writeFile} from 'node:fs/promises'; -import {dirname, resolve} from 'node:path'; -import glob from 'glob'; -import {asyncify, eachLimit, retry} from 'async'; -import {XMLParser} from 'fast-xml-parser'; -import {Session} from '@yandex-cloud/nodejs-sdk/dist/session'; -import {TranslationServiceClient} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/service_clients'; -import { - TranslateRequest_Format as Format, - TranslateRequest, -} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/ai/translate/v2/translation_service'; +import {mkdir} from 'node:fs/promises'; +import {dirname, extname, join, resolve} from 'node:path'; +import {asyncify, eachLimit} from 'async'; -import {compose, extract} from '@diplodoc/markdown-translation'; import {LogLevel, Logger, Writer} from '~/logger'; +import { + AuthError, + Defer, + LimitExceed, + RequestError, + bytes, + compose, + dumpFile, + extract, + loadFile, + resolveSchemas, +} from './utils'; +import axios, { AxiosError, AxiosResponse } from 'axios'; +import { logger } from '~/utils'; +import { green, red } from 'chalk'; const REQUESTS_LIMIT = 20; const BYTES_LIMIT = 10000; const RETRY_LIMIT = 3; -const MTRANS_LOCALE = 'MTRANS'; class TranslateLogger extends Logger { translating: Writer; @@ -49,179 +54,318 @@ export class Provider { this.logger = new TranslateLogger(config); } - async translate(config: TranslateConfig & YandexTranslationConfig) { - const {input, oauthToken, folderId, glossaryPairs, sourceLanguage, targetLanguage} = config; - const files = glob.sync('**/.md', {cwd: input, nodir: true}); - - const session = new Session({oauthToken}); - const client = session.client(TranslationServiceClient); - const request = (texts: string[]) => () => - client - .translate( - TranslateRequest.fromPartial({ - texts, - folderId, - sourceLanguageCode: sourceLanguage, - targetLanguageCode: targetLanguage, - glossaryConfig: { - glossaryData: {glossaryPairs}, - }, - format: Format.PLAIN_TEXT, + async translate(files: string[], config: TranslateConfig & YandexTranslationConfig) { + const {input, output, auth, folder, source, target: targets, dryRun} = config; + + try { + for (const target of targets) { + const translatorParams = { + input, + output, + auth, + sourceLanguage: source.language, + targetLanguage: target.language, + // yandexCloudTranslateGlossaryPairs, + folderId: folder, + dryRun, + }; + + const cache = new Map(); + const request = requester(translatorParams, cache); + const split = splitter(request, cache); + const translate = translator(translatorParams, split); + + await eachLimit( + files, + REQUESTS_LIMIT, + asyncify(async function (file: string) { + try { + await translate(file); + } catch (error: any) { + if (error instanceof TranslateError) { + logger.error(file, `${error.message}`, error.code); + + if (error.fatal) { + process.exit(1); + } + } else { + logger.error(file, error.message); + } + } }), - ) - .then((results) => results.translations.map(({text}) => text)); + ); - await eachLimit(files, REQUESTS_LIMIT, (path) => this.translateFile(path, request, config)); + console.log( + green('PROCESSED'), + `bytes: ${request.stat.bytes} chunks: ${request.stat.chunks}`, + ); + } + } catch (error: any) { + if (error instanceof TranslateError) { + console.error(red(error.code), error.message); + } else { + console.error(error); + } + + process.exit(1); + } } +} - private async translateFile( - mdPath: string, - request: (texts: string[]) => () => Promise, - config: TranslateConfig & YandexTranslationConfig, - ) { - const {input, output, sourceLanguage, targetLanguage} = config; +type TranslatorParams = { + input: string; + output: string; + sourceLanguage: string; + targetLanguage: string; + // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[]; +}; - try { - this.logger.translating(mdPath); - - const md = await readFile(resolve(input, mdPath), {encoding: 'utf-8'}); - - const {xlf, skeleton} = extract({ - source: { - language: sourceLanguage, - locale: 'US', - }, - target: { - language: targetLanguage, - locale: 'US', - }, - markdown: md, - markdownPath: mdPath, - skeletonPath: '', - }); - - const texts = parseSourcesFromXLIFF(xlf); - - const parts = await Promise.all( - texts.reduce( - ( - { - promises, - buffer, - bufferSize, - }: { - promises: Promise[]; - buffer: string[]; - bufferSize: number; - }, - text, - index, - ) => { - if (text.length >= BYTES_LIMIT) { - this.logger.warn( - mdPath, - 'Skip document part for translation. Part is too big.', - ); - promises.push(Promise.resolve([text])); - return {promises, buffer, bufferSize}; - } +type RequesterParams = { + auth: string; + folderId: string | undefined; + sourceLanguage: string; + targetLanguage: string; + dryRun: boolean; +}; - if (bufferSize + text.length > BYTES_LIMIT || index === texts.length - 1) { - promises.push(backoff(request(buffer))); - buffer = []; - bufferSize = 0; - } +type Request = { + (texts: string[]): () => Promise; + stat: { + bytes: number; + chunks: number; + }; +}; - buffer.push(text); - bufferSize += text.length; - - return {promises, buffer, bufferSize}; - }, - { - promises: [], - buffer: [], - bufferSize: 0, - }, - ).promises, - ); - - const translations = ([] as string[]).concat(...parts); - - const translatedXLIFF = createXLIFFDocument({ - sourceLanguage: sourceLanguage + '-' + MTRANS_LOCALE, - targetLanguage: targetLanguage + '-' + MTRANS_LOCALE, - sources: texts, - targets: translations, - }); - - const composed = await compose({ - xlf: translatedXLIFF, - skeleton, - }); - - const outputPath = mdPath.replace(input, output); - - await mkdir(dirname(outputPath), {recursive: true}); - await writeFile(outputPath, composed); - - this.logger.translated(mdPath); - } catch (err) { - if (err instanceof Error) { - throw new TranslateError(err.toString(), mdPath); - } - } +type Split = (path: string, texts: string[]) => Promise; + +type Cache = Map; + +type Translations = { + translations: { + text: string; + }[]; +}; + +function scheduler(limit: number, interval: number) { + const scheduled: Defer[] = []; + + let processing = 0; + + function idle() { + const defer = new Defer(); + + scheduled.push(defer); + + return defer.promise; + } + + async function queue() { + processing++; + await wait(interval); + processing--; + unqueue(); + } + + async function unqueue() { + scheduled.shift()?.resolve(); } + + return async function (action: Function): Promise { + if (processing >= limit) { + await idle(); + } + + queue(); + + return action(); + }; } -function backoff(action: () => Promise): Promise { - return retry( - { - times: RETRY_LIMIT, - interval: (count: number) => { - // eslint-disable-next-line no-bitwise - return (1 << count) * 1000; +function requester(params: RequesterParams, cache: Cache) { + const {auth, folderId, sourceLanguage, targetLanguage, dryRun} = params; + const schedule = scheduler(REQUESTS_LIMIT, 1000); + + const request = function request(texts: string[]) { + const resolve = (text: string, index: number) => { + const defer = cache.get(texts[index]); + if (defer) { + defer.resolve(text); + } + }; + + request.stat.bytes += bytes(texts); + request.stat.chunks++; + + return async function () { + if (dryRun) { + texts.forEach(resolve); + } + + try { + const {data} = await schedule>(() => + axios({ + method: 'POST', + url: 'https://translate.api.cloud.yandex.net/translate/v2/translate', + timeout: 5000, + maxRedirects: 0, + headers: { + Authorization: auth, + 'Content-Type': 'application/json', + 'User-Agent': 'github.com/diplodoc-platform/cli', + }, + data: { + folderId, + texts, + sourceLanguageCode: sourceLanguage, + targetLanguageCode: targetLanguage, + format: 'HTML', + }, + }), + ); + + return data.translations.map(({text}) => text).forEach(resolve); + } catch (error: any) { + if (error instanceof AxiosError) { + const {response} = error; + const {status, statusText, data} = response as AxiosResponse; + + switch (true) { + case LimitExceed.is(data.message): + throw new LimitExceed(data.message); + case AuthError.is(data.message): + throw new AuthError(data.message); + default: + throw new RequestError(status, statusText, data); + } + } + + throw new RequestError(0, error.message, {fatal: true}); + } + }; + }; + + request.stat = { + bytes: 0, + chunks: 0, + }; + + return request; +} + +function translator(params: TranslatorParams, split: Split) { + const {input, output, sourceLanguage, targetLanguage} = params; + const inputRoot = resolve(input); + const outputRoot = resolve(output); + + return async (path: string) => { + const ext = extname(path); + if (!['.yaml', '.json', '.md'].includes(ext)) { + return; + } + + const inputPath = join(inputRoot, path); + const outputPath = join(outputRoot, path.replace(sourceLanguage, targetLanguage)); + const content = await loadFile(inputPath); + + await mkdir(dirname(outputPath), {recursive: true}); + + if (!content) { + await dumpFile(outputPath, content); + return; + } + + const schemas = await resolveSchemas(path); + if (['.yaml', '.json'].includes(ext) && !schemas.length) { + return; + } + + const {units, skeleton} = extract(content, { + compact: true, + source: { + language: sourceLanguage, + locale: 'RU', + }, + target: { + language: targetLanguage, + locale: 'US', }, - }, - asyncify(action), - ); + schemas, + }); + + if (!units.length) { + await dumpFile(outputPath, content); + return; + } + + const parts = await split(path, units); + const composed = compose(skeleton, parts, {useSource: true, schemas}); + + await dumpFile(outputPath, composed); + }; } -function parseSourcesFromXLIFF(xliff: string): string[] { - const parser = new XMLParser(); +function splitter(request: Request, cache: Cache): Split { + return async function (path: string, texts: string[]) { + const promises: Promise[] = []; + const requests: Promise[] = []; + let buffer: string[] = []; + let bufferSize = 0; + + const release = () => { + requests.push(backoff(request(buffer))); + buffer = []; + bufferSize = 0; + }; + + for (const text of texts) { + if (text.length >= BYTES_LIMIT) { + logger.warn(path, 'Skip document part for translation. Part is too big.'); + promises.push(Promise.resolve(text)); + } else { + const defer = cache.get(text) || new Defer(); + promises.push(defer.promise); + + if (!cache.get(text)) { + if (bufferSize + text.length > BYTES_LIMIT) { + release(); + } + + buffer.push(text); + bufferSize += text.length; + } + + cache.set(text, defer); + } + } + + if (bufferSize) { + release(); + } - const inputs = parser.parse(xliff)?.xliff?.file?.body['trans-unit'] ?? []; + await Promise.all(requests); - return Array.isArray(inputs) - ? inputs.map(({source}: {source: string}) => source) - : [inputs.source]; + return Promise.all(promises); + }; } -export type CreateXLIFFDocumentParams = { - sourceLanguage: string; - targetLanguage: string; - sources: string[]; - targets: string[]; -}; +function wait(interval: number) { + const defer = new Defer(); + setTimeout(() => defer.resolve(), interval); + return defer.promise; +} -function createXLIFFDocument(params: CreateXLIFFDocumentParams) { - const {sourceLanguage, targetLanguage, sources, targets} = params; - - const unit = (text: string, i: number): string => ` - - ${sources[i]} - ${text} -`; - - const doc = ` - - - -
- -
- ${targets.map(unit)} -
-
`; - - return doc; +async function backoff(action: () => Promise): Promise { + let retry = 0; + + while (++retry < RETRY_LIMIT) { + try { + await action(); + } catch (error: any) { + if (RequestError.canRetry(error)) { + await wait(Math.pow(2, retry) * 1000); + } else { + throw error; + } + } + } } diff --git a/src/cmd/translation/providers/yandex/utils/errors.ts b/src/cmd/translation/providers/yandex/utils/errors.ts new file mode 100644 index 000000000..0cf2c8457 --- /dev/null +++ b/src/cmd/translation/providers/yandex/utils/errors.ts @@ -0,0 +1,100 @@ +export class TranslateError extends Error { + code: string; + + fatal: boolean; + + constructor(message: string, code: string, fatal = false) { + super(message); + + this.code = code; + this.fatal = fatal; + } +} + +export class RequestError extends TranslateError { + static canRetry(error: any) { + if (error instanceof RequestError) { + switch (true) { + case error.status === 429: + return true; + case error.status === 500: + return true; + case error.status === 503: + return true; + case error.status === 504: + return true; + default: + return false; + } + } + + return false; + } + + status: number; + + constructor( + status: number, + statusText: string, + info: {code?: number; message?: string; fatal?: boolean} = {}, + ) { + super(`${statusText}\n${info.message || ''}`, 'REQUEST_ERROR', info.fatal); + + this.status = status; + } +} + +const INACTIVE_CLOUD = /^The cloud .*? is inactive/; +const WRONG_APIKEY = /^Unknown api key/; +const WRONG_TOKEN = /^The token is invalid/; +const EXPIRED_TOKEN = /^The token has expired/; + +export class AuthError extends TranslateError { + static is(message: string) { + return Boolean(AuthError.reason(message)); + } + + static reason(message: string) { + switch (true) { + case INACTIVE_CLOUD.test(message): + return 'INACTIVE_CLOUD'; + case WRONG_APIKEY.test(message): + return 'WRONG_APIKEY'; + case WRONG_TOKEN.test(message): + return 'WRONG_TOKEN'; + case EXPIRED_TOKEN.test(message): + return 'EXPIRED_TOKEN'; + default: + return null; + } + } + + constructor(message: string) { + super(message, AuthError.reason(message) || 'AUTH_ERROR', true); + } +} + +const LIMIT_EXCEED_RX = /^limit on units was exceeded. (.*)$/; + +export class LimitExceed extends TranslateError { + static is(message: string) { + return Boolean(LIMIT_EXCEED_RX.test(message)); + } + + constructor(message: string) { + const [, desc] = LIMIT_EXCEED_RX.exec(message) || []; + super(desc, 'TRANSLATE_LIMIT_EXCEED', true); + } +} + +export class ExtractError extends TranslateError { + constructor(error: Error) { + super(error?.message || String(error), 'EXTRACT_ERROR'); + } +} + +export class ComposeError extends TranslateError { + constructor(error: Error) { + super(error?.message || String(error), 'COMPOSE_ERROR'); + } +} diff --git a/src/cmd/translation/providers/yandex/utils/fs.ts b/src/cmd/translation/providers/yandex/utils/fs.ts new file mode 100644 index 000000000..4313590ed --- /dev/null +++ b/src/cmd/translation/providers/yandex/utils/fs.ts @@ -0,0 +1,96 @@ +import {JSONValue, resolveRefs} from '@diplodoc/translation'; +import {dirname, join} from 'node:path'; +import {readFile, writeFile} from 'node:fs/promises'; +import {dump, load} from 'js-yaml'; + +const ROOT = dirname(require.resolve('#package')); + +function last(array: T[]): T | undefined { + return array[array.length - 1]; +} + +function ext(path: string) { + const parts = path.split('.'); + + if (last(parts) === 'skl') { + parts.pop(); + } + + return last(parts); +} + +function parseFile(text: string, path: string): JSONValue | string { + if (typeof text !== 'string') { + return text; + } + + switch (ext(path)) { + case 'yaml': + return load(text) as object; + case 'json': + return JSON.parse(text); + default: + return text; + } +} + +function stringifyFile(content: JSONValue | string, path: string): string { + if (typeof content === 'string') { + return content; + } + + switch (ext(path)) { + case 'yaml': + return dump(content); + case 'json': + return JSON.stringify(content); + default: + return content as unknown as string; + } +} + +export async function loadFile(path: string, resolve = true): Promise { + const text = await readFile(path, 'utf8'); + + let content = parseFile(text, path); + + if (content && typeof content === 'object' && resolve) { + content = await resolveRefs(content, path, parseFile); + } + + return content as T; +} + +export async function dumpFile(path: string, content: string | JSONValue) { + const text = stringifyFile(content, path); + + await writeFile(path, text, 'utf8'); +} + +/** + * Takes toc schema if file matched as toc. + * Takes leading schema if file matched as leading page. + * Takes presets schema if file matched as presets. + * Any way translation inner logic will search `$schema` attribute with high priority. + * If `$schema` attribute not found anc precise schema not resolved, + * we think that current yaml is a part of complex toc.yaml + */ +export async function resolveSchemas(path: string) { + if (path.endsWith('toc.yaml')) { + return [await loadFile(join(ROOT, 'schemas/toc-schema.yaml'), false)]; + } + + if (path.endsWith('index.yaml')) { + return [await loadFile(join(ROOT, 'schemas/leading-schema.yaml'), false)]; + } + + if (path.endsWith('presets.yaml')) { + return [await loadFile(join(ROOT, 'schemas/presets-schema.yaml'), false)]; + } + + if (path.endsWith('redirects.yaml')) { + return []; + } + + return [await loadFile(join(ROOT, 'schemas/toc-schema.yaml'), false)]; +} diff --git a/src/cmd/translation/providers/yandex/utils/index.ts b/src/cmd/translation/providers/yandex/utils/index.ts new file mode 100644 index 000000000..878dc4a78 --- /dev/null +++ b/src/cmd/translation/providers/yandex/utils/index.ts @@ -0,0 +1,158 @@ +import {ok} from 'assert'; +import {basename, dirname, extname, resolve} from 'path'; +import {readFileSync} from 'node:fs'; +import glob from 'glob'; + +export {dumpFile, loadFile, resolveSchemas} from './fs'; +export {extract, compose} from './translate'; +export {TranslateError, LimitExceed, RequestError, AuthError} from './errors'; + +type TranslateArgs = { + input: string; + output?: string; + source?: string; + sourceLanguage?: string; + sourceLanguageLocale?: string; + target?: string; + targetLanguage?: string; + targetLanguageLocale?: string; + auth?: string; + folder?: string; + glossary?: string; + include?: string[] | string; + exclude?: string[] | string; + dryRun?: boolean; + useSource?: boolean; +} & { + [prop: string]: any; +}; + +export type TranslateParams = { + input: string; + output: string; + source: [string, string]; + targets: [string, string][]; + auth?: string; + folder?: string; + glossary?: string; + files: string[]; + dryRun: boolean; + useSource: boolean; +}; + +export function normalizeParams(params: TranslateArgs, exts = EXTS): TranslateParams { + const source = normalizeLocale(params, 'source')[0]; + const targets = normalizeLocale(params, 'target'); + const {input, files} = normalizeInput(params, source[0], exts); + const {output, auth, folder, dryRun, useSource} = params; + + ok(input, 'Required param input is not configured'); + + return { + input, + output: output || input, + auth, + folder, + source, + targets, + files: [...new Set(files)], + dryRun: Boolean(dryRun), + useSource: Boolean(useSource), + }; +} + +function normalizeLocale(params: TranslateArgs, scope: 'source' | 'target'): [string, string][] { + const _scopeLanguage = params[scope + 'Language']; + const _scopeLanguageLocale = params[scope + 'LanguageLocale']; + const _scope = params[scope] || _scopeLanguageLocale || _scopeLanguage; + + if (!_scope) { + return [['', '']]; + } + + if (Array.isArray(_scope)) { + return _scope.map((_scope) => _scope.split('-')); + } + + return [_scope.split('-') as [string, string]]; +} + +const EXTS = ['.md', '.yaml', '.json']; + +function normalizeInput(params: TranslateArgs, language: string, exts: string[]) { + let {input, include = [], exclude = []} = params; + + include = ([] as string[]).concat(include); + exclude = ([] as string[]).concat(exclude); + + let files: string[] | null = null; + if (extname(input) === '.list') { + const list = readFileSync(input, 'utf8').split('\n'); + input = dirname(input); + files = list.map((file) => { + const absPath = resolve(input, file); + + if (!absPath.startsWith(input)) { + throw new Error(`Insecure access to file out of project scope. (file: ${absPath})`); + } + + if (!exts.includes(extname(file))) { + throw new Error(`Unhandles file extension. (file: ${absPath})`); + } + + return absPath; + }); + } else { + if (exts.includes(extname(input))) { + files = [basename(input)]; + input = dirname(input); + } + + if (!include.length) { + include.push('...'); + } + + include = include.reduce((acc, item) => { + if (item === '...') { + acc.push(...exts.map((ext) => (language || '.') + '/**/*' + ext)); + } else { + acc.push(item); + } + + return acc; + }, [] as string[]); + + files = + files || + ([] as string[]).concat( + ...include.map((match) => + glob.sync(match, { + cwd: input, + ignore: exclude, + nodir: true, + }), + ), + ); + } + + return {input, files}; +} + +export class Defer { + resolve!: (text: T) => void; + + reject!: (error: any) => void; + + promise: Promise; + + constructor() { + this.promise = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); + } +} + +export function bytes(texts: string[]) { + return texts.reduce((sum, text) => sum + text.length, 0); +} diff --git a/src/cmd/translation/providers/yandex/utils/translate.ts b/src/cmd/translation/providers/yandex/utils/translate.ts new file mode 100644 index 000000000..5163719ce --- /dev/null +++ b/src/cmd/translation/providers/yandex/utils/translate.ts @@ -0,0 +1,26 @@ +import type {ComposeOptions, ExtractOptions} from '@diplodoc/translation'; +import {compose as _compose, extract as _extract} from '@diplodoc/translation'; +import {ComposeError, ExtractError} from './errors'; + +type Content = Parameters[0]; + +export function extract(content: Content, options: ExtractOptions) { + try { + const {units, skeleton} = _extract(content, options); + + return {units, skeleton}; + } catch (error: any) { + throw new ExtractError(error); + } +} + +type Skeleton = Parameters[0]; +type Xliff = Parameters[1]; + +export function compose(skeleton: Skeleton, xliff: Xliff, options: ComposeOptions) { + try { + return _compose(skeleton, xliff, options); + } catch (error: any) { + throw new ComposeError(error); + } +} diff --git a/src/cmd/translation/utils.ts b/src/cmd/translation/utils.ts new file mode 100644 index 000000000..b8e30579b --- /dev/null +++ b/src/cmd/translation/utils.ts @@ -0,0 +1,126 @@ +import {ok} from 'assert'; +import {extname, resolve} from 'path'; +import {readFileSync, statSync} from 'node:fs'; +import glob from 'glob'; +import { defined } from '~/config'; + +type PartialLocale = { + language: string; + locale?: string; +}; + +export type Locale = { + language: string; + locale: string; +}; + +type SourceLocaleConfig = { + source?: string | PartialLocale; + sourceLanguage?: string; + sourceLanguageLocale?: string; +}; + +export function resolveSource(config: SourceLocaleConfig, args: SourceLocaleConfig): Locale { + const value = defined('source', args, config); + if (value) { + ok(typeof value === 'string' || typeof value === 'object' && value.language, + `Field 'source' should be string or locale.` + ); + + return parseLocale(value); + } + + return {language: '', locale: ''}; +} + +type TargetLocaleConfig = { + target?: string | PartialLocale | (string | PartialLocale)[]; + targetLanguage?: string | string[]; + targetLanguageLocale?: string | string[]; +}; + +export function resolveTargets(config: TargetLocaleConfig, args: TargetLocaleConfig) { + const value = defined('target', args, config); + + if (value) { + ok( + ['string', 'object'].includes(typeof value) || Array.isArray(value), + `Field 'target' should be string, locale or array.` + ); + + if (Array.isArray(value)) { + return value.map(parseLocale); + } else { + return [parseLocale(value)]; + } + } + + return [{language: '', locale: ''}]; +} + +function parseLocale(raw: string | Locale) { + if (typeof raw === 'object') { + raw.locale = raw.locale || ''; + + return raw; + } + + const [language, locale = ''] = raw.split('-'); + + return {language, locale}; +} + +function listResolver(path: string) { + if (extname(path) !== '.list') { + return [path]; + } + + const list = readFileSync(path, 'utf8') + .split('\n') + // Remove comments + .filter((line) => !line.match(/^#/)) + // Remove empty lines + .filter(Boolean); + + return list.map((file) => { + const absPath = resolve(path, file); + ok(absPath.startsWith(path), `Insecure access to file out of project scope. (file: ${absPath})`); + + const stat = statSync(absPath); + ok(stat.isFile(), `${absPath} is not a file.`); + + return absPath; + }); +} + +export function withListResolver(actor: (value: string | string[], previous: any) => string[]) { + return (value: string, previous: any) => actor(listResolver(value), previous); +} + +export function resolveFiles(input: string, include: string[], exclude: string[], lang: string, exts: string[]) { + if (!include.length) { + include.push('...'); + } + + include = include.reduce((acc, item) => { + if (item === '...') { + acc.push(...exts.map((ext) => (lang || '.') + '/**/*' + ext)); + } else { + acc.push(item); + } + + return acc; + }, [] as string[]); + + const files = ([] as string[]).concat( + ...include.map((match) => + glob.sync(match, { + cwd: input, + ignore: exclude, + nodir: true, + }), + ), + ); + + return [...new Set(files)]; +} diff --git a/src/config/index.ts b/src/config/index.ts index c1e3fd9b8..46d4b2824 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -28,16 +28,16 @@ export function trim(string: string | TemplateStringsArray): string { return lines.join('\n').trim(); } -export function toArray(value: string, previous: string | string[]) { +export function toArray(value: string | string[], previous: string | string[]) { + value = ([]as string[]).concat(value); + if (previous) { - if (Array.isArray(previous)) { - return previous.concat(value); - } else { - return [previous, value]; - } + previous = ([]as string[]).concat(previous); + + return [...new Set([...previous, ...value])]; } - return [value]; + return value; } export type OptionInfo = { @@ -50,6 +50,7 @@ export type OptionInfo = { defaultInfo?: any; required?: boolean; choices?: string[]; + variadic?: boolean; hidden?: boolean; deprecated?: string; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -357,9 +358,15 @@ export class Command extends BaseCommand { if (o.isBoolean() && (o as ExtendedOption)[OptionSource]) { const original = (o as ExtendedOption)[OptionSource]; + const flags = original.flags + // replace short flags with void + .replace(/(^|\s+|,)-\w+\s*($|,?\s*)/g, '') + // add negation to long options + .replace(/--/g, '--no-'); + const negated = { ...original, - flags: original.flags.replace('--', '--no-'), + flags, desc: 'auto negation', hidden: true, }; diff --git a/src/program/config.ts b/src/program/config.ts index a31622997..bdd4cfe19 100644 --- a/src/program/config.ts +++ b/src/program/config.ts @@ -55,11 +55,13 @@ const input = (defaultPath?: string) => default: defaultPath, }); -const output = option({ - flags: '-o, --output ', - desc: `Configure path to {{PROGRAM}} output directory.`, - required: true, -}); +const output = (defaultPath?: string) => + option({ + flags: '-o, --output ', + desc: `Configure path to {{PROGRAM}} output directory.`, + required: true, + default: defaultPath, + }); const config = (defaultConfig: string) => option({ diff --git a/src/program/index.ts b/src/program/index.ts index 26498e1b8..144a0c1f7 100644 --- a/src/program/index.ts +++ b/src/program/index.ts @@ -5,7 +5,7 @@ import {SyncWaterfallHook} from 'tapable'; import {Command, configRoot} from '~/config'; import {YFM_CONFIG_FILENAME} from '~/constants'; -import {Build, Publish, Translate, xliff} from '~/cmd'; +import {Build, Publish, Translate} from '~/cmd'; import {NAME, USAGE, options} from './config'; import {HandledError, isRelative} from './utils';