From 082d472b07deeb7f3c07ea2c8ce7eb71e3c63a9e Mon Sep 17 00:00:00 2001 From: jy95 Date: Tue, 5 Oct 2021 22:40:09 +0200 Subject: [PATCH 01/10] feat: add keySeparator option support for custom separator in output & input --- src/cmds/diff.ts | 1 + src/cmds/diff/detectChanges.ts | 26 +++++++++++++++------- src/cmds/export_cmds/export_commons.ts | 30 +++++++++++++++++--------- src/cmds/export_cmds/export_csv.ts | 1 + src/cmds/export_cmds/export_xlsx.ts | 1 + src/cmds/import_cmds/import_commons.ts | 16 +++++++++----- src/cmds/import_cmds/import_csv.ts | 7 +++++- src/cmds/import_cmds/import_xlsx.ts | 7 +++++- src/commons/commandBuilder.ts | 10 +++++++++ src/commons/enhancedGet.ts | 16 ++++++++++++++ src/commons/enhancedSet.ts | 17 +++++++++++++++ src/commons/getLeavesPathes.ts | 11 +++++++--- src/types/diffTypes.ts | 2 ++ src/types/exportTypes.ts | 2 ++ src/types/importTypes.ts | 2 ++ 15 files changed, 121 insertions(+), 28 deletions(-) create mode 100644 src/commons/enhancedGet.ts create mode 100644 src/commons/enhancedSet.ts diff --git a/src/cmds/diff.ts b/src/cmds/diff.ts index 1f7021f..9eb3fa3 100644 --- a/src/cmds/diff.ts +++ b/src/cmds/diff.ts @@ -65,6 +65,7 @@ export const builder = function (y: Argv) { new CommonDiffYargsBuilder(y) .addFilenameOption() .addOutputDirOption() + .addKeySeparatorOption() .addOutputFormatOption() .addFilesOption() .addSettingConfig() diff --git a/src/cmds/diff/detectChanges.ts b/src/cmds/diff/detectChanges.ts index 7239888..805f15c 100644 --- a/src/cmds/diff/detectChanges.ts +++ b/src/cmds/diff/detectChanges.ts @@ -6,10 +6,12 @@ import type { DelOperation, PutOperation, } from '../../types/diffTypes'; + +// Own methods import getLeavesPathes from '../../commons/getLeavesPathes'; +import get from '../../commons/enhancedGet'; // lodash method -import get from 'lodash/get'; import intersection from 'lodash/intersection'; import isEqual from 'lodash/isEqual'; import difference from 'lodash/difference'; @@ -24,7 +26,8 @@ function createChangeOperation( technicalKey: string, op: ChangesOps, file1: fileParam, - file2: fileParam + file2: fileParam, + keySeparator: string ): ChangeOperations { // common part let obj: CommonChangeOperation = { @@ -37,13 +40,15 @@ function createChangeOperation( if ([ChangesOps.DEL, ChangesOps.PUT].some((o) => o === op)) { (obj as DelOperation | PutOperation).oldValue = get( file1.obj, - technicalKey + technicalKey, + keySeparator ); } if ([ChangesOps.ADD, ChangesOps.PUT].some((o) => o === op)) { (obj as AddOperation | PutOperation).newValue = get( file2.obj, - technicalKey + technicalKey, + keySeparator ); } // return result @@ -55,6 +60,7 @@ export default function detectChanges( argv: CommonDiffArguments ): ChangeOperations[] { let result: ChangeOperations[] = []; + let keySeparator: string = argv.keySeparator; // Fetch keys let files: fileParam[] = argv.files.map((file, idx) => ({ @@ -78,26 +84,30 @@ export default function detectChanges( // Computes changes of values let sameKeys = intersection(file1.keys, file2.keys); let modifiedKeys = sameKeys.filter( - (key) => !isEqual(get(file1.obj, key), get(file2.obj, key)) + (key) => + !isEqual( + get(file1.obj, key, keySeparator), + get(file2.obj, key, keySeparator) + ) ); result.push( ...modifiedKeys.map((key) => - createChangeOperation(key, ChangesOps.PUT, file1, file2) + createChangeOperation(key, ChangesOps.PUT, file1, file2, keySeparator) ) ); // Computes deleted keys result.push( ...difference(file1.keys, file2.keys).map((key) => - createChangeOperation(key, ChangesOps.DEL, file1, file2) + createChangeOperation(key, ChangesOps.DEL, file1, file2, keySeparator) ) ); // Computes new keys result.push( ...difference(file2.keys, file1.keys).map((key) => - createChangeOperation(key, ChangesOps.ADD, file1, file2) + createChangeOperation(key, ChangesOps.ADD, file1, file2, keySeparator) ) ); } diff --git a/src/cmds/export_cmds/export_commons.ts b/src/cmds/export_cmds/export_commons.ts index 7651243..f08a875 100644 --- a/src/cmds/export_cmds/export_commons.ts +++ b/src/cmds/export_cmds/export_commons.ts @@ -3,7 +3,9 @@ import fs, { PathLike } from 'fs'; // lodash methodes import groupBy from 'lodash/groupBy'; import flattenDeep from 'lodash/flattenDeep'; -import get from 'lodash/get'; + +// "Enhanced get" +import get from '../../commons/enhancedGet'; // For typings import { @@ -77,7 +79,11 @@ export function merge_i18n_files( return new Promise((resolve, reject) => { Promise // Read files and convert them to useful obj - .all(Object.entries(argv.files).map((entry) => readFile(entry))) + .all( + Object.entries(argv.files).map((entry) => + readFile(entry, argv.keySeparator) + ) + ) // merge results .then((results) => mergeResults(results)) .then((data) => resolve(data)) @@ -88,27 +94,31 @@ export function merge_i18n_files( // merge_i18n_files sub functions // read file and turning into a useful array of objects -function readFile([locale, file_path]: [ - string, - PathLike -]): Promise { +function readFile( + [locale, file_path]: [string, PathLike], + keySeparator: string +): Promise { return new Promise((resolve, reject) => { fs.promises .readFile(file_path, 'utf8') .then((jsonData) => Promise.resolve(JSON.parse(jsonData))) - .then((json) => i18n_to_result_format(json, locale)) + .then((json) => i18n_to_result_format(json, locale, keySeparator)) .then((result) => resolve(result)) .catch(/* istanbul ignore next */ (err) => reject(err)); }); } // turns i18n object to usable format -function i18n_to_result_format(obj: I18N_Object, locale: string): I18N_Result { - let leafPaths = getLeavesPathes(obj); +function i18n_to_result_format( + obj: I18N_Object, + locale: string, + keySeparator: string +): I18N_Result { + let leafPaths = getLeavesPathes(obj, keySeparator); return leafPaths.map((leafPath) => ({ locale: locale, technical_key: leafPath, - label: get(obj, leafPath) as string, + label: get(obj, leafPath, keySeparator) as string, })); } diff --git a/src/cmds/export_cmds/export_csv.ts b/src/cmds/export_cmds/export_csv.ts index 407b1d1..41ba0ed 100644 --- a/src/cmds/export_cmds/export_csv.ts +++ b/src/cmds/export_cmds/export_csv.ts @@ -99,6 +99,7 @@ export const builder = function (y: Argv) { .addFilenameOption() .addOutputDirOption() .addSettingConfig() + .addKeySeparatorOption() .addColumnsOption() .addDelimiterOption() .addRowDelimiterOption() diff --git a/src/cmds/export_cmds/export_xlsx.ts b/src/cmds/export_cmds/export_xlsx.ts index d938098..138a1c8 100644 --- a/src/cmds/export_cmds/export_xlsx.ts +++ b/src/cmds/export_cmds/export_xlsx.ts @@ -67,6 +67,7 @@ export const builder = function (y: Argv) { .addFilenameOption() .addOutputDirOption() .addSettingConfig() + .addKeySeparatorOption() .addColumnsOption() .addWorksheetCustomizerOption() .addWorksheetNameOption() diff --git a/src/cmds/import_cmds/import_commons.ts b/src/cmds/import_cmds/import_commons.ts index 34bf0d8..fc1d0c9 100644 --- a/src/cmds/import_cmds/import_commons.ts +++ b/src/cmds/import_cmds/import_commons.ts @@ -2,9 +2,11 @@ import fs from 'fs'; import { resolve as pathResolve } from 'path'; // lodash methodes -import set from 'lodash/set'; import groupBy from 'lodash/groupBy'; +// Own method +import set from '../../commons/enhancedSet'; + // For typings import { CommonImportArguments, @@ -66,7 +68,8 @@ export function generate_i18n_filepaths(argv: CommonImportArguments) { // extractedTranslation[] to i18n file(s) export function extractedTranslations_to_i18n_files( files: { [x: string]: string }, - translations: extractedTranslation[] + translations: extractedTranslation[], + keySeparator: string ) { let groupBy_locales = groupBy(translations, 'locale'); return Promise.all( @@ -74,7 +77,7 @@ export function extractedTranslations_to_i18n_files( write_new_i18n_file( locale, files[locale], - translations_2_i18n_object(translations) + translations_2_i18n_object(translations, keySeparator) ) ) ); @@ -99,10 +102,13 @@ function write_new_i18n_file( } // Turns array for a given lang into a i18n js object -function translations_2_i18n_object(translations: extractedTranslation[]) { +function translations_2_i18n_object( + translations: extractedTranslation[], + keySeparator: string +) { let result = {}; translations.forEach((item) => { - set(result, item['technical_key'], item['label']); + set(result, item['technical_key'], item['label'], keySeparator); }); return result; } diff --git a/src/cmds/import_cmds/import_csv.ts b/src/cmds/import_cmds/import_csv.ts index dad4bc6..40b1f44 100644 --- a/src/cmds/import_cmds/import_csv.ts +++ b/src/cmds/import_cmds/import_csv.ts @@ -83,6 +83,7 @@ export const builder = function (y: Argv) { .addInputOption() .addLocalesOption() .addOutputDirOption(true) + .addKeySeparatorOption() .addSuffixOption() .addColumnsOption() .addDelimiterOption() @@ -100,7 +101,11 @@ export const handler = async function (argv: CSVImportArguments) { try { const translations = await csv_2_translation_objects(argv); const files = generate_i18n_filepaths(argv); - await extractedTranslations_to_i18n_files(files, translations); + await extractedTranslations_to_i18n_files( + files, + translations, + argv.keySeparator + ); console.log('Successfully exported found locale(s) to i18n json file(s)'); return Promise.resolve(undefined); } catch (error) { diff --git a/src/cmds/import_cmds/import_xlsx.ts b/src/cmds/import_cmds/import_xlsx.ts index da26ac0..11e4f5f 100644 --- a/src/cmds/import_cmds/import_xlsx.ts +++ b/src/cmds/import_cmds/import_xlsx.ts @@ -46,6 +46,7 @@ export const builder = function (y: Argv) { .addInputOption() .addLocalesOption() .addOutputDirOption(true) + .addKeySeparatorOption() .addSuffixOption() .addColumnsOption() .addSettingConfig() @@ -59,7 +60,11 @@ export const handler = async function (argv: XLSXImportArguments) { try { const translations = await xlsx_2_translation_objects(argv); const files = generate_i18n_filepaths(argv); - await extractedTranslations_to_i18n_files(files, translations); + await extractedTranslations_to_i18n_files( + files, + translations, + argv.keySeparator + ); console.log('Successfully exported found locale(s) to i18n json file(s)'); return Promise.resolve(undefined); } catch (error) { diff --git a/src/commons/commandBuilder.ts b/src/commons/commandBuilder.ts index 98a3bac..3e69b57 100644 --- a/src/commons/commandBuilder.ts +++ b/src/commons/commandBuilder.ts @@ -37,6 +37,16 @@ export default class CommandBuilder { return this; } + addKeySeparatorOption() { + this.y = this.y.option('keySeparator', { + type: 'string', + alias: 'ks', + describe: 'Char to separate i18n keys', + default: '.', + }); + return this; + } + build() { return this.y; } diff --git a/src/commons/enhancedGet.ts b/src/commons/enhancedGet.ts new file mode 100644 index 0000000..54bb3d7 --- /dev/null +++ b/src/commons/enhancedGet.ts @@ -0,0 +1,16 @@ +// lodash methods +import get from 'lodash/get'; + +// "Enhance" Lodash get, to deal with custom separator +export default function enhancedGet( + obj: any, + key: string, + keySeparator: string +) { + // compute path that use dot (or custom) separator + square brack notation + let path = key + // handle square brack notation - eg: a[10] should be translated as a.10 + .replace(/\[(\d+)\]/g, `${keySeparator}$1`) + .split(keySeparator); + return get(obj, path); +} diff --git a/src/commons/enhancedSet.ts b/src/commons/enhancedSet.ts new file mode 100644 index 0000000..3c788a2 --- /dev/null +++ b/src/commons/enhancedSet.ts @@ -0,0 +1,17 @@ +// lodash methods +import set from 'lodash/set'; + +// "Enhance" Lodash set, to deal with custom separator +export default function enhancedSet( + obj: any, + key: string, + val: any, + keySeparator: string +) { + // compute path that use dot (or custom) separator + square brack notation + let path = key + // handle square brack notation - eg: a[10] should be translated as a.10 + .replace(/\[(\d+)\]/g, `${keySeparator}$1`) + .split(keySeparator); + return set(obj, path, val); +} diff --git a/src/commons/getLeavesPathes.ts b/src/commons/getLeavesPathes.ts index c98a0d2..b95e785 100644 --- a/src/commons/getLeavesPathes.ts +++ b/src/commons/getLeavesPathes.ts @@ -4,17 +4,22 @@ import reduce from 'lodash/reduce'; // Get all leaves paths of a object // Typescript code inspired by https://stackoverflow.com/a/55381003/6149867 -export default function getLeavesPathes(dataObj: any): string[] { +export default function getLeavesPathes( + dataObj: any, + keySeparator = '.' +): string[] { const reducer = (aggregator: string[], val: any, key: string) => { let paths = [key]; if (isObject(val)) { paths = reduce(val, reducer, []); - paths = paths.map((path) => key + '.' + path); + paths = paths.map((path) => key + keySeparator + path); } aggregator.push(...paths); return aggregator; }; - const arrayIndexRegEx = /\.(\d+)(?!\w+)/gi; + // Need to double escape stuff when using this constructor + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + const arrayIndexRegEx = new RegExp(`\\${keySeparator}(\d+)(?!\w+)`, 'gi'); let paths = reduce(dataObj, reducer, []); paths = paths.map((path) => path.replace(arrayIndexRegEx, '[$1]')); diff --git a/src/types/diffTypes.ts b/src/types/diffTypes.ts index 8eb9014..2169b3c 100644 --- a/src/types/diffTypes.ts +++ b/src/types/diffTypes.ts @@ -12,6 +12,8 @@ export interface CommonDiffArguments extends Argv { files: [any, any, ...any[]]; // output format outputFormat: 'JSON'; + // https://github.com/jy95/i18n-tools/issues/25 + keySeparator: '.' | string; } // for exporter(s) diff --git a/src/types/exportTypes.ts b/src/types/exportTypes.ts index 66bee81..e5cb3c9 100644 --- a/src/types/exportTypes.ts +++ b/src/types/exportTypes.ts @@ -19,6 +19,8 @@ export interface CommonExportArguments extends Argv { filename: string; outputDir: string; resultsFilter?: string | ((x: I18N_Merged_Data) => I18N_Merged_Data); + // https://github.com/jy95/i18n-tools/issues/25 + keySeparator: '.' | string; } // Yargs export arguments for TO_XLSX command export interface XLSXExportArguments extends CommonExportArguments { diff --git a/src/types/importTypes.ts b/src/types/importTypes.ts index 6e9522f..6bcf845 100644 --- a/src/types/importTypes.ts +++ b/src/types/importTypes.ts @@ -7,6 +7,8 @@ export interface CommonImportArguments extends Argv { outputDir: string; suffix: string; locales: string[]; + // https://github.com/jy95/i18n-tools/issues/25 + keySeparator: '.' | string; } // Yargs import arguments for FROM_XLSX command From ee88794871b6a8e6a12e693a0ea6019b9f58af12 Mon Sep 17 00:00:00 2001 From: jy95 Date: Tue, 5 Oct 2021 23:14:39 +0200 Subject: [PATCH 02/10] fix: regression in getLeavesPathes Nasty RegExp constructor ^^ --- src/commons/getLeavesPathes.ts | 2 +- test/getLeavesPathes.test.ts | 38 +++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/commons/getLeavesPathes.ts b/src/commons/getLeavesPathes.ts index b95e785..e794d1b 100644 --- a/src/commons/getLeavesPathes.ts +++ b/src/commons/getLeavesPathes.ts @@ -19,7 +19,7 @@ export default function getLeavesPathes( }; // Need to double escape stuff when using this constructor // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping - const arrayIndexRegEx = new RegExp(`\\${keySeparator}(\d+)(?!\w+)`, 'gi'); + const arrayIndexRegEx = new RegExp(`\\${keySeparator}(\\d+)(?!\\w+)`, 'gi'); let paths = reduce(dataObj, reducer, []); paths = paths.map((path) => path.replace(arrayIndexRegEx, '[$1]')); diff --git a/test/getLeavesPathes.test.ts b/test/getLeavesPathes.test.ts index 9282ceb..51c6783 100644 --- a/test/getLeavesPathes.test.ts +++ b/test/getLeavesPathes.test.ts @@ -63,7 +63,31 @@ const PATH_SCENARIOS: [string, any, string[]][] = [ ], ]; -describe('[commons - getLeavesPathes]', () => { +// scenarios for custom separator +// Might be lazy but better to support same tests that the dot separator XD +let CUST_SEPARATOR = '_'; +let expectedResult: string[][] = [ + ['key', 'someArray[0]', 'someArray[1]', 'someArray[2]'], + [ + `commons${CUST_SEPARATOR}firstNestedKey`, + `commons${CUST_SEPARATOR}units${CUST_SEPARATOR}secondNestedKey`, + ], + [`commons${CUST_SEPARATOR}units${CUST_SEPARATOR}5ml`], + ['Key with spaces'], + [ + `someArray${CUST_SEPARATOR}0${CUST_SEPARATOR}type`, + `someArray${CUST_SEPARATOR}0${CUST_SEPARATOR}message`, + `someArray${CUST_SEPARATOR}1${CUST_SEPARATOR}type`, + `someArray${CUST_SEPARATOR}1${CUST_SEPARATOR}message`, + ], +]; +const PATH_SCENARIOS_2: [string, any, string[]][] = PATH_SCENARIOS.map( + (entry, index) => { + return [entry[0], entry[1], expectedResult[index]]; + } +); + +describe('[commons - getLeavesPathes] dot separator', () => { test.each(PATH_SCENARIOS)( '%s', async (_title: string, obj: any, expectedArray: string[]) => { @@ -72,3 +96,15 @@ describe('[commons - getLeavesPathes]', () => { } ); }); + +// Might be lazy but better to support same tests that the dot separator XD +// Later WET instead of DRY ? +describe('[commons - getLeavesPathes] custom separator', () => { + test.each(PATH_SCENARIOS_2)( + '%s', + async (_title: string, obj: any, expectedArray: string[]) => { + const paths = getLeavesPathes(obj, CUST_SEPARATOR); + expect(paths.sort()).toEqual(expectedArray.sort()); + } + ); +}); From 4cba804a61610d3f2835b168d1dcbcb0af96df67 Mon Sep 17 00:00:00 2001 From: jy95 Date: Wed, 6 Oct 2021 00:06:19 +0200 Subject: [PATCH 03/10] test: add keySeparator options tests --- src/checks/commons/keySeparator_check.ts | 9 +++++++++ src/checks/diff_checks.ts | 8 +++++++- src/checks/export/export_common_checks.ts | 10 +++++++++- src/checks/import/import_common_checks.ts | 5 ++++- test/diff.test.ts | 7 +++++++ test/export/export-csv.test.ts | 12 ++++++++++++ test/export/export-xlsx.test.ts | 12 ++++++++++++ test/import/import-csv.test.ts | 13 +++++++++++++ test/import/import-xlsx.test.ts | 13 +++++++++++++ 9 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 src/checks/commons/keySeparator_check.ts diff --git a/src/checks/commons/keySeparator_check.ts b/src/checks/commons/keySeparator_check.ts new file mode 100644 index 0000000..12c44a9 --- /dev/null +++ b/src/checks/commons/keySeparator_check.ts @@ -0,0 +1,9 @@ +const KEYSEPARATOR_CHECK = async (argv: any) => { + let keySeparator = argv.keySeparator as any; + if (keySeparator.length !== 1) { + return new Error(`Option keySeparator should be a not-empty char`); + } else { + return true; + } +}; +export default KEYSEPARATOR_CHECK; diff --git a/src/checks/diff_checks.ts b/src/checks/diff_checks.ts index 029db92..4d2d8ed 100644 --- a/src/checks/diff_checks.ts +++ b/src/checks/diff_checks.ts @@ -1,5 +1,7 @@ // reuse check function from export command import { FILENAME_CHECK } from './export/index'; +// key separator check +import KEYSEPARATOR_CHECK from './commons/keySeparator_check'; // check if at least two paths were provided // For that, we will use "backupKey" from backupPaths : "paths" @@ -12,4 +14,8 @@ export const AT_LEAST_2_PATHS_CHECK = async (argv: any) => { }; // export checks in expected order into a single array -export const CHECKS = [FILENAME_CHECK, AT_LEAST_2_PATHS_CHECK]; +export const CHECKS = [ + KEYSEPARATOR_CHECK, + FILENAME_CHECK, + AT_LEAST_2_PATHS_CHECK, +]; diff --git a/src/checks/export/export_common_checks.ts b/src/checks/export/export_common_checks.ts index 30c4910..5e155bb 100644 --- a/src/checks/export/export_common_checks.ts +++ b/src/checks/export/export_common_checks.ts @@ -7,6 +7,9 @@ import isFunction from 'lodash/isFunction'; import isEmpty from 'lodash/isEmpty'; import uniq from 'lodash/uniq'; +// key separator check +import KEYSEPARATOR_CHECK from '../commons/keySeparator_check'; + // validation for filename option export const FILENAME_CHECK = async (argv: any) => { let filename: unknown = argv['filename']; @@ -81,4 +84,9 @@ export const RESULTSFILTER_CHECK = async (argv: any) => { }; // export checks in expected order into a single array -export const CHECKS = [FILENAME_CHECK, FILES_CHECK, RESULTSFILTER_CHECK]; +export const CHECKS = [ + KEYSEPARATOR_CHECK, + FILENAME_CHECK, + FILES_CHECK, + RESULTSFILTER_CHECK, +]; diff --git a/src/checks/import/import_common_checks.ts b/src/checks/import/import_common_checks.ts index 9aff7ba..f4d72cb 100644 --- a/src/checks/import/import_common_checks.ts +++ b/src/checks/import/import_common_checks.ts @@ -1,6 +1,9 @@ // lodash methodes import uniq from 'lodash/uniq'; +// key separator check +import KEYSEPARATOR_CHECK from '../commons/keySeparator_check'; + // validation for locales option export const LOCALES_CHECK = async (argv: any) => { const locales = argv.locales as any[]; @@ -14,4 +17,4 @@ export const LOCALES_CHECK = async (argv: any) => { }; // export checks in expected order into a single array -export const CHECKS = [LOCALES_CHECK]; +export const CHECKS = [KEYSEPARATOR_CHECK, LOCALES_CHECK]; diff --git a/test/diff.test.ts b/test/diff.test.ts index e890e0e..2ec6bb6 100644 --- a/test/diff.test.ts +++ b/test/diff.test.ts @@ -220,6 +220,13 @@ const VALIDATIONS_SCENARIOS: [ [[TEST_FILE_FILE1]], 'At least two paths must be provided', ], + [ + // Test out the message : "Option keySeparator should be a not-empty char" + 'Option keySeparator - Invalid separator should be rejected', + [[TEST_FILE_FILE1, TEST_FILE_FILE2], '--keySeparator', `"HACKERMAN"`], + 'keySeparator', + 'not-empty char', + ], ]; // E2E scenarios for JSON reporter diff --git a/test/export/export-csv.test.ts b/test/export/export-csv.test.ts index 683ede9..9b83285 100644 --- a/test/export/export-csv.test.ts +++ b/test/export/export-csv.test.ts @@ -375,6 +375,18 @@ const VALIDATIONS_SCENARIOS: [string, string[], ...string[]][] = [ 'test.csv', 'extension', ], + [ + // Test out the message : "Option keySeparator should be a not-empty char" + 'Option keySeparator - Invalid separator should be rejected', + [ + TEST_FILE_FILES, + TEST_FILE_EXPORT_COLUMNS, + '--keySeparator', + `"HACKERMAN"`, + ], + 'keySeparator', + 'not-empty char', + ], [ // Test out the message : "Option files is not a JSON Object" 'Option files - unexpected file should be rejected', diff --git a/test/export/export-xlsx.test.ts b/test/export/export-xlsx.test.ts index b7dbc7d..9a1be3c 100644 --- a/test/export/export-xlsx.test.ts +++ b/test/export/export-xlsx.test.ts @@ -459,6 +459,18 @@ const VALIDATIONS_SCENARIOS: [string, string[], ...string[]][] = [ 'test.xlsx', 'extension', ], + [ + // Test out the message : "Option keySeparator should be a not-empty char" + 'Option keySeparator - Invalid separator should be rejected', + [ + TEST_FILE_FILES, + TEST_FILE_EXPORT_COLUMNS, + '--keySeparator', + `"HACKERMAN"`, + ], + 'keySeparator', + 'not-empty char', + ], [ // Test out the message : "Option files is not a JSON Object" 'Option files - unexpected file should be rejected', diff --git a/test/import/import-csv.test.ts b/test/import/import-csv.test.ts index 31a1ce8..4901b7c 100644 --- a/test/import/import-csv.test.ts +++ b/test/import/import-csv.test.ts @@ -151,6 +151,19 @@ const VALIDATIONS_SCENARIOS: [ // I have to disable the error message check as yargs is buggy atm //"doesn't contain uniq values" ], + [ + // Test out the message : "Option keySeparator should be a not-empty char" + 'Option keySeparator - Invalid separator should be rejected', + [ + TEST_FILE_INPUT, + TEST_FILE_COLUMNS, + ['FR', 'NL'], + '--keySeparator', + `"HACKERMAN"`, + ], + 'keySeparator', + 'not-empty char', + ], [ // Test out the message : 'columns is not a JSON Object' 'Option columns - unexpected file should be rejected', diff --git a/test/import/import-xlsx.test.ts b/test/import/import-xlsx.test.ts index 124bb5b..e23e373 100644 --- a/test/import/import-xlsx.test.ts +++ b/test/import/import-xlsx.test.ts @@ -153,6 +153,19 @@ const VALIDATIONS_SCENARIOS: [ // I have to disable the error message check as yargs is buggy atm //"doesn't contain uniq values" ], + [ + // Test out the message : "Option keySeparator should be a not-empty char" + 'Option keySeparator - Invalid separator should be rejected', + [ + TEST_FILE_INPUT, + TEST_FILE_COLUMNS, + ['FR', 'NL'], + '--keySeparator', + `"HACKERMAN"`, + ], + 'keySeparator', + 'not-empty char', + ], [ // Test out the message : 'columns is not a JSON Object' 'Option columns - unexpected file should be rejected', From 34296be4ba18a366041deb6b119b23822597d7d0 Mon Sep 17 00:00:00 2001 From: jy95 Date: Wed, 6 Oct 2021 22:08:00 +0200 Subject: [PATCH 04/10] feat: enhance keySeparator option to allow false Last possibility to support i18next --- src/checks/commons/keySeparator_check.ts | 8 +++-- src/cmds/diff/detectChanges.ts | 4 +-- src/cmds/export_cmds/export_commons.ts | 4 +-- src/cmds/import_cmds/import_commons.ts | 4 +-- src/commons/commandBuilder.ts | 16 +++++---- src/commons/enhancedGet.ts | 12 ++++--- src/commons/enhancedSet.ts | 12 ++++--- src/commons/getLeavesPathes.ts | 16 +++++---- src/middlewares/middlewares.ts | 12 +++++++ src/types/diffTypes.ts | 2 +- src/types/exportTypes.ts | 2 +- src/types/importTypes.ts | 2 +- test/getLeavesPathes.test.ts | 43 ++++++++++++++++++++++-- 13 files changed, 102 insertions(+), 35 deletions(-) diff --git a/src/checks/commons/keySeparator_check.ts b/src/checks/commons/keySeparator_check.ts index 12c44a9..d367cf3 100644 --- a/src/checks/commons/keySeparator_check.ts +++ b/src/checks/commons/keySeparator_check.ts @@ -1,7 +1,11 @@ const KEYSEPARATOR_CHECK = async (argv: any) => { let keySeparator = argv.keySeparator as any; - if (keySeparator.length !== 1) { - return new Error(`Option keySeparator should be a not-empty char`); + let check = [ + () => keySeparator.length !== 1, + () => keySeparator === true, + ].some((pred) => pred()); + if (check) { + return new Error(`Option keySeparator should be a not-empty char or false`); } else { return true; } diff --git a/src/cmds/diff/detectChanges.ts b/src/cmds/diff/detectChanges.ts index 805f15c..cc539e9 100644 --- a/src/cmds/diff/detectChanges.ts +++ b/src/cmds/diff/detectChanges.ts @@ -27,7 +27,7 @@ function createChangeOperation( op: ChangesOps, file1: fileParam, file2: fileParam, - keySeparator: string + keySeparator: string | false ): ChangeOperations { // common part let obj: CommonChangeOperation = { @@ -60,7 +60,7 @@ export default function detectChanges( argv: CommonDiffArguments ): ChangeOperations[] { let result: ChangeOperations[] = []; - let keySeparator: string = argv.keySeparator; + let keySeparator: string | false = argv.keySeparator; // Fetch keys let files: fileParam[] = argv.files.map((file, idx) => ({ diff --git a/src/cmds/export_cmds/export_commons.ts b/src/cmds/export_cmds/export_commons.ts index f08a875..47e5f89 100644 --- a/src/cmds/export_cmds/export_commons.ts +++ b/src/cmds/export_cmds/export_commons.ts @@ -96,7 +96,7 @@ export function merge_i18n_files( // read file and turning into a useful array of objects function readFile( [locale, file_path]: [string, PathLike], - keySeparator: string + keySeparator: string | false ): Promise { return new Promise((resolve, reject) => { fs.promises @@ -112,7 +112,7 @@ function readFile( function i18n_to_result_format( obj: I18N_Object, locale: string, - keySeparator: string + keySeparator: string | false ): I18N_Result { let leafPaths = getLeavesPathes(obj, keySeparator); return leafPaths.map((leafPath) => ({ diff --git a/src/cmds/import_cmds/import_commons.ts b/src/cmds/import_cmds/import_commons.ts index fc1d0c9..dc3f987 100644 --- a/src/cmds/import_cmds/import_commons.ts +++ b/src/cmds/import_cmds/import_commons.ts @@ -69,7 +69,7 @@ export function generate_i18n_filepaths(argv: CommonImportArguments) { export function extractedTranslations_to_i18n_files( files: { [x: string]: string }, translations: extractedTranslation[], - keySeparator: string + keySeparator: string | false ) { let groupBy_locales = groupBy(translations, 'locale'); return Promise.all( @@ -104,7 +104,7 @@ function write_new_i18n_file( // Turns array for a given lang into a i18n js object function translations_2_i18n_object( translations: extractedTranslation[], - keySeparator: string + keySeparator: string | false ) { let result = {}; translations.forEach((item) => { diff --git a/src/commons/commandBuilder.ts b/src/commons/commandBuilder.ts index 3e69b57..6e08d39 100644 --- a/src/commons/commandBuilder.ts +++ b/src/commons/commandBuilder.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'fs'; import { extname, resolve } from 'path'; +import { parseUnknownToFalse } from '../middlewares/middlewares'; // For typing // eslint-disable-next-line @@ -38,12 +39,15 @@ export default class CommandBuilder { } addKeySeparatorOption() { - this.y = this.y.option('keySeparator', { - type: 'string', - alias: 'ks', - describe: 'Char to separate i18n keys', - default: '.', - }); + this.y = this.y + .option('keySeparator', { + type: 'string', + alias: 'ks', + describe: 'Char to separate i18n keys', + default: '.', + }) + // parse false values + .middleware(parseUnknownToFalse('keySeparator'), true); return this; } diff --git a/src/commons/enhancedGet.ts b/src/commons/enhancedGet.ts index 54bb3d7..21697f3 100644 --- a/src/commons/enhancedGet.ts +++ b/src/commons/enhancedGet.ts @@ -5,12 +5,14 @@ import get from 'lodash/get'; export default function enhancedGet( obj: any, key: string, - keySeparator: string + keySeparator: string | false ) { // compute path that use dot (or custom) separator + square brack notation - let path = key - // handle square brack notation - eg: a[10] should be translated as a.10 - .replace(/\[(\d+)\]/g, `${keySeparator}$1`) - .split(keySeparator); + let path = keySeparator + ? key + // handle square brack notation - eg: a[10] should be translated as a.10 + .replace(/\[(\d+)\]/g, `${keySeparator}$1`) + .split(keySeparator) + : [key]; return get(obj, path); } diff --git a/src/commons/enhancedSet.ts b/src/commons/enhancedSet.ts index 3c788a2..d5317bb 100644 --- a/src/commons/enhancedSet.ts +++ b/src/commons/enhancedSet.ts @@ -6,12 +6,14 @@ export default function enhancedSet( obj: any, key: string, val: any, - keySeparator: string + keySeparator: string | false ) { // compute path that use dot (or custom) separator + square brack notation - let path = key - // handle square brack notation - eg: a[10] should be translated as a.10 - .replace(/\[(\d+)\]/g, `${keySeparator}$1`) - .split(keySeparator); + let path = keySeparator + ? key + // handle square brack notation - eg: a[10] should be translated as a.10 + .replace(/\[(\d+)\]/g, `${keySeparator}$1`) + .split(keySeparator) + : [key]; return set(obj, path, val); } diff --git a/src/commons/getLeavesPathes.ts b/src/commons/getLeavesPathes.ts index e794d1b..6b188dc 100644 --- a/src/commons/getLeavesPathes.ts +++ b/src/commons/getLeavesPathes.ts @@ -6,22 +6,26 @@ import reduce from 'lodash/reduce'; // Typescript code inspired by https://stackoverflow.com/a/55381003/6149867 export default function getLeavesPathes( dataObj: any, - keySeparator = '.' + keySeparator: string | false = '.' ): string[] { const reducer = (aggregator: string[], val: any, key: string) => { let paths = [key]; if (isObject(val)) { paths = reduce(val, reducer, []); - paths = paths.map((path) => key + keySeparator + path); + // In theory, no flatten i18n should arrive here + // Better prevent that cure, let have a backup scenario + paths = paths.map((path) => key + (keySeparator || '.') + path); } aggregator.push(...paths); return aggregator; }; - // Need to double escape stuff when using this constructor - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping - const arrayIndexRegEx = new RegExp(`\\${keySeparator}(\\d+)(?!\\w+)`, 'gi'); let paths = reduce(dataObj, reducer, []); - paths = paths.map((path) => path.replace(arrayIndexRegEx, '[$1]')); + if (keySeparator) { + // Need to double escape stuff when using this constructor + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + const arrayIndexRegEx = new RegExp(`\\${keySeparator}(\\d+)(?!\\w+)`, 'gi'); + paths = paths.map((path) => path.replace(arrayIndexRegEx, '[$1]')); + } return paths; } diff --git a/src/middlewares/middlewares.ts b/src/middlewares/middlewares.ts index ae88652..758dfaa 100644 --- a/src/middlewares/middlewares.ts +++ b/src/middlewares/middlewares.ts @@ -72,3 +72,15 @@ export function parsePathToFunction(prop: string) { return argv; }; } + +// Turn unknown into false, if possible +export function parseUnknownToFalse(prop: string) { + return async (argv: any) => { + let param = argv[prop] as unknown; + let check = ['false', false].some((pred) => pred === param); + if (check) { + argv[prop] = false; + } + return argv; + }; +} diff --git a/src/types/diffTypes.ts b/src/types/diffTypes.ts index 2169b3c..7ace514 100644 --- a/src/types/diffTypes.ts +++ b/src/types/diffTypes.ts @@ -13,7 +13,7 @@ export interface CommonDiffArguments extends Argv { // output format outputFormat: 'JSON'; // https://github.com/jy95/i18n-tools/issues/25 - keySeparator: '.' | string; + keySeparator: '.' | string | false; } // for exporter(s) diff --git a/src/types/exportTypes.ts b/src/types/exportTypes.ts index e5cb3c9..7716e22 100644 --- a/src/types/exportTypes.ts +++ b/src/types/exportTypes.ts @@ -20,7 +20,7 @@ export interface CommonExportArguments extends Argv { outputDir: string; resultsFilter?: string | ((x: I18N_Merged_Data) => I18N_Merged_Data); // https://github.com/jy95/i18n-tools/issues/25 - keySeparator: '.' | string; + keySeparator: '.' | string | false; } // Yargs export arguments for TO_XLSX command export interface XLSXExportArguments extends CommonExportArguments { diff --git a/src/types/importTypes.ts b/src/types/importTypes.ts index 6bcf845..7aace98 100644 --- a/src/types/importTypes.ts +++ b/src/types/importTypes.ts @@ -8,7 +8,7 @@ export interface CommonImportArguments extends Argv { suffix: string; locales: string[]; // https://github.com/jy95/i18n-tools/issues/25 - keySeparator: '.' | string; + keySeparator: '.' | string | false; } // Yargs import arguments for FROM_XLSX command diff --git a/test/getLeavesPathes.test.ts b/test/getLeavesPathes.test.ts index 51c6783..594d5f9 100644 --- a/test/getLeavesPathes.test.ts +++ b/test/getLeavesPathes.test.ts @@ -87,6 +87,37 @@ const PATH_SCENARIOS_2: [string, any, string[]][] = PATH_SCENARIOS.map( } ); +// Scenarios for keySeparator is set to false +const PATH_SCENARIOS_3: [string, any, string[]][] = [ + [ + 'Simple keys', + { + key: 42, + verylooooogKey: 'Hello world', + }, + ['key', 'verylooooogKey'], + ], + [ + 'Keys with special characters', + { + 'Hello.world !': 42, + '$x.y_42-z~5!': 'jy95', + }, + ['Hello.world !', '$x.y_42-z~5!'], + ], + [ + 'Nested JSON with separator set to false - backup strategy', + { + lol: { + test: { + world: 42, + }, + }, + }, + ['lol.test.world'], + ], +]; + describe('[commons - getLeavesPathes] dot separator', () => { test.each(PATH_SCENARIOS)( '%s', @@ -97,8 +128,6 @@ describe('[commons - getLeavesPathes] dot separator', () => { ); }); -// Might be lazy but better to support same tests that the dot separator XD -// Later WET instead of DRY ? describe('[commons - getLeavesPathes] custom separator', () => { test.each(PATH_SCENARIOS_2)( '%s', @@ -108,3 +137,13 @@ describe('[commons - getLeavesPathes] custom separator', () => { } ); }); + +describe('[commons - getLeavesPathes] separator set to false', () => { + test.each(PATH_SCENARIOS_3)( + '%s', + async (_title: string, obj: any, expectedArray: string[]) => { + const paths = getLeavesPathes(obj, false); + expect(paths.sort()).toEqual(expectedArray.sort()); + } + ); +}); From 1bffd469f16bfb5a909c048e4ff3323f7af80271 Mon Sep 17 00:00:00 2001 From: jy95 Date: Wed, 6 Oct 2021 22:43:22 +0200 Subject: [PATCH 05/10] test: Update diff.test.ts to deal with keySeparator set to false --- test/diff.test.ts | 53 +++++++++++++++++++++++++++++++++++- test/getLeavesPathes.test.ts | 3 +- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/test/diff.test.ts b/test/diff.test.ts index 2ec6bb6..90fa24d 100644 --- a/test/diff.test.ts +++ b/test/diff.test.ts @@ -96,6 +96,9 @@ const test_files_list = [ 'file1.json', 'file2.json', 'file3.json', + // flat json, to test + 'flat_file1.json', + 'flat_file2.json', // to test out the json reporter 'settings1-JSON.json', 'settings2-JSON.json', @@ -106,6 +109,8 @@ const [ TEST_FILE_FILE1, TEST_FILE_FILE2, TEST_FILE_FILE3, + TEST_FILE_FLAT_FILE1, + TEST_FILE_FLAT_FILE2, TEST_FILE_JSON_SETTINGS1, TEST_FILE_JSON_SETTINGS2, TEST_FILE_JSON_SETTINGS3, @@ -118,7 +123,7 @@ const TEST_FILES: { [x in test_files_type]: string } = test_files_list.reduce( acc[curr] = path.resolve( TEMP_FOLDER, ROOT_TEST_FOLDER, - idx < 7 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, + idx < 9 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, curr ); return acc; @@ -189,6 +194,23 @@ const structure: fsify_structure = [ ), }), }, + // flat files + { + type: fsify.FILE, + name: TEST_FILE_FLAT_FILE1, + contents: JSON.stringify({ + 'unchanged.key_with-special-char!': 'unchanged', + 'changed.key_test$': 'Hello', + }), + }, + { + type: fsify.FILE, + name: TEST_FILE_FLAT_FILE2, + contents: JSON.stringify({ + 'unchanged.key_with-special-char!': 'unchanged', + 'changed.key_test$': 'Bonjour', + }), + }, // js file { type: fsify.FILE, @@ -383,6 +405,35 @@ const E2E_JSON_REPORTER: [ ], }, ], + [ + 'should work with flat json', + [ + [TEST_FILE_FLAT_FILE1, TEST_FILE_FLAT_FILE2], + '--filename', + `"diff_flat_inline-JSON"`, + '--outputDir', + `"${TEMP_FOLDER}"`, + "--keySeparator", + `"false"`, + ], + path.resolve(TEMP_FOLDER, 'diff_flat_inline-JSON.json'), + { + files: { + file1: TEST_FILES[TEST_FILE_FLAT_FILE1], + file2: TEST_FILES[TEST_FILE_FLAT_FILE2], + }, + changes: [ + { + from: 'file1', + key: 'changed.key_test$', + newValue: 'Bonjour', + oldValue: 'Hello', + to: 'file2', + type: 'REPLACED', + }, + ], + }, + ], ]; beforeAll(() => { diff --git a/test/getLeavesPathes.test.ts b/test/getLeavesPathes.test.ts index 594d5f9..c89a59f 100644 --- a/test/getLeavesPathes.test.ts +++ b/test/getLeavesPathes.test.ts @@ -102,8 +102,9 @@ const PATH_SCENARIOS_3: [string, any, string[]][] = [ { 'Hello.world !': 42, '$x.y_42-z~5!': 'jy95', + '[Hello].[World]|42': 'Hello', }, - ['Hello.world !', '$x.y_42-z~5!'], + ['Hello.world !', '$x.y_42-z~5!', '[Hello].[World]|42'], ], [ 'Nested JSON with separator set to false - backup strategy', From 05c7957a42b9a932a24d133262d5c9b16e415295 Mon Sep 17 00:00:00 2001 From: jy95 Date: Wed, 6 Oct 2021 23:11:36 +0200 Subject: [PATCH 06/10] test: Update export-xlsx.test.ts to deal with keySeparator set to false --- test/diff.test.ts | 2 +- test/export/export-xlsx.test.ts | 62 +++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/test/diff.test.ts b/test/diff.test.ts index 90fa24d..b9da267 100644 --- a/test/diff.test.ts +++ b/test/diff.test.ts @@ -413,7 +413,7 @@ const E2E_JSON_REPORTER: [ `"diff_flat_inline-JSON"`, '--outputDir', `"${TEMP_FOLDER}"`, - "--keySeparator", + '--keySeparator', `"false"`, ], path.resolve(TEMP_FOLDER, 'diff_flat_inline-JSON.json'), diff --git a/test/export/export-xlsx.test.ts b/test/export/export-xlsx.test.ts index 9a1be3c..7211e08 100644 --- a/test/export/export-xlsx.test.ts +++ b/test/export/export-xlsx.test.ts @@ -84,11 +84,13 @@ const test_files_list = [ // correct files 'columns.json', 'files.json', + 'files-flat.json', 'resultsFilter.js', 'settings1.json', 'settings2.json', 'settings3.json', 'settings4.js', + 'settings5.js', // wrong files 'emptyObject.json', 'emptyArray.json', @@ -108,11 +110,13 @@ const test_files_list = [ const [ TEST_FILE_EXPORT_COLUMNS, TEST_FILE_FILES, + TEST_FILE_FLAT_FILES, TEST_FILE_RESULTSFILTER, TEST_FILE_SETTINGS1, TEST_FILE_SETTINGS2, TEST_FILE_SETTINGS3, TEST_FILE_SETTINGS4, + TEST_FILE_SETTINGS5, TEST_FILE_EMPTY_OBJECT, TEST_FILE_EMPTY_ARRAY, TEST_FILE_FILES_DUP, @@ -139,12 +143,21 @@ const structure: fsify_structure = [ type: fsify.DIRECTORY, name: VALID_TEST_FOLDER, contents: flat([ - // 3 i18n files + // 3 i18n files (deep) TRANSLATIONS_KEYS.map((locale) => ({ type: fsify.FILE, name: `${locale.toLowerCase()}.json`, contents: JSON.stringify(generate_i18n(locale)), })), + // 3 i18n files (flat) + TRANSLATIONS_KEYS.map((locale) => ({ + type: fsify.FILE, + name: `${locale.toLowerCase()}-flat.json`, + contents: JSON.stringify({ + 'unchanged.key_with-special-char!': locale, + 'changed.key_test$': locale, + }), + })), // the columns.json { type: fsify.FILE, @@ -166,6 +179,21 @@ const structure: fsify_structure = [ ) ), }, + // the files-flat.json + { + type: fsify.FILE, + name: TEST_FILE_FLAT_FILES, + contents: JSON.stringify( + generate_files(TRANSLATIONS_KEYS, (locale) => + path.resolve( + TEMP_FOLDER, + ROOT_TEST_FOLDER, + VALID_TEST_FOLDER, + `${locale.toLowerCase()}-flat.json` + ) + ) + ), + }, // resultsFilter.js { type: fsify.FILE, @@ -277,6 +305,35 @@ const structure: fsify_structure = [ } `, }, + // Second format of settings.js (keySeparator) + { + type: fsify.FILE, + name: TEST_FILE_SETTINGS5, + // As fsify uses fs.writeFile, we need to double backslash stuff again + contents: `module.exports = { + "files": "${path + .resolve( + TEMP_FOLDER, + ROOT_TEST_FOLDER, + VALID_TEST_FOLDER, + TEST_FILE_FLAT_FILES + ) + .replace(/\\/g, '\\\\')}", + "columns": "${path + .resolve( + TEMP_FOLDER, + ROOT_TEST_FOLDER, + VALID_TEST_FOLDER, + TEST_FILE_EXPORT_COLUMNS + ) + .replace(/\\/g, '\\\\')}", + "worksheetName": "Settings 5 - Worksheet", + "filename": 'settings5-output', + "outputDir": "${TEMP_FOLDER.replace(/\\/g, '\\\\')}", + "keySeparator": false + } + `, + }, ]), }, // In this folder, files used for validations @@ -385,7 +442,7 @@ const TEST_FILES: { [x in test_files_type]: string } = test_files_list.reduce( let arr = [ TEMP_FOLDER, ROOT_TEST_FOLDER, - idx < 7 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, + idx < 9 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, curr, ]; acc[curr] = path.resolve(...arr); @@ -657,6 +714,7 @@ describe('[export_xlsx command]', () => { 'settings.js (Include worksheetCustomizer / resultsFilter as fct)', TEST_FILE_SETTINGS4, ], + ['settings.js (keySeparator set to false)', TEST_FILE_SETTINGS5], ])('%s', async (_title: string, settingsFile: test_files_type) => { let test_cmd = concat_cmd([ '--settings', From 4584738736883d948b01f4e76f00c8172283988e Mon Sep 17 00:00:00 2001 From: jy95 Date: Wed, 6 Oct 2021 23:41:42 +0200 Subject: [PATCH 07/10] test: Update export-xlsx.csv.ts to deal with keySeparator set to false --- test/export/export-csv.test.ts | 59 ++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/test/export/export-csv.test.ts b/test/export/export-csv.test.ts index 9b83285..976c9bc 100644 --- a/test/export/export-csv.test.ts +++ b/test/export/export-csv.test.ts @@ -84,9 +84,11 @@ const test_files_list = [ // correct files 'columns.json', 'files.json', + 'files-flat.json', 'settings1.json', 'settings2.json', 'settings3.js', + 'settings4.js', // wrong files 'emptyObject.json', 'emptyArray.json', @@ -100,9 +102,11 @@ const test_files_list = [ const [ TEST_FILE_EXPORT_COLUMNS, TEST_FILE_FILES, + TEST_FILE_FLAT_FILES, TEST_FILE_SETTINGS1, TEST_FILE_SETTINGS2, TEST_FILE_SETTINGS3, + TEST_FILE_SETTINGS4, TEST_FILE_EMPTY_OBJECT, TEST_FILE_EMPTY_ARRAY, TEST_FILE_FILES_DUP, @@ -125,12 +129,21 @@ const structure: fsify_structure = [ type: fsify.DIRECTORY, name: VALID_TEST_FOLDER, contents: flat([ - // 3 i18n files + // 3 i18n files (deep) TRANSLATIONS_KEYS.map((locale) => ({ type: fsify.FILE, name: `${locale.toLowerCase()}.json`, contents: JSON.stringify(generate_i18n(locale)), })), + // 3 i18n files (flat) + TRANSLATIONS_KEYS.map((locale) => ({ + type: fsify.FILE, + name: `${locale.toLowerCase()}-flat.json`, + contents: JSON.stringify({ + 'unchanged.key_with-special-char!': locale, + 'changed.key_test$': locale, + }), + })), // the columns.json { type: fsify.FILE, @@ -152,6 +165,21 @@ const structure: fsify_structure = [ ) ), }, + // the files-flat.json + { + type: fsify.FILE, + name: TEST_FILE_FLAT_FILES, + contents: JSON.stringify( + generate_files(TRANSLATIONS_KEYS, (locale) => + path.resolve( + TEMP_FOLDER, + ROOT_TEST_FOLDER, + VALID_TEST_FOLDER, + `${locale.toLowerCase()}-flat.json` + ) + ) + ), + }, // First format of settings.json (Path) { type: fsify.FILE, @@ -217,6 +245,32 @@ const structure: fsify_structure = [ "outputDir": "${TEMP_FOLDER.replace(/\\/g, '\\\\')}" }`, }, + // Second format of settings.js (keySeparator) + { + type: fsify.FILE, + name: TEST_FILE_SETTINGS4, + contents: `module.exports = { + "files": "${path + .resolve( + TEMP_FOLDER, + ROOT_TEST_FOLDER, + VALID_TEST_FOLDER, + TEST_FILE_FLAT_FILES + ) + .replace(/\\/g, '\\\\')}", + "columns": "${path + .resolve( + TEMP_FOLDER, + ROOT_TEST_FOLDER, + VALID_TEST_FOLDER, + TEST_FILE_EXPORT_COLUMNS + ) + .replace(/\\/g, '\\\\')}", + "filename": 'settings4-output', + "outputDir": "${TEMP_FOLDER.replace(/\\/g, '\\\\')}", + "keySeparator": false + }`, + }, ]), }, // In this folder, files used for validations @@ -301,7 +355,7 @@ const TEST_FILES: { [x in test_files_type]: string } = test_files_list.reduce( let arr = [ TEMP_FOLDER, ROOT_TEST_FOLDER, - idx < 5 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, + idx < 7 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, curr, ]; acc[curr] = path.resolve(...arr); @@ -522,6 +576,7 @@ describe('[export_csv command]', () => { ['settings.json (Paths)', TEST_FILE_SETTINGS1], ['settings.json (Object/Array instead of Paths)', TEST_FILE_SETTINGS2], ['settings.js (Include resultsFilter as fct)', TEST_FILE_SETTINGS3], + ['settings.js (keySeparator set to false)', TEST_FILE_SETTINGS4], ])('%s', async (_title: string, settingsFile: test_files_type) => { let test_cmd = concat_cmd([ '--settings', From 4b3dd6aa870c194b34cad402f518133d63274848 Mon Sep 17 00:00:00 2001 From: jy95 Date: Thu, 7 Oct 2021 00:05:30 +0200 Subject: [PATCH 08/10] test: Update import-xlsx.test.ts to deal with keySeparator set to false --- .../import-xlsx/export-flat-xlsx.xlsx | Bin 0 -> 6555 bytes test/import/import-xlsx.test.ts | 33 ++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/import-xlsx/export-flat-xlsx.xlsx diff --git a/test/fixtures/import-xlsx/export-flat-xlsx.xlsx b/test/fixtures/import-xlsx/export-flat-xlsx.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..61b5c6d2b2ba6efc43d741430120b5625503aadc GIT binary patch literal 6555 zcmai22Q=L4(!1CqD1ekUV;!riB7cWH4!yhM9n6M7Hrhj z^R2j&+}wM=|9j5v*|R&(J9D0y-^}czu7rY$jf8}RiG*G(rM17fyxL6}c=&kePDh{7bfyl7^$h5!lH%oxyttnu(^dtuzITuj|S zXHwT{89>s4qZrgHKP2<^m%P|fNd*NE1W$cg%&?V3!zbV2?|X)L|ebl+${JdqmnhNlNbNnH^#PKcO- zHQNeuc$;Uo@(6a)jgUihywat~w-B5G&Xv?@P!ciFSZ588!lsAUV7Y`6lt;lW!1Q&p z4A}wW>{M0xrIDN%6gIHSl#v46+WxUV(N;JW-idwHqyX%6KfG39kF{Q6)?BszJTSB;`-q`Gfj*y&bY_;?H!H4YYCfXd9+rBr~NFJ z%2bhoUK6mVVS3BCtm-2x#=0MxPsNL}YrK@X%(w33>db%-<&nZT6q_vB;D}6 z@n%M75WS}Xmc=R07_;DrFVYnw|6~mX!N?=~pOpR60PE7=-7@cFxc+mNw^}~wwsIaF6<1u8gA%es0d7O1qR+Xf}@1g)+^g9SPMFw!F4F*Z(`vkL@W4dhiy;$F7Cq1r!}uJ^I*wX(4ffcuE^P*I z!M{aj_5j^9SU6_&NHK&VJ8Y1XB1^;)?VhnEP5g&NJ*79UA78s3KNG_*HNqu`*8Sur zeKHJ`{MHhAbVxe5n8;;Hx7kOxb7P6`Saztl_=eZ^;yTTL6ODgK&;>QF)*z7EpJnP5 zL5`1#AaWkjRLK8f`vby#!DmXcF?g1iBn$>;BZT9fzr9C1UD0Itp2=Zs!hMFnb4u>D z1g8Ah#s^pkUqX_chEK0BXNwpHzNKen7L%P#R**!|HcQ^!gKMns1%=OcPanA|Meep# zz6vv0c4MPzWoxsftm3Rl86)OfHC8o(4Fz#C8%{e5UZeNXwymOqlh1?+TQF93<*C*% zL=~1TmK6Dfhn_`XOw<8hqE@e4!r)B0wp&Edy8c(eM$8iCADmBK@(5lN2_{u4NP(hj zDT-7de%yO-q-il-x&0(M*c0>Pw5OuJlllqKOlx3UrPuH>ZdO#8rhezvL}ZG)Dr`|U zT~pW6Y|G-oDM*3Qu5IibO8}a0YZ#};my4=vQfhJ@Z&$3*Z{wEXgGA+m?55$Mw1!Hp zr^=kPNXiIrdCP8(N$nexW`DRvvH1G1tI0Z9Io@Dn`QxIcvT)1?+2m^X1n%v^$5gi4 z#RpsyiCX11ZPzp`B0MdXCAy#Bp1!e)k#TT&BQ#NX@xddV`wZ7YofVp&t1- zTa3LEEKG2RxdF0HHTn|Ynpbh7t+#6`7>jE43oUKdilE z$6Tv~LuK^8?7;sg;!kHB-EAV;MAe9E(rHD}Y_XX1Ca zvC^fxbWJ89Bmo@m$S1mO8=mj$NP0*`DT$@otY{krtTG~GZDef=tf;{$w^q!mw2O$U zgup<8RHbwuSqt>p1dLB~B|78xi#ySiy9%%x%ZwGc%?_VDa3yriA0>%-Sc9>`EkyYm zWzevCbp3LB!A8-EMpung8r#Zq^_N(9d>vNuWn27nBY$K5o!l&T~XR zyfpjq{^P0JG*AK03Xr$K^az_8cG5JNUugBj&O18H&^n9m$t%_NgaObJRTpMQs4;NK zC-ItbryRn~i~>og3QqW^MeM0Y+N<;?!-1K$PnqV`X@!Mu?Tf+4pm0VT({|^ZJZ|zK z`C{w(`=$Ii32(npLr!De{e@nQ)yHcK?Qks;t?6ssDyPh(G47>grKOX)6SsuEr5~ud z9p<1!R#@HAZvp6R<|d=lL?+%FELO-N)qOX=F4AS~ zF*;2`V$F@T$j6?AS}Xb1upEgK&Z3vSs3JXO7J(dbG{t>?9=z|9#$yHbt9TD~ZYZw1Zozm}%}W91Bt z3UMpSD-rfU_6K|-`b-R^;-yX7V~o1IV22>y9eKenM$5D1<|^9(MbBApMXtuXj5W+_lIgE zO~rinH_80Z4ktpOfJga@;d~5$dB%Izap9kyDWKhmyNu%%M*&SsPemXFQ8CGMr@K+bJgEfylRk#h{==$;NyBJ$$zSi1M zBmFPu3}JJHa=EPm$;$Jfs&7-wL>eq!y#$!t+tiwdQ80XWv^3wc5t*o_CmATb`ShTN z^LcdaBNNv2W!^w(Q3CZe-seFMF8X28p${ICDCz?<1h?;g9dcL0Ge4Bx)4mVc0NP=5Y2l zTM@gI;G0GNR9gm*%e}%Zlc3CxNDs$HWAx@I7Lcf23qbjt-|PcW^`2>7-@p(;H{REV zaJn9~-Q`Ym0#%n>&)2iSK7YrQwZMb(sLbP4+OVKf!<@9H$R0?+n3w2fQ~z?^ZfRN3 zREqm#7IxOKjfjJIhq@6av#H(02Z~~}c-bdJoCYz<@@@JspD*kDkJqkk5RJDi+qE8@ zFMNGEni?JLGeqb)tM0>YNS^mRwr-2>CQhT!mcE5hSf>SfuyQ&d%i!q?i$>O6_s|*R zJ-^mlYwP?)Ge zezH8)T22)`)wV|jJ!1XjY3jgrB|@ul~a1;&D8DGh~< z?|Fu)<(A^B%rsZ+6u8()mNuE=6r=7N%FpVD(EEQwzL{hQYl$0rUpHJ|P`@`XGpjxM z4%}zk++0|z@!7B2Ye2QW1=8!ZFV%;!fu#LPS(KUJh}sgJH5P#Bvacx0HfipHTHoL< zNo@9gv%1tc1&hqj`~3p6Eb4o);yYshUP8Z>iPhP#jQYB0O_BTO&arXSfoQW%SeaG! zif_0EvprffX*p0U7d^Rrz;dxRh39`Jw1+Lqjs$QMbxZjr)#geh>yU$Wl8w4Yfb@#O; z*`1nxbIQ=G_{rovnmWI!oDsg<{`ZlLCK_Pkfmr5H5K{&3uaWHLWe@r>iwh7NTqf6# z4UQeEg@VT(8DMJ@rCprK%Ae%%H9(>480E%zg31X@6deF?pD8@(@2!k`wtLLmxR!s1 zbCB!PMg%(eMGQg7(rmJYnaz{fW{z4)LLygMFxOtc^lBWUQJu zMREnZ(4}0aqD)S?F{LE$jaqcwzV2)Hinvcb7fkWVg-*3XS`5?R8l_{phOpZRIT)Y+ ziG~wDy7Du4+(uSzIBm=@(;nW$yscK3&eMZM7Et7alu)cN*g-{I6O-eJ;nXTSP$h>YhzdseIf zS)8XM?Pu->w$SZ|z2WHbAbQ$?m}sgYNfpDWcOt=Oe&0I|8sW0!C(rUa;5&Ab-yU*j z`RpEw?Ch{$e%Q8mC;Kv*4qt_|ho}WT;GwyFJ5AZ?HOD4+SVxFq^&CXB^ax;i+*Sj< zD$sx5j4Z(x>MmfXKmU{zCeb@YBGyV?#NR|hL>>P$GjnpfSSz!Vq|_D>G6(F0jU8%K zR28k8Vo}RlCd8hniSyJdOTi*xD#0pZ=4@Z6m|`rHxIvIt?VahH*tU;?b*o&7I1*{m zUQB4-l-MhtjlOf|*=NZ?63dZfi|s|L&gJtW4%k%=!EJ zx|)tP`k~euZgm!exvkrihAQDAXbvN7a=jQyHvG5u-$+ABN$eG0VAXkpS4Bhotn8c; zVzwSl=YAnXYxK9VBhx%Z>!M~(qq>Wc{bkgH9o@L0XH+cwQ2P3rrPYHRsulaux&pL$~l#bMX8kc_m zVym+o-v{>MT+Qd_jCYmY%Y_!NW+}u3k+wxd3nKbGZ5Ch`(2u-n4qV7nEpWD5 z(BLFmI+9S@y6Z)^bXZ^LfGb*2(|YPR3h4YzSEP_qw7{aGHH(jhjIv934w1j+>*4^^ zZUF)o4CQs$-+I(wuP`{hO@2_ITSbc}^m34f$5FH5c8j4yM;O$~}mUA7(=zTou%!gS@Od!qVyXJJ&tRFgIAXH%n{B5vB?OC=UTOzXtAl#Wm zu0{n=G%5y3dRM=ac^qxVFSY~jt?A@~4z&;ZBmX@a#S06gmWX5oB9cLHISYQ*M<9tx zY6uOg7v>wq0x#dpRxi&AL9!v^i$q>wcm81Q1yq0^9=zI&3VF2w==C_=g(~QNjn-X4 zm5{f?&VXnsb~v%+3zO`*!UeqZ8SXwr)tbB`RY|zonPxJrV3IhZ_rVbM8A-j z7z+(7JQ5cgI<0o5cdV1?F=4$?NtFy3+8yfDe|Lag)MmLG7&Gf{eY>%tQ`b4>Mdn-o z-G?@bptHAMN)oZ#j*XZH8$AnWIidisrG2mS1hKFjXdmGEUu$!I2hm;?QyB-U^h498 zyl>^0L!v~r`A4=@$ZGfoSUOXQPA3P57xi!(>av=gYVwT)^M;7VfyV*l+svpu%=9}D zNE7{hT*MQJKH6`vwdFZj+f`6Lq;JoeWjKC-RYk?+)Q<2O_jB#65r0 zz^+2S>|B4UU{`1)B%Oa?e`#XB82ficxTbDVKK&!8?JPOzlZ;) zlJLv%$nUUUy23BNmy!RuIR3ie7J|x)$bYVpzo7ps%YVP z8u)*r|C}_x(7Ax$oLPl!oo;5NJm C+^jbM literal 0 HcmV?d00001 diff --git a/test/import/import-xlsx.test.ts b/test/import/import-xlsx.test.ts index e23e373..9f72a6c 100644 --- a/test/import/import-xlsx.test.ts +++ b/test/import/import-xlsx.test.ts @@ -95,11 +95,13 @@ async function expectError(cmd: string, ...messages: string[]) { const test_files_list = [ // inpput file 'export-xlsx.xlsx', + 'export-flat-xlsx.xlsx', // correct files 'columns.json', 'settings1.json', 'settings2.json', 'settings3.js', + 'settings4.js', // wrong files 'emptyObject.json', 'emptyArray.json', @@ -110,10 +112,12 @@ const test_files_list = [ ] as const; const [ TEST_FILE_INPUT, + TEST_FILE_FLAT_INPUT, TEST_FILE_COLUMNS, TEST_FILE_SETTINGS1, TEST_FILE_SETTINGS2, TEST_FILE_SETTINGS3, + TEST_FILE_SETTINGS4, TEST_FILE_EMPTY_OBJECT, TEST_FILE_EMPTY_ARRAY, TEST_FILE_COLUMNS_TKNS, @@ -126,12 +130,12 @@ type test_files_type = typeof test_files_list[number]; const TEST_FILES: { [x in test_files_type]: string } = test_files_list.reduce( (acc: any, curr: test_files_type, idx: number) => { let arr = - idx === 0 + idx === 0 || idx === 1 ? [__dirname, '..', 'fixtures', 'import-xlsx', curr] : [ TEMP_FOLDER, ROOT_TEST_FOLDER, - idx > 0 && idx < 5 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, + idx > 0 && idx < 7 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, curr, ]; acc[curr] = path.resolve(...arr); @@ -280,6 +284,30 @@ const structure: fsify_structure = [ "suffix": '_settings3', }`, }, + // Second format of settings.js (keySeparator) + { + type: fsify.FILE, + name: TEST_FILE_SETTINGS4, + // As fsify uses fs.writeFile, we need to double backslash stuff again + contents: `module.exports = { + "input": "${TEST_FILES[TEST_FILE_FLAT_INPUT].replace( + /\\/g, + '\\\\' + )}", + "columns": { + "technical_key": 'Technical Key', + "locales": { + "FR": 'French translation', + "NL": 'Dutch translation', + "DE": 'German translation' + } + }, + "locales": ['FR', 'NL', 'DE'], + "outputDir": "${TEMP_FOLDER.replace(/\\/g, '\\\\')}", + "suffix": '_settings4', + "keySeparator": false + }`, + }, ], }, // In this folder, files used for validations @@ -416,6 +444,7 @@ describe('[import_xlsx command]', () => { ['(Paths)', TEST_FILE_SETTINGS1], ['(Object/Array instead of Paths)', TEST_FILE_SETTINGS2], ['should work with js config file', TEST_FILE_SETTINGS3], + ['(keySeparator set to false)', TEST_FILE_SETTINGS4], ])('settings %s', async (_title: string, settingsFile: test_files_type) => { let test_cmd = concat_cmd([ '--settings', From 4af6e2b0330f1bf7879f605cff3b8a14336354a9 Mon Sep 17 00:00:00 2001 From: jy95 Date: Thu, 7 Oct 2021 00:16:00 +0200 Subject: [PATCH 09/10] test: Update import-csv.test.ts to deal with keySeparator set to false --- test/fixtures/import-csv/export-flat-csv.csv | 3 ++ test/import/import-csv.test.ts | 29 ++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/import-csv/export-flat-csv.csv diff --git a/test/fixtures/import-csv/export-flat-csv.csv b/test/fixtures/import-csv/export-flat-csv.csv new file mode 100644 index 0000000..c79a5ef --- /dev/null +++ b/test/fixtures/import-csv/export-flat-csv.csv @@ -0,0 +1,3 @@ +Technical Key;French translation;Dutch translation;German translation +changed.key_test$;FR;NL;DE +unchanged.key_with-special-char!;FR;NL;DE \ No newline at end of file diff --git a/test/import/import-csv.test.ts b/test/import/import-csv.test.ts index 4901b7c..3427bc6 100644 --- a/test/import/import-csv.test.ts +++ b/test/import/import-csv.test.ts @@ -95,10 +95,12 @@ async function expectError(cmd: string, ...messages: string[]) { const test_files_list = [ // inpput file 'export-csv.csv', + 'export-flat-csv.csv', // correct files 'columns.json', 'settings1.json', 'settings2.json', + 'settings3.json', // wrong files 'emptyObject.json', 'emptyArray.json', @@ -109,9 +111,11 @@ const test_files_list = [ ] as const; const [ TEST_FILE_INPUT, + TEST_FILE_FLAT_INPUT, TEST_FILE_COLUMNS, TEST_FILE_SETTINGS1, TEST_FILE_SETTINGS2, + TEST_FILE_SETTINGS3, TEST_FILE_EMPTY_OBJECT, TEST_FILE_EMPTY_ARRAY, TEST_FILE_COLUMNS_TKNS, @@ -124,12 +128,12 @@ type test_files_type = typeof test_files_list[number]; const TEST_FILES: { [x in test_files_type]: string } = test_files_list.reduce( (acc: any, curr: test_files_type, idx: number) => { let arr = - idx === 0 + idx === 0 || idx === 1 ? [__dirname, '..', 'fixtures', 'import-csv', curr] : [ TEMP_FOLDER, ROOT_TEST_FOLDER, - idx > 0 && idx < 4 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, + idx > 0 && idx < 6 ? VALID_TEST_FOLDER : USELESS_TEST_FOLDER, curr, ]; acc[curr] = path.resolve(...arr); @@ -258,6 +262,26 @@ const structure: fsify_structure = [ suffix: '_settings2', }), }, + // Third format of settings.json (keySeparator) + { + type: fsify.FILE, + name: TEST_FILE_SETTINGS3, + contents: JSON.stringify({ + input: TEST_FILES[TEST_FILE_FLAT_INPUT], + columns: { + technical_key: 'Technical Key', + locales: { + FR: 'French translation', + NL: 'Dutch translation', + DE: 'German translation', + }, + }, + locales: ['FR', 'NL', 'DE'], + outputDir: path.resolve(TEMP_FOLDER, ROOT_TEST_FOLDER), + suffix: '_settings3', + keySeparator: false, + }), + }, ], }, // In this folder, files used for validations @@ -393,6 +417,7 @@ describe('[import_csv command]', () => { test.each([ ['(Paths)', TEST_FILE_SETTINGS1], ['(Object/Array instead of Paths)', TEST_FILE_SETTINGS2], + ['(keySeparator set to false)', TEST_FILE_SETTINGS3], ])( 'settings.json %s', async (_title: string, settingsFile: test_files_type) => { From bc6016d78b444708780226ea5cdc4e104a856e32 Mon Sep 17 00:00:00 2001 From: jy95 Date: Thu, 7 Oct 2021 00:21:07 +0200 Subject: [PATCH 10/10] docs: Update keySeparator option description --- src/commons/commandBuilder.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commons/commandBuilder.ts b/src/commons/commandBuilder.ts index 6e08d39..9fb5915 100644 --- a/src/commons/commandBuilder.ts +++ b/src/commons/commandBuilder.ts @@ -43,7 +43,8 @@ export default class CommandBuilder { .option('keySeparator', { type: 'string', alias: 'ks', - describe: 'Char to separate i18n keys', + describe: + 'Char to separate i18n keys. If working with flat JSON, set this to false', default: '.', }) // parse false values