feat: add export to_csv & import from_csv (#10)

* feat: add export CSV

* test: update files with new export format

* docs: export to_csv

* feat: add import CSV

* docs: import from_csv

* test: import from_csv

* test: test from_csv

Fix imports

* test: fix imports

* test: fix imports

* fix: import from_csv

Cannot use fast-csv option "headers" = true with exceljs

README.md

@@ -1,9 +1,9 @@
-# @jy95/i18n-tools [![codecov](https://codecov.io/gh/jy95/i18n-tools/branch/master/graph/badge.svg?token=PQDE2R2GYR)](https://codecov.io/gh/jy95/i18n-tools)
+# @jy95/i18n-tools [![codecov](https://codecov.io/gh/jy95/i18n-tools/branch/master/graph/badge.svg?token=PQDE2R2GYR)](https://codecov.io/gh/jy95/i18n-tools) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/95593519673143d6a1e475c1d2c4332c)](https://www.codacy.com/gh/jy95/i18n-tools)
 
 CLI to make common operations around i18n files simpler.
 
-- 👩‍💻 Export i18n files into something else (xlsx, ...)
-- ✨ Turn a file (xlsx, ...) to i18n file(s)
+- 👩‍💻 Export i18n files into something else (xlsx, csv, ...)
+- ✨ Turn a file (xlsx, csv, ...) to i18n file(s)
 - 📜 Compare at least two i18n files and generate a report
 - ...
 

package.json

@@ -70,11 +70,11 @@
     "@types/yargs": "^17.0.2",
     "fsify": "^4.0.2",
     "husky": "^7.0.2",
+    "semantic-release": "^17.4.7",
     "size-limit": "^5.0.3",
     "tsdx": "^0.14.1",
     "tslib": "^2.3.1",
-    "typescript": "^4.3.5",
-    "semantic-release": "^17.4.7"
+    "typescript": "^4.4.3"
   },
   "dependencies": {
     "exceljs": "^4.2.1",

src/checks/export/index.ts

@@ -2,4 +2,8 @@
 export * from "./export_common_checks";
 
 // check for xlsx sub command
-export * as XLSX from "./export_xlsx_checks";
+export * as XLSX from "./export_xlsx_checks";
+
+// check for csv sub command
+// as it is identical (at that time) to xlsx, simply re-export same module
+export * as CSV from "./export_xlsx_checks"; 

src/checks/import/index.ts

@@ -2,4 +2,8 @@
 export * from "./import_common_checks";
 
 // check for xlsx sub command
-export * as XLSX from "./import_xlsx_checks";
+export * as XLSX from "./import_xlsx_checks";
+
+// check for csv sub command
+// as it is identical (at that time) to xlsx, simply re-export same module
+export * as CSV from "./import_xlsx_checks";

src/cmds/export.ts

@@ -1,5 +1,6 @@
 // export command
 import export_xlsx from './export_cmds/export_xlsx';
+import export_csv from './export_cmds/export_csv';
 
 // named exports
 export const command = 'export <command>';
@@ -10,6 +11,7 @@ export const builder = function(y: any) {
     y
       // commandDir doesn't work very well in Typescript
       .command(export_xlsx)
+      .command(export_csv)
   );
 };
 /* istanbul ignore next */

src/cmds/export_cmds/export_csv.ts

@@ -0,0 +1,139 @@
+// for fs ops
+import path from 'path';
+import Excel from 'exceljs';
+
+// common fct
+import { merge_i18n_files, setUpCommonsOptions } from './export_commons';
+import { parsePathToJSON } from '../../middlewares/middlewares';
+
+// checks import
+import { resolveChecksInOrder, EXPORT_CHECKS } from '../../checks/index';
+
+// For typing
+// eslint-disable-next-line
+import { Argv } from "yargs";
+import { CSVExportArguments, I18N_Merged_Data } from '../../types/exportTypes';
+
+// checks for this command
+const CHECKS = [...EXPORT_CHECKS.CHECKS, ...EXPORT_CHECKS.CSV.CHECKS];
+
+// named exports
+export const command = 'to_csv';
+export const description = 'Export i18n files into a csv file';
+
+export const builder = function(y: Argv) {
+  return (
+    setUpCommonsOptions(y) // set up common options for export
+      .option('columns', {
+        description:
+          'Absolute path to a JSON array of objects, to control the columns. Example : [{ "locale": "FR", "label": "French translation" }]',
+        demandOption: true,
+      })
+      .option('delimiter', {
+        description: 'Specify an field delimiter such as | or \\t',
+        choices: [',', ';', '\t', ' ', '|'],
+        default: ';',
+      })
+      .option('rowDelimiter', {
+        description: 'Specify an alternate row delimiter (i.e \\r\\n)',
+        type: 'string',
+        default: '\n',
+      })
+      .option('quote', {
+        description: 'String to quote fields that contain a delimiter',
+        type: 'string',
+        default: '"',
+      })
+      .option('escape', {
+        description:
+          'The character to use when escaping a value that is quoted and contains a quote character that is not the end of the field',
+        type: 'string',
+        default: '"',
+      })
+      .option('writeBOM', {
+        description:
+          'Set to true if you want the first character written to the stream to be a utf-8 BOM character.',
+        type: 'boolean',
+        default: false,
+      })
+      .option('quoteHeaders', {
+        description: 'If true then all headers will be quoted',
+        type: 'boolean',
+        default: true,
+      })
+      // coerce columns into Object
+      .middleware(parsePathToJSON('columns'), true)
+      // validations
+      .check(resolveChecksInOrder(CHECKS))
+  );
+};
+
+export const handler = async function(argv: CSVExportArguments) {
+  try {
+    let data: I18N_Merged_Data = await merge_i18n_files(argv);
+    const CSV_FILE = path.resolve(argv.outputDir, argv.filename + '.csv');
+    await export_as_csv(CSV_FILE, argv, data);
+    console.log(`${CSV_FILE} successfully written`);
+    return Promise.resolve(undefined);
+  } catch (/* istanbul ignore next */ err) {
+    return Promise.reject(err);
+  }
+};
+
+// write
+async function export_as_csv(
+  CSV_FILE: string,
+  argv: CSVExportArguments,
+  data: I18N_Merged_Data
+) {
+  console.log('Preparing CSV file ...');
+
+  // prepare data
+  const workbook = new Excel.Workbook();
+  let worksheet = workbook.addWorksheet();
+
+  // Set up columns
+  worksheet.columns = [
+    { header: 'Technical Key', key: 'technical_key' },
+  ].concat(
+    argv.columns.map(({ label, locale }) => ({
+      header: label,
+      key: `labels.${locale}`,
+    }))
+  );
+
+  // workaround as Exceljs doesn't support nested key
+  worksheet.addRows(
+    data.map(item =>
+      argv.columns.reduce(
+        (acc: { [x: string]: string }, { locale }) => {
+          acc[`labels.${locale}`] = item['labels'][locale] || '';
+          return acc;
+        },
+        { technical_key: item['technical_key'] }
+      )
+    )
+  );
+
+  // finally write this file
+  const options = {
+    // https://c2fo.io/fast-csv/docs/formatting/options
+    formatterOptions: {
+      delimiter: argv.delimiter,
+      rowDelimiter: argv.rowDelimiter,
+      quote: argv.quote,
+      escape: argv.escape,
+      writeBOM: argv.writeBOM,
+      quoteHeaders: argv.quoteHeaders,
+    },
+  };
+  return workbook.csv.writeFile(CSV_FILE, options);
+}
+
+// default export
+export default {
+  command: command,
+  description: description,
+  builder: builder,
+  handler: handler,
+};

src/cmds/import.ts

@@ -1,5 +1,6 @@
 // import command
 import import_xlsx from './import_cmds/import_xlsx';
+import import_csv from './import_cmds/import_csv';
 
 // named exports
 export const command = 'import <command>';
@@ -10,6 +11,7 @@ export const builder = function(y: any) {
     y
       // commandDir doesn't work very well in Typescript
       .command(import_xlsx)
+      .command(import_csv)
   );
 };
 

src/cmds/import_cmds/import_csv.ts

@@ -0,0 +1,141 @@
+import Excel from 'exceljs';
+
+// common fct
+import { setUpCommonsOptions, generate_i18n_filepaths, extractedTranslations_to_i18n_files } from "./import_commons";
+import { parsePathToJSON } from "../../middlewares/middlewares";
+
+// lodash methods
+import flattenDeep from "lodash/flattenDeep";
+
+// checks import
+import { 
+    resolveChecksInOrder,
+    IMPORT_CHECKS
+} from "../../checks/index";
+
+// For typing
+// eslint-disable-next-line
+import type { Argv } from "yargs";
+import { CSVImportArguments } from "../../types/importTypes";
+
+// checks for this command
+const CHECKS = [...IMPORT_CHECKS.CHECKS, ...IMPORT_CHECKS.CSV.CHECKS];
+
+// named exports
+export const command = "from_csv";
+export const description = "Turn a csv file to i18n file(s)";
+
+export const builder = function (y : Argv) {
+    return setUpCommonsOptions(y) // set up common options for import
+        .options("columns", {
+            describe: "Absolute path to a JSON object that describe headers of the excel columns used to store translations",
+            demandOption: true
+        })
+        .option('delimiter', {
+            description: 'Specify an field delimiter such as | or \\t',
+            choices: [',', ';', '\t', ' ', '|'],
+            default: ';',
+        })
+        .option('quote', {
+            description: 'String used to quote fields that contain a delimiter',
+            type: 'string',
+            default: '"',
+          })
+        .option('escape', {
+            description:
+                'The character used when escaping a value that is quoted and contains a quote character that is not the end of the field',
+            type: 'string',
+            default: '"',
+        })
+        .option('encoding', {
+            description: "Input file encoding",
+            choices: ['utf8', 'utf16le', 'latin1'],
+            default: 'utf8'
+        })
+        // coerce columns into Object
+        .middleware(parsePathToJSON("columns"), true)
+        // validations
+        .check(resolveChecksInOrder(CHECKS))
+}
+
+export const handler = async function (argv : CSVImportArguments) {
+    try {
+        const translations = await csv_2_translation_objects(argv);
+        const files = generate_i18n_filepaths(argv);
+        await extractedTranslations_to_i18n_files(files, translations);
+        console.log("Successfully exported found locale(s) to i18n json file(s)");
+        return Promise.resolve(undefined);
+    } catch (error) {
+        return Promise.reject(error);
+    }
+}
+
+// Extract translations from csv file
+async function csv_2_translation_objects(argv : CSVImportArguments) {
+    const options = {
+        // https://c2fo.io/fast-csv/docs/parsing/options
+        parserOptions: {
+            delimiter: argv.delimiter,
+            quote: argv.quote,
+            escape: argv.escape,
+            encoding: argv.encoding
+        }
+    };
+    const workbook = new Excel.Workbook();
+    const worksheet = await workbook.csv.readFile(argv.input, options);
+    let rowCount = worksheet.rowCount;
+
+    // columns properties to load
+    let columns = argv.columns;
+
+    // retrieve the headers of the table
+    // Warning : Exceljs put for some reason a undefined value at the 0 index
+    let headers = worksheet.getRow(1).values as (undefined | string)[];
+    // retrieve data of the table
+    let data = (worksheet.getRows(2, rowCount-1) || /* istanbul ignore next */ []).map(item => item.values);
+
+    // find out where the technical key is
+    const technical_key_index = headers.findIndex(h => (h || '').includes(columns.technical_key));
+
+    if (technical_key_index === -1) {
+        return Promise.reject(new Error("Couldn't find index for technical_key with provided label"));
+    }
+
+    // find out where the translations are positioned in the value
+    const locales_index = Object
+        .entries(columns.locales)
+        .map( ([key, value]) => ({ [key]: headers.findIndex(h => (h || '').includes(value)) }))
+        .reduce( (prev, curr) => Object.assign(prev, curr), {})
+
+    // Warn users if some locale translations couldn't be found
+    let missing_indexes = Object
+        .entries(locales_index)
+        .filter( ([_, idx]) => idx === -1);
+
+    for(let [locale, ] of missing_indexes) {
+        /* istanbul ignore next Not worthy to create a test case for that*/
+        console.warn(`Couldn't find index for ${locale} locale with provided label`)
+    }
+
+    // build results
+    let results = data.map(
+        (row : any) => Object
+            .entries(locales_index)
+            // skip translation(s) where index couldn't be found
+            .filter( ([_, idx]) => idx !== -1)
+            .map( ([locale, localeIndex]) => ({
+                "technical_key": row[technical_key_index],
+                "label": row[localeIndex],
+                "locale": locale
+            }))
+    )
+    return Promise.resolve(flattenDeep(results));
+}
+
+// default export
+export default {
+    command : command,
+    description: description,
+    builder : builder,
+    handler: handler
+}