From ed53d52c80d5deb263ddc443b6bb8690c620f674 Mon Sep 17 00:00:00 2001 From: Michael Rodrigues Date: Mon, 31 May 2021 05:58:11 +0000 Subject: [PATCH 1/2] Add `unwindIgnoreEmptyArrays` option. In order to convert the generated CSV back to the most accurate JSON possible, the module must still insert `[]` for unwound empty array values. However, there are some legitimate cases where users want to be able to have the module ignore these values so that the generated CSV is a bit more user-friendly, depending on the data being converted. This new option allows users the ability to achieve this without impacting existing use cases, as the user has opted to specify an option which may impact the ability to convert the CSV back to JSON later on. Fixes #168 --- lib/constants.json | 1 + lib/json2csv.js | 5 ++-- lib/utils.js | 8 ++--- test/config/testCsvFilesList.js | 1 + test/config/testJsonFilesList.js | 1 + test/data/csv/unwindIgnoreEmptyArrays.csv | 3 ++ test/data/json/unwindIgnoreEmptyArrays.json | 33 +++++++++++++++++++++ test/json2csv.js | 19 ++++++++---- 8 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 test/data/csv/unwindIgnoreEmptyArrays.csv create mode 100644 test/data/json/unwindIgnoreEmptyArrays.json diff --git a/lib/constants.json b/lib/constants.json index ddc4c7c..b49cb64 100644 --- a/lib/constants.json +++ b/lib/constants.json @@ -33,6 +33,7 @@ "checkSchemaDifferences": false, "expandArrayObjects": false, "unwindArrays": false, + "unwindIgnoreEmptyArrays": false, "useDateIso8601Format": false, "useLocaleFormat": false, "parseValue": null, diff --git a/lib/json2csv.js b/lib/json2csv.js index 008e18d..797f24e 100755 --- a/lib/json2csv.js +++ b/lib/json2csv.js @@ -13,7 +13,8 @@ const Json2Csv = function(options) { deeksOptions = { expandArrayObjects: expandingWithoutUnwinding, ignoreEmptyArraysWhenExpanding: expandingWithoutUnwinding, - escapeNestedDots: true + escapeNestedDots: true, + ignoreEmptyArrays: options.unwindIgnoreEmptyArrays }; /** HEADER FIELD FUNCTIONS **/ @@ -201,7 +202,7 @@ const Json2Csv = function(options) { // Unwind each of the documents at the given headerField params.headerFields.forEach((headerField) => { - params.records = utils.unwind(params.records, headerField); + params.records = utils.unwind(params.records, headerField, options); }); return retrieveHeaderFields(params.records) diff --git a/lib/utils.js b/lib/utils.js index 3671df6..ac3744a 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -216,7 +216,7 @@ function getNCharacters(str, start, n) { * @param item {any} * @param fieldPath {String} */ -function unwindItem(accumulator, item, fieldPath) { +function unwindItem(accumulator, item, fieldPath, options) { const valueToUnwind = path.evaluatePath(item, fieldPath); let cloned = deepCopy(item); @@ -227,7 +227,7 @@ function unwindItem(accumulator, item, fieldPath) { }); } else if (Array.isArray(valueToUnwind) && valueToUnwind.length === 0) { // Push an empty string so the value is empty since there are no values - path.setPath(cloned, fieldPath, ''); + path.setPath(cloned, fieldPath, options.unwindIgnoreEmptyArrays ? [] : ''); accumulator.push(cloned); } else { accumulator.push(cloned); @@ -240,10 +240,10 @@ function unwindItem(accumulator, item, fieldPath) { * @param field {String} * @returns {Array} */ -function unwind(array, field) { +function unwind(array, field, options) { const result = []; array.forEach((item) => { - unwindItem(result, item, field); + unwindItem(result, item, field, options); }); return result; } diff --git a/test/config/testCsvFilesList.js b/test/config/testCsvFilesList.js index cb115ec..117f4b9 100644 --- a/test/config/testCsvFilesList.js +++ b/test/config/testCsvFilesList.js @@ -32,6 +32,7 @@ const fs = require('fs'), {key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv'}, {key: 'unwind', file: '../data/csv/unwind.csv'}, {key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv'}, + {key: 'unwindIgnoreEmptyArrays', file: '../data/csv/unwindIgnoreEmptyArrays.csv'}, {key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv'}, {key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv'}, {key: 'localeFormat', file: '../data/csv/localeFormat.csv'}, diff --git a/test/config/testJsonFilesList.js b/test/config/testJsonFilesList.js index fb2f351..ef34bdd 100644 --- a/test/config/testJsonFilesList.js +++ b/test/config/testJsonFilesList.js @@ -25,6 +25,7 @@ module.exports = { csvEmptyLastValue: require('../data/json/csvEmptyLastValue'), unwind: require('../data/json/unwind'), unwindEmptyArray: require('../data/json/unwindEmptyArray'), + unwindIgnoreEmptyArrays: require('../data/json/unwindIgnoreEmptyArrays'), localeFormat: require('../data/json/localeFormat'), invalidParsedValues: require('../data/json/invalidParsedValues'), firstColumnWrapCRLF: require('../data/json/firstColumnWrapCRLF.json'), diff --git a/test/data/csv/unwindIgnoreEmptyArrays.csv b/test/data/csv/unwindIgnoreEmptyArrays.csv new file mode 100644 index 0000000..a4302a1 --- /dev/null +++ b/test/data/csv/unwindIgnoreEmptyArrays.csv @@ -0,0 +1,3 @@ +entities.id,entities.name,entities.siblings.entity.name,entities.siblings.relationship_set.relationships.relationship.name +4,Zeus,Exo,Senior +4,Zeus,Chrono, \ No newline at end of file diff --git a/test/data/json/unwindIgnoreEmptyArrays.json b/test/data/json/unwindIgnoreEmptyArrays.json new file mode 100644 index 0000000..0c77e43 --- /dev/null +++ b/test/data/json/unwindIgnoreEmptyArrays.json @@ -0,0 +1,33 @@ +[ + { + "entities": + { + "id": 4, + "name": "Zeus", + "siblings": [ + { + "entity": { + "name": "Exo" + }, + "relationship_set": { + "relationships": [ + { + "relationship": { + "name": "Senior" + } + } + ] + } + }, + { + "entity": { + "name": "Chrono" + }, + "relationship_set": { + "relationships": [] + } + } + ] + } + } +] \ No newline at end of file diff --git a/test/json2csv.js b/test/json2csv.js index d0a4530..5835046 100644 --- a/test/json2csv.js +++ b/test/json2csv.js @@ -656,15 +656,24 @@ function runTests(jsonTestData, csvTestData) { }); }); + // Test case for #168 + it('should ignore empty arrays when specified when unwinding arrays', (done) => { + converter.json2csv(jsonTestData.unwindIgnoreEmptyArrays, (err, csv) => { + if (err) done(err); + csv.should.equal(csvTestData.unwindIgnoreEmptyArrays); + done(); + }, { + expandArrayObjects: true, + unwindArrays: true, + unwindIgnoreEmptyArrays: true + }); + }); + // Test case for #184 it('should handle keys with nested dots when expanding and unwinding arrays', (done) => { converter.json2csv(jsonTestData.nestedDotKeysWithArrayExpandedUnwound, (err, csv) => { if (err) done(err); - - // Replace raw boolean values with quoted versions - let expectedCsv = csvTestData.nestedDotKeysWithArrayExpandedUnwound; - - csv.should.equal(expectedCsv); + csv.should.equal(csvTestData.nestedDotKeysWithArrayExpandedUnwound); done(); }, { expandArrayObjects: true, From 7424e7a21aa16bfca88d5f0abff67bf104bcb318 Mon Sep 17 00:00:00 2001 From: Michael Rodrigues Date: Tue, 1 Jun 2021 03:13:46 +0000 Subject: [PATCH 2/2] chore(release): 3.15.0 --- package-lock.json | 8 ++++---- package.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0b1ae81..b7fc59e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "json-2-csv", - "version": "3.14.2", + "version": "3.15.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -937,9 +937,9 @@ "dev": true }, "deeks": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/deeks/-/deeks-2.4.1.tgz", - "integrity": "sha512-18Zd9f6dqyAFDoPttDdD8w5u7xy9M1IoxhpSsaatFwI4O+wZyUK+rDtwul4s6iYSfUfZBXPfUbxZrKHDqpEQYA==" + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/deeks/-/deeks-2.5.0.tgz", + "integrity": "sha512-5QrXgMaxVnO4iwY2WYJI6MqpPH89Am6K/PAN7u7VKIkAUOuygJb/bqk6clflw+gZN8Kjexfo8xx40mo5UHrx5Q==" }, "deep-is": { "version": "0.1.3", diff --git a/package.json b/package.json index cc16c59..26cc36f 100755 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ }, "name": "json-2-csv", "description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.", - "version": "3.14.2", + "version": "3.15.0", "homepage": "https://mrodrig.github.io/json-2-csv", "repository": { "type": "git", @@ -39,7 +39,7 @@ "cli" ], "dependencies": { - "deeks": "2.4.1", + "deeks": "2.5.0", "doc-path": "3.0.1" }, "devDependencies": {