From 8118b860e8c17d9acf45a10d782e0be80c1d9b97 Mon Sep 17 00:00:00 2001 From: jzlai Date: Mon, 2 Dec 2019 14:55:55 +0100 Subject: [PATCH 1/5] allow exclusion of collections by path and regex --- build/index.js | 39 +++++++++++++++++++++++++++++++-------- index.js | 31 +++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/build/index.js b/build/index.js index 5fda656..fb7191c 100755 --- a/build/index.js +++ b/build/index.js @@ -43,6 +43,16 @@ var _FirestoreDocument = require('./lib/FirestoreDocument'); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } +function commaSeparatedList(value) { + return value.split(','); +} + +function commaSeparatedListAndRegExp(value) { + return value.split(',').map(function (entry) { + return new RegExp(entry); + }); +} + var accountCredentialsPathParamKey = 'accountCredentials'; var accountCredentialsPathParamDescription = 'Google Cloud account credentials JSON file'; @@ -60,6 +70,11 @@ var stableParamParamDescription = 'JSON backups done with stable-stringify'; var plainJSONBackupParamKey = 'plainJSONBackup'; var plainJSONBackupParamDescription = 'JSON backups done without preserving any type information\n - Lacks full fidelity restore to Firestore\n - Can be used for other export purposes'; +var excludeCollectionParamKey = 'excludeCollection'; +var excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]'; + +var excludePatternParamKey = 'excludePattern'; +var excludePatternParamDescription = 'Patterns to match against when backung up, e.g. [regex1],[regex2]'; var packagePath = __dirname.includes('/build') ? '..' : '.'; @@ -72,7 +87,7 @@ try { // or they can be merged with existing ones var mergeData = false; -_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' ', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' ', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' ', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).parse(_process2.default.argv); +_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' ', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' ', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' ', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).option('-e, --' + excludeCollectionParamKey + ' ', excludeCollectionParamDescription, commaSeparatedList).option('-e2, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp).parse(_process2.default.argv); var accountCredentialsPath = _commander2.default[accountCredentialsPathParamKey]; if (accountCredentialsPath && !_fs2.default.existsSync(accountCredentialsPath)) { @@ -101,6 +116,10 @@ var stable = _commander2.default[stableParamKey] !== undefined && _commander2.de var plainJSONBackup = _commander2.default[plainJSONBackupParamKey] !== undefined && _commander2.default[plainJSONBackupParamKey] !== null; +var excludeCollections = _commander2.default[excludeCollectionParamKey] || []; + +var excludePatterns = _commander2.default[excludePatternParamKey] || []; + var accountApp = accountCredentialsPath ? (0, _FirestoreFunctions.getFireApp)(accountCredentialsPath) : {}; try { @@ -159,13 +178,17 @@ var backupDocument = function backupDocument(document, backupPath, logPath) { }; var backupCollection = function backupCollection(collection, backupPath, logPath) { - console.log("Backing up Collection '" + logPath + collection.id + "'"); - - // TODO: implement feature to skip certain Collections - // if (collection.id.toLowerCase().indexOf('geotrack') > 0) { - // console.log(`Skipping ${collection.id}`); - // return promiseSerial([() => Promise.resolve()]); - // } + var collectionPath = logPath + collection.id; + console.log("Backing up Collection '" + collectionPath + "'"); + + if (excludeCollections.includes(collectionPath) || excludePatterns.some(function (pattern) { + return pattern.test(collectionPath); + })) { + console.log('Skipping ' + collection.id); + return promiseSerial([function () { + return Promise.resolve(); + }]); + } try { _mkdirp2.default.sync(backupPath); diff --git a/index.js b/index.js index 2bb5f85..abfc6cc 100755 --- a/index.js +++ b/index.js @@ -17,6 +17,14 @@ import { saveDocument } from './lib/FirestoreDocument'; +function commaSeparatedList(value) { + return value.split(',') +} + +function commaSeparatedListAndRegExp(value) { + return value.split(',').map(entry => new RegExp(entry)) +} + const accountCredentialsPathParamKey = 'accountCredentials'; const accountCredentialsPathParamDescription = 'Google Cloud account credentials JSON file'; @@ -38,6 +46,11 @@ const plainJSONBackupParamKey = 'plainJSONBackup'; const plainJSONBackupParamDescription = `JSON backups done without preserving any type information - Lacks full fidelity restore to Firestore - Can be used for other export purposes`; +const excludeCollectionParamKey = 'excludeCollection'; +const excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]'; + +const excludePatternParamKey = 'excludePattern'; +const excludePatternParamDescription = 'Exclude patterns to match against when backung up, e.g. [regex1],[regex2]' const packagePath = __dirname.includes('/build') ? '..' : '.'; @@ -65,6 +78,8 @@ commander .option('-S, --' + stableParamKey, stableParamParamDescription) .option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription) + .option('-e, --' + excludeCollectionParamKey + ' ', excludeCollectionParamDescription, commaSeparatedList) + .option('-e2, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp) .parse(process.argv); const accountCredentialsPath = commander[accountCredentialsPathParamKey]; @@ -116,6 +131,10 @@ const plainJSONBackup = commander[plainJSONBackupParamKey] !== undefined && commander[plainJSONBackupParamKey] !== null; +const excludeCollections = commander[excludeCollectionParamKey] || []; + +const excludePatterns = commander[excludePatternParamKey] || []; + const accountApp: Object = accountCredentialsPath ? getFireApp(accountCredentialsPath) : {}; @@ -219,13 +238,13 @@ const backupCollection = ( backupPath: string, logPath: string ): Promise => { - console.log("Backing up Collection '" + logPath + collection.id + "'"); + const collectionPath = logPath + collection.id + console.log("Backing up Collection '" + collectionPath + "'"); - // TODO: implement feature to skip certain Collections - // if (collection.id.toLowerCase().indexOf('geotrack') > 0) { - // console.log(`Skipping ${collection.id}`); - // return promiseSerial([() => Promise.resolve()]); - // } + if (excludeCollections.includes(collectionPath) || excludePatterns.some(pattern => pattern.test(collectionPath))) { + console.log(`Skipping ${collection.id}`); + return promiseSerial([() => Promise.resolve()]); + } try { mkdirp.sync(backupPath); From ad0e3828466762ef8a119cf30348ea935e5991d3 Mon Sep 17 00:00:00 2001 From: jzlai Date: Mon, 2 Dec 2019 15:01:03 +0100 Subject: [PATCH 2/5] git commit --amend --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index abfc6cc..eee1330 100755 --- a/index.js +++ b/index.js @@ -46,11 +46,11 @@ const plainJSONBackupParamKey = 'plainJSONBackup'; const plainJSONBackupParamDescription = `JSON backups done without preserving any type information - Lacks full fidelity restore to Firestore - Can be used for other export purposes`; -const excludeCollectionParamKey = 'excludeCollection'; +const excludeCollectionParamKey = 'excludeCollections'; const excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]'; const excludePatternParamKey = 'excludePattern'; -const excludePatternParamDescription = 'Exclude patterns to match against when backung up, e.g. [regex1],[regex2]' +const excludePatternParamDescription = 'Exclude patterns to match against when backing up, e.g. [regex1],[regex2]' const packagePath = __dirname.includes('/build') ? '..' : '.'; @@ -79,7 +79,7 @@ commander .option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription) .option('-e, --' + excludeCollectionParamKey + ' ', excludeCollectionParamDescription, commaSeparatedList) - .option('-e2, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp) + .option('-e2, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp) .parse(process.argv); const accountCredentialsPath = commander[accountCredentialsPathParamKey]; From 03ac890042a8cd8423a8e1dd2d22ea7a86545a93 Mon Sep 17 00:00:00 2001 From: jzlai Date: Mon, 2 Dec 2019 15:02:08 +0100 Subject: [PATCH 3/5] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0125bf6..8ec3a09 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ Options: * `-S`, `--stable` JSON backups done with stable-stringify. * `-J`, `--plainJSONBackup` JSON backups done without preserving any type information. - Lacks full fidelity restore to Firestore. - Can be used for other export purposes. * `-h`, `--help` output usage information +* `-e`, `--excludeCollections` Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[/collection3] +* `-e2`, `--excludePattern` Exclude patterns to match against when backing up, e.g. [regex1],[regex2] ### Backup: @@ -66,7 +68,7 @@ As of version 1.2, the default is to save files with each field converted to a ` Example backup: ```sh -firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase +firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludePattern '.*/collection1.*' ``` ### Clone: From 8c93bc67164e388a4d57f13318c153c4efbf9e4f Mon Sep 17 00:00:00 2001 From: jzlai Date: Mon, 2 Dec 2019 16:08:06 +0100 Subject: [PATCH 4/5] add pattern test for documents --- README.md | 2 +- build/index.js | 15 ++++++++++++--- index.js | 7 ++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8ec3a09..2fd4461 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Options: * `-J`, `--plainJSONBackup` JSON backups done without preserving any type information. - Lacks full fidelity restore to Firestore. - Can be used for other export purposes. * `-h`, `--help` output usage information * `-e`, `--excludeCollections` Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[/collection3] -* `-e2`, `--excludePattern` Exclude patterns to match against when backing up, e.g. [regex1],[regex2] +* `-E`, `--excludePattern` Exclude patterns to match against when backing up, e.g. [regex1],[regex2] ### Backup: diff --git a/build/index.js b/build/index.js index fb7191c..4c444b4 100755 --- a/build/index.js +++ b/build/index.js @@ -70,11 +70,11 @@ var stableParamParamDescription = 'JSON backups done with stable-stringify'; var plainJSONBackupParamKey = 'plainJSONBackup'; var plainJSONBackupParamDescription = 'JSON backups done without preserving any type information\n - Lacks full fidelity restore to Firestore\n - Can be used for other export purposes'; -var excludeCollectionParamKey = 'excludeCollection'; +var excludeCollectionParamKey = 'excludeCollections'; var excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]'; var excludePatternParamKey = 'excludePattern'; -var excludePatternParamDescription = 'Patterns to match against when backung up, e.g. [regex1],[regex2]'; +var excludePatternParamDescription = 'Exclude patterns to match against when backing up, e.g. [regex1],[regex2]'; var packagePath = __dirname.includes('/build') ? '..' : '.'; @@ -87,7 +87,7 @@ try { // or they can be merged with existing ones var mergeData = false; -_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' ', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' ', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' ', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).option('-e, --' + excludeCollectionParamKey + ' ', excludeCollectionParamDescription, commaSeparatedList).option('-e2, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp).parse(_process2.default.argv); +_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' ', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' ', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' ', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).option('-e, --' + excludeCollectionParamKey + ' ', excludeCollectionParamDescription, commaSeparatedList).option('-E, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp).parse(_process2.default.argv); var accountCredentialsPath = _commander2.default[accountCredentialsPathParamKey]; if (accountCredentialsPath && !_fs2.default.existsSync(accountCredentialsPath)) { @@ -145,6 +145,15 @@ var promiseSerial = function promiseSerial(funcs) { var backupDocument = function backupDocument(document, backupPath, logPath) { console.log("Backing up Document '" + logPath + document.id + "'" + (plainJSONBackup === true ? ' with -J --plainJSONBackup' : ' with type information')); + if (excludePatterns.some(function (pattern) { + return pattern.test(logPath + document.id); + })) { + console.log('Skipping ' + document.id); + return promiseSerial([function () { + return Promise.resolve(); + }]); + } + try { _mkdirp2.default.sync(backupPath); var fileContents = void 0; diff --git a/index.js b/index.js index eee1330..e584c6e 100755 --- a/index.js +++ b/index.js @@ -79,7 +79,7 @@ commander .option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription) .option('-e, --' + excludeCollectionParamKey + ' ', excludeCollectionParamDescription, commaSeparatedList) - .option('-e2, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp) + .option('-E, --' + excludePatternParamKey + ' ', excludePatternParamDescription, commaSeparatedListAndRegExp) .parse(process.argv); const accountCredentialsPath = commander[accountCredentialsPathParamKey]; @@ -181,6 +181,11 @@ const backupDocument = ( : ' with type information') ); + if(excludePatterns.some(pattern => pattern.test(logPath + document.id))){ + console.log(`Skipping ${document.id}`); + return promiseSerial([() => Promise.resolve()]) + } + try { mkdirp.sync(backupPath); let fileContents: string; From 38ce73ea9661760502e69d4e1fc390ba9ba49be2 Mon Sep 17 00:00:00 2001 From: jzlai Date: Mon, 2 Dec 2019 16:16:36 +0100 Subject: [PATCH 5/5] update readme --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2fd4461..eff820b 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ As of version 1.2, the default is to save files with each field converted to a ` Example backup: ```sh -firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludePattern '.*/collection1.*' +firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase ``` ### Clone: @@ -204,6 +204,25 @@ Document saved with type information (Default) identifier: { value: 'provider', type: 'string' } ``` +### Exclude collections + +The optional parameter allows `--excludeCollections` skipping of provided collections. This parameter accepts a comma seperated list of collections. + +Example: + +```sh +firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludeCollections /collection1/document/subcollectionToIgnore,/collectionToIgnore +``` + +### Exclude paths by regex +Skips documents or collections by pattern matching. All subpaths of matched paths will also be excluded. This parameter accepts a comma seperated list of regular expressions. + +Example: + +```sh +firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludePattern '^/collectionToIgnore,^/[^/]*/[^/]*/subcollectionToIgnore' +``` + ## Contributions Feel free to report bugs in the [Issue Tracker](https://github.com/willhlaw/node-firestore-backup-restore/issues), fork and create pull requests!