Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Exclude collections and documents #59

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Exclude collections and documents #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8118b86
allow exclusion of collections by path and regex
Dec 2, 2019
ad0e382
git commit --amend
Dec 2, 2019
03ac890
update readme
Dec 2, 2019
8c93bc6
add pattern test for documents
Dec 2, 2019
38ce73e
update readme
Dec 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ Options:
* `-S`, `--stable` JSON backups done with stable-stringify.
* `-J`, `--plainJSONBackup` JSON backups done without preserving any type information. - Lacks full fidelity restore to Firestore. - Can be used for other export purposes.
* `-h`, `--help` output usage information
* `-e`, `--excludeCollections` Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[/collection3]
* `-E`, `--excludePattern` Exclude patterns to match against when backing up, e.g. [regex1],[regex2]

### Backup:

Expand Down Expand Up @@ -202,6 +204,25 @@ Document saved with type information (Default)
identifier: { value: 'provider', type: 'string' }
```

### Exclude collections

The optional parameter allows `--excludeCollections` skipping of provided collections. This parameter accepts a comma seperated list of collections.

Example:

```sh
firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludeCollections /collection1/document/subcollectionToIgnore,/collectionToIgnore
```

### Exclude paths by regex
Skips documents or collections by pattern matching. All subpaths of matched paths will also be excluded. This parameter accepts a comma seperated list of regular expressions.

Example:

```sh
firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludePattern '^/collectionToIgnore,^/[^/]*/[^/]*/subcollectionToIgnore'
```

## Contributions

Feel free to report bugs in the [Issue Tracker](https://github.com/willhlaw/node-firestore-backup-restore/issues), fork and create pull requests!
48 changes: 40 additions & 8 deletions build/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ var _FirestoreDocument = require('./lib/FirestoreDocument');

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function commaSeparatedList(value) {
return value.split(',');
}

function commaSeparatedListAndRegExp(value) {
return value.split(',').map(function (entry) {
return new RegExp(entry);
});
}

var accountCredentialsPathParamKey = 'accountCredentials';
var accountCredentialsPathParamDescription = 'Google Cloud account credentials JSON file';

Expand All @@ -60,6 +70,11 @@ var stableParamParamDescription = 'JSON backups done with stable-stringify';

var plainJSONBackupParamKey = 'plainJSONBackup';
var plainJSONBackupParamDescription = 'JSON backups done without preserving any type information\n - Lacks full fidelity restore to Firestore\n - Can be used for other export purposes';
var excludeCollectionParamKey = 'excludeCollections';
var excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]';

var excludePatternParamKey = 'excludePattern';
var excludePatternParamDescription = 'Exclude patterns to match against when backing up, e.g. [regex1],[regex2]';

var packagePath = __dirname.includes('/build') ? '..' : '.';

Expand All @@ -72,7 +87,7 @@ try {
// or they can be merged with existing ones
var mergeData = false;

_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).parse(_process2.default.argv);
_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).option('-e, --' + excludeCollectionParamKey + ' <collections>', excludeCollectionParamDescription, commaSeparatedList).option('-E, --' + excludePatternParamKey + ' <regex>', excludePatternParamDescription, commaSeparatedListAndRegExp).parse(_process2.default.argv);

var accountCredentialsPath = _commander2.default[accountCredentialsPathParamKey];
if (accountCredentialsPath && !_fs2.default.existsSync(accountCredentialsPath)) {
Expand Down Expand Up @@ -101,6 +116,10 @@ var stable = _commander2.default[stableParamKey] !== undefined && _commander2.de

var plainJSONBackup = _commander2.default[plainJSONBackupParamKey] !== undefined && _commander2.default[plainJSONBackupParamKey] !== null;

var excludeCollections = _commander2.default[excludeCollectionParamKey] || [];

var excludePatterns = _commander2.default[excludePatternParamKey] || [];

var accountApp = accountCredentialsPath ? (0, _FirestoreFunctions.getFireApp)(accountCredentialsPath) : {};

try {
Expand All @@ -126,6 +145,15 @@ var promiseSerial = function promiseSerial(funcs) {
var backupDocument = function backupDocument(document, backupPath, logPath) {
console.log("Backing up Document '" + logPath + document.id + "'" + (plainJSONBackup === true ? ' with -J --plainJSONBackup' : ' with type information'));

if (excludePatterns.some(function (pattern) {
return pattern.test(logPath + document.id);
})) {
console.log('Skipping ' + document.id);
return promiseSerial([function () {
return Promise.resolve();
}]);
}

try {
_mkdirp2.default.sync(backupPath);
var fileContents = void 0;
Expand Down Expand Up @@ -159,13 +187,17 @@ var backupDocument = function backupDocument(document, backupPath, logPath) {
};

var backupCollection = function backupCollection(collection, backupPath, logPath) {
console.log("Backing up Collection '" + logPath + collection.id + "'");

// TODO: implement feature to skip certain Collections
// if (collection.id.toLowerCase().indexOf('geotrack') > 0) {
// console.log(`Skipping ${collection.id}`);
// return promiseSerial([() => Promise.resolve()]);
// }
var collectionPath = logPath + collection.id;
console.log("Backing up Collection '" + collectionPath + "'");

if (excludeCollections.includes(collectionPath) || excludePatterns.some(function (pattern) {
return pattern.test(collectionPath);
})) {
console.log('Skipping ' + collection.id);
return promiseSerial([function () {
return Promise.resolve();
}]);
}

try {
_mkdirp2.default.sync(backupPath);
Expand Down
36 changes: 30 additions & 6 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ import {
saveDocument
} from './lib/FirestoreDocument';

function commaSeparatedList(value) {
return value.split(',')
}

function commaSeparatedListAndRegExp(value) {
return value.split(',').map(entry => new RegExp(entry))
}

const accountCredentialsPathParamKey = 'accountCredentials';
const accountCredentialsPathParamDescription =
'Google Cloud account credentials JSON file';
Expand All @@ -38,6 +46,11 @@ const plainJSONBackupParamKey = 'plainJSONBackup';
const plainJSONBackupParamDescription = `JSON backups done without preserving any type information
- Lacks full fidelity restore to Firestore
- Can be used for other export purposes`;
const excludeCollectionParamKey = 'excludeCollections';
const excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]';

const excludePatternParamKey = 'excludePattern';
const excludePatternParamDescription = 'Exclude patterns to match against when backing up, e.g. [regex1],[regex2]'

const packagePath = __dirname.includes('/build') ? '..' : '.';

Expand Down Expand Up @@ -65,6 +78,8 @@ commander
.option('-S, --' + stableParamKey, stableParamParamDescription)

.option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription)
.option('-e, --' + excludeCollectionParamKey + ' <collections>', excludeCollectionParamDescription, commaSeparatedList)
.option('-E, --' + excludePatternParamKey + ' <regex>', excludePatternParamDescription, commaSeparatedListAndRegExp)
.parse(process.argv);

const accountCredentialsPath = commander[accountCredentialsPathParamKey];
Expand Down Expand Up @@ -116,6 +131,10 @@ const plainJSONBackup =
commander[plainJSONBackupParamKey] !== undefined &&
commander[plainJSONBackupParamKey] !== null;

const excludeCollections = commander[excludeCollectionParamKey] || [];

const excludePatterns = commander[excludePatternParamKey] || [];

const accountApp: Object = accountCredentialsPath
? getFireApp(accountCredentialsPath)
: {};
Expand Down Expand Up @@ -162,6 +181,11 @@ const backupDocument = (
: ' with type information')
);

if(excludePatterns.some(pattern => pattern.test(logPath + document.id))){
console.log(`Skipping ${document.id}`);
return promiseSerial([() => Promise.resolve()])
}

try {
mkdirp.sync(backupPath);
let fileContents: string;
Expand Down Expand Up @@ -219,13 +243,13 @@ const backupCollection = (
backupPath: string,
logPath: string
): Promise<void> => {
console.log("Backing up Collection '" + logPath + collection.id + "'");
const collectionPath = logPath + collection.id
console.log("Backing up Collection '" + collectionPath + "'");

// TODO: implement feature to skip certain Collections
// if (collection.id.toLowerCase().indexOf('geotrack') > 0) {
// console.log(`Skipping ${collection.id}`);
// return promiseSerial([() => Promise.resolve()]);
// }
if (excludeCollections.includes(collectionPath) || excludePatterns.some(pattern => pattern.test(collectionPath))) {
console.log(`Skipping ${collection.id}`);
return promiseSerial([() => Promise.resolve()]);
}

try {
mkdirp.sync(backupPath);
Expand Down