Skip to content

Commit

Permalink
Merge pull request #20 from jeremylorino/parallel
Browse files Browse the repository at this point in the history 
Parallel
  • Loading branch information
@yoiang
yoiang authored Mar 20, 2018
2 parents 8f4a40d + 8927d2e commit 424c97d
Show file tree
Hide file tree
Showing 18 changed files with 4,124 additions and 1,853 deletions.
3 changes: 2 additions & 1 deletion .babelrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@
"presets": [
"flow",
"env"
]
],
"plugins": ["transform-class-properties"]
}
1 change: 1 addition & 0 deletions .flowconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ lib/.*
[libs]

[options]
munge_underscores=true

[lints]
30 changes: 25 additions & 5 deletions bin/firestore-backup.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,18 @@ var backupPathParamDescription = 'Path to store backup.'
var prettyPrintParamKey = 'prettyPrint'
var prettyPrintParamDescription = 'JSON backups done with pretty-printing.'

var databaseStartPathParamKey = 'databaseStartPath'
var databaseStartPathParamDescription = 'The database collection or document path to begin backup.'

var requestCountLimitParamKey = 'requestCountLimit'
var requestCountLimitParamDescription = 'The maximum number of requests to be made in parallel.'

commander.version('1.0.1')
.option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription)
.option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription)
.option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription)
.option('-S, --' + databaseStartPathParamKey + ' <path>', databaseStartPathParamDescription)
.option('-L, --' + requestCountLimitParamKey + ' <number>', requestCountLimitParamDescription)
.parse(process.argv)

const accountCredentialsPath = commander[accountCredentialsPathParamKey]
Expand All @@ -43,14 +51,26 @@ if (!backupPath) {
process.exit(1)
}

const prettyPrint = commander[prettyPrintParamKey] !== undefined && commander[prettyPrintParamKey] !== null
const prettyPrintJSON = commander[prettyPrintParamKey] !== undefined && commander[prettyPrintParamKey] !== null

const databaseStartPath = (commander[databaseStartPathParamKey] || '').replace(/^\//, '')

const requestCountLimit = parseInt(commander[requestCountLimitParamKey] || '1', 10)

var firestoreBackup = require('../dist/index.js')
try {
firestoreBackup.default(accountCredentialsPath, backupPath, prettyPrint)
.then(() => {
console.log(colors.bold(colors.green('All done 💫')))
})
console.time('backuptime')
firestoreBackup.default({
accountCredentials: accountCredentialsPath,
databaseStartPath,
backupPath,
prettyPrintJSON,
requestCountLimit
})
.then(() => {
console.log(colors.bold(colors.green('All done 💫')))
console.timeEnd('backuptime')
})
} catch (error) {
console.log(colors.red(error))
process.exit(1)
Expand Down
163 changes: 105 additions & 58 deletions dist/firestore.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.backupRootCollections = exports.backupCollection = exports.backupDocument = exports.constructDocumentValue = exports.constructReferenceUrl = undefined;
exports.FirestoreBackup = exports.constructDocumentValue = exports.constructReferenceUrl = undefined;

var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();

var _types = require('./types');

Expand All @@ -19,6 +21,8 @@ var _mkdirp2 = _interopRequireDefault(_mkdirp);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }

function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }

var constructReferenceUrl = exports.constructReferenceUrl = function constructReferenceUrl(reference) {
Expand Down Expand Up @@ -104,68 +108,111 @@ var constructDocumentValue = exports.constructDocumentValue = function construct
return documentDataToStore;
};

var backupDocument = exports.backupDocument = function backupDocument(document, backupPath, logPath, prettyPrintJSON) {
console.log('Backing up Document \'' + logPath + document.id + '\'');
try {
_mkdirp2.default.sync(backupPath);
} catch (error) {
throw new Error('Unable to create backup path for Document \'' + document.id + '\': ' + error);
}
var defaultBackupOptions = {
databaseStartPath: '',
requestCountLimit: 1
};

var fileContents = void 0;
try {
var documentData = document.data();
var keys = Object.keys(documentData);
var documentDataToStore = {};
documentDataToStore = Object.assign({}, constructDocumentValue(documentDataToStore, keys, documentData));
if (prettyPrintJSON === true) {
fileContents = JSON.stringify(documentDataToStore, null, 2);
} else {
fileContents = JSON.stringify(documentDataToStore);
var FirestoreBackup = exports.FirestoreBackup = function () {
function FirestoreBackup(options) {
_classCallCheck(this, FirestoreBackup);

this.options = Object.assign({}, defaultBackupOptions, options);

if (this.options.requestCountLimit > 1) {
this.documentRequestLimit = 3; // 3 is the max before diminishing returns
}
} catch (error) {
throw new Error('Unable to serialize Document \'' + document.id + '\': ' + error);
}
try {
_fs2.default.writeFileSync(backupPath + '/' + document.id + '.json', fileContents);
} catch (error) {
throw new Error('Unable to write Document \'' + document.id + '\': ' + error);
}

return document.ref.getCollections().then(function (collections) {
return (0, _utility.promiseSerial)(collections.map(function (collection) {
return function () {
return backupCollection(collection, backupPath + '/' + collection.id, logPath + document.id + '/', prettyPrintJSON);
};
}));
});
};
_createClass(FirestoreBackup, [{
key: 'backup',
value: function backup() {
var _this = this;

var backupCollection = exports.backupCollection = function backupCollection(collection, backupPath, logPath, prettyPrintJSON) {
console.log('Backing up Collection \'' + logPath + collection.id + '\'');
try {
_mkdirp2.default.sync(backupPath);
} catch (error) {
throw new Error('Unable to create backup path for Collection \'' + collection.id + '\': ' + error);
}
if ((0, _types.isDocumentPath)(this.options.databaseStartPath)) {
var databaseDocument = this.options.database.doc(this.options.databaseStartPath);
return databaseDocument.get().then(function (document) {
return _this.backupDocument(document, _this.options.backupPath + '/' + document.ref.path, '/');
});
}

return collection.get().then(function (documentSnapshots) {
var backupFunctions = [];
documentSnapshots.forEach(function (document) {
backupFunctions.push(function () {
return backupDocument(document, backupPath + '/' + document.id, logPath + collection.id + '/', prettyPrintJSON);
if ((0, _types.isCollectionPath)(this.options.databaseStartPath)) {
var databaseCollection = this.options.database.collection(this.options.databaseStartPath);
return this.backupCollection(databaseCollection, this.options.backupPath + '/' + databaseCollection.path, '/');
}

return this.backupRootCollections();
}
}, {
key: 'backupRootCollections',
value: function backupRootCollections() {
var _this2 = this;

return this.options.database.getCollections().then(function (collections) {
return (0, _utility.promiseParallel)(collections, function (collection) {
return _this2.backupCollection(collection, _this2.options.backupPath + '/' + collection.id, '/');
}, 1);
});
});
return (0, _utility.promiseSerial)(backupFunctions);
});
};
}
}, {
key: 'backupCollection',
value: function backupCollection(collection, backupPath, logPath) {
var _this3 = this;

console.log('Backing up Collection \'' + logPath + collection.id + '\'');
try {
_mkdirp2.default.sync(backupPath);
} catch (error) {
throw new Error('Unable to create backup path for Collection \'' + collection.id + '\': ' + error);
}

var backupRootCollections = exports.backupRootCollections = function backupRootCollections(database, backupPath, prettyPrintJSON) {
return database.getCollections().then(function (collections) {
return (0, _utility.promiseSerial)(collections.map(function (collection) {
return function () {
return backupCollection(collection, backupPath + '/' + collection.id, '/', prettyPrintJSON);
};
}));
});
};
return collection.get().then(function (documentSnapshots) {
return documentSnapshots.docs;
}).then(function (docs) {
return (0, _utility.promiseParallel)(docs, function (document) {
return _this3.backupDocument(document, backupPath + '/' + document.id, logPath + collection.id + '/');
}, _this3.options.requestCountLimit);
});
}
}, {
key: 'backupDocument',
value: function backupDocument(document, backupPath, logPath) {
var _this4 = this;

console.log('Backing up Document \'' + logPath + document.id + '\'');
try {
_mkdirp2.default.sync(backupPath);
} catch (error) {
throw new Error('Unable to create backup path for Document \'' + document.id + '\': ' + error);
}

var fileContents = void 0;
try {
var documentData = document.data();
var keys = Object.keys(documentData);
var documentDataToStore = {};
documentDataToStore = Object.assign({}, constructDocumentValue(documentDataToStore, keys, documentData));
if (this.prettyPrintJSON === true) {
fileContents = JSON.stringify(documentDataToStore, null, 2);
} else {
fileContents = JSON.stringify(documentDataToStore);
}
} catch (error) {
throw new Error('Unable to serialize Document \'' + document.id + '\': ' + error);
}
try {
_fs2.default.writeFileSync(backupPath + '/' + document.id + '.json', fileContents);
} catch (error) {
throw new Error('Unable to write Document \'' + document.id + '\': ' + error);
}

return document.ref.getCollections().then(function (collections) {
return (0, _utility.promiseParallel)(collections, function (collection) {
return _this4.backupCollection(collection, backupPath + '/' + collection.id, logPath + document.id + '/');
}, _this4.documentRequestLimit);
});
}
}]);

return FirestoreBackup;
}();
Loading

1 comment on commit 424c97d

@yoiang
Copy link
Member Author

@yoiang yoiang commented on 424c97d Apr 1, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Completes #16

Please sign in to comment.