From 820d385fdb0a3d1b34edbf9e5b286881ffaccfe5 Mon Sep 17 00:00:00 2001 From: Sarvesh Jain Date: Thu, 28 Jun 2018 15:54:32 +0530 Subject: [PATCH 1/6] Sync service to sync data from geth leveldb to local folder --- config/error/general.json | 10 +++++ services/manifest.js | 10 +++++ services/sync/sync.js | 94 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 services/sync/sync.js diff --git a/config/error/general.json b/config/error/general.json index ef7d9ef..c095126 100644 --- a/config/error/general.json +++ b/config/error/general.json @@ -189,5 +189,15 @@ "http_code": "422", "code": "UNPROCESSABLE_ENTITY", "message": "No new blocks on chain." + }, + "sync_source_undefined": { + "http_code": "422", + "code": "REQUIRED_ARGUMENT_MISSING", + "message": "Sync source is not defined." + }, + "sync_destination_undefined": { + "http_code": "422", + "code": "REQUIRED_ARGUMENT_MISSING", + "message": "Sync destination is not defined." } } \ No newline at end of file diff --git a/services/manifest.js b/services/manifest.js index 465bcb3..1ff8cf8 100644 --- a/services/manifest.js +++ b/services/manifest.js @@ -42,6 +42,8 @@ const rootPrefix = ".." , RegisterBrandedTokenInterComm = require(rootPrefix + '/services/inter_comm/register_branded_token') , StakeAndMintInterCommKlass = require(rootPrefix + '/services/inter_comm/stake_and_mint') , StakeAndMintProcessorInterCommKlass = require(rootPrefix + '/services/inter_comm/stake_and_mint_processor') + + , SyncKlass = require(rootPrefix + '/services/sync/sync') ; /** @@ -145,7 +147,15 @@ ServiceManifestKlass.prototype = { registerBrandedToken: RegisterBrandedTokenInterComm, stakeAndMint: StakeAndMintInterCommKlass, stakeAndMintProcessor: StakeAndMintProcessorInterCommKlass + }, + + /** + * Service to sync chaindata level db folder of geth + */ + sync: { + SyncKlass: SyncKlass } + }; module.exports = new ServiceManifestKlass(); \ No newline at end of file diff --git a/services/sync/sync.js b/services/sync/sync.js new file mode 100644 index 0000000..8297369 --- /dev/null +++ b/services/sync/sync.js @@ -0,0 +1,94 @@ +const Rsync = require('rsync'); + +const rootPrefix = "../.." + , basicHelper = require(rootPrefix + '/helpers/basic_helper') + , responseHelper = require(rootPrefix + '/lib/formatter/response') + , logger = require(rootPrefix + '/helpers/custom_console_logger') +; + +/** + * + * @param source folder path of chainData + * @param destination folder path of chainData + * @constructor + */ +function SyncKlass(source, destination) { + let oThis = this; + + oThis.source = source; + oThis.destination = destination; + oThis.rsync = new Rsync() + .flags('az') + .source(oThis.source) + .destination(oThis.destination); +} + +SyncKlass.prototype = { + /** + * Sync source folder to destination folder + * @return {Promise} + */ + perform: function () { + let oThis = this; + + oThis._validate(); + return oThis._sync(); + }, + /** + * Sync source folder to destination folder + * @return {Promise} + * @private + */ + _sync: function () { + let oThis = this; + + return new Promise(function (resolve, reject) { + oThis.rsync.execute(function (error, code, cmd) { + if (error) { + + logger.error(error); + let responseError = responseHelper.error({ + internal_error_identifier: 's_s_sync_validate_1', + api_error_identifier: 'exception', + error_config: basicHelper.fetchErrorConfig(), + debug: {error} + }); + reject(responseError); + } + resolve(responseHelper.successWithData({ + statusCode: code, + cmd: cmd + })) + }); + }); + }, + /** + * Input validations + * @private + */ + _validate: async function () { + let oThis = this; + + let errorConf = { + internal_error_identifier: "s_s_val_validate_2", + debug_options: {}, + error_config: basicHelper.fetchErrorConfig() + }; + + if (!oThis.source) { + logger.error("Sync source is not defined"); + errorConf.api_error_identifier = "sync_source_undefined"; + let errorResponse = responseHelper.error(errorConf); + return Promise.reject(errorResponse); + } + + if (!oThis.destination) { + logger.error("Sync destination is not defined"); + errorConf.api_error_identifier = "sync_destination_undefined"; + let errorResponse = responseHelper.error(errorConf); + return Promise.reject(errorResponse); + } + } +}; + +module.exports = SyncKlass; From 6765e44acb76a8e80ccfb1c0ab0e162e8edfe37b Mon Sep 17 00:00:00 2001 From: Sarvesh Jain Date: Mon, 2 Jul 2018 19:54:41 +0530 Subject: [PATCH 2/6] Added Rsync service logic for remote sync aswell --- package.json | 1 + services/sync/sync.js | 43 +++++++++++++++------ test/services/sync_test.js | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 test/services/sync_test.js diff --git a/package.json b/package.json index d81865a..753b74a 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "edit-json-file": "1.0.8", "node-cmd": "3.0.0", "readline": "1.3.0", + "rsync": "^0.6.1", "shell-source": "1.1.0", "shelljs": "0.8.1", "shortid": "2.2.8", diff --git a/services/sync/sync.js b/services/sync/sync.js index 8297369..41329a0 100644 --- a/services/sync/sync.js +++ b/services/sync/sync.js @@ -7,20 +7,16 @@ const rootPrefix = "../.." ; /** - * - * @param source folder path of chainData - * @param destination folder path of chainData + * For remote Sync specify user and host in sourceConfig/destinationConfig, for local sync dont add user and host key + * @param sourceConfig object {user:'source_user', host:'10.1.2.2', path:'~/temp'} + * @param destinationConfig object {user:'destination_user', host:'10.1.2.3', path:'~/temp'} * @constructor */ -function SyncKlass(source, destination) { +function SyncKlass(sourceConfig, destinationConfig) { let oThis = this; - oThis.source = source; - oThis.destination = destination; - oThis.rsync = new Rsync() - .flags('az') - .source(oThis.source) - .destination(oThis.destination); + oThis.source = oThis._formatPath(sourceConfig); + oThis.destination = oThis._formatPath(destinationConfig); } SyncKlass.prototype = { @@ -31,7 +27,6 @@ SyncKlass.prototype = { perform: function () { let oThis = this; - oThis._validate(); return oThis._sync(); }, /** @@ -39,9 +34,16 @@ SyncKlass.prototype = { * @return {Promise} * @private */ - _sync: function () { + _sync: async function () { let oThis = this; + await oThis._validate(); + + oThis.rsync = new Rsync() + .flags('az') + .source(oThis.source) + .destination(oThis.destination); + return new Promise(function (resolve, reject) { oThis.rsync.execute(function (error, code, cmd) { if (error) { @@ -88,7 +90,24 @@ SyncKlass.prototype = { let errorResponse = responseHelper.error(errorConf); return Promise.reject(errorResponse); } + }, + + /** + * @param origin source/destination + * @return {string} formatted path of source/destination + * @private + */ + _formatPath: function (origin) { + + let user = origin.user; + let host = origin.host; + let path = origin.path; + if (!user || !host) { + return path; + } + return path ? `${user}@${host}:${path}` : path; } + }; module.exports = SyncKlass; diff --git a/test/services/sync_test.js b/test/services/sync_test.js new file mode 100644 index 0000000..aac238b --- /dev/null +++ b/test/services/sync_test.js @@ -0,0 +1,79 @@ +const assert = require('assert'); + +const rootPrefix = '../..' + , SyncKlass = require(rootPrefix + '/services/sync/sync') +; + + +describe +('Sync service', function () { + + it('should not sync data is source path is missing ', async function () { + let sourceConfig = { + user: "user", + host: "10.1.1.1", + }, + destinationConfig = { + path: "~/tmp" + }; + + let syncService = new SyncKlass(sourceConfig, destinationConfig); + + try { + await syncService.perform(); + } catch (error) { + assert.equal(error.apiErrorIdentifier, 'sync_source_undefined') + } + }); + + + it('should not sync data is destination path is missing ', async function () { + let sourceConfig = { + user: "user", + host: "10.1.1.1", + path: "~/tmp" + }, + destinationConfig = {}; + let syncService = new SyncKlass(sourceConfig, destinationConfig); + + try { + await syncService.perform(); + } catch (error) { + assert.equal(error.apiErrorIdentifier, 'sync_destination_undefined') + } + }); + + + it('should format path for local path', async function () { + let path = "~/tmp"; + let sourceConfig = { + path: path + }, + destinationConfig = {}; + + let syncService = new SyncKlass(sourceConfig, destinationConfig); + + let formatPath = syncService._formatPath(sourceConfig); + assert.equal(formatPath, path); + + }); + + it('should format path for remote path', async function () { + let path = "~/tmp" + , host = "10.1.1.1" + , user = "user"; + let sourceConfig = { + user: user, + host: host, + path: path + }, + destinationConfig = {}; + + let syncService = new SyncKlass(sourceConfig, destinationConfig); + let formatPath = syncService._formatPath(sourceConfig); + assert.equal(formatPath, `${user}@${host}:${path}`); + }); + + +}) +; \ No newline at end of file From 69fb11c092e59d137edd0d28fe0f4b8e486737ec Mon Sep 17 00:00:00 2001 From: Sarvesh Jain Date: Tue, 3 Jul 2018 15:15:21 +0530 Subject: [PATCH 3/6] Added comments, unit tests and test dependency --- config/error/general.json | 5 ++ package.json | 4 +- services/sync/sync.js | 37 +++++++++++++-- test/services/sync_test.js | 94 +++++++++++++++++++++++++++++++++++--- 4 files changed, 129 insertions(+), 11 deletions(-) diff --git a/config/error/general.json b/config/error/general.json index c095126..10da3d1 100644 --- a/config/error/general.json +++ b/config/error/general.json @@ -199,5 +199,10 @@ "http_code": "422", "code": "REQUIRED_ARGUMENT_MISSING", "message": "Sync destination is not defined." + }, + "rsync_failed": { + "http_code": "422", + "code": "FAILED_DURING_RSYNC", + "message": "Rsync failed" } } \ No newline at end of file diff --git a/package.json b/package.json index 753b74a..3b41710 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,8 @@ "chai": "4.1.2", "ink-docstrap": "1.3.2", "jsdoc": "3.5.5", - "mocha": "5.0.0" + "mocha": "5.0.0", + "sinon": "6.0.1", + "mock-require": "3.0.2" } } diff --git a/services/sync/sync.js b/services/sync/sync.js index 41329a0..1089d62 100644 --- a/services/sync/sync.js +++ b/services/sync/sync.js @@ -6,11 +6,35 @@ const rootPrefix = "../.." , logger = require(rootPrefix + '/helpers/custom_console_logger') ; +//az is rsync flag where z represents compression and a stands for "archive" and syncs recursively and +// preserves symbolic links, special and device files, modification times, group, owner, and permissions +const defaultFlag = 'az'; + /** - * For remote Sync specify user and host in sourceConfig/destinationConfig, for local sync dont add user and host key + * @notice For remote Sync specify user,host and path in sourceConfig/destinationConfig, for local sync dont add user and host key + * + * @dev Key pair based ssh needs to setup for remote rsync + * * @param sourceConfig object {user:'source_user', host:'10.1.2.2', path:'~/temp'} * @param destinationConfig object {user:'destination_user', host:'10.1.2.3', path:'~/temp'} + * * @constructor + * + * Example: + * + * let sourceConfig = { + * user: "user", + * host: "10.1.1.1", + * path: "~/tmp" + * }, + * destinationConfig = { + * path: "~/tmp" + * }, + * let syncInstance = new SyncKlass(sourceConfig, destinationConfig); + * syncInstance.perform( result =>{ + * //Logic on success + * }); + * //if source / destination is local then user or path key in config can be skipped */ function SyncKlass(sourceConfig, destinationConfig) { let oThis = this; @@ -22,6 +46,7 @@ function SyncKlass(sourceConfig, destinationConfig) { SyncKlass.prototype = { /** * Sync source folder to destination folder + * * @return {Promise} */ perform: function () { @@ -31,6 +56,7 @@ SyncKlass.prototype = { }, /** * Sync source folder to destination folder + * * @return {Promise} * @private */ @@ -39,8 +65,8 @@ SyncKlass.prototype = { await oThis._validate(); - oThis.rsync = new Rsync() - .flags('az') + oThis.rsync = new Rsync(); + oThis.rsync.flags(defaultFlag) .source(oThis.source) .destination(oThis.destination); @@ -51,7 +77,7 @@ SyncKlass.prototype = { logger.error(error); let responseError = responseHelper.error({ internal_error_identifier: 's_s_sync_validate_1', - api_error_identifier: 'exception', + api_error_identifier: 'rsync_failed', error_config: basicHelper.fetchErrorConfig(), debug: {error} }); @@ -66,6 +92,7 @@ SyncKlass.prototype = { }, /** * Input validations + * * @private */ _validate: async function () { @@ -94,7 +121,9 @@ SyncKlass.prototype = { /** * @param origin source/destination + * * @return {string} formatted path of source/destination + * * @private */ _formatPath: function (origin) { diff --git a/test/services/sync_test.js b/test/services/sync_test.js index aac238b..d2fa506 100644 --- a/test/services/sync_test.js +++ b/test/services/sync_test.js @@ -1,12 +1,31 @@ -const assert = require('assert'); +const assert = require('assert') + , sinon = require('sinon') + , mock = require('mock-require') +; const rootPrefix = '../..' - , SyncKlass = require(rootPrefix + '/services/sync/sync') ; -describe -('Sync service', function () { +describe('Sync service', function () { + let RSyncStub, cb, errorCode; + + before(function () { + + RSyncStub = sinon.spy(); + let instance = new RSyncStub() + RSyncStub.prototype.flags = sinon.stub().returns(instance); + RSyncStub.prototype.source = sinon.stub().returns(instance); + RSyncStub.prototype.destination = sinon.stub().returns(instance); + RSyncStub.prototype.execute = function (cb) { + cb(errorCode, 0); + }; + mock('rsync', RSyncStub); + SyncKlass = require(rootPrefix + '/services/sync/sync') + cb = function (error, code, cmd) { + //placeholder callback function + }; + }); it('should not sync data is source path is missing ', async function () { let sourceConfig = { @@ -74,6 +93,69 @@ describe assert.equal(formatPath, `${user}@${host}:${path}`); }); + it('should form correct rsync command', async function () { + + let sourceConfig = { + user: "user", + host: "10.1.1.1", + path: "~/tmp" + }, + destinationConfig = { + path: "~/tmp" + }, + syncService = new SyncKlass(sourceConfig, destinationConfig); + + await syncService.perform(); + assert.equal(RSyncStub.prototype.flags.called, true); + assert.equal(RSyncStub.prototype.source.called, true); + assert.equal(RSyncStub.prototype.destination.called, true); + }); + + + it('should fail if rync fails ', async function () { + + let sourceConfig = { + user: "user", + host: "10.1.1.1", + path: "~/tmp" + }, + destinationConfig = { + path: "~/tmp" + }, + syncService = new SyncKlass(sourceConfig, destinationConfig); + errorCode = 1; + + try { + await syncService.perform(); + } + catch (error) { + + assert.equal(error.apiErrorIdentifier, 'rsync_failed'); + assert.equal(RSyncStub.prototype.flags.called, true); + assert.equal(RSyncStub.prototype.source.called, true); + assert.equal(RSyncStub.prototype.destination.called, true); + } + }); + + + it('should return success if rsync was success ', async function () { -}) -; \ No newline at end of file + let sourceConfig = { + user: "user", + host: "10.1.1.1", + path: "~/tmp" + }, + destinationConfig = { + path: "~/tmp" + }, + syncService = new SyncKlass(sourceConfig, destinationConfig); + errorCode = undefined; + + let result = await syncService.perform(); + + assert.equal(result.success, true); + assert.equal(RSyncStub.prototype.flags.called, true); + assert.equal(RSyncStub.prototype.source.called, true); + assert.equal(RSyncStub.prototype.destination.called, true); + }); +}); \ No newline at end of file From a31207154a2537d960c85974f74166acd8975d3e Mon Sep 17 00:00:00 2001 From: Sarvesh Jain Date: Thu, 5 Jul 2018 17:21:59 +0530 Subject: [PATCH 4/6] Refactoring - Directly exposing the class and Added comments on path returns --- services/manifest.js | 4 +--- services/sync/sync.js | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/manifest.js b/services/manifest.js index 1ff8cf8..a7f23f6 100644 --- a/services/manifest.js +++ b/services/manifest.js @@ -152,9 +152,7 @@ ServiceManifestKlass.prototype = { /** * Service to sync chaindata level db folder of geth */ - sync: { - SyncKlass: SyncKlass - } + Sync: SyncKlass }; diff --git a/services/sync/sync.js b/services/sync/sync.js index 1089d62..a0ad5a4 100644 --- a/services/sync/sync.js +++ b/services/sync/sync.js @@ -132,6 +132,7 @@ SyncKlass.prototype = { let host = origin.host; let path = origin.path; if (!user || !host) { + //return local directory path return path; } return path ? `${user}@${host}:${path}` : path; From c886532f220eebb020dbe8027f1c8dec9237313e Mon Sep 17 00:00:00 2001 From: Sarvesh Jain Date: Thu, 5 Jul 2018 19:02:55 +0530 Subject: [PATCH 5/6] Added example to call service on CONFIGURE.md --- CONFIGURE.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CONFIGURE.md b/CONFIGURE.md index fe800d9..8b41e5f 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -359,4 +359,25 @@ serviceObj.perform().then(function(response) { }); ``` +* Sync service +``` +const platform = require('@openstfoundation/openst-platform'); +let source = { + path: '~/source/', + user: 'user', + host: '172.16.0.201' +}; +let destination = { + path: '~/destination' +}; +let syncService = new platform.services.Sync(source, destination); + +syncService.perform().then((response) => { + console.log(response); +}); + +``` +Note: source and destination can be remote as well as local machines + + For complete implementation details of OpenST Platform, please refer [API documentation](http://docs.openst.org/). From 7452873e53d1c27a38e9802c814d1716f5862109 Mon Sep 17 00:00:00 2001 From: Sarvesh Jain Date: Thu, 5 Jul 2018 20:29:39 +0530 Subject: [PATCH 6/6] Freezing version of dependency in package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3b41710..e776f38 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "edit-json-file": "1.0.8", "node-cmd": "3.0.0", "readline": "1.3.0", - "rsync": "^0.6.1", + "rsync": "0.6.1", "shell-source": "1.1.0", "shelljs": "0.8.1", "shortid": "2.2.8",