Mosa 130/sync chain data

config/error/general.json

@@ -189,5 +189,20 @@
     "http_code": "422",
     "code": "UNPROCESSABLE_ENTITY",
     "message": "No new blocks on chain."
+  },
+  "sync_source_undefined": {
+    "http_code": "422",
+    "code": "REQUIRED_ARGUMENT_MISSING",
+    "message": "Sync source is not defined."
+  },
+  "sync_destination_undefined": {
+    "http_code": "422",
+    "code": "REQUIRED_ARGUMENT_MISSING",
+    "message": "Sync destination is not defined."
+  },
+  "rsync_failed": {
+    "http_code": "422",
+    "code": "FAILED_DURING_RSYNC",
+    "message": "Rsync failed"
   }
 }

package.json

@@ -39,6 +39,7 @@
     "edit-json-file": "1.0.8",
     "node-cmd": "3.0.0",
     "readline": "1.3.0",
+    "rsync": "^0.6.1",
     "shell-source": "1.1.0",
     "shelljs": "0.8.1",
     "shortid": "2.2.8",
@@ -49,6 +50,8 @@
     "chai": "4.1.2",
     "ink-docstrap": "1.3.2",
     "jsdoc": "3.5.5",
-    "mocha": "5.0.0"
+    "mocha": "5.0.0",
+    "sinon": "6.0.1",
+    "mock-require": "3.0.2"
   }
 }

services/manifest.js

@@ -42,6 +42,8 @@ const rootPrefix = ".."
   , RegisterBrandedTokenInterComm = require(rootPrefix + '/services/inter_comm/register_branded_token')
   , StakeAndMintInterCommKlass = require(rootPrefix + '/services/inter_comm/stake_and_mint')
   , StakeAndMintProcessorInterCommKlass = require(rootPrefix + '/services/inter_comm/stake_and_mint_processor')
+
+  , SyncKlass = require(rootPrefix + '/services/sync/sync')
 ;
 
 /**
@@ -145,7 +147,15 @@ ServiceManifestKlass.prototype = {
     registerBrandedToken: RegisterBrandedTokenInterComm,
     stakeAndMint: StakeAndMintInterCommKlass,
     stakeAndMintProcessor: StakeAndMintProcessorInterCommKlass
+  },
+
+  /**
+   * Service to sync chaindata level db folder of geth
+   */
+  sync: {
+    SyncKlass: SyncKlass
   }
+
 };
 
 module.exports = new ServiceManifestKlass();

services/sync/sync.js

@@ -0,0 +1,142 @@
+const Rsync = require('rsync');
+
+const rootPrefix = "../.."
+  , basicHelper = require(rootPrefix + '/helpers/basic_helper')
+  , responseHelper = require(rootPrefix + '/lib/formatter/response')
+  , logger = require(rootPrefix + '/helpers/custom_console_logger')
+;
+
+//az is rsync flag where z represents compression and a stands for "archive" and syncs recursively and
+// preserves symbolic links, special and device files, modification times, group, owner, and permissions
+const defaultFlag = 'az';
+
+/**
+ * @notice For remote Sync specify user,host and path in sourceConfig/destinationConfig, for local sync dont add user and host key
+ *
+ * @dev Key pair based ssh needs to setup for remote rsync
+ *
+ * @param sourceConfig object {user:'source_user', host:'10.1.2.2', path:'~/temp'}
+ * @param destinationConfig object {user:'destination_user', host:'10.1.2.3', path:'~/temp'}
+ *
+ * @constructor
+ *
+ * Example:
+ *
+ * let sourceConfig = {
+ *       user: "user",
+ *       host: "10.1.1.1",
+ *       path: "~/tmp"
+ *     },
+ * destinationConfig = {
+ *       path: "~/tmp"
+ *     },
+ *  let syncInstance = new SyncKlass(sourceConfig, destinationConfig);
+ *  syncInstance.perform( result =>{
+ *     //Logic on success
+ *  });
+ *   //if source / destination is local then user or path key in config can be skipped
+ */
+function SyncKlass(sourceConfig, destinationConfig) {
+  let oThis = this;
+
+  oThis.source = oThis._formatPath(sourceConfig);
+  oThis.destination = oThis._formatPath(destinationConfig);
+}
+
+SyncKlass.prototype = {
+  /**
+   * Sync source folder to destination folder
+   *
+   * @return {Promise<result>}
+   */
+  perform: function () {
+    let oThis = this;
+
+    return oThis._sync();
+  },
+  /**
+   * Sync source folder to destination folder
+   *
+   * @return {Promise<result>}
+   * @private
+   */
+  _sync: async function () {
+    let oThis = this;
+
+    await oThis._validate();
+
+    oThis.rsync = new Rsync();
+    oThis.rsync.flags(defaultFlag)
+      .source(oThis.source)
+      .destination(oThis.destination);
+
+    return new Promise(function (resolve, reject) {
+      oThis.rsync.execute(function (error, code, cmd) {
+        if (error) {
+
+          logger.error(error);
+          let responseError = responseHelper.error({
+            internal_error_identifier: 's_s_sync_validate_1',
+            api_error_identifier: 'rsync_failed',
+            error_config: basicHelper.fetchErrorConfig(),
+            debug: {error}
+          });
+          reject(responseError);
+        }
+        resolve(responseHelper.successWithData({
+          statusCode: code,
+          cmd: cmd
+        }))
+      });
+    });
+  },
+  /**
+   * Input validations
+   *
+   * @private
+   */
+  _validate: async function () {
+    let oThis = this;
+
+    let errorConf = {
+      internal_error_identifier: "s_s_val_validate_2",
+      debug_options: {},
+      error_config: basicHelper.fetchErrorConfig()
+    };
+
+    if (!oThis.source) {
+      logger.error("Sync source is not defined");
+      errorConf.api_error_identifier = "sync_source_undefined";
+      let errorResponse = responseHelper.error(errorConf);
+      return Promise.reject(errorResponse);
+    }
+
+    if (!oThis.destination) {
+      logger.error("Sync destination is not defined");
+      errorConf.api_error_identifier = "sync_destination_undefined";
+      let errorResponse = responseHelper.error(errorConf);
+      return Promise.reject(errorResponse);
+    }
+  },
+
+  /**
+   * @param origin source/destination
+   *
+   * @return {string} formatted path of source/destination
+   *
+   * @private
+   */
+  _formatPath: function (origin) {
+
+    let user = origin.user;
+    let host = origin.host;
+    let path = origin.path;
+    if (!user || !host) {
+      return path;
+    }
+    return path ? `${user}@${host}:${path}` : path;
+  }
+
+};
+
+module.exports = SyncKlass;

test/services/sync_test.js

@@ -0,0 +1,161 @@
+const assert = require('assert')
+
+  , sinon = require('sinon')
+  , mock = require('mock-require')
+;
+const rootPrefix = '../..'
+;
+
+
+describe('Sync service', function () {
+  let RSyncStub, cb, errorCode;
+
+  before(function () {
+
+    RSyncStub = sinon.spy();
+    let instance = new RSyncStub()
+    RSyncStub.prototype.flags = sinon.stub().returns(instance);
+    RSyncStub.prototype.source = sinon.stub().returns(instance);
+    RSyncStub.prototype.destination = sinon.stub().returns(instance);
+    RSyncStub.prototype.execute = function (cb) {
+      cb(errorCode, 0);
+    };
+    mock('rsync', RSyncStub);
+    SyncKlass = require(rootPrefix + '/services/sync/sync')
+    cb = function (error, code, cmd) {
+      //placeholder callback function
+    };
+  });
+
+  it('should not sync data is source path is missing ', async function () {
+    let sourceConfig = {
+        user: "user",
+        host: "10.1.1.1",
+      },
+      destinationConfig = {
+        path: "~/tmp"
+      };
+
+    let syncService = new SyncKlass(sourceConfig, destinationConfig);
+
+    try {
+      await syncService.perform();
+    } catch (error) {
+      assert.equal(error.apiErrorIdentifier, 'sync_source_undefined')
+    }
+  });
+
+
+  it('should not sync data is destination path is missing ', async function () {
+    let sourceConfig = {
+        user: "user",
+        host: "10.1.1.1",
+        path: "~/tmp"
+      },
+      destinationConfig = {};
+    let syncService = new SyncKlass(sourceConfig, destinationConfig);
+
+    try {
+      await syncService.perform();
+    } catch (error) {
+      assert.equal(error.apiErrorIdentifier, 'sync_destination_undefined')
+    }
+  });
+
+
+  it('should format path for local path', async function () {
+    let path = "~/tmp";
+    let sourceConfig = {
+        path: path
+      },
+      destinationConfig = {};
+
+    let syncService = new SyncKlass(sourceConfig, destinationConfig);
+
+    let formatPath = syncService._formatPath(sourceConfig);
+    assert.equal(formatPath, path);
+
+  });
+
+  it('should format path for remote path', async function () {
+    let path = "~/tmp"
+      , host = "10.1.1.1"
+      , user = "user";
+    let sourceConfig = {
+        user: user,
+        host: host,
+        path: path
+      },
+      destinationConfig = {};
+
+    let syncService = new SyncKlass(sourceConfig, destinationConfig);
+    let formatPath = syncService._formatPath(sourceConfig);
+    assert.equal(formatPath, `${user}@${host}:${path}`);
+  });
+
+  it('should form correct rsync command', async function () {
+
+    let sourceConfig = {
+        user: "user",
+        host: "10.1.1.1",
+        path: "~/tmp"
+      },
+      destinationConfig = {
+        path: "~/tmp"
+      },
+      syncService = new SyncKlass(sourceConfig, destinationConfig);
+
+    await syncService.perform();
+    assert.equal(RSyncStub.prototype.flags.called, true);
+    assert.equal(RSyncStub.prototype.source.called, true);
+    assert.equal(RSyncStub.prototype.destination.called, true);
+  });
+
+
+  it('should fail if rync fails ', async function () {
+
+    let sourceConfig = {
+        user: "user",
+        host: "10.1.1.1",
+        path: "~/tmp"
+      },
+      destinationConfig = {
+        path: "~/tmp"
+      },
+      syncService = new SyncKlass(sourceConfig, destinationConfig);
+    errorCode = 1;
+
+    try {
+      await syncService.perform();
+    }
+    catch (error) {
+
+      assert.equal(error.apiErrorIdentifier, 'rsync_failed');
+      assert.equal(RSyncStub.prototype.flags.called, true);
+      assert.equal(RSyncStub.prototype.source.called, true);
+      assert.equal(RSyncStub.prototype.destination.called, true);
+    }
+  });
+
+
+  it('should return success if rsync was success ', async function () {
+
+    let sourceConfig = {
+        user: "user",
+        host: "10.1.1.1",
+        path: "~/tmp"
+      },
+      destinationConfig = {
+        path: "~/tmp"
+      },
+      syncService = new SyncKlass(sourceConfig, destinationConfig);
+    errorCode = undefined;
+
+    let result = await syncService.perform();
+
+    assert.equal(result.success, true);
+    assert.equal(RSyncStub.prototype.flags.called, true);
+    assert.equal(RSyncStub.prototype.source.called, true);
+    assert.equal(RSyncStub.prototype.destination.called, true);
+  });
+});