Issue #35: Reduce index build race conditions (#36)

Motivation
----------
In some cases with partitioned indexes and replicas all index nodes are
not ready to build indexes immediately.  Issuing the BUILD INDEX command
can then cause them to only partially build at first.

Modifications
-------------
Add a 3 second delay between creating all indexes and issuing the BUILD
INDEX command.

Results
-------
Race conditions should be significantly reduced.

app/plan.js

@@ -5,6 +5,7 @@ import chalk from 'chalk';
  * @typedef PlanOptions
  * @property {?logger} logger logger for output
  * @property {?number} buildTimeout Milliseconds to wait for indexes to build
+ * @property {?number} buildDelay Milliseconds to wait before building indexes
  */
 
  /**
@@ -29,6 +30,11 @@ function addMutationsByPhase(mutations, currentMutations) {
     return currentMutations;
 }
 
+const defaultOptions = {
+    logger: console,
+    buildDelay: 3000,
+};
+
 /**
  * Represents a planned set of mutations for synchronization
  *
@@ -44,7 +50,7 @@ export class Plan {
      */
     constructor(manager, mutations, options) {
         this.manager = manager;
-        this.options = extend({logger: console}, options);
+        this.options = extend(defaultOptions, options);
 
         this.mutations = addMutationsByPhase(mutations, []);
     }
@@ -116,6 +122,14 @@ export class Plan {
 
             this.options.logger.info(
                 chalk.greenBright('Building indexes...'));
+
+            // Wait 3 seconds for index nodes to synchronize before building
+            // This helps to reduce race conditions
+            // https://github.com/brantburnett/couchbase-index-manager/issues/35
+            if (this.options.buildDelay > 0) {
+                await new Promise((resolve) =>
+                    setTimeout(resolve, this.options.buildDelay));
+            }
             await this.manager.buildDeferredIndexes();
 
             if (!await this.manager.waitForIndexBuild(

test/plan.spec.js

@@ -18,13 +18,14 @@ const mockManager = {
     waitForIndexBuild: () => Promise.resolve(),
 };
 
-const noLogging = {
+const planOptions = {
     logger: {
         log: () => {},
         error: () => {},
         warn: () => {},
         info: () => {},
     },
+    buildDelay: 0, // don't delay during unit tests
 };
 
 describe('execute', function() {
@@ -41,7 +42,7 @@ describe('execute', function() {
         }));
         let secondStub = stub(second, 'execute').returns(Promise.resolve());
 
-        let plan = new Plan(mockManager, [first, second], noLogging);
+        let plan = new Plan(mockManager, [first, second], planOptions);
 
         await plan.execute();
 
@@ -71,7 +72,7 @@ describe('execute', function() {
         }));
         let thirdStub = stub(third, 'execute').returns(Promise.resolve());
 
-        let plan = new Plan(mockManager, [first, second, third], noLogging);
+        let plan = new Plan(mockManager, [first, second, third], planOptions);
 
         await plan.execute();
 
@@ -96,7 +97,7 @@ describe('execute', function() {
         }));
         let secondStub = stub(second, 'execute').returns(Promise.resolve());
 
-        let plan = new Plan(mockManager, [first, second], noLogging);
+        let plan = new Plan(mockManager, [first, second], planOptions);
 
         await plan.execute()
             .catch(() => {});
@@ -119,7 +120,7 @@ describe('execute', function() {
         second.phase = 2;
         let secondStub = stub(second, 'execute').returns(Promise.resolve());
 
-        let plan = new Plan(mockManager, [first, second], noLogging);
+        let plan = new Plan(mockManager, [first, second], planOptions);
 
         await plan.execute()
             .catch(() => {});