Merge pull request #205 from tbranch227/feature/clone-retry

Adding retry logic to git clone operation.  KP is having issues where…

bin/stampede-worker.js

@@ -53,6 +53,8 @@ const conf = require("rc")("stampede", {
   logQueuePath: null,
   // Heartbeat
   heartbeatInterval: 15000,
+  cloneRetryInterval: 1 * 60 * 1000,  // retry every minute must be specified in milliseconds
+  cloneRetryAttempts: 3 // retry 3 tiems
 });
 
 // Configure winston logging

lib/workingDirectory.js

@@ -3,6 +3,7 @@ const fs = require("fs");
 const { exec } = require("child_process");
 const git = require("git-last-commit");
 
+let retries = 0;
 /**
  * prepare the working directory
  * @param {*} taskExecutionConfig
@@ -79,7 +80,7 @@ async function prepareWorkingDirectory(taskExecutionConfig, conf, logger) {
 
   // Perform the clone
   if (gitOperations.clone != null) {
-    const cloneResult = await cloneRepo(
+    let cloneResult = await cloneRepo(
       gitRepoURL,
       gitOperations.clone,
       dir,
@@ -88,11 +89,36 @@ async function prepareWorkingDirectory(taskExecutionConfig, conf, logger) {
       logger,
       conf.workspaceRoot
     );
-    if (cloneResult === false) {
+    if (conf.cloneRetryAttempts && cloneResult !== true) {
+      if (retries > conf.cloneRetryAttempts) {
+        while (true) {
+          await delay(conf.cloneRetryInterval);
+          cloneResult = await cloneRepo(
+            gitRepoURL,
+            gitOperations.clone,
+            dir,
+            gitOperations.depth,
+            taskExecutionConfig.gitCloneOptions,
+            logger,
+            conf.workspaceRoot
+          );
+          retries++;
+          if (retries > conf.cloneRetryAttempts || cloneResult === true) {
+            break; // we run until we get a positive clone result or we've run out of retries
+          }
+        }
+      }
+    }
+    retries = 0; // reset our retry attempts
+    if (cloneResult !== true) {
+      retries++;
+      let err = "Unable to clone the repository, please contact the service desk and report the issue.\n" 
+        + cloneResult + "\n"
+        + "Directory: " + dir + "\n"
+        + "Repo URL: " + gitRepoURL + "\n"
       return {
         error: "clone-error",
-        message:
-          "Unable to clone the repository, please contact the service desk and report the issue.",
+        message: err,
       };
     }
   }
@@ -111,7 +137,7 @@ async function prepareWorkingDirectory(taskExecutionConfig, conf, logger) {
   // Perform the merge
   if (gitOperations.merge == true) {
     const mergeResult = await gitMerge(gitOperations.mergeBase, dir, logger);
-    if (mergeResult === false) {
+    if (mergeResult == false) {
       return {
         error: "merge-error",
         message:
@@ -138,6 +164,13 @@ async function prepareWorkingDirectory(taskExecutionConfig, conf, logger) {
   };
 }
 
+/**
+ * Pauses execution for specified ms
+ * @param {*} ms 
+ * @returns 
+ */
+const delay = ms => new Promise(res => setTimeout(res, ms));
+
 /**
  * Clone the repository to our working directory
  * @param {*} cloneUrl
@@ -171,30 +204,34 @@ async function cloneRepo(
     workingDirectory;
   logger.verbose("clone: " + cloneCommand);
   return new Promise((resolve) => {
+    let success = true;
     exec(cloneCommand, { cwd: workspaceRoot }, (error, stdout, stderr) => {
       if (error) {
+        success = false;
         logger.error(`cloneRepo error: ${error}`);
+        const cloneErrorLog =
+          "cloneRepo error: " +
+          error +
+          "\n" +
+          "stdout: " +
+          stdout +
+          "\n" +
+          "stderr: " +
+          stderr +
+          "\n";
         try {
-          const cloneErrorLog =
-            "cloneRepo error: " +
-            error +
-            "\n" +
-            "stdout: " +
-            stdout +
-            "\n" +
-            "stderr: " +
-            stderr +
-            "\n";
           fs.writeFileSync(workingDirectory + "/cloneerror.log", cloneErrorLog);
         } catch (e) {
           logger.error(`error writing out clone log: ` + e);
         }
-        resolve(false);
+        resolve(cloneErrorLog); // no retries attempted
         return;
       }
       logger.verbose(`stdout: ${stdout}`);
       logger.verbose(`stderr: ${stderr}`);
-      resolve(true);
+      if (success) {
+        resolve(true); // original attempt or a retry succeeded
+      }
     });
   });
 }