Launcher watchdog implementation

Akkadius · Jan 5, 2020 · d9908e7 · d9908e7
1 parent d37b91e
commit d9908e7
Show file tree

Hide file tree

Showing 5 changed files with 1,268 additions and 2,724 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.4] - 2020-1-4
+* Fix edge case in launcher stability where in the event that the launcher loses connection with world or the main launcher loop hangs, zones stop booting. This situation was rarer but a watchdog has been implemented to kill the launcher process in the event that this occurs. Requires the launcher process to be ran in a loop
+
 ## [1.0.3] - 2019-11-3
 * Launcher stability
 

diff --git a/app/commands/server-launcher.js b/app/commands/server-launcher.js
@@ -2,10 +2,18 @@
  * @type {{check: module.exports.check}}
  */
 module.exports = {
+
+  /**
+   * @param options
+   */
   serverLauncher: function (options) {
     require('../core/server-process-manager').start(options);
   },
+
+  /**
+   * @param options
+   */
   stopServer: function (options) {
     require('../core/server-process-manager').stopServer(options);
   },
-};
+};
diff --git a/app/core/server-process-manager.js b/app/core/server-process-manager.js
@@ -11,6 +11,14 @@ const util              = require('util')
 const psList            = require('ps-list');
 const debug             = require('debug')('eqemu-admin:process-manager');
 
+/**
+ * Constants
+ * @type {number}
+ */
+const MAX_PROCESS_POLL_DELTA_SECONDS_BEFORE_KILL = 60;
+const WATCHDOG_TIMER                             = 1000;
+const LAUNCHER_MAIN_LOOP_TIMER                   = 5 * 1000;
+
 /**
  * @type {{check: module.exports.check}}
  */
@@ -23,6 +31,8 @@ module.exports = {
   serverProcessNames: ['zone', 'world', 'ucs', 'queryserv', 'loginserver'],
   systemProcessList: {},
   launchOptions: {},
+  watchDogTimer: null,
+  lastProcessPollTime: Math.floor(new Date() / 1000),
 
   /**
    * Launcher initialization
@@ -39,13 +49,58 @@ module.exports = {
     });
   },
 
+  /**
+   * @returns {number}
+   */
+  getWatchDogPollDelta() {
+    return (Math.floor(new Date() / 1000) - this.lastProcessPollTime)
+  },
+
   /**
    * @param options
    * @returns {Promise<void>}
    */
   start: async function (options = []) {
     this.init(options);
 
+    /**
+     * Start watchdog
+     * @type {module.exports}
+     */
+    let self           = this;
+    this.watchDogTimer = setInterval(function () {
+
+      let processStatus = {
+        'Zone Processes': self.processCount['zone'],
+        'Booted Zone Processes': self.zoneBootedProcessCount,
+        'World Processes': self.processCount['world'],
+        'Loginserver Processes': self.processCount['loginserver'],
+        'UCS Processes': self.processCount['ucs'],
+        'Watchdog Polling Delta': self.getWatchDogPollDelta() + ' / ' + MAX_PROCESS_POLL_DELTA_SECONDS_BEFORE_KILL
+      };
+
+      console.table(processStatus);
+
+      if (self.getWatchDogPollDelta() > MAX_PROCESS_POLL_DELTA_SECONDS_BEFORE_KILL) {
+        console.log('HIT WATCHDOG THRESHOLD. TERMINATING...');
+
+        const data    = new Date() + ' watchdog killed launcher\n';
+        const logFile = pathManager.getEmuServerPath() + '/logs/launcher-watchdog-kill.log';
+
+        if (fs.existsSync(logFile)) {
+          fs.appendFileSync(logFile, data);
+        } else {
+          fs.writeFileSync(logFile, data);
+        }
+
+        process.exit()
+      }
+
+    }, WATCHDOG_TIMER);
+
+    /**
+     * Main launcher loop
+     */
     while (1) {
       await this.pollProcessList();
       await this.getBootedZoneCount();
@@ -54,12 +109,12 @@ module.exports = {
        * @type {exports}
        */
       let self = this;
-      ['loginserver', 'ucs', 'world', 'queryserv'].forEach(async function (process_name) {
+      for (const process_name of ['loginserver', 'ucs', 'world', 'queryserv']) {
         if (self.doesProcessNeedToBoot(process_name) && !await self.isProcessRunning(process_name)) {
-          debug("starting unique process [" + process_name + "]")
+          debug('starting unique process [' + process_name + ']')
           self.startProcess(process_name);
         }
-      });
+      }
 
       /**
        * Zone
@@ -68,13 +123,9 @@ module.exports = {
         await self.startProcess('zone');
       }
 
-      console.log('Zone Processes [%s]', this.processCount['zone']);
-      console.log('World Process [%s]', this.processCount['world']);
-      console.log('Loginserver Process [%s]', this.processCount['loginserver']);
-      console.log('UCS Process [%s]', this.processCount['ucs']);
-      console.log('Booted Zone Processes [%s]', this.zoneBootedProcessCount);
+      await this.sleep(LAUNCHER_MAIN_LOOP_TIMER);
 
-      await this.sleep(5000);
+      this.lastProcessPollTime = Math.floor(new Date() / 1000);
     }
   },
 
@@ -97,12 +148,12 @@ module.exports = {
 
       return true;
     }
+
     if (process_name === 'loginserver') {
       return this.launchOptions && this.launchOptions.withLoginserver && this.processCount[process_name] === 0;
-
     }
 
-    debug("[doesProcessNeedToBoot] returning [%s]", this.processCount[process_name] === 0)
+    debug('[doesProcessNeedToBoot] returning [%s]', this.processCount[process_name] === 0)
 
     return this.processCount[process_name] === 0;
   },
@@ -168,16 +219,14 @@ module.exports = {
     // TODO: Windows
     if (process.platform === 'linux') {
       startProcessString = util.format(
-        'PKG_EXECPATH=; nohup %s server-launcher %s &',
+        'while : ; do PKG_EXECPATH=; nohup %s server-launcher %s && sleep 1 ; done &',
+        //'PKG_EXECPATH=; nohup %s server-launcher %s &',
         pathManager.getEqemuAdminEntrypoint(),
         argString
       );
     }
 
-    const command = util.format('rm -rf %s*', this.getLogRedirectDir());
-    exec(command, { cwd: pathManager.emuServerPath });
-    exec("rm -rf logs/zone/*.log", { cwd: pathManager.emuServerPath });
-    exec("rm -rf logs/*.log", { cwd: pathManager.emuServerPath });
+    this.purgeServerLogs();
 
     debug('start string [%s]', startProcessString);
     debug('cwd [%s]', path.join(path.dirname(process.argv[0]), '../'));
@@ -245,15 +294,6 @@ module.exports = {
     return this.zoneBootedProcessCount;
   },
 
-  getLogRedirectDir() {
-    const logRedirectDir = path.join(os.tmpdir(), 'admin-process-logs');
-    if (!fs.existsSync(logRedirectDir)) {
-      fs.mkdirSync(logRedirectDir);
-    }
-
-    return logRedirectDir;
-  },
-
   /**
    * @param process_name
    * @param args
@@ -271,17 +311,13 @@ module.exports = {
     if (is_windows) {
       start_process_string = process_name + '.exe';
     }
+
     if (is_linux) {
       start_process_string =
         util.format('./bin/%s %s &',
           process_name,
           argString
         );
-
-      if (process_name !== 'zone') {
-        const command = util.format('rm -rf %s*', path.join(this.getLogRedirectDir(), process_name));
-        exec(command, { cwd: pathManager.emuServerPath });
-      }
     }
 
     debug('Starting process [%s] command [%s] path [%s]',
@@ -303,26 +339,6 @@ module.exports = {
     // console.log('stdout ', child);
   },
 
-  /**
-   * @param process_name
-   * @param args
-   * @param data
-   */
-  handleProcessError: function (process_name, args, data) {
-    console.error('[Error] Process [%s] via path: [%s], with args failed to start [%s]\n%s',
-      process_name,
-      pathManager.emuServerPath,
-      args,
-      data
-    );
-
-    if (typeof this.erroredStartsCount[process_name] === 'undefined') {
-      this.erroredStartsCount[process_name] = 0;
-    }
-
-    this.erroredStartsCount[process_name]++;
-  },
-
   /**
    * @param process_id
    */
@@ -385,18 +401,25 @@ module.exports = {
       }
     })();
 
-
     const stdout = require('child_process').execSync(cmd).toString();
 
     let foundProcess = false;
-    stdout.split("\n").forEach(function(row) {
+    stdout.split('\n').forEach(function (row) {
       if (row.toLowerCase().trim() === processName.toLowerCase().trim()) {
         foundProcess = true;
       }
     });
 
-    debug("[isProcessRunning] found process [%s] [%s]", processName, foundProcess)
+    debug('[isProcessRunning] found process [%s] [%s]', processName, foundProcess)
 
     return foundProcess;
+  },
+
+  /**
+   * Purges server logs
+   */
+  purgeServerLogs() {
+    exec('rm -rf logs/zone/*.log', { cwd: pathManager.emuServerPath });
+    exec('rm -rf logs/*.log', { cwd: pathManager.emuServerPath });
   }
 };