ZENKO-2909: add heap profiler

countItems.js

@@ -1,8 +1,22 @@
 const cluster = require('cluster');
 
+const { parseEnvInt } = require('./utils');
+
+const HEAP_PROFILER_INTERVAL_MS =
+    parseEnvInt(process.env.HEAP_PROFILER_INTERVAL_MS, 10 * 60 * 1000);
+const HEAP_PROFILER_PATH = process.env.HEAP_PROFILER_PATH;
+
 if (cluster.isMaster) {
+    require('./utils/heapProfiler')(
+        `${HEAP_PROFILER_PATH}/master`,
+        HEAP_PROFILER_INTERVAL_MS
+    );
     require('./CountItems/masterProcess');
 } else if (cluster.isWorker) {
+    require('./utils/heapProfiler')(
+        `${HEAP_PROFILER_PATH}/worker/${cluster.worker.id}`,
+        HEAP_PROFILER_INTERVAL_MS
+    );
     require('./CountItems/workerProcess');
 }
 

package.json

@@ -30,6 +30,7 @@
     "arsenal": "github:scality/arsenal#81d05b6",
     "async": "^2.6.1",
     "aws-sdk": "^2.333.0",
+    "heapdump": "^0.3.15",
     "node-uuid": "^1.4.8",
     "werelogs": "scality/werelogs",
     "zenkoclient": "scality/zenkoclient#5c7f655"

stalled.js

@@ -14,26 +14,15 @@ const {
     StalledRequestHandler,
 } = require('./StalledRetry/StalledRequestHandler');
 
+const { parseEnvInt } = require('./utils');
+
 const ENDPOINT = process.env.ENDPOINT;
 const ACCESS_KEY = process.env.ACCESS_KEY;
 const SECRET_KEY = process.env.SECRET_KEY;
 const MONGODB_REPLICASET = process.env.MONGODB_REPLICASET;
 const MONGODB_DATABASE = process.env.MONGODB_DATABASE || 'metadata';
 const DRY_RUN = process.env.DRY_RUN && process.env.DRY_RUN !== '0';
 
-/**
- * parse environment variable as a integer number
- * @param {string} env - environment variable value
- * @param {int} defValue - default value if environment cannot be found or is
- *      not int parseable
- * @returns {int} - return integer value
- */
-function parseEnvInt(env, defValue) {
-    return (env && !Number.isNaN(env))
-        ? Number.parseInt(env, 10)
-        : defValue;
-}
-
 const BATCH_SIZE = parseEnvInt(process.env.REQUEST_BATCH_SIZE, 10);
 const QUEUE_LIMIT = parseEnvInt(process.env.QUEUE_LIMIT, 1000);
 const CONCURRENT_REQUESTS = parseEnvInt(process.env.CONCURRENT_REQUESTS, 5);
@@ -54,6 +43,12 @@ if (!SECRET_KEY) {
 if (!MONGODB_REPLICASET) {
     throw new Error('MONGODB_REPLICASET not defined');
 }
+
+const HEAP_PROFILER_INTERVAL_MS =
+    parseEnvInt(process.env.HEAP_PROFILER_INTERVAL_MS, 10 * 60 * 1000);
+const HEAP_PROFILER_PATH = process.env.HEAP_PROFILER_PATH;
+require('./utils/heapProfiler')(HEAP_PROFILER_PATH, HEAP_PROFILER_INTERVAL_MS);
+
 const log = new Logger('S3Utils::Stalled');
 
 function wrapperFactory(bucketName, cmpDate, cursor, log) {
@@ -97,7 +92,7 @@ const config = {
     writeConcern: 'majority',
     replicaSet: 'rs0',
     readPreference: 'primary',
-    database: 'metadata',
+    database: MONGODB_DATABASE,
     replicationGroupId: 'RG001',
     logger: log,
     cursorWrapperFactory: wrapperFactory,

utils/heapProfiler.js

@@ -0,0 +1,43 @@
+const fs = require('fs');
+const path = require('path');
+const werelogs = require('werelogs');
+const heapdump = require('heapdump');
+
+const logger = new werelogs.Logger('Utils::HeapProfiler');
+
+function generateFilename(dir) {
+    return path.join(dir, `${Date.now()}.heapsnapshot`);
+}
+
+function writeSnapshot(filename) {
+    heapdump.writeSnapshot(filename, (err, filename) => {
+        if (err) {
+            logger.error(`unable to write heapdump to ${filename}:`, {
+                error: err,
+            });
+        } else {
+            logger.debug(`heapdump written to ${filename}:`);
+        }
+    });
+}
+
+module.exports = (dir, interval) => {
+    if (process.env.ENABLE_HEAP_PROFILER &&
+        process.env.ENABLE_HEAP_PROFILER !== '0') {
+        if (!fs.existsSync(dir)) {
+            try {
+                fs.mkdirSync(dir, { recursive: true });
+            } catch (err) {
+                logger.error('unable to create directory for heap profiler', {
+                    error: err,
+                });
+                logger.error('skipping heap profiling');
+                return;
+            }
+        }
+
+        writeSnapshot(generateFilename(dir));
+        setInterval(() => writeSnapshot(generateFilename(dir)), interval);
+        process.on('exit', () => writeSnapshot(generateFilename(dir)));
+    }
+};

utils/index.js

@@ -0,0 +1,17 @@
+
+/**
+ * parse environment variable as a integer number
+ * @param {string} env - environment variable value
+ * @param {int} defValue - default value if environment cannot be found or is
+ *      not int parseable
+ * @returns {int} - return integer value
+ */
+function parseEnvInt(env, defValue) {
+    return (env && !Number.isNaN(env))
+        ? Number.parseInt(env, 10)
+        : defValue;
+}
+
+module.exports = {
+    parseEnvInt,
+};

yarn.lock

@@ -273,7 +273,7 @@ are-we-there-yet@~1.1.2:
     delegates "^1.0.0"
     readable-stream "^2.0.6"
 
-argparse@^1.0.7:
+argparse@^1.0.7, argparse@~1.0.2:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
   integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==
@@ -1015,10 +1015,12 @@ combined-stream@~1.0.6:
   dependencies:
     delayed-stream "~1.0.0"
 
-[email protected]:
-  version "2.19.0"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
-  integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==
+[email protected]:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-1.3.2.tgz#8a8f30ec670a6fdd64af52f1914b907d79ead5b5"
+  integrity sha1-io8w7GcKb91kr1LxkUuQfXnq1bU=
+  dependencies:
+    keypress "0.1.x"
 
 commander@~2.17.1:
   version "2.17.1"
@@ -1512,8 +1514,8 @@ eslint-config-airbnb@^6.2.0:
   version "8.1.0"
   resolved "https://codeload.github.com/scality/Guidelines/tar.gz/e49d1a2abfbbfb2c766c92d04a695b9171d511ff"
   dependencies:
-    commander "2.19.0"
-    markdownlint "0.11.0"
+    commander "1.3.2"
+    markdownlint "0.0.8"
 
 eslint-plugin-jest@^23.6.0:
   version "23.6.0"
@@ -2148,6 +2150,13 @@ hdclient@scality/hdclient#5145e04e5ed33e85106765b1caa90cd245ef482b:
   dependencies:
     werelogs scality/werelogs#GA7.2.0.5
 
+heapdump@^0.3.15:
+  version "0.3.15"
+  resolved "https://registry.yarnpkg.com/heapdump/-/heapdump-0.3.15.tgz#631a8a2585588ea64778d8ec80a64c6c025f6a08"
+  integrity sha512-n8aSFscI9r3gfhOcAECAtXFaQ1uy4QSke6bnaL+iymYZ/dWs9cqDqHM+rALfsHUwukUbxsdlECZ0pKmJdQ/4OA==
+  dependencies:
+    nan "^2.13.2"
+
 home-or-tmp@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-2.0.0.tgz#e36c3f2d2cae7d746a857e38d18d5f32a7882db8"
@@ -3144,6 +3153,11 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
+[email protected]:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/keypress/-/keypress-0.1.0.tgz#4a3188d4291b66b4f65edb99f806aa9ae293592a"
+  integrity sha1-SjGI1CkbZrT2XtuZ+AaqmuKTWSo=
+
 kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
@@ -3314,10 +3328,10 @@ levn@^0.3.0, levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
-linkify-it@^2.0.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-2.2.0.tgz#e3b54697e78bf915c70a38acd78fd09e0058b1cf"
-  integrity sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==
+linkify-it@~1.2.0:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-1.2.4.tgz#0773526c317c8fd13bd534ee1d180ff88abf881a"
+  integrity sha1-B3NSbDF8j9E71TTuHRgP+Iq/iBo=
   dependencies:
     uc.micro "^1.0.1"
 
@@ -3434,23 +3448,23 @@ map-visit@^1.0.0:
   dependencies:
     object-visit "^1.0.0"
 
-markdown-it@8.4.2:
-  version "8.4.2"
-  resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-8.4.2.tgz#386f98998dc15a37722aa7722084f4020bdd9b54"
-  integrity sha512-GcRz3AWTqSUphY3vsUqQSFMbgR38a4Lh3GWlHRh/7MRwz8mcu9n2IO7HOh+bXHrR9kOPDl5RNCaEsrneb+xhHQ==
+markdown-it@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-4.4.0.tgz#3df373dbea587a9a7fef3e56311b68908f75c414"
+  integrity sha1-PfNz2+pYepp/7z5WMRtokI91xBQ=
   dependencies:
-    argparse "^1.0.7"
+    argparse "~1.0.2"
     entities "~1.1.1"
-    linkify-it "^2.0.0"
-    mdurl "^1.0.1"
-    uc.micro "^1.0.5"
+    linkify-it "~1.2.0"
+    mdurl "~1.0.0"
+    uc.micro "^1.0.0"
 
-markdownlint@0.11.0:
-  version "0.11.0"
-  resolved "https://registry.yarnpkg.com/markdownlint/-/markdownlint-0.11.0.tgz#3858bbdbc1ab78abf0c098d841c72b63dd3206a0"
-  integrity sha512-wE5WdKD6zW2DQaPQ5TFBTXh5j76DnWd/IFffnDQgHmi6Y61DJXBDfLftZ/suJHuv6cwPjM6gKw2GaRLJMOR+Mg==
+markdownlint@0.0.8:
+  version "0.0.8"
+  resolved "https://registry.yarnpkg.com/markdownlint/-/markdownlint-0.0.8.tgz#ed2a0fc6cba3aae6af09ae08bdbfa4de178a4677"
+  integrity sha1-7SoPxsujquavCa4Ivb+k3heKRnc=
   dependencies:
-    markdown-it "8.4.2"
+    markdown-it "^4.4.0"
 
 math-random@^1.0.1:
   version "1.0.1"
@@ -3465,7 +3479,7 @@ [email protected]:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-mdurl@^1.0.1:
+mdurl@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
   integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=
@@ -3655,6 +3669,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.5.tgz#8fbfabb0a98a253d3184331f9e8deb7372fac6c0"
   integrity sha1-j7+rsKmKJT0xhDMfno3rc3L6xsA=
 
+nan@^2.13.2:
+  version "2.14.2"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
+  integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
+
 nan@^2.14.0:
   version "2.14.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
@@ -5209,16 +5228,16 @@ typewiselite@~1.0.0:
   resolved "https://registry.yarnpkg.com/typewiselite/-/typewiselite-1.0.0.tgz#c8882fa1bb1092c06005a97f34ef5c8508e3664e"
   integrity sha1-yIgvobsQksBgBal/NO9chQjjZk4=
 
+uc.micro@^1.0.0:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
+  integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==
+
 uc.micro@^1.0.1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.5.tgz#0c65f15f815aa08b560a61ce8b4db7ffc3f45376"
   integrity sha512-JoLI4g5zv5qNyT09f4YAvEZIIV1oOjqnewYg5D38dkQljIzpPT296dbIGvKro3digYI1bkb7W6EP1y4uDlmzLg==
 
-uc.micro@^1.0.5:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
-  integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==
-
 uglify-js@^3.1.4:
   version "3.4.9"
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.4.9.tgz#af02f180c1207d76432e473ed24a28f4a782bae3"