Merge pull request #420 from metrico/filter_nan

write nan values as NaN on ingestion
metrico · Jan 3, 2024 · 7089fc3 · 7089fc3
2 parents e0d6b4c + e10df0b
commit 7089fc3
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 5 deletions.
diff --git a/lib/db/clickhouse.js b/lib/db/clickhouse.js
@@ -38,7 +38,11 @@ const { parseLabels, hashLabels, isCustomSamplesOrderingRule, isOmitTablesCreati
 
 const { Worker, isMainThread } = require('worker_threads')
 
-const jsonSerializer = (k, val) => typeof val === 'bigint' ? val.toString() : val
+const jsonSerializer = (k, val) => typeof val === 'bigint'
+  ? val.toString()
+  : typeof val === 'number' && isNaN(val)
+    ? 'NaN'
+    : val
 
 const createCsvArrayWriter = require('csv-writer').createArrayCsvStringifier
 

diff --git a/lib/handlers/common.js b/lib/handlers/common.js
@@ -59,6 +59,18 @@ module.exports.checkCustomPlugins = async (options) => {
   }
 }
 
+/**
+ *
+ * @param value {number}
+ * @returns {[number,boolean]} new value and if I should ingest it
+ */
+module.exports.checkNanValue = (value) => {
+  if (typeof value === 'number' && isNaN(value)) {
+    return [NaN, true]
+  }
+  return [value, true]
+}
+
 const newResponse = () => {
   let head = null
   let onWriteHead = new EventEmitter()

diff --git a/lib/handlers/influx_write.js b/lib/handlers/influx_write.js
@@ -43,6 +43,7 @@ const DATABASE = require('../db/clickhouse')
 const { bulk_labels, bulk, labels } = DATABASE.cache
 const { fingerPrint } = require('../utils')
 const { readonly } = require('../../common')
+const { checkNanValue } = require('./common')
 
 async function handler (req, res) {
   if (!req.body && !req.body.metrics) {
@@ -108,13 +109,14 @@ async function handler (req, res) {
             asyncLogError('no bulkable data', req.log)
             return res.code(204).send('')
           }
-          if (typeof value === 'number' && isNaN(value)) {
+          const [_value, ingest] = checkNanValue(value)
+          if (!ingest) {
             return
           }
           const values = [
             finger,
             BigInt(pad('0000000000000000000', timestamp, true)),
-            value || 0,
+            _value || 0,
             key || ''
           ]
           bulk.add([values])

diff --git a/lib/handlers/prom_push.js b/lib/handlers/prom_push.js
@@ -17,6 +17,7 @@ const DATABASE = require('../db/clickhouse')
 const { bulk_labels, bulk, labels } = DATABASE.cache
 const { fingerPrint } = require('../utils')
 const { readonly } = require('../../common')
+const { checkNanValue } = require('./common')
 
 async function handler (req, res) {
   const self = this
@@ -59,14 +60,15 @@ async function handler (req, res) {
               asyncLogError({ entry }, req.log)
               return
             }
-            if (isNaN(entry.value)) {
+            const [value, ingest] = checkNanValue(entry.value)
+            if (!ingest) {
               return
             }
             const ts = BigInt(entry.timestamp)
             const values = [
               finger,
               ts,
-              entry.value,
+              value,
               JSONLabels.__name__ || 'undefined'
             ]
             dates[