fix: error handling when payload contains toString as key

.gitignore

@@ -132,3 +132,6 @@ dist
 
 # Others
 **/.DS_Store
+
+
+.idea

src/warehouse/config/helpers.js

@@ -1,12 +1,18 @@
 const _ = require('lodash');
 const get = require('get-value');
+const logger = require('../../logger');
 
 const isNull = (x) => {
   return x === null || x === undefined;
 };
 
 const isBlank = (value) => {
-  return _.isEmpty(_.toString(value));
+  try {
+    return _.isEmpty(_.toString(value));
+  } catch (e) {
+    logger.error(`Error in isBlank: ${e.message}`);
+    return false;
+  }
 };
 
 const getFirstValidValue = (message, props) => {
@@ -27,6 +33,7 @@ function isDataLakeProvider(provider) {
 
 module.exports = {
   isNull,
+  isBlank,
   getFirstValidValue,
   isDataLakeProvider,
 };

src/warehouse/index.js

@@ -5,7 +5,6 @@ const { v4: uuidv4 } = require('uuid');
 
 const {
   isObject,
-  isBlank,
   isValidJsonPathKey,
   isValidLegacyJsonPathKey,
   keysFromJsonPaths,
@@ -24,7 +23,7 @@ const whPageColumnMappingRules = require('./config/WHPageConfig.js');
 const whScreenColumnMappingRules = require('./config/WHScreenConfig.js');
 const whGroupColumnMappingRules = require('./config/WHGroupConfig.js');
 const whAliasColumnMappingRules = require('./config/WHAliasConfig.js');
-const { isDataLakeProvider } = require('./config/helpers');
+const {isDataLakeProvider, isBlank} = require('./config/helpers');
 const { InstrumentationError } = require('@rudderstack/integrations-lib');
 const whExtractEventTableColumnMappingRules = require('./config/WHExtractEventTableConfig.js');
 

src/warehouse/util.js

@@ -4,6 +4,7 @@ const get = require('get-value');
 const v0 = require('./v0/util');
 const v1 = require('./v1/util');
 const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib');
+const {isBlank} = require('./config/helpers');
 
 const minTimeInMs = Date.parse('0001-01-01T00:00:00Z');
 const maxTimeInMs = Date.parse('9999-12-31T23:59:59.999Z');
@@ -22,10 +23,6 @@ const isValidLegacyJsonPathKey = (eventType, key, level, jsonKeys = {}) => {
   return eventType === 'track' && jsonKeys[key] === level;
 };
 
-const isBlank = (value) => {
-  return _.isEmpty(_.toString(value));
-};
-
 /*
 This function takes in an array of json paths and returns an object with keys as the json path and value as the position of the key in the json path
 Example:
@@ -97,6 +94,9 @@ const timestampRegex = new RegExp(
 );
 
 function validTimestamp(input) {
+  if (typeof input !== 'string') {
+    return false;
+  }
   if (timestampRegex.test(input)) {
     // check if date value lies in between min time and max time. if not then it's not a valid timestamp
     const d = new Date(input);
@@ -147,7 +147,6 @@ const getRecordIDForExtract = (message) => {
 
 module.exports = {
   isObject,
-  isBlank,
   isValidJsonPathKey,
   isValidLegacyJsonPathKey,
   keysFromJsonPaths,

test/__tests__/data/warehouse/events.js

@@ -60,7 +60,17 @@ const sampleEvents = {
         originalTimestamp: "2020-01-24T06:29:02.364Z",
         properties: {
           currency: "USD",
-          revenue: 50
+          revenue: 50,
+          stack: {
+            history: {
+              errorDetails: [
+                {
+                  "message": "Cannot set headers after they are sent to the client",
+                  "toString": "[function]"
+                }
+              ]
+            }
+          }
         },
         receivedAt: "2020-01-24T11:59:02.403+05:30",
         request_ip: "[::1]:53708",
@@ -180,6 +190,12 @@ const sampleEvents = {
           data: {
             currency: "USD",
             revenue: 50,
+            stack_history_error_details: [
+              {
+                "message": "Cannot set headers after they are sent to the client",
+                "toString": "[function]"
+              }
+            ],
             context_app_build: "1.0.0",
             context_app_name: "RudderLabs JavaScript SDK",
             context_app_namespace: "com.rudderlabs.javascript",
@@ -314,6 +330,12 @@ const sampleEvents = {
           data: {
             CURRENCY: "USD",
             REVENUE: 50,
+            STACK_HISTORY_ERROR_DETAILS: [
+              {
+                "message": "Cannot set headers after they are sent to the client",
+                "toString": "[function]"
+              }
+            ],
             CONTEXT_APP_BUILD: "1.0.0",
             CONTEXT_APP_NAME: "RudderLabs JavaScript SDK",
             CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript",
@@ -340,7 +362,7 @@ const sampleEvents = {
             RECEIVED_AT: "2020-01-24T06:29:02.403Z",
             ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z",
             CHANNEL: "web",
-            EVENT: "button_clicked"
+            EVENT: "button_clicked",
           }
         }
       ],
@@ -448,6 +470,12 @@ const sampleEvents = {
           data: {
             currency: "USD",
             revenue: 50,
+            stack_history_error_details: [
+              {
+                "message": "Cannot set headers after they are sent to the client",
+                "toString": "[function]"
+              }
+            ],
             context_app_build: "1.0.0",
             context_app_name: "RudderLabs JavaScript SDK",
             context_app_namespace: "com.rudderlabs.javascript",
@@ -2053,7 +2081,6 @@ const sampleEvents = {
               id: "string",
               user_id: "string",
               received_at: "datetime",
-              event: "string"
             },
             receivedAt: "2020-01-24T11:59:02.403+05:30"
           },
@@ -2121,7 +2148,6 @@ const sampleEvents = {
               id: "string",
               user_id: "string",
               received_at: "datetime",
-              event: "string",
               boolean_property: "boolean",
             },
             receivedAt: "2020-01-24T11:59:02.403+05:30"

test/__tests__/warehouse.test.js

@@ -20,6 +20,7 @@ const {
 const {
   validTimestamp
 } = require("../../src/warehouse/util.js");
+const {isBlank} = require("../../src/warehouse/config/helpers.js");
 
 const version = "v0";
 const integrations = [
@@ -1096,54 +1097,117 @@ describe("Integration options", () => {
 describe("validTimestamp", () => {
   const testCases = [
     {
+      name: "undefined input should return false",
       input: undefined,
       expected: false,
     },
     {
+      name: "negative year and time input should return false #1",
       input: '-0001-11-30T00:00:00+0000',
       expected: false,
     },
     {
+      name: "negative year and time input should return false #2",
       input: '-2023-06-14T05:23:59.244Z',
       expected: false,
     },
     {
+      name: "negative year and time input should return false #3",
+      input: '-1900-06-14T05:23:59.244Z',
+      expected: false,
+    },
+    {
+      name: "positive year and time input should return false",
       input: '+2023-06-14T05:23:59.244Z',
       expected: false,
     },
     {
+      name: "valid timestamp input should return true",
       input: '2023-06-14T05:23:59.244Z',
       expected: true,
     },
     {
-      input: '-1900-06-14T05:23:59.244Z',
-      expected: false,
-    },
-    {
+      name: "non-date string input should return false",
       input: 'abc',
       expected: false,
     },
     {
-      input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein',
+      name: "malicious string input should return false",
+      input: '%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216%u002e%u002e%u2216Windows%u2216win%u002ein',
       expected: false,
     },
     {
+      name: "empty string input should return false",
       input: '',
       expected: false,
     },
     {
+      name: "valid date input should return true",
       input: '2023-06-14',
       expected: true,
     },
     {
+      name: "time-only input should return false",
       input: '05:23:59.244Z',
       expected: false,
-    }
-  ]
+    },
+    {
+      name: "non-string input should return false",
+      input: { abc: 123 },
+      expected: false,
+    },
+    {
+      name: "object with toString method input should return false",
+      input: {
+        toString: '2023-06-14T05:23:59.244Z'
+      },
+      expected: false,
+    },
+  ];
 
   for (const testCase of testCases) {
-    it(`should return ${testCase.expected} for ${testCase.input}`, () => {
+    it(`should return ${testCase.expected} for ${testCase.name}`, () => {
       expect(validTimestamp(testCase.input)).toEqual(testCase.expected);
     });
   }
 });
+
+
+
+describe("isBlank", () => {
+  const testCases = [
+    {
+      name: "null",
+      input: null,
+      expected: true
+    },
+    {
+      name: "empty string",
+      input: "",
+      expected: true
+    },
+    {
+      name: "non-empty string",
+      input: "test",
+      expected: false
+    },
+    {
+      name: "numeric value",
+      input: 1634762544,
+      expected: false
+    },
+    {
+      name: "object with toString property",
+      input: {
+        toString: '2023-06-14T05:23:59.244Z'
+      },
+      expected: false
+    },
+  ];
+
+  for (const testCase of testCases) {
+    it(`should return ${testCase.expected} for ${testCase.name}`, () => {
+      expect(isBlank(testCase.input)).toEqual(testCase.expected);
+    });
+  }
+});