Merge branch 'develop' into salesforce-oauth-proxy-v1

rudderlabs · Mar 11, 2024 · bc3a235 · bc3a235
2 parents 90446d8 + 01d460c
commit bc3a235
Show file tree

Hide file tree

Showing 13 changed files with 309 additions and 37 deletions.
diff --git a/.gitignore b/.gitignore
@@ -137,4 +137,5 @@ dist
 .idea
 
 # component test report
-test_reports/
+test_reports/
+temp/
diff --git a/src/util/prometheus.js b/src/util/prometheus.js
@@ -710,7 +710,7 @@ class Prometheus {
         name: 'get_libraries_code_time',
         help: 'get_libraries_code_time',
         type: 'histogram',
-        labelNames: ['libraryVersionId', 'versionId', 'type'],
+        labelNames: ['libraryVersionId', 'versionId', 'type', 'version'],
       },
       {
         name: 'isolate_cpu_time',

diff --git a/src/util/redis/redisConnector.test.js b/src/util/redis/redisConnector.test.js
@@ -70,7 +70,7 @@ describe(`Redis Class Get Tests`, () => {
   data.forEach((dataPoint, index) => {
     it(`${index}. Redis Get- ${dataPoint.description}`, async () => {
       try {
-        const output = await RedisDB.getVal(dataPoint.input.value, (isObjExpected = false));
+        const output = await RedisDB.getVal(dataPoint.input.value, false);
         expect(output).toEqual(dataPoint.output);
       } catch (error) {
         expect(error.message).toEqual(dataPoint.output.error);

diff --git a/src/util/redis/testData/shopify_source.json b/src/util/redis/testData/shopify_source.json
@@ -5,7 +5,8 @@
       "user_id": "rudder01",
       "id": "shopify_test_get_items_fail",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test_get_items_fail",
       "email": "[email protected]",
@@ -115,7 +116,8 @@
     "input": {
       "cart_token": "shopifyGetAnonymousId",
       "query_parameters": {
-        "topic": ["checkouts_delete"]
+        "topic": ["checkouts_delete"],
+        "writeKey": ["wr"]
       },
       "line_items": [],
       "note": null,
@@ -154,7 +156,8 @@
     "input": {
       "id": "shopify_test3",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test3",
       "line_items": [],
@@ -252,7 +255,8 @@
       "user_id": "rudder01",
       "id": "shopify_test_cart",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test_cart",
       "email": "[email protected]",
@@ -1256,7 +1260,8 @@
     "input": {
       "id": "shopify_test4",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test4",
       "line_items": [],

diff --git a/src/v0/destinations/clevertap/data/CleverTapIdentify.json b/src/v0/destinations/clevertap/data/CleverTapIdentify.json
@@ -1,7 +1,7 @@
 [
   {
     "destKey": "Email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

diff --git a/src/v0/sources/shopify/shopify_redis.util.test.js b/src/v0/sources/shopify/shopify_redis.util.test.js
@@ -1,5 +1,9 @@
 const { getAnonymousIdAndSessionId, checkAndUpdateCartItems } = require('./util');
 jest.mock('ioredis', () => require('../../../../test/__mocks__/redis'));
+const metricMetadata = {
+  writeKey: 'dummyKey',
+  source: 'src',
+};
 describe('Shopify Utils Test', () => {
   describe('Check for valid cart update event test cases', () => {
     it('Event containing token and nothing is retreived from redis and less than req. time difference between created_at and uadated_at', async () => {
@@ -14,7 +18,7 @@ describe('Shopify Utils Test', () => {
         created_at: '2023-02-10T12:05:04.402Z',
       };
       const expectedOutput = false;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
     it('Event containing token and nothing is retreived from redis', async () => {
@@ -28,7 +32,7 @@ describe('Shopify Utils Test', () => {
         ],
       };
       const expectedOutput = true;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
 
@@ -44,7 +48,7 @@ describe('Shopify Utils Test', () => {
       };
 
       const expectedOutput = true;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
 
@@ -60,7 +64,7 @@ describe('Shopify Utils Test', () => {
       };
 
       const expectedOutput = false;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
 
@@ -76,7 +80,7 @@ describe('Shopify Utils Test', () => {
       };
 
       const expectedOutput = true;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
   });

diff --git a/src/v0/sources/shopify/transform.js b/src/v0/sources/shopify/transform.js
@@ -143,7 +143,7 @@ const processEvent = async (inputEvent, metricMetadata) => {
       break;
     case 'carts_update':
       if (useRedisDatabase) {
-        redisData = await getDataFromRedis(event.id || event.token);
+        redisData = await getDataFromRedis(event.id || event.token, metricMetadata);
         const isValidEvent = await checkAndUpdateCartItems(inputEvent, redisData, metricMetadata);
         if (!isValidEvent) {
           return NO_OPERATION_SUCCESS;
@@ -155,7 +155,8 @@ const processEvent = async (inputEvent, metricMetadata) => {
       if (!SUPPORTED_TRACK_EVENTS.includes(shopifyTopic)) {
         stats.increment('invalid_shopify_event', {
           event: shopifyTopic,
-          ...metricMetadata,
+          source: metricMetadata.source,
+          shopifyTopic: metricMetadata.shopifyTopic,
         });
         return NO_OPERATION_SUCCESS;
       }
@@ -215,7 +216,8 @@ const processIdentifierEvent = async (event, metricMetadata) => {
       stats.increment('shopify_redis_calls', {
         type: 'set',
         field: 'itemsHash',
-        ...metricMetadata,
+        source: metricMetadata.source,
+        writeKey: metricMetadata.writeKey,
       });
       /* cart_token: {
            anonymousId: 'anon_id1',
@@ -236,14 +238,16 @@ const processIdentifierEvent = async (event, metricMetadata) => {
       stats.increment('shopify_redis_calls', {
         type: 'set',
         field,
-        ...metricMetadata,
+        source: metricMetadata.source,
+        writeKey: metricMetadata.writeKey,
       });
       await RedisDB.setVal(`${event.cartToken}`, value);
     } catch (e) {
       logger.debug(`{{SHOPIFY::}} cartToken map set call Failed due redis error ${e}`);
       stats.increment('shopify_redis_failures', {
         type: 'set',
-        ...metricMetadata,
+        source: metricMetadata.source,
+        writeKey: metricMetadata.writeKey,
       });
     }
   }

diff --git a/src/v0/sources/shopify/util.js b/src/v0/sources/shopify/util.js
@@ -29,23 +29,26 @@ const getDataFromRedis = async (key, metricMetadata) => {
     stats.increment('shopify_redis_calls', {
       type: 'get',
       field: 'all',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
     const redisData = await RedisDB.getVal(key);
     if (
       redisData === null ||
       (typeof redisData === 'object' && Object.keys(redisData).length === 0)
     ) {
       stats.increment('shopify_redis_no_val', {
-        ...metricMetadata,
+        writeKey: metricMetadata.writeKey,
+        source: metricMetadata.source,
       });
     }
     return redisData;
   } catch (e) {
     logger.debug(`{{SHOPIFY::}} Get call Failed due redis error ${e}`);
     stats.increment('shopify_redis_failures', {
       type: 'get',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
   }
   return null;
@@ -166,7 +169,9 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n
   if (isDefinedAndNotNull(anonymousId) && isDefinedAndNotNull(sessionId)) {
     stats.increment('shopify_anon_id_resolve', {
       method: 'note_attributes',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
+      shopifyTopic: metricMetadata.shopifyTopic,
     });
     return { anonymousId, sessionId };
   }
@@ -198,7 +203,9 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n
     // and for how many
     stats.increment('shopify_anon_id_resolve', {
       method: 'database',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
+      shopifyTopic: metricMetadata.shopifyTopic,
     });
   }
   return { anonymousId, sessionId };
@@ -215,14 +222,16 @@ const updateCartItemsInRedis = async (cartToken, newCartItemsHash, metricMetadat
     stats.increment('shopify_redis_calls', {
       type: 'set',
       field: 'itemsHash',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
     await RedisDB.setVal(`${cartToken}`, value);
   } catch (e) {
     logger.debug(`{{SHOPIFY::}} itemsHash set call Failed due redis error ${e}`);
     stats.increment('shopify_redis_failures', {
       type: 'set',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
   }
 };

diff --git a/test/integrations/destinations/clevertap/processor/data.ts b/test/integrations/destinations/clevertap/processor/data.ts
@@ -122,6 +122,130 @@ export const data = [
       },
     },
   },
+  {
+    name: 'clevertap',
+    description: 'Should not load email from externalId',
+    feature: 'processor',
+    module: 'destination',
+    version: 'v0',
+    input: {
+      request: {
+        body: [
+          {
+            destination: {
+              Config: {
+                passcode: 'sample_passcode',
+                accountId: '476550467',
+                trackAnonymous: true,
+                enableObjectIdMapping: false,
+              },
+            },
+            message: {
+              channel: 'web',
+              messageId: '84e26acc-56a5-4835-8233-591137fca468',
+              session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22',
+              originalTimestamp: '2019-10-14T09:03:17.562Z',
+              anonymousId: 'anon_id',
+              type: 'identify',
+              traits: {
+                anonymousId: 'anon_id',
+                name: 'James Doe',
+                phone: '92374162212',
+                gender: 'M',
+                employed: true,
+                birthday: '1614775793',
+                education: 'Science',
+                graduate: true,
+                married: true,
+                customerType: 'Prime',
+                msg_push: true,
+                msgSms: true,
+                msgemail: true,
+                msgwhatsapp: false,
+                custom_tags: ['Test_User', 'Interested_User', 'DIY_Hobby'],
+                custom_mappings: {
+                  Office: 'Trastkiv',
+                  Country: 'Russia',
+                },
+                address: {
+                  city: 'kolkata',
+                  country: 'India',
+                  postalCode: 789223,
+                  state: 'WB',
+                  street: '',
+                },
+                'category-unsubscribe': { email: ['Marketing', 'Transactional'] },
+              },
+              context: {
+                externalId: [{ type: 'someId', id: 'someID' }],
+              },
+              integrations: {
+                All: true,
+              },
+              sentAt: '2019-10-14T09:03:22.563Z',
+            },
+          },
+        ],
+      },
+    },
+    output: {
+      response: {
+        status: 200,
+        body: [
+          {
+            output: {
+              version: '1',
+              type: 'REST',
+              method: 'POST',
+              endpoint: 'https://api.clevertap.com/1/upload',
+              headers: {
+                'X-CleverTap-Account-Id': '476550467',
+                'X-CleverTap-Passcode': 'sample_passcode',
+                'Content-Type': 'application/json',
+              },
+              params: {},
+              body: {
+                JSON: {
+                  d: [
+                    {
+                      type: 'profile',
+                      profileData: {
+                        Name: 'James Doe',
+                        Phone: '92374162212',
+                        Gender: 'M',
+                        Employed: true,
+                        DOB: '1614775793',
+                        Education: 'Science',
+                        Married: true,
+                        'Customer Type': 'Prime',
+                        graduate: true,
+                        msg_push: true,
+                        msgSms: true,
+                        msgemail: true,
+                        msgwhatsapp: false,
+                        custom_mappings: '{"Office":"Trastkiv","Country":"Russia"}',
+                        custom_tags: '["Test_User","Interested_User","DIY_Hobby"]',
+                        address:
+                          '{"city":"kolkata","country":"India","postalCode":789223,"state":"WB","street":""}',
+                        'category-unsubscribe': { email: ['Marketing', 'Transactional'] },
+                      },
+                      identity: 'anon_id',
+                    },
+                  ],
+                },
+                JSON_ARRAY: {},
+                XML: {},
+                FORM: {},
+              },
+              files: {},
+              userId: '',
+            },
+            statusCode: 200,
+          },
+        ],
+      },
+    },
+  },
   {
     name: 'clevertap',
     description: 'Test 1',

diff --git a/test/integrations/destinations/salesforce/network.ts b/test/integrations/destinations/salesforce/network.ts
@@ -466,5 +466,5 @@ export const networkCallsData = [
   ...tfProxyMocksData,
   ...transformationMocksData,
   ...businessMockData,
-  ...otherMocksData
+  ...otherMocksData,
 ];