Merge branch 'develop' into test-refactor.fbPixel

.github/workflows/prepare-for-staging-deploy.yml

@@ -101,7 +101,7 @@ jobs:
           cd rudder-devops
           BRANCH_NAME="shared-transformer-$TAG_NAME"
           echo $BRANCH_NAME
-          if [ -n `git ls-remote --heads origin $BRANCH_NAME 2>/dev/null` ]
+          if [ -n "$(git ls-remote --heads origin $BRANCH_NAME 2>/dev/null)" ]
           then
             echo "Staging deployment branch already exists!"
           else

.gitignore

@@ -137,4 +137,5 @@ dist
 .idea
 
 # component test report
-test_reports/
+test_reports/
+temp/

CHANGELOG.md

@@ -2,6 +2,32 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04)
+
+
+### Features
+
+* add support for interaction events in sfmc ([#3109](https://github.com/rudderlabs/rudder-transformer/issues/3109)) ([0486049](https://github.com/rudderlabs/rudder-transformer/commit/0486049ba2ad96b50d8f29e96b46b96a8a5c9f76))
+* add support of custom page/screen event name in mixpanel ([#3098](https://github.com/rudderlabs/rudder-transformer/issues/3098)) ([0eb2393](https://github.com/rudderlabs/rudder-transformer/commit/0eb2393939fba2452ef7f07a1d149d87f18290c3))
+* consent mode support for google adwords remarketing list  ([#3143](https://github.com/rudderlabs/rudder-transformer/issues/3143)) ([7532c90](https://github.com/rudderlabs/rudder-transformer/commit/7532c90d7e1feac00f12961c56da18757010f44a))
+* **facebook:** update content_type mapping logic for fb pixel and fb conversions ([#3113](https://github.com/rudderlabs/rudder-transformer/issues/3113)) ([aea417c](https://github.com/rudderlabs/rudder-transformer/commit/aea417cd2691547399010c034cadbc5db6b0c6ee))
+* klaviyo profile mapping ([#3105](https://github.com/rudderlabs/rudder-transformer/issues/3105)) ([2761786](https://github.com/rudderlabs/rudder-transformer/commit/2761786ff3fc99ed6d4d3b7a6c2400226b1cfb12))
+* onboard new destination ninetailed ([#3106](https://github.com/rudderlabs/rudder-transformer/issues/3106)) ([0e2588e](https://github.com/rudderlabs/rudder-transformer/commit/0e2588ecd87f3b2c6877a099aa1cbf2d5325966c))
+
+
+### Bug Fixes
+
+* add error handling for tiktok ads ([#3144](https://github.com/rudderlabs/rudder-transformer/issues/3144)) ([e93e47f](https://github.com/rudderlabs/rudder-transformer/commit/e93e47f33e098104fb532916932fe38bbfeaa4a1))
+* **algolia:** added check for objectIds or filters to be non empty ([#3126](https://github.com/rudderlabs/rudder-transformer/issues/3126)) ([d619c97](https://github.com/rudderlabs/rudder-transformer/commit/d619c9769cd270cb2d16dad0865683ff4beb2d19))
+* clevertap remove stringification of array object properties ([#3048](https://github.com/rudderlabs/rudder-transformer/issues/3048)) ([69e43b6](https://github.com/rudderlabs/rudder-transformer/commit/69e43b6ffadeaec87b7440da34a341890ceba252))
+* convert to string from null in hs ([#3136](https://github.com/rudderlabs/rudder-transformer/issues/3136)) ([75e9f46](https://github.com/rudderlabs/rudder-transformer/commit/75e9f462b0ff9b9a8abab3c78dc7d147926e9e5e))
+* event fix and added utility ([#3142](https://github.com/rudderlabs/rudder-transformer/issues/3142)) ([9b705b7](https://github.com/rudderlabs/rudder-transformer/commit/9b705b71a9d3a595ea0fbf532602c3941b0a18db))
+* metadata structure correction ([#3119](https://github.com/rudderlabs/rudder-transformer/issues/3119)) ([8351b5c](https://github.com/rudderlabs/rudder-transformer/commit/8351b5cbbf81bbc14b2f884feaae4ad3ca59a39a))
+* one_signal:  Encode external_id in endpoint ([#3140](https://github.com/rudderlabs/rudder-transformer/issues/3140)) ([8a20886](https://github.com/rudderlabs/rudder-transformer/commit/8a2088608d6da4b35bbb506db2fc3df1e4d41f3b))
+* rakuten: sync property mapping sourcekeys to rudderstack standard spec ([#3129](https://github.com/rudderlabs/rudder-transformer/issues/3129)) ([2ebff95](https://github.com/rudderlabs/rudder-transformer/commit/2ebff956ff2aa74b008a8de832a31d8774d2d47e))
+* reddit revenue mapping for floating point values ([#3118](https://github.com/rudderlabs/rudder-transformer/issues/3118)) ([41f4078](https://github.com/rudderlabs/rudder-transformer/commit/41f4078011ef54334bb9ecc11a7b2ccc8831a4aa))
+* version deprecation failure false positive ([#3104](https://github.com/rudderlabs/rudder-transformer/issues/3104)) ([657b780](https://github.com/rudderlabs/rudder-transformer/commit/657b7805eb01da25a007d978198d5debf03917fd))
+
 ### [1.57.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.0...v1.57.1) (2024-03-04)
 
 

github-release.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  gitRawCommitsOpts: {
+    merges: null,
+  },
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.57.1",
+  "version": "1.58.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -49,7 +49,7 @@
     "commit-msg": "commitlint --edit",
     "prepare": "node ./scripts/skipPrepareScript.js || husky install",
     "release": "npx standard-version",
-    "release:github": "DEBUG=conventional-github-releaser npx conventional-github-releaser -p angular -v",
+    "release:github": "DEBUG=conventional-github-releaser npx conventional-github-releaser -p angular --config github-release.config.js",
     "clean:node": "modclean",
     "check:lint": "eslint . -f json -o reports/eslint.json || exit 0"
   },

src/util/prometheus.js

@@ -710,7 +710,7 @@ class Prometheus {
         name: 'get_libraries_code_time',
         help: 'get_libraries_code_time',
         type: 'histogram',
-        labelNames: ['libraryVersionId', 'versionId', 'type'],
+        labelNames: ['libraryVersionId', 'versionId', 'type', 'version'],
       },
       {
         name: 'isolate_cpu_time',

src/util/redis/redisConnector.test.js

@@ -70,7 +70,7 @@ describe(`Redis Class Get Tests`, () => {
   data.forEach((dataPoint, index) => {
     it(`${index}. Redis Get- ${dataPoint.description}`, async () => {
       try {
-        const output = await RedisDB.getVal(dataPoint.input.value, (isObjExpected = false));
+        const output = await RedisDB.getVal(dataPoint.input.value, false);
         expect(output).toEqual(dataPoint.output);
       } catch (error) {
         expect(error.message).toEqual(dataPoint.output.error);

src/util/redis/testData/shopify_source.json

@@ -5,7 +5,8 @@
       "user_id": "rudder01",
       "id": "shopify_test_get_items_fail",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test_get_items_fail",
       "email": "[email protected]",
@@ -115,7 +116,8 @@
     "input": {
       "cart_token": "shopifyGetAnonymousId",
       "query_parameters": {
-        "topic": ["checkouts_delete"]
+        "topic": ["checkouts_delete"],
+        "writeKey": ["wr"]
       },
       "line_items": [],
       "note": null,
@@ -154,7 +156,8 @@
     "input": {
       "id": "shopify_test3",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test3",
       "line_items": [],
@@ -252,7 +255,8 @@
       "user_id": "rudder01",
       "id": "shopify_test_cart",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test_cart",
       "email": "[email protected]",
@@ -1256,7 +1260,8 @@
     "input": {
       "id": "shopify_test4",
       "query_parameters": {
-        "topic": ["carts_update"]
+        "topic": ["carts_update"],
+        "writeKey": ["wr"]
       },
       "token": "shopify_test4",
       "line_items": [],

src/v0/destinations/clevertap/data/CleverTapIdentify.json

@@ -1,7 +1,7 @@
 [
   {
     "destKey": "Email",
-    "sourceKeys": "email",
+    "sourceKeys": "emailOnly",
     "required": false,
     "sourceFromGenericMap": true
   },

src/v0/sources/shopify/shopify_redis.util.test.js

@@ -1,5 +1,9 @@
 const { getAnonymousIdAndSessionId, checkAndUpdateCartItems } = require('./util');
 jest.mock('ioredis', () => require('../../../../test/__mocks__/redis'));
+const metricMetadata = {
+  writeKey: 'dummyKey',
+  source: 'src',
+};
 describe('Shopify Utils Test', () => {
   describe('Check for valid cart update event test cases', () => {
     it('Event containing token and nothing is retreived from redis and less than req. time difference between created_at and uadated_at', async () => {
@@ -14,7 +18,7 @@ describe('Shopify Utils Test', () => {
         created_at: '2023-02-10T12:05:04.402Z',
       };
       const expectedOutput = false;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
     it('Event containing token and nothing is retreived from redis', async () => {
@@ -28,7 +32,7 @@ describe('Shopify Utils Test', () => {
         ],
       };
       const expectedOutput = true;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
 
@@ -44,7 +48,7 @@ describe('Shopify Utils Test', () => {
       };
 
       const expectedOutput = true;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
 
@@ -60,7 +64,7 @@ describe('Shopify Utils Test', () => {
       };
 
       const expectedOutput = false;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
 
@@ -76,7 +80,7 @@ describe('Shopify Utils Test', () => {
       };
 
       const expectedOutput = true;
-      const output = await checkAndUpdateCartItems(input);
+      const output = await checkAndUpdateCartItems(input, null, metricMetadata);
       expect(output).toEqual(expectedOutput);
     });
   });

src/v0/sources/shopify/transform.js

@@ -143,7 +143,7 @@ const processEvent = async (inputEvent, metricMetadata) => {
       break;
     case 'carts_update':
       if (useRedisDatabase) {
-        redisData = await getDataFromRedis(event.id || event.token);
+        redisData = await getDataFromRedis(event.id || event.token, metricMetadata);
         const isValidEvent = await checkAndUpdateCartItems(inputEvent, redisData, metricMetadata);
         if (!isValidEvent) {
           return NO_OPERATION_SUCCESS;
@@ -155,7 +155,8 @@ const processEvent = async (inputEvent, metricMetadata) => {
       if (!SUPPORTED_TRACK_EVENTS.includes(shopifyTopic)) {
         stats.increment('invalid_shopify_event', {
           event: shopifyTopic,
-          ...metricMetadata,
+          source: metricMetadata.source,
+          shopifyTopic: metricMetadata.shopifyTopic,
         });
         return NO_OPERATION_SUCCESS;
       }
@@ -215,7 +216,8 @@ const processIdentifierEvent = async (event, metricMetadata) => {
       stats.increment('shopify_redis_calls', {
         type: 'set',
         field: 'itemsHash',
-        ...metricMetadata,
+        source: metricMetadata.source,
+        writeKey: metricMetadata.writeKey,
       });
       /* cart_token: {
            anonymousId: 'anon_id1',
@@ -236,14 +238,16 @@ const processIdentifierEvent = async (event, metricMetadata) => {
       stats.increment('shopify_redis_calls', {
         type: 'set',
         field,
-        ...metricMetadata,
+        source: metricMetadata.source,
+        writeKey: metricMetadata.writeKey,
       });
       await RedisDB.setVal(`${event.cartToken}`, value);
     } catch (e) {
       logger.debug(`{{SHOPIFY::}} cartToken map set call Failed due redis error ${e}`);
       stats.increment('shopify_redis_failures', {
         type: 'set',
-        ...metricMetadata,
+        source: metricMetadata.source,
+        writeKey: metricMetadata.writeKey,
       });
     }
   }

src/v0/sources/shopify/util.js

@@ -29,23 +29,26 @@ const getDataFromRedis = async (key, metricMetadata) => {
     stats.increment('shopify_redis_calls', {
       type: 'get',
       field: 'all',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
     const redisData = await RedisDB.getVal(key);
     if (
       redisData === null ||
       (typeof redisData === 'object' && Object.keys(redisData).length === 0)
     ) {
       stats.increment('shopify_redis_no_val', {
-        ...metricMetadata,
+        writeKey: metricMetadata.writeKey,
+        source: metricMetadata.source,
       });
     }
     return redisData;
   } catch (e) {
     logger.debug(`{{SHOPIFY::}} Get call Failed due redis error ${e}`);
     stats.increment('shopify_redis_failures', {
       type: 'get',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
   }
   return null;
@@ -166,7 +169,9 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n
   if (isDefinedAndNotNull(anonymousId) && isDefinedAndNotNull(sessionId)) {
     stats.increment('shopify_anon_id_resolve', {
       method: 'note_attributes',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
+      shopifyTopic: metricMetadata.shopifyTopic,
     });
     return { anonymousId, sessionId };
   }
@@ -198,7 +203,9 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n
     // and for how many
     stats.increment('shopify_anon_id_resolve', {
       method: 'database',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
+      shopifyTopic: metricMetadata.shopifyTopic,
     });
   }
   return { anonymousId, sessionId };
@@ -215,14 +222,16 @@ const updateCartItemsInRedis = async (cartToken, newCartItemsHash, metricMetadat
     stats.increment('shopify_redis_calls', {
       type: 'set',
       field: 'itemsHash',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
     await RedisDB.setVal(`${cartToken}`, value);
   } catch (e) {
     logger.debug(`{{SHOPIFY::}} itemsHash set call Failed due redis error ${e}`);
     stats.increment('shopify_redis_failures', {
       type: 'set',
-      ...metricMetadata,
+      writeKey: metricMetadata.writeKey,
+      source: metricMetadata.source,
     });
   }
 };