Merge branch 'develop' into chore.update-gaec-api

rudderlabs · Aug 26, 2024 · c71fab8 · c71fab8
2 parents 3d04d59 + 78e7a95
commit c71fab8
Show file tree

Hide file tree

Showing 29 changed files with 3,250 additions and 34 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -21,3 +21,5 @@ src/v0/destinations/personalize/scripts/
 test/integrations/destinations/testTypes.d.ts
 *.config*.js
 scripts/skipPrepareScript.js
+package-lock.json
+package.json
diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml
@@ -107,7 +107,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -124,7 +124,7 @@ jobs:
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -160,7 +160,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -177,7 +177,7 @@ jobs:
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.5.0
+        uses: docker/build-push-action@v6.7.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}

diff --git a/.prettierignore b/.prettierignore
@@ -8,3 +8,4 @@ test/**/*.js
 src/util/lodash-es-core.js
 src/util/url-search-params.min.js
 dist
+.eslintignore
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14)
+
+
+### Bug Fixes
+
+* add validation for concurrent_modification error ([#3654](https://github.com/rudderlabs/rudder-transformer/issues/3654)) ([62cdc46](https://github.com/rudderlabs/rudder-transformer/commit/62cdc4641d44d79e21949722660173df4c749f24))
+* clevertap bugsnag issue ([#3656](https://github.com/rudderlabs/rudder-transformer/issues/3656)) ([6c51487](https://github.com/rudderlabs/rudder-transformer/commit/6c51487183b6b0b755620aac6cd51c2ffc966102))
+* snapchat conversion bugsnag issue ([#3657](https://github.com/rudderlabs/rudder-transformer/issues/3657)) ([31b03fc](https://github.com/rudderlabs/rudder-transformer/commit/31b03fc022ace0cda8df798c50e4764a9703c23b))
+* validation for iterable object of HS ([#3653](https://github.com/rudderlabs/rudder-transformer/issues/3653)) ([1cb3f86](https://github.com/rudderlabs/rudder-transformer/commit/1cb3f86c894f30bdf04df870484c6df3a956f36e))
+
+## [1.75.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.1...v1.75.0) (2024-08-12)
+
+
+### Features
+
+* move hubspot to transformer proxy to enable partial batch handling ([#3308](https://github.com/rudderlabs/rudder-transformer/issues/3308)) ([8450021](https://github.com/rudderlabs/rudder-transformer/commit/8450021672c51ac798ec0aeab422f5fceea5e53e))
+
+
+### Bug Fixes
+
+* handle attentive tag null, undefined properties ([#3647](https://github.com/rudderlabs/rudder-transformer/issues/3647)) ([9327925](https://github.com/rudderlabs/rudder-transformer/commit/932792561c98833baf9881a83ee36ae5000e37b4))
+* handle null values for braze dedupe ([#3638](https://github.com/rudderlabs/rudder-transformer/issues/3638)) ([0a9b681](https://github.com/rudderlabs/rudder-transformer/commit/0a9b68118241158623d45ee28896377296bafd91))
+
 ### [1.74.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.0...v1.74.1) (2024-08-08)
 
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.74.1",
+  "version": "1.75.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -71,7 +71,7 @@
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",
-    "axios": "^1.6.4",
+    "axios": "^1.7.3",
     "btoa": "^1.2.1",
     "component-each": "^0.2.6",
     "crypto-js": "^4.2.0",

diff --git a/src/v0/destinations/attentive_tag/transform.js b/src/v0/destinations/attentive_tag/transform.js
@@ -16,7 +16,7 @@ const {
 const {
   getDestinationItemProperties,
   getExternalIdentifiersMapping,
-  getPropertiesKeyValidation,
+  arePropertiesValid,
   validateTimestamp,
 } = require('./util');
 const { JSON_MIME_TYPE } = require('../../util/constant');
@@ -137,9 +137,9 @@ const trackResponseBuilder = (message, { Config }) => {
       payload = constructPayload(message, mappingConfig[ConfigCategory.TRACK.name]);
       endpoint = ConfigCategory.TRACK.endpoint;
       payload.type = get(message, 'event');
-      if (!getPropertiesKeyValidation(payload)) {
+      if (!arePropertiesValid(payload.properties)) {
         throw new InstrumentationError(
-          '[Attentive Tag]:The event name contains characters which is not allowed',
+          '[Attentive Tag]:The properties contains characters which is not allowed',
         );
       }
   }

diff --git a/src/v0/destinations/attentive_tag/util.js b/src/v0/destinations/attentive_tag/util.js
@@ -14,12 +14,19 @@ const { mappingConfig, ConfigCategory } = require('./config');
  * The keys should not contain any of the values inside the validationArray.
  * STEP 1: Storing keys in the array.
  * Checking for the non-valid characters inside the keys of properties.
+ * ref: https://docs.attentivemobile.com/openapi/reference/tag/Custom-Events/
  * @param {*} payload
  * @returns
  */
-const getPropertiesKeyValidation = (payload) => {
+const arePropertiesValid = (properties) => {
+  if (!isDefinedAndNotNullAndNotEmpty(properties)) {
+    return true;
+  }
+  if (typeof properties !== 'object') {
+    return false;
+  }
   const validationArray = [`'`, `"`, `{`, `}`, `[`, `]`, ',', `,`];
-  const keys = Object.keys(payload.properties);
+  const keys = Object.keys(properties);
   for (const key of keys) {
     for (const validationChar of validationArray) {
       if (key.includes(validationChar)) {
@@ -134,6 +141,6 @@ const getDestinationItemProperties = (message, isItemsRequired) => {
 module.exports = {
   getDestinationItemProperties,
   getExternalIdentifiersMapping,
-  getPropertiesKeyValidation,
+  arePropertiesValid,
   validateTimestamp,
 };
diff --git a/src/v0/destinations/attentive_tag/util.test.js b/src/v0/destinations/attentive_tag/util.test.js
@@ -0,0 +1,51 @@
+const { arePropertiesValid } = require('./util');
+
+describe('arePropertiesValid', () => {
+  // returns true for valid properties object with no special characters in keys
+  it('should return true when properties object has no special characters in keys', () => {
+    const properties = { key1: 'value1', key2: 'value2' };
+    const result = arePropertiesValid(properties);
+    expect(result).toBe(true);
+  });
+
+  // returns true for null properties input
+  it('should return true when properties input is null', () => {
+    const properties = null;
+    const result = arePropertiesValid(properties);
+    expect(result).toBe(true);
+  });
+
+  // returns false for properties object with keys containing special characters
+  it('should return false for properties object with keys containing special characters', () => {
+    const properties = {
+      key1: 'value1',
+      'key,2': 'value2',
+      key3: 'value3',
+    };
+    expect(arePropertiesValid(properties)).toBe(false);
+  });
+
+  // returns true for empty properties object
+  it('should return true for empty properties object', () => {
+    const properties = {};
+    expect(arePropertiesValid(properties)).toBe(true);
+  });
+
+  // returns true for undefined properties input
+  it('should return true for undefined properties input', () => {
+    const result = arePropertiesValid(undefined);
+    expect(result).toBe(true);
+  });
+
+  // returns true for empty string properties input
+  it('should return true for empty string properties input', () => {
+    const result = arePropertiesValid('');
+    expect(result).toBe(true);
+  });
+
+  // returns false for empty string properties input
+  it('should return false for non object properties input', () => {
+    const result = arePropertiesValid('1234');
+    expect(result).toBe(false);
+  });
+});
diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js
@@ -791,6 +791,136 @@ describe('dedup utility tests', () => {
       const result = BrazeDedupUtility.deduplicate(userData, store);
       expect(result).toBeNull();
     });
+
+    test('deduplicates user data correctly when user data is null and it doesnt exist in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is null and it is same in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+          nullProperty: null,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is null and it is different in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+          nullProperty: 'valid data',
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual({
+        external_id: '123',
+        nullProperty: null,
+      });
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it doesnt exist in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it is same in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          undefinedProperty: undefined,
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it is defined in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          undefinedProperty: 'defined data',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual({
+        external_id: '123',
+        undefinedProperty: undefined,
+      });
+    });
   });
 });
 

diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js
@@ -288,7 +288,14 @@ const BrazeDedupUtility = {
 
     if (keys.length > 0) {
       keys.forEach((key) => {
-        if (!_.isEqual(userData[key], storedUserData[key])) {
+        // ref: https://www.braze.com/docs/user_guide/data_and_analytics/custom_data/custom_attributes/#adding-descriptions
+        // null is a valid value in braze for unsetting, so we need to compare the values only if the key is present in the stored user data
+        // in case of keys having null values only compare if the key is present in the stored user data
+        if (userData[key] === null) {
+          if (isDefinedAndNotNull(storedUserData[key])) {
+            deduplicatedUserData[key] = userData[key];
+          }
+        } else if (!_.isEqual(userData[key], storedUserData[key])) {
           deduplicatedUserData[key] = userData[key];
         }
       });

diff --git a/src/v0/destinations/clevertap/transform.js b/src/v0/destinations/clevertap/transform.js
@@ -316,7 +316,7 @@ const responseBuilderSimple = (message, category, destination) => {
     // For 'Order Completed' type of events we are mapping it as 'Charged'
     // Special event in Clevertap.
     // Source: https://developer.clevertap.com/docs/concepts-events#recording-customer-purchases
-    if (get(message.event) && get(message.event).toLowerCase() === 'order completed') {
+    if (get(message.event) && get(message.event).toString().toLowerCase() === 'order completed') {
       eventPayload = {
         evtName: 'Charged',
         evtData: constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.ECOM.name]),