chore: conflict fix from main to develop (#3670)

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14)
+
+
+### Bug Fixes
+
+* add validation for concurrent_modification error ([#3654](https://github.com/rudderlabs/rudder-transformer/issues/3654)) ([62cdc46](https://github.com/rudderlabs/rudder-transformer/commit/62cdc4641d44d79e21949722660173df4c749f24))
+* clevertap bugsnag issue ([#3656](https://github.com/rudderlabs/rudder-transformer/issues/3656)) ([6c51487](https://github.com/rudderlabs/rudder-transformer/commit/6c51487183b6b0b755620aac6cd51c2ffc966102))
+* snapchat conversion bugsnag issue ([#3657](https://github.com/rudderlabs/rudder-transformer/issues/3657)) ([31b03fc](https://github.com/rudderlabs/rudder-transformer/commit/31b03fc022ace0cda8df798c50e4764a9703c23b))
+* validation for iterable object of HS ([#3653](https://github.com/rudderlabs/rudder-transformer/issues/3653)) ([1cb3f86](https://github.com/rudderlabs/rudder-transformer/commit/1cb3f86c894f30bdf04df870484c6df3a956f36e))
+
+## [1.75.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.1...v1.75.0) (2024-08-12)
+
+
+### Features
+
+* move hubspot to transformer proxy to enable partial batch handling ([#3308](https://github.com/rudderlabs/rudder-transformer/issues/3308)) ([8450021](https://github.com/rudderlabs/rudder-transformer/commit/8450021672c51ac798ec0aeab422f5fceea5e53e))
+
+
+### Bug Fixes
+
+* handle attentive tag null, undefined properties ([#3647](https://github.com/rudderlabs/rudder-transformer/issues/3647)) ([9327925](https://github.com/rudderlabs/rudder-transformer/commit/932792561c98833baf9881a83ee36ae5000e37b4))
+* handle null values for braze dedupe ([#3638](https://github.com/rudderlabs/rudder-transformer/issues/3638)) ([0a9b681](https://github.com/rudderlabs/rudder-transformer/commit/0a9b68118241158623d45ee28896377296bafd91))
+
 ### [1.74.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.0...v1.74.1) (2024-08-08)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.74.1",
+  "version": "1.75.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/v0/destinations/braze/braze.util.test.js

@@ -791,6 +791,136 @@ describe('dedup utility tests', () => {
       const result = BrazeDedupUtility.deduplicate(userData, store);
       expect(result).toBeNull();
     });
+
+    test('deduplicates user data correctly when user data is null and it doesnt exist in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is null and it is same in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+          nullProperty: null,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is null and it is different in stored data', () => {
+      const userData = {
+        external_id: '123',
+        nullProperty: null,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+          nullProperty: 'valid data',
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual({
+        external_id: '123',
+        nullProperty: null,
+      });
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it doesnt exist in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it is same in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          undefinedProperty: undefined,
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual(null);
+    });
+
+    test('deduplicates user data correctly when user data is undefined and it is defined in stored data', () => {
+      const userData = {
+        external_id: '123',
+        undefinedProperty: undefined,
+        color: 'green',
+        age: 30,
+      };
+      const storeData = {
+        external_id: '123',
+        custom_attributes: {
+          color: 'green',
+          undefinedProperty: 'defined data',
+          age: 30,
+        },
+      };
+      store.set('123', storeData);
+      const result = BrazeDedupUtility.deduplicate(userData, store);
+      expect(store.size).toBe(1);
+      expect(result).toEqual({
+        external_id: '123',
+        undefinedProperty: undefined,
+      });
+    });
   });
 });
 

src/v0/destinations/braze/util.js

@@ -288,7 +288,14 @@ const BrazeDedupUtility = {
 
     if (keys.length > 0) {
       keys.forEach((key) => {
-        if (!_.isEqual(userData[key], storedUserData[key])) {
+        // ref: https://www.braze.com/docs/user_guide/data_and_analytics/custom_data/custom_attributes/#adding-descriptions
+        // null is a valid value in braze for unsetting, so we need to compare the values only if the key is present in the stored user data
+        // in case of keys having null values only compare if the key is present in the stored user data
+        if (userData[key] === null) {
+          if (isDefinedAndNotNull(storedUserData[key])) {
+            deduplicatedUserData[key] = userData[key];
+          }
+        } else if (!_.isEqual(userData[key], storedUserData[key])) {
           deduplicatedUserData[key] = userData[key];
         }
       });

src/v0/destinations/clevertap/transform.js

@@ -316,7 +316,7 @@ const responseBuilderSimple = (message, category, destination) => {
     // For 'Order Completed' type of events we are mapping it as 'Charged'
     // Special event in Clevertap.
     // Source: https://developer.clevertap.com/docs/concepts-events#recording-customer-purchases
-    if (get(message.event) && get(message.event).toLowerCase() === 'order completed') {
+    if (get(message.event) && get(message.event).toString().toLowerCase() === 'order completed') {
       eventPayload = {
         evtName: 'Charged',
         evtData: constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.ECOM.name]),

src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js

@@ -1,7 +1,7 @@
 const { NetworkError } = require('@rudderstack/integrations-lib');
+const get = require('get-value');
 const { prepareProxyRequest, handleHttpRequest } = require('../../../adapters/network');
 const { isHttpStatusSuccess } = require('../../util/index');
-
 const {
   processAxiosResponse,
   getDynamicErrorType,
@@ -148,9 +148,15 @@ const gaAudienceProxyRequest = async (request) => {
 };
 
 const gaAudienceRespHandler = (destResponse, stageMsg) => {
-  const { status, response } = destResponse;
-  // const respAttributes = response["@attributes"] || null;
-  // const { stat, err_code: errorCode } = respAttributes;
+  let { status } = destResponse;
+  const { response } = destResponse;
+
+  if (
+    status === 400 &&
+    get(response, 'error.details.0.errors.0.errorCode.databaseError') === 'CONCURRENT_MODIFICATION'
+  ) {
+    status = 500;
+  }
 
   throw new NetworkError(
     `${JSON.stringify(response)} ${stageMsg}`,

src/v0/destinations/hs/transform.js

@@ -19,6 +19,7 @@ const {
   fetchFinalSetOfTraits,
   getProperties,
   validateDestinationConfig,
+  convertToResponseFormat,
 } = require('./util');
 
 const processSingleMessage = async ({ message, destination, metadata }, propertyMap) => {
@@ -106,6 +107,7 @@ const processBatchRouter = async (inputs, reqMetadata) => {
       errorRespList: tempInputs.map((input) =>
         handleRtTfSingleEventError(input, error, reqMetadata),
       ),
+      dontBatchEvents: [],
     };
   }
 
@@ -147,20 +149,38 @@ const processBatchRouter = async (inputs, reqMetadata) => {
     }),
   );
 
-  if (successRespList.length > 0) {
+  const dontBatchTrueResponses = [];
+  const dontBatchFalseOrUndefinedResponses = [];
+  // segregating successRepList depending on dontbatch value
+  successRespList.forEach((successResp) => {
+    if (successResp.metadata?.dontBatch) {
+      dontBatchTrueResponses.push(successResp);
+    } else {
+      dontBatchFalseOrUndefinedResponses.push(successResp);
+    }
+  });
+
+  // batch implementation
+  if (dontBatchFalseOrUndefinedResponses.length > 0) {
     if (destination.Config.apiVersion === API_VERSION.v3) {
-      batchedResponseList = batchEvents(successRespList);
+      batchedResponseList = batchEvents(dontBatchFalseOrUndefinedResponses);
     } else {
-      batchedResponseList = legacyBatchEvents(successRespList);
+      batchedResponseList = legacyBatchEvents(dontBatchFalseOrUndefinedResponses);
     }
   }
-  return { batchedResponseList, errorRespList };
+  return {
+    batchedResponseList,
+    errorRespList,
+    // if there are any events where dontbatch set to true we need to update them according to the response format
+    dontBatchEvents: convertToResponseFormat(dontBatchTrueResponses),
+  };
 };
 // we are batching by default at routerTransform
 const processRouterDest = async (inputs, reqMetadata) => {
   const tempNewInputs = batchEventsInOrder(inputs);
   const batchedResponseList = [];
   const errorRespList = [];
+  const dontBatchEvents = [];
   const promises = tempNewInputs.map(async (inputEvents) => {
     const response = await processBatchRouter(inputEvents, reqMetadata);
     return response;
@@ -171,8 +191,9 @@ const processRouterDest = async (inputs, reqMetadata) => {
   results.forEach((response) => {
     errorRespList.push(...response.errorRespList);
     batchedResponseList.push(...response.batchedResponseList);
+    dontBatchEvents.push(...response.dontBatchEvents);
   });
-  return [...batchedResponseList, ...errorRespList];
+  return [...batchedResponseList, ...errorRespList, ...dontBatchEvents];
 };
 
 module.exports = { process, processRouterDest };

src/v0/destinations/hs/util.js

@@ -22,6 +22,9 @@ const {
   getValueFromMessage,
   isNull,
   validateEventName,
+  defaultBatchRequestConfig,
+  defaultPostRequestConfig,
+  getSuccessRespEvents,
 } = require('../../util');
 const {
   CONTACT_PROPERTY_MAP_ENDPOINT,
@@ -844,6 +847,42 @@ const addExternalIdToHSTraits = (message) => {
   set(getFieldValueFromMessage(message, 'traits'), externalIdObj.identifierType, externalIdObj.id);
 };
 
+const convertToResponseFormat = (successRespListWithDontBatchTrue) => {
+  const response = [];
+  if (Array.isArray(successRespListWithDontBatchTrue)) {
+    successRespListWithDontBatchTrue.forEach((event) => {
+      const { message, metadata, destination } = event;
+      const endpoint = get(message, 'endpoint');
+
+      const batchedResponse = defaultBatchRequestConfig();
+      batchedResponse.batchedRequest.headers = message.headers;
+      batchedResponse.batchedRequest.endpoint = endpoint;
+      batchedResponse.batchedRequest.body = message.body;
+      batchedResponse.batchedRequest.params = message.params;
+      batchedResponse.batchedRequest.method = defaultPostRequestConfig.requestMethod;
+      batchedResponse.metadata = [metadata];
+      batchedResponse.destination = destination;
+
+      response.push(
+        getSuccessRespEvents(
+          batchedResponse.batchedRequest,
+          batchedResponse.metadata,
+          batchedResponse.destination,
+        ),
+      );
+    });
+  }
+  return response;
+};
+
+const isIterable = (obj) => {
+  // checks for null and undefined
+  if (obj == null) {
+    return false;
+  }
+  return typeof obj[Symbol.iterator] === 'function';
+};
+
 module.exports = {
   validateDestinationConfig,
   addExternalIdToHSTraits,
@@ -863,4 +902,6 @@ module.exports = {
   getObjectAndIdentifierType,
   extractIDsForSearchAPI,
   getRequestData,
+  convertToResponseFormat,
+  isIterable,
 };