Merge branch 'develop' into ga4.refactor.proxy

.github/workflows/prepare-for-staging-deploy.yml

@@ -101,7 +101,7 @@ jobs:
           cd rudder-devops
           BRANCH_NAME="shared-transformer-$TAG_NAME"
           echo $BRANCH_NAME
-          if [ `git ls-remote --heads origin $BRANCH_NAME 2>/dev/null` ]
+          if [ -n "$(git ls-remote --heads origin $BRANCH_NAME 2>/dev/null)" ]
           then
             echo "Staging deployment branch already exists!"
           else

.gitignore

@@ -137,4 +137,5 @@ dist
 .idea
 
 # component test report
-test_reports/
+test_reports/
+temp/

CHANGELOG.md

@@ -2,6 +2,52 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.58.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.58.0) (2024-03-04)
+
+
+### Features
+
+* add support for interaction events in sfmc ([#3109](https://github.com/rudderlabs/rudder-transformer/issues/3109)) ([0486049](https://github.com/rudderlabs/rudder-transformer/commit/0486049ba2ad96b50d8f29e96b46b96a8a5c9f76))
+* add support of custom page/screen event name in mixpanel ([#3098](https://github.com/rudderlabs/rudder-transformer/issues/3098)) ([0eb2393](https://github.com/rudderlabs/rudder-transformer/commit/0eb2393939fba2452ef7f07a1d149d87f18290c3))
+* consent mode support for google adwords remarketing list  ([#3143](https://github.com/rudderlabs/rudder-transformer/issues/3143)) ([7532c90](https://github.com/rudderlabs/rudder-transformer/commit/7532c90d7e1feac00f12961c56da18757010f44a))
+* **facebook:** update content_type mapping logic for fb pixel and fb conversions ([#3113](https://github.com/rudderlabs/rudder-transformer/issues/3113)) ([aea417c](https://github.com/rudderlabs/rudder-transformer/commit/aea417cd2691547399010c034cadbc5db6b0c6ee))
+* klaviyo profile mapping ([#3105](https://github.com/rudderlabs/rudder-transformer/issues/3105)) ([2761786](https://github.com/rudderlabs/rudder-transformer/commit/2761786ff3fc99ed6d4d3b7a6c2400226b1cfb12))
+* onboard new destination ninetailed ([#3106](https://github.com/rudderlabs/rudder-transformer/issues/3106)) ([0e2588e](https://github.com/rudderlabs/rudder-transformer/commit/0e2588ecd87f3b2c6877a099aa1cbf2d5325966c))
+
+
+### Bug Fixes
+
+* add error handling for tiktok ads ([#3144](https://github.com/rudderlabs/rudder-transformer/issues/3144)) ([e93e47f](https://github.com/rudderlabs/rudder-transformer/commit/e93e47f33e098104fb532916932fe38bbfeaa4a1))
+* **algolia:** added check for objectIds or filters to be non empty ([#3126](https://github.com/rudderlabs/rudder-transformer/issues/3126)) ([d619c97](https://github.com/rudderlabs/rudder-transformer/commit/d619c9769cd270cb2d16dad0865683ff4beb2d19))
+* clevertap remove stringification of array object properties ([#3048](https://github.com/rudderlabs/rudder-transformer/issues/3048)) ([69e43b6](https://github.com/rudderlabs/rudder-transformer/commit/69e43b6ffadeaec87b7440da34a341890ceba252))
+* convert to string from null in hs ([#3136](https://github.com/rudderlabs/rudder-transformer/issues/3136)) ([75e9f46](https://github.com/rudderlabs/rudder-transformer/commit/75e9f462b0ff9b9a8abab3c78dc7d147926e9e5e))
+* event fix and added utility ([#3142](https://github.com/rudderlabs/rudder-transformer/issues/3142)) ([9b705b7](https://github.com/rudderlabs/rudder-transformer/commit/9b705b71a9d3a595ea0fbf532602c3941b0a18db))
+* metadata structure correction ([#3119](https://github.com/rudderlabs/rudder-transformer/issues/3119)) ([8351b5c](https://github.com/rudderlabs/rudder-transformer/commit/8351b5cbbf81bbc14b2f884feaae4ad3ca59a39a))
+* one_signal:  Encode external_id in endpoint ([#3140](https://github.com/rudderlabs/rudder-transformer/issues/3140)) ([8a20886](https://github.com/rudderlabs/rudder-transformer/commit/8a2088608d6da4b35bbb506db2fc3df1e4d41f3b))
+* rakuten: sync property mapping sourcekeys to rudderstack standard spec ([#3129](https://github.com/rudderlabs/rudder-transformer/issues/3129)) ([2ebff95](https://github.com/rudderlabs/rudder-transformer/commit/2ebff956ff2aa74b008a8de832a31d8774d2d47e))
+* reddit revenue mapping for floating point values ([#3118](https://github.com/rudderlabs/rudder-transformer/issues/3118)) ([41f4078](https://github.com/rudderlabs/rudder-transformer/commit/41f4078011ef54334bb9ecc11a7b2ccc8831a4aa))
+* version deprecation failure false positive ([#3104](https://github.com/rudderlabs/rudder-transformer/issues/3104)) ([657b780](https://github.com/rudderlabs/rudder-transformer/commit/657b7805eb01da25a007d978198d5debf03917fd))
+
+### [1.57.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.0...v1.57.1) (2024-03-04)
+
+
+### Bug Fixes
+
+* amplitude fix for user operations ([7f2364e](https://github.com/rudderlabs/rudder-transformer/commit/7f2364e41167611c41003559de65cee1fece5464))
+* amplitude fix for user operations ([#3153](https://github.com/rudderlabs/rudder-transformer/issues/3153)) ([31869fb](https://github.com/rudderlabs/rudder-transformer/commit/31869fb114bb141d545de01c56f57b97e5aa54a6))
+
+## [1.57.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.56.1...v1.57.0) (2024-02-29)
+
+
+### Features
+
+* add event mapping support for branch destination ([#3135](https://github.com/rudderlabs/rudder-transformer/issues/3135)) ([cc94bba](https://github.com/rudderlabs/rudder-transformer/commit/cc94bba682f667877a721f63627adc6ff6a7386a))
+
+
+### Bug Fixes
+
+* marketo bulk upload zero and null value allowed ([#3134](https://github.com/rudderlabs/rudder-transformer/issues/3134)) ([4dcbf8f](https://github.com/rudderlabs/rudder-transformer/commit/4dcbf8fb189a39bb40b950742425a0b9da2d8d7c))
+
 ### [1.56.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.56.0...v1.56.1) (2024-02-21)
 
 

github-release.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  gitRawCommitsOpts: {
+    merges: null,
+  },
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.56.1",
+  "version": "1.58.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -49,7 +49,7 @@
     "commit-msg": "commitlint --edit",
     "prepare": "node ./scripts/skipPrepareScript.js || husky install",
     "release": "npx standard-version",
-    "release:github": "DEBUG=conventional-github-releaser npx conventional-github-releaser -p angular -v",
+    "release:github": "DEBUG=conventional-github-releaser npx conventional-github-releaser -p angular --config github-release.config.js",
     "clean:node": "modclean",
     "check:lint": "eslint . -f json -o reports/eslint.json || exit 0"
   },

src/services/destination/nativeIntegration.ts

@@ -209,7 +209,11 @@ export class NativeIntegrationDestinationService implements DestinationService {
         const jobStates = (deliveryRequest as ProxyV1Request).metadata.map(
           (metadata) =>
             ({
-              error: JSON.stringify(v0Response.destinationResponse?.response),
+              error: JSON.stringify(
+                v0Response.destinationResponse?.response === undefined
+                  ? v0Response.destinationResponse
+                  : v0Response.destinationResponse?.response,
+              ),
               statusCode: v0Response.status,
               metadata,
             }) as DeliveryJobState,

src/v0/destinations/am/transform.js

@@ -268,7 +268,7 @@ const updateConfigProperty = (message, payload, mappingJson, validatePayload, Co
     }
   });
 };
-const identifyBuilder = (message, destination, rawPayload) => {
+const userPropertiesHandler = (message, destination, rawPayload) => {
   // update payload user_properties from userProperties/traits/context.traits/nested traits of Rudder message
   // traits like address converted to top level user properties (think we can skip this extra processing as AM supports nesting upto 40 levels)
   let traits = getFieldValueFromMessage(message, 'traits');
@@ -335,14 +335,15 @@ const getDefaultResponseData = (message, rawPayload, evType, groupInfo) => {
   const groups = groupInfo && cloneDeep(groupInfo);
   return { groups, rawPayload };
 };
+
 const getResponseData = (evType, destination, rawPayload, message, groupInfo) => {
   let groups;
 
   switch (evType) {
     case EventType.IDENTIFY:
       // event_type for identify event is $identify
       rawPayload.event_type = IDENTIFY_AM;
-      rawPayload = identifyBuilder(message, destination, rawPayload);
+      rawPayload = userPropertiesHandler(message, destination, rawPayload);
       break;
     case EventType.GROUP:
       // event_type for identify event is $identify
@@ -357,8 +358,15 @@ const getResponseData = (evType, destination, rawPayload, message, groupInfo) =>
     case EventType.ALIAS:
       break;
     default:
+      if (destination.Config.enableEnhancedUserOperations) {
+        // handle all other events like track, page, screen for user properties
+        rawPayload = userPropertiesHandler(message, destination, rawPayload);
+      }
       ({ groups, rawPayload } = getDefaultResponseData(message, rawPayload, evType, groupInfo));
   }
+  if (destination.Config.enableEnhancedUserOperations) {
+    rawPayload = AMUtils.userPropertiesPostProcess(rawPayload);
+  }
   return { rawPayload, groups };
 };
 

src/v0/destinations/am/util.test.js

@@ -1,4 +1,4 @@
-const { getUnsetObj, validateEventType } = require('./utils');
+const { getUnsetObj, validateEventType, userPropertiesPostProcess } = require('./utils');
 
 describe('getUnsetObj', () => {
   it("should return undefined when 'message.integrations.Amplitude.fieldsToUnset' is not array", () => {
@@ -97,3 +97,70 @@ describe('validateEventType', () => {
     );
   });
 });
+
+describe('userPropertiesPostProcess', () => {
+  // The function correctly removes duplicate keys found in both operation keys and root level keys.
+  it('should remove duplicate keys from user_properties', () => {
+    // Arrange
+    const rawPayload = {
+      user_properties: {
+        $setOnce: {
+          key1: 'value1',
+        },
+        $add: {
+          key2: 'value2',
+        },
+        key3: 'value3',
+        key1: 'value4',
+      },
+    };
+
+    // Act
+    const result = userPropertiesPostProcess(rawPayload);
+
+    // Assert
+    expect(result.user_properties).toEqual({
+      $setOnce: {
+        key1: 'value1',
+      },
+      $add: {
+        key2: 'value2',
+      },
+      $set: {
+        key3: 'value3',
+      },
+    });
+  });
+
+  // The function correctly moves root level properties to $set operation.
+  it('should move root level properties to $set operation when they dont belong to any other operation', () => {
+    // Arrange
+    const rawPayload = {
+      user_properties: {
+        $setOnce: {
+          key1: 'value1',
+        },
+        $add: {
+          key2: 'value2',
+        },
+        key3: 'value3',
+      },
+    };
+
+    // Act
+    const result = userPropertiesPostProcess(rawPayload);
+
+    // Assert
+    expect(result.user_properties).toEqual({
+      $set: {
+        key3: 'value3',
+      },
+      $setOnce: {
+        key1: 'value1',
+      },
+      $add: {
+        key2: 'value2',
+      },
+    });
+  });
+});

src/v0/destinations/am/utils.js

@@ -122,6 +122,60 @@ const validateEventType = (evType) => {
     );
   }
 };
+
+const userPropertiesPostProcess = (rawPayload) => {
+  const operationList = [
+    '$setOnce',
+    '$add',
+    '$unset',
+    '$append',
+    '$prepend',
+    '$preInsert',
+    '$postInsert',
+    '$remove',
+  ];
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  const { user_properties } = rawPayload;
+  const userPropertiesKeys = Object.keys(user_properties).filter(
+    (key) => !operationList.includes(key),
+  );
+  const duplicatekeys = new Set();
+  // eslint-disable-next-line no-restricted-syntax, guard-for-in
+  for (const key of userPropertiesKeys) {
+    // check if any of the keys are present in the user_properties $setOnce, $add, $unset, $append, $prepend, $preInsert, $postInsert, $remove keys as well as root level
+
+    if (
+      operationList.some(
+        (operation) => user_properties[operation] && user_properties[operation][key],
+      )
+    ) {
+      duplicatekeys.add(key);
+    }
+  }
+  // eslint-disable-next-line no-restricted-syntax, guard-for-in
+  for (const key of duplicatekeys) {
+    delete user_properties[key];
+  }
+
+  // Moving root level properties that doesn't belong to any operation under $set
+  const setProps = {};
+  // eslint-disable-next-line no-restricted-syntax
+  for (const [key, value] of Object.entries(user_properties)) {
+    if (!operationList.includes(key)) {
+      setProps[key] = value;
+      delete user_properties[key];
+    }
+  }
+
+  if (Object.keys(setProps).length > 0) {
+    user_properties.$set = setProps;
+  }
+
+  // eslint-disable-next-line no-param-reassign
+  rawPayload.user_properties = user_properties;
+  return rawPayload;
+};
+
 module.exports = {
   getOSName,
   getOSVersion,
@@ -132,4 +186,5 @@ module.exports = {
   getEventId,
   getUnsetObj,
   validateEventType,
+  userPropertiesPostProcess,
 };

src/v0/destinations/branch/transform.js

@@ -13,6 +13,7 @@ const {
   simpleProcessRouterDest,
 } = require('../../util');
 const { JSON_MIME_TYPE } = require('../../util/constant');
+const { getMappedEventNameFromConfig } = require('./utils');
 
 function responseBuilder(payload, message, destination, category) {
   const response = defaultRequestConfig();
@@ -213,24 +214,21 @@ function getCommonPayload(message, category, evName) {
   return rawPayload;
 }
 
-// function getTrackPayload(message) {
-//   const rawPayload = {};
-//   const { name, category } = getCategoryAndName(message.event);
-//   rawPayload.name = name;
-//
-//   return commonPayload(message, rawPayload, category);
-// }
-
 function processMessage(message, destination) {
   let evName;
   let category;
   switch (message.type) {
-    case EventType.TRACK:
+    case EventType.TRACK: {
       if (!message.event) {
         throw new InstrumentationError('Event name is required');
       }
       ({ evName, category } = getCategoryAndName(message.event));
+      const eventNameFromConfig = getMappedEventNameFromConfig(message, destination);
+      if (eventNameFromConfig) {
+        evName = eventNameFromConfig;
+      }
       break;
+    }
     case EventType.IDENTIFY:
       ({ evName, category } = getCategoryAndName(message.userId));
       break;

src/v0/destinations/branch/utils.js

@@ -0,0 +1,24 @@
+const { getHashFromArray } = require('../../util');
+
+/**
+ * Retrieves the mapped event name from the given config.
+ *
+ * @param {object} message - The message object containing the event.
+ * @param {object} destination - The destination object containing the events mapping configuration.
+ * @returns {string} - The mapped event name, or undefined if not found.
+ */
+const getMappedEventNameFromConfig = (message, destination) => {
+  let eventName;
+  const { event } = message;
+  const { eventsMapping } = destination.Config;
+
+  // if event is mapped on dashboard, use the mapped event name
+  if (Array.isArray(eventsMapping) && eventsMapping.length > 0) {
+    const keyMap = getHashFromArray(eventsMapping, 'from', 'to', false);
+    eventName = keyMap[event];
+  }
+
+  return eventName;
+};
+
+module.exports = { getMappedEventNameFromConfig };

src/v0/destinations/branch/utils.test.js

@@ -0,0 +1,18 @@
+const { getMappedEventNameFromConfig } = require('./utils');
+describe('getMappedEventNameFromConfig', () => {
+  it('should return the mapped event name when it exists in the events mapping configuration', () => {
+    const message = { event: 'Order Completed' };
+    const destination = {
+      Config: { eventsMapping: [{ from: 'Order Completed', to: 'PURCHASE' }] },
+    };
+    const result = getMappedEventNameFromConfig(message, destination);
+    expect(result).toBe('PURCHASE');
+  });
+
+  it('should return undefined when the event mapping is not created', () => {
+    const message = { event: 'Order Completed' };
+    const destination = { Config: {} };
+    const result = getMappedEventNameFromConfig(message, destination);
+    expect(result).toBeUndefined();
+  });
+});