Merge branch 'develop' into chore.clean-depricated-files

.github/workflows/build-push-docker-image.yml

@@ -96,15 +96,15 @@ jobs:
         with:
           ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
-
-      - name: Setup Docker Buildx
-        uses: docker/[email protected]
-
+
       - name: Login to DockerHub
         uses: docker/[email protected]
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}    
+
+      - name: Setup Docker Buildx
+        uses: docker/[email protected]
 
       - name: Build Docker Image
         uses: docker/[email protected]
@@ -149,16 +149,16 @@ jobs:
         with:
           ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
-
-      - name: Setup Docker Buildx
-        uses: docker/[email protected]
-
+
       - name: Login to DockerHub
         uses: docker/[email protected]
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
+      - name: Setup Docker Buildx
+        uses: docker/[email protected]
+
       - name: Build Docker Image
         uses: docker/[email protected]
         with:
@@ -198,15 +198,15 @@ jobs:
     needs: [build-transformer-image-amd64, build-transformer-image-arm64]
 
     steps:
-      - name: Set up Docker Buildx
-        uses: docker/[email protected]
-
       - name: Login to DockerHub
         uses: docker/[email protected]
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
       - name: Create multi-arch manifest
         run: |
           docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Checkout
         uses: actions/[email protected]
         with:
-          fetch-depth: 1
+          fetch-depth: 0
 
       - name: Setup Node
         uses: actions/[email protected]

.github/workflows/prepare-for-staging-deploy.yml

@@ -78,6 +78,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [generate-tag-names, build-transformer-image, build-user-transformer-image]
     env:
+      TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer
       TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }}
       UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
     steps:
@@ -113,6 +114,7 @@ jobs:
             git checkout -b $BRANCH_NAME
 
             cd helm-charts/shared-services/per-az/environment/staging
+            yq eval -i ".rudder-transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" staging.yaml
             yq eval -i ".rudder-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml
             git add staging.yaml

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.86.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.1...v1.86.0) (2024-12-09)
+
+
+### Features
+
+* add support for getting anonymousId by note attributes array ([53d2bef](https://github.com/rudderlabs/rudder-transformer/commit/53d2bef38ad5d7bc5504111ec797b3c3973546dd))
+* add support for getting anonymousId by note attributes array ([#3893](https://github.com/rudderlabs/rudder-transformer/issues/3893)) ([d7f390c](https://github.com/rudderlabs/rudder-transformer/commit/d7f390cb471e44afb276484b8b804d1f257c539c))
+
+
+### Bug Fixes
+
+* braze subscription group fixes ([#3901](https://github.com/rudderlabs/rudder-transformer/issues/3901)) ([ebcf84e](https://github.com/rudderlabs/rudder-transformer/commit/ebcf84e07bf121d882c99df973af265a915a1ce1))
+* remove redundant ids and userIdentifier when gbraid or wbraid are there ([#3910](https://github.com/rudderlabs/rudder-transformer/issues/3910)) ([313710c](https://github.com/rudderlabs/rudder-transformer/commit/313710ca725538e5ffe357216d9c88e444f995c8))
+* skipping users events for snowpipe streaming ([#3836](https://github.com/rudderlabs/rudder-transformer/issues/3836)) ([12621c8](https://github.com/rudderlabs/rudder-transformer/commit/12621c8eee641f5a03a997e95ed016cff0eefde7))
+
 ### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.85.1",
+  "version": "1.86.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {
@@ -36,11 +36,11 @@
     "test": "jest -c jest.config.js --detectOpenHandles",
     "test:ci": "npm run test -- --coverage --expand --maxWorkers=50%",
     "test:js": "jest -c jest.default.config.js --detectOpenHandles",
-    "test:js:silent": "npm run test:js -- --silent",
+    "test:js:silent": "export LOG_LEVEL=silent && npm run test:js -- --silent",
     "test:js:ci": "npm run test:js -- --coverage --expand --maxWorkers=50%",
     "test:ts": "jest -c jest.config.typescript.js --detectOpenHandles",
     "test:ts:component:generateNwMocks": "npm run test:ts -- component --generate=true",
-    "test:ts:silent": "npm run test:ts -- --silent",
+    "test:ts:silent": "export LOG_LEVEL=silent && npm run test:ts -- --silent",
     "test:ts:ci": "npm run test:ts -- --coverage --expand --maxWorkers=50%",
     "test:ut:integration": "jest \"user_transformation.integration.test.js\"  --detectOpenHandles --notify",
     "test:ut:integration:silent": "npm run test:ut:integration -- --silent",

src/cdk/v2/destinations/pinterest_tag/utils.js

@@ -1,8 +1,8 @@
-/* eslint-disable no-param-reassign */
 const sha256 = require('sha256');
 const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib');
 
 const { API_VERSION } = require('./config');
+const { CommonUtils } = require('../../../../util/common');
 
 const VALID_ACTION_SOURCES = ['app_android', 'app_ios', 'web', 'offline'];
 
@@ -21,9 +21,15 @@ const ecomEventMaps = [
   },
 ];
 
-const USER_NON_ARRAY_PROPERTIES = ['client_user_agent', 'client_ip_address'];
+const USER_NON_ARRAY_PROPERTIES = [
+  'client_user_agent',
+  'client_ip_address',
+  'click_id',
+  'partner_id',
+];
 
-const getHashedValue = (key, value) => {
+const transformValue = (key, value) => {
+  const arrayValue = CommonUtils.toArray(value);
   switch (key) {
     case 'em':
     case 'ct':
@@ -32,33 +38,18 @@ const getHashedValue = (key, value) => {
     case 'ln':
     case 'fn':
     case 'ge':
-      value = Array.isArray(value)
-        ? value.map((val) => val.toString().trim().toLowerCase())
-        : value.toString().trim().toLowerCase();
-      break;
+      return arrayValue.map((val) => val.toString().trim().toLowerCase());
     case 'ph':
-      // phone numbers should only contain digits & should not contain leading zeros
-      value = Array.isArray(value)
-        ? value.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, ''))
-        : value.toString().replace(/\D/g, '').replace(/^0+/, '');
-      break;
+      return arrayValue.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, ''));
     case 'zp':
-      // zip fields should only contain digits
-      value = Array.isArray(value)
-        ? value.map((val) => val.toString().trim().replace(/\D/g, ''))
-        : value.toString().replace(/\D/g, '');
-      break;
-    case 'hashed_maids':
-    case 'external_id':
-    case 'db':
-      // no action needed on value
-      break;
+      return arrayValue.map((val) => val.toString().trim().replace(/\D/g, ''));
     default:
-      return String(value);
+      return arrayValue;
   }
-  return Array.isArray(value) ? value.map((val) => sha256(val)) : [sha256(value)];
 };
 
+const getHashedValue = (key, value) => transformValue(key, value).map((val) => sha256(val));
+
 /**
  *
  * @param {*} userPayload Payload mapped from user fields
@@ -67,10 +58,15 @@ const getHashedValue = (key, value) => {
  * Ref: https://s.pinimg.com/ct/docs/conversions_api/dist/v3.html
  */
 const processUserPayload = (userPayload) => {
-  Object.keys(userPayload).forEach((key) => {
-    userPayload[key] = getHashedValue(key, userPayload[key]);
+  const newPayload = { ...userPayload };
+  Object.keys(newPayload).forEach((key) => {
+    if (USER_NON_ARRAY_PROPERTIES.includes(key)) {
+      newPayload[key] = String(newPayload[key]);
+    } else {
+      newPayload[key] = getHashedValue(key, newPayload[key]);
+    }
   });
-  return userPayload;
+  return newPayload;
 };
 
 /**
@@ -99,22 +95,16 @@ const processHashedUserPayload = (userPayload, message) => {
   const processedHashedUserPayload = {};
   Object.keys(userPayload).forEach((key) => {
     if (!USER_NON_ARRAY_PROPERTIES.includes(key)) {
-      if (Array.isArray(userPayload[key])) {
-        processedHashedUserPayload[key] = [...userPayload[key]];
-      } else {
-        processedHashedUserPayload[key] = [userPayload[key]];
-      }
+      processedHashedUserPayload[key] = CommonUtils.toArray(userPayload[key]);
     } else {
       processedHashedUserPayload[key] = userPayload[key];
     }
   });
   // multiKeyMap will works on only specific values like m, male, MALE, f, F, Female
   // if hashed data is sent from the user, it is directly set over here
   const gender = message.traits?.gender || message.context?.traits?.gender;
-  if (gender && Array.isArray(gender)) {
-    processedHashedUserPayload.ge = [...gender];
-  } else if (gender) {
-    processedHashedUserPayload.ge = [gender];
+  if (gender) {
+    processedHashedUserPayload.ge = CommonUtils.toArray(gender);
   }
   return processedHashedUserPayload;
 };

src/cdk/v2/destinations/reddit/rtWorkflow.yaml

@@ -36,7 +36,11 @@ steps:
     description: Batches the successfulEvents using endpoint
     condition: $.outputs.successfulEvents.length
     template: |
-      let batches = $.batchEvents($.outputs.successfulEvents);
+      const dontBatchTrueEvents =  $.outputs.successfulEvents{.metadata.dontBatch}[];
+      const dontBatchFalseEvents = $.outputs.successfulEvents{!.metadata.dontBatch}[];
+      const dontBatchTrueEventsChunks = $.chunk(dontBatchTrueEvents, 1);
+      
+      let batches = [...$.batchEvents(dontBatchFalseEvents), ...$.batchEventChunks(dontBatchTrueEventsChunks)];
       batches@batch.({
         "batchedRequest": {
           "body": {

src/cdk/v2/destinations/reddit/utils.js

@@ -81,6 +81,7 @@ const populateRevenueField = (eventType, properties) => {
 };
 module.exports = {
   batchEvents,
+  batchEventChunks,
   populateRevenueField,
   calculateDefaultRevenue,
 };

src/features.ts

@@ -93,6 +93,7 @@ const defaultFeaturesConfig: FeaturesConfig = {
     AMAZON_AUDIENCE: true,
     INTERCOM_V2: true,
     LINKEDIN_AUDIENCE: true,
+    TOPSORT: true,
   },
   regulations: [
     'BRAZE',

src/util/prometheus.js

@@ -520,6 +520,24 @@ class Prometheus {
         type: 'counter',
         labelNames: ['destination_id'],
       },
+      {
+        name: 'braze_batch_subscription_size',
+        help: 'braze_batch_subscription_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'braze_batch_subscription_combined_size',
+        help: 'braze_batch_subscription_combined_size',
+        type: 'gauge',
+        labelNames: ['destination_id'],
+      },
+      {
+        name: 'mailjet_packing_size',
+        help: 'mailjet_packing_size',
+        type: 'gauge',
+        labelNames: ['group'],
+      },
       {
         name: 'hs_batch_size',
         help: 'hs_batch_size',

src/v0/destinations/amazon_audience/utils.js

@@ -21,7 +21,8 @@ const buildResponseWithUsers = (users, action, config, jobIdList, secret) => {
   if (!secret?.clientId) {
     throw new OAuthSecretError('OAuth - Client Id not found');
   }
-  const externalId = `Rudderstack_${sha256(`${jobIdList}`)}`;
+  const jobIdHash = sha256(String(jobIdList));
+  const externalId = `Rudderstack_${jobIdHash}`;
   const response = defaultRequestConfig();
   response.endpoint = '';
   response.method = defaultPostRequestConfig.requestMethod;