chore(release): pull main into develop post release v1.76.0

.eslintignore

@@ -21,3 +21,8 @@ src/v0/destinations/personalize/scripts/
 test/integrations/destinations/testTypes.d.ts
 *.config*.js
 scripts/skipPrepareScript.js
+*.yaml
+*.yml
+.eslintignore
+.prettierignore
+*.json

.eslintrc.json

@@ -19,7 +19,8 @@
   "parserOptions": {
     "ecmaVersion": 12,
     "sourceType": "module",
-    "project": "./tsconfig.json"
+    "project": "./tsconfig.json",
+    "extraFileExtensions": [".yaml"]
   },
   "rules": {
     "unicorn/filename-case": [

.prettierignore

@@ -8,3 +8,5 @@ test/**/*.js
 src/util/lodash-es-core.js
 src/util/url-search-params.min.js
 dist
+.eslintignore
+.prettierignore

CHANGELOG.md

@@ -2,6 +2,32 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.76.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.76.0...v1.76.1) (2024-08-29)
+
+
+### Bug Fixes
+
+* reddit authorisation failed case handling ([#3690](https://github.com/rudderlabs/rudder-transformer/issues/3690)) ([f24759a](https://github.com/rudderlabs/rudder-transformer/commit/f24759aebbeb560f0de9d4920ae2ed0cdc7bfa3f))
+
+## [1.76.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.1...v1.76.0) (2024-08-20)
+
+
+### Features
+
+* klaviyo onboard unsubscribe profile support ([#3646](https://github.com/rudderlabs/rudder-transformer/issues/3646)) ([474f2bd](https://github.com/rudderlabs/rudder-transformer/commit/474f2bddc58e1962206e39d92514827f29f84c83))
+* onboard sfmc with vdm for rETL ([#3655](https://github.com/rudderlabs/rudder-transformer/issues/3655)) ([d987d1f](https://github.com/rudderlabs/rudder-transformer/commit/d987d1fc9afb9e1dc7482b2fe1458573f0f2699e))
+* onboard smartly destination ([#3660](https://github.com/rudderlabs/rudder-transformer/issues/3660)) ([474a36e](https://github.com/rudderlabs/rudder-transformer/commit/474a36ec385abf9ff83596b062d4d8e4c24469b8))
+* add bloomreach retl support ([#3619](https://github.com/rudderlabs/rudder-transformer/issues/3619)) ([6b1a23a](https://github.com/rudderlabs/rudder-transformer/commit/6b1a23af845084d6f2f5fd14656e4a1d11a7e34b))
+
+
+### Bug Fixes
+
+* add alias support in case alias details are present  ([#3579](https://github.com/rudderlabs/rudder-transformer/issues/3579)) ([cb67262](https://github.com/rudderlabs/rudder-transformer/commit/cb672628b312f20ea0fcc27a60ec8ab5692f8b06))
+* attentive tag bugsnag issue ([#3663](https://github.com/rudderlabs/rudder-transformer/issues/3663)) ([866dbf3](https://github.com/rudderlabs/rudder-transformer/commit/866dbf3e81754e71ff8ac08b258b359ec5cc6889))
+* fixing facebook utils ([#3664](https://github.com/rudderlabs/rudder-transformer/issues/3664)) ([1a61675](https://github.com/rudderlabs/rudder-transformer/commit/1a6167584a5780ab50beda13cc5ef6bf4e283e38))
+* reserved properties for braze ([#3573](https://github.com/rudderlabs/rudder-transformer/issues/3573)) ([413e9ce](https://github.com/rudderlabs/rudder-transformer/commit/413e9ce56f8f6569bbeb188bff4f43d400ea71b1))
+* source transformation integration test generation ([#3645](https://github.com/rudderlabs/rudder-transformer/issues/3645)) ([23196ec](https://github.com/rudderlabs/rudder-transformer/commit/23196ec42acf35f314e1953f339f6acbb72edd70))
+
 ### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.75.1",
+  "version": "1.76.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/bloomreach_catalog/config.ts

@@ -0,0 +1,31 @@
+export const MAX_PAYLOAD_SIZE = 10000000;
+export const MAX_ITEMS = 5000;
+
+// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-update-catalog-item
+export const getCreateBulkCatalogItemEndpoint = (
+  apiBaseUrl: string,
+  projectToken: string,
+  catalogId: string,
+): string => `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items`;
+
+// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-partial-update-catalog-item
+export const getUpdateBulkCatalogItemEndpoint = (
+  apiBaseUrl: string,
+  projectToken: string,
+  catalogId: string,
+): string =>
+  `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items/partial-update`;
+
+// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-delete-catalog-items
+export const getDeleteBulkCatalogItemEndpoint = (
+  apiBaseUrl: string,
+  projectToken: string,
+  catalogId: string,
+): string =>
+  `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items/bulk-delete`;
+
+export const CatalogAction = {
+  INSERT: 'insert',
+  UPDATE: 'update',
+  DELETE: 'delete',
+};

src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml

@@ -0,0 +1,42 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - name: processRecordInputs
+    path: ./transformRecord
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - name: InstrumentationError
+    path: '@rudderstack/integrations-lib'
+
+steps:
+  - name: validateConfig
+    template: |
+      const config = ^[0].destination.Config
+      $.assertConfig(config.apiBaseUrl, "API Base URL is not present. Aborting");
+      $.assertConfig(config.apiKey, "API Key is not present . Aborting");
+      $.assertConfig(config.apiSecret, "API Secret is not present. Aborting");
+      $.assertConfig(config.projectToken, "Project Token is not present. Aborting");
+      $.assertConfig(config.catalogID, "Catalog Id is not present. Aborting");
+
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: processRecordEvents
+    template: |
+      $.processRecordInputs(^.{.message.type === $.EventType.RECORD}[], ^[0].destination)
+
+  - name: failOtherEvents
+    template: |
+      const otherEvents = ^.{.message.type !== $.EventType.RECORD}[]
+      let failedEvents = otherEvents.map(
+        function(event) {
+        const error = new $.InstrumentationError("Event type " + event.message.type + " is not supported");
+          $.handleRtTfSingleEventError(event, error, {})
+        }
+      )
+      failedEvents ?? []
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.processRecordEvents, ...$.outputs.failOtherEvents]

src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts

@@ -0,0 +1,93 @@
+import { InstrumentationError } from '@rudderstack/integrations-lib';
+import { CatalogAction } from './config';
+import { batchResponseBuilder } from './utils';
+
+import { handleRtTfSingleEventError, isEmptyObject } from '../../../../v0/util';
+
+const prepareCatalogInsertOrUpdatePayload = (fields: any): any => {
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  const { item_id, ...properties } = fields;
+  return { item_id, properties };
+};
+
+const processEvent = (event: any) => {
+  const { message } = event;
+  const { fields, action } = message;
+  const response = {
+    action,
+    payload: null,
+  };
+  if (isEmptyObject(fields)) {
+    throw new InstrumentationError('`fields` cannot be empty');
+  }
+  if (!fields.item_id) {
+    throw new InstrumentationError('`item_id` cannot be empty');
+  }
+  if (action === CatalogAction.INSERT || action === CatalogAction.UPDATE) {
+    response.payload = prepareCatalogInsertOrUpdatePayload(fields);
+  } else if (action === CatalogAction.DELETE) {
+    response.payload = fields.item_id;
+  } else {
+    throw new InstrumentationError(
+      `Invalid action type ${action}. You can only add, update or remove items from the catalog`,
+    );
+  }
+  return response;
+};
+
+const getEventChunks = (
+  input: any,
+  insertItemRespList: any[],
+  updateItemRespList: any[],
+  deleteItemRespList: any[],
+) => {
+  switch (input.response.action) {
+    case CatalogAction.INSERT:
+      insertItemRespList.push({ payload: input.response.payload, metadata: input.metadata });
+      break;
+    case CatalogAction.UPDATE:
+      updateItemRespList.push({ payload: input.response.payload, metadata: input.metadata });
+      break;
+    case CatalogAction.DELETE:
+      deleteItemRespList.push({ payload: input.response.payload, metadata: input.metadata });
+      break;
+    default:
+      throw new InstrumentationError(`Invalid action type ${input.response.action}`);
+  }
+};
+
+export const processRecordInputs = (inputs: any[], destination: any) => {
+  const insertItemRespList: any[] = [];
+  const updateItemRespList: any[] = [];
+  const deleteItemRespList: any[] = [];
+  const batchErrorRespList: any[] = [];
+
+  if (!inputs || inputs.length === 0) {
+    return [];
+  }
+
+  inputs.forEach((input) => {
+    try {
+      getEventChunks(
+        {
+          response: processEvent(input),
+          metadata: input.metadata,
+        },
+        insertItemRespList,
+        updateItemRespList,
+        deleteItemRespList,
+      );
+    } catch (error) {
+      const errRespEvent = handleRtTfSingleEventError(input, error, {});
+      batchErrorRespList.push(errRespEvent);
+    }
+  });
+
+  const batchSuccessfulRespList = batchResponseBuilder(
+    insertItemRespList,
+    updateItemRespList,
+    deleteItemRespList,
+    destination,
+  );
+  return [...batchSuccessfulRespList, ...batchErrorRespList];
+};

src/cdk/v2/destinations/bloomreach_catalog/utils.ts

@@ -0,0 +1,147 @@
+import { BatchUtils } from '@rudderstack/workflow-engine';
+import { base64Convertor } from '@rudderstack/integrations-lib';
+import {
+  getCreateBulkCatalogItemEndpoint,
+  getDeleteBulkCatalogItemEndpoint,
+  getUpdateBulkCatalogItemEndpoint,
+  MAX_ITEMS,
+  MAX_PAYLOAD_SIZE,
+} from './config';
+
+const buildBatchedRequest = (
+  payload: string,
+  method: string,
+  endpoint: string,
+  headers: any,
+  metadata: any,
+  destination: any,
+) => ({
+  batchedRequest: {
+    body: {
+      JSON: {},
+      JSON_ARRAY: { batch: payload },
+      XML: {},
+      FORM: {},
+    },
+    version: '1',
+    type: 'REST',
+    method,
+    endpoint,
+    headers,
+    params: {},
+    files: {},
+  },
+  metadata,
+  batched: true,
+  statusCode: 200,
+  destination,
+});
+
+const getHeaders = (destination: any) => ({
+  'Content-Type': 'application/json',
+  Authorization: `Basic ${base64Convertor(`${destination.Config.apiKey}:${destination.Config.apiSecret}`)}`,
+});
+
+// returns merged metadata for a batch
+const getMergedMetadata = (batch: any[]) => batch.map((input) => input.metadata);
+
+// returns merged payload for a batch
+const getMergedEvents = (batch: any[]) => batch.map((input) => input.payload);
+
+// builds final batched response for insert action records
+const insertItemBatchResponseBuilder = (insertItemRespList: any[], destination: any) => {
+  const insertItemBatchedResponse: any[] = [];
+
+  const method = 'PUT';
+  const endpoint = getCreateBulkCatalogItemEndpoint(
+    destination.Config.apiBaseUrl,
+    destination.Config.projectToken,
+    destination.Config.catalogID,
+  );
+  const headers = getHeaders(destination);
+
+  const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(insertItemRespList, {
+    maxSizeInBytes: MAX_PAYLOAD_SIZE,
+    maxItems: MAX_ITEMS,
+  });
+  batchesOfEvents.items.forEach((batch: any) => {
+    const mergedPayload = JSON.stringify(getMergedEvents(batch));
+    const mergedMetadata = getMergedMetadata(batch);
+    insertItemBatchedResponse.push(
+      buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination),
+    );
+  });
+  return insertItemBatchedResponse;
+};
+
+// builds final batched response for update action records
+const updateItemBatchResponseBuilder = (updateItemRespList: any[], destination: any) => {
+  const updateItemBatchedResponse: any[] = [];
+
+  const method = 'POST';
+  const endpoint = getUpdateBulkCatalogItemEndpoint(
+    destination.Config.apiBaseUrl,
+    destination.Config.projectToken,
+    destination.Config.catalogID,
+  );
+  const headers = getHeaders(destination);
+
+  const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(updateItemRespList, {
+    maxSizeInBytes: MAX_PAYLOAD_SIZE,
+    maxItems: MAX_ITEMS,
+  });
+  batchesOfEvents.items.forEach((batch: any) => {
+    const mergedPayload = JSON.stringify(getMergedEvents(batch));
+    const mergedMetadata = getMergedMetadata(batch);
+    updateItemBatchedResponse.push(
+      buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination),
+    );
+  });
+  return updateItemBatchedResponse;
+};
+
+// builds final batched response for delete action records
+const deleteItemBatchResponseBuilder = (deleteItemRespList: any[], destination: any) => {
+  const deleteItemBatchedResponse: any[] = [];
+
+  const method = 'DELETE';
+  const endpoint = getDeleteBulkCatalogItemEndpoint(
+    destination.Config.apiBaseUrl,
+    destination.Config.projectToken,
+    destination.Config.catalogID,
+  );
+  const headers = getHeaders(destination);
+
+  const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(deleteItemRespList, {
+    maxSizeInBytes: MAX_PAYLOAD_SIZE,
+    maxItems: MAX_ITEMS,
+  });
+  batchesOfEvents.items.forEach((batch: any) => {
+    const mergedPayload = JSON.stringify(getMergedEvents(batch));
+    const mergedMetadata = getMergedMetadata(batch);
+    deleteItemBatchedResponse.push(
+      buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination),
+    );
+  });
+  return deleteItemBatchedResponse;
+};
+
+// returns final batched response
+export const batchResponseBuilder = (
+  insertItemRespList: any,
+  updateItemRespList: any,
+  deleteItemRespList: any,
+  destination: any,
+) => {
+  const response: any[] = [];
+  if (insertItemRespList.length > 0) {
+    response.push(...insertItemBatchResponseBuilder(insertItemRespList, destination));
+  }
+  if (updateItemRespList.length > 0) {
+    response.push(...updateItemBatchResponseBuilder(updateItemRespList, destination));
+  }
+  if (deleteItemRespList.length > 0) {
+    response.push(...deleteItemBatchResponseBuilder(deleteItemRespList, destination));
+  }
+  return response;
+};