diff --git a/.eslintignore b/.eslintignore index ce54730f4b..b1bbed4416 100644 --- a/.eslintignore +++ b/.eslintignore @@ -21,3 +21,8 @@ src/v0/destinations/personalize/scripts/ test/integrations/destinations/testTypes.d.ts *.config*.js scripts/skipPrepareScript.js +*.yaml +*.yml +.eslintignore +.prettierignore +*.json \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json index 556470697d..144b90e348 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -19,7 +19,8 @@ "parserOptions": { "ecmaVersion": 12, "sourceType": "module", - "project": "./tsconfig.json" + "project": "./tsconfig.json", + "extraFileExtensions": [".yaml"] }, "rules": { "unicorn/filename-case": [ diff --git a/.prettierignore b/.prettierignore index 99747b29bb..ac5f1fc409 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,3 +8,5 @@ test/**/*.js src/util/lodash-es-core.js src/util/url-search-params.min.js dist +.eslintignore +.prettierignore diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e4a636b4a..0ec77cc89e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.76.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.76.0...v1.76.1) (2024-08-29) + + +### Bug Fixes + +* reddit authorisation failed case handling ([#3690](https://github.com/rudderlabs/rudder-transformer/issues/3690)) ([f24759a](https://github.com/rudderlabs/rudder-transformer/commit/f24759aebbeb560f0de9d4920ae2ed0cdc7bfa3f)) + +## [1.76.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.1...v1.76.0) (2024-08-20) + + +### Features + +* klaviyo onboard unsubscribe profile support ([#3646](https://github.com/rudderlabs/rudder-transformer/issues/3646)) ([474f2bd](https://github.com/rudderlabs/rudder-transformer/commit/474f2bddc58e1962206e39d92514827f29f84c83)) +* onboard sfmc with vdm for rETL ([#3655](https://github.com/rudderlabs/rudder-transformer/issues/3655)) ([d987d1f](https://github.com/rudderlabs/rudder-transformer/commit/d987d1fc9afb9e1dc7482b2fe1458573f0f2699e)) +* onboard smartly destination ([#3660](https://github.com/rudderlabs/rudder-transformer/issues/3660)) ([474a36e](https://github.com/rudderlabs/rudder-transformer/commit/474a36ec385abf9ff83596b062d4d8e4c24469b8)) +* add bloomreach retl support ([#3619](https://github.com/rudderlabs/rudder-transformer/issues/3619)) ([6b1a23a](https://github.com/rudderlabs/rudder-transformer/commit/6b1a23af845084d6f2f5fd14656e4a1d11a7e34b)) + + +### Bug Fixes + +* add alias support in case alias details are present ([#3579](https://github.com/rudderlabs/rudder-transformer/issues/3579)) ([cb67262](https://github.com/rudderlabs/rudder-transformer/commit/cb672628b312f20ea0fcc27a60ec8ab5692f8b06)) +* attentive tag bugsnag issue ([#3663](https://github.com/rudderlabs/rudder-transformer/issues/3663)) ([866dbf3](https://github.com/rudderlabs/rudder-transformer/commit/866dbf3e81754e71ff8ac08b258b359ec5cc6889)) +* fixing facebook utils ([#3664](https://github.com/rudderlabs/rudder-transformer/issues/3664)) ([1a61675](https://github.com/rudderlabs/rudder-transformer/commit/1a6167584a5780ab50beda13cc5ef6bf4e283e38)) +* reserved properties for braze ([#3573](https://github.com/rudderlabs/rudder-transformer/issues/3573)) ([413e9ce](https://github.com/rudderlabs/rudder-transformer/commit/413e9ce56f8f6569bbeb188bff4f43d400ea71b1)) +* source transformation integration test generation ([#3645](https://github.com/rudderlabs/rudder-transformer/issues/3645)) ([23196ec](https://github.com/rudderlabs/rudder-transformer/commit/23196ec42acf35f314e1953f339f6acbb72edd70)) + ### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14) diff --git a/package-lock.json b/package-lock.json index 8c7377da3e..80e057dcf8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.75.1", + "version": "1.76.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.75.1", + "version": "1.76.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index c14e6b9ef5..7aefc0e363 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.75.1", + "version": "1.76.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/cdk/v2/destinations/bloomreach_catalog/config.ts b/src/cdk/v2/destinations/bloomreach_catalog/config.ts new file mode 100644 index 0000000000..8b469c3cf9 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/config.ts @@ -0,0 +1,31 @@ +export const MAX_PAYLOAD_SIZE = 10000000; +export const MAX_ITEMS = 5000; + +// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-update-catalog-item +export const getCreateBulkCatalogItemEndpoint = ( + apiBaseUrl: string, + projectToken: string, + catalogId: string, +): string => `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items`; + +// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-partial-update-catalog-item +export const getUpdateBulkCatalogItemEndpoint = ( + apiBaseUrl: string, + projectToken: string, + catalogId: string, +): string => + `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items/partial-update`; + +// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-delete-catalog-items +export const getDeleteBulkCatalogItemEndpoint = ( + apiBaseUrl: string, + projectToken: string, + catalogId: string, +): string => + `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items/bulk-delete`; + +export const CatalogAction = { + INSERT: 'insert', + UPDATE: 'update', + DELETE: 'delete', +}; diff --git a/src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml b/src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml new file mode 100644 index 0000000000..55809350eb --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml @@ -0,0 +1,42 @@ +bindings: + - name: EventType + path: ../../../../constants + - name: processRecordInputs + path: ./transformRecord + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - name: InstrumentationError + path: '@rudderstack/integrations-lib' + +steps: + - name: validateConfig + template: | + const config = ^[0].destination.Config + $.assertConfig(config.apiBaseUrl, "API Base URL is not present. Aborting"); + $.assertConfig(config.apiKey, "API Key is not present . Aborting"); + $.assertConfig(config.apiSecret, "API Secret is not present. Aborting"); + $.assertConfig(config.projectToken, "Project Token is not present. Aborting"); + $.assertConfig(config.catalogID, "Catalog Id is not present. Aborting"); + + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: processRecordEvents + template: | + $.processRecordInputs(^.{.message.type === $.EventType.RECORD}[], ^[0].destination) + + - name: failOtherEvents + template: | + const otherEvents = ^.{.message.type !== $.EventType.RECORD}[] + let failedEvents = otherEvents.map( + function(event) { + const error = new $.InstrumentationError("Event type " + event.message.type + " is not supported"); + $.handleRtTfSingleEventError(event, error, {}) + } + ) + failedEvents ?? [] + + - name: finalPayload + template: | + [...$.outputs.processRecordEvents, ...$.outputs.failOtherEvents] diff --git a/src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts b/src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts new file mode 100644 index 0000000000..68277448d0 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts @@ -0,0 +1,93 @@ +import { InstrumentationError } from '@rudderstack/integrations-lib'; +import { CatalogAction } from './config'; +import { batchResponseBuilder } from './utils'; + +import { handleRtTfSingleEventError, isEmptyObject } from '../../../../v0/util'; + +const prepareCatalogInsertOrUpdatePayload = (fields: any): any => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { item_id, ...properties } = fields; + return { item_id, properties }; +}; + +const processEvent = (event: any) => { + const { message } = event; + const { fields, action } = message; + const response = { + action, + payload: null, + }; + if (isEmptyObject(fields)) { + throw new InstrumentationError('`fields` cannot be empty'); + } + if (!fields.item_id) { + throw new InstrumentationError('`item_id` cannot be empty'); + } + if (action === CatalogAction.INSERT || action === CatalogAction.UPDATE) { + response.payload = prepareCatalogInsertOrUpdatePayload(fields); + } else if (action === CatalogAction.DELETE) { + response.payload = fields.item_id; + } else { + throw new InstrumentationError( + `Invalid action type ${action}. You can only add, update or remove items from the catalog`, + ); + } + return response; +}; + +const getEventChunks = ( + input: any, + insertItemRespList: any[], + updateItemRespList: any[], + deleteItemRespList: any[], +) => { + switch (input.response.action) { + case CatalogAction.INSERT: + insertItemRespList.push({ payload: input.response.payload, metadata: input.metadata }); + break; + case CatalogAction.UPDATE: + updateItemRespList.push({ payload: input.response.payload, metadata: input.metadata }); + break; + case CatalogAction.DELETE: + deleteItemRespList.push({ payload: input.response.payload, metadata: input.metadata }); + break; + default: + throw new InstrumentationError(`Invalid action type ${input.response.action}`); + } +}; + +export const processRecordInputs = (inputs: any[], destination: any) => { + const insertItemRespList: any[] = []; + const updateItemRespList: any[] = []; + const deleteItemRespList: any[] = []; + const batchErrorRespList: any[] = []; + + if (!inputs || inputs.length === 0) { + return []; + } + + inputs.forEach((input) => { + try { + getEventChunks( + { + response: processEvent(input), + metadata: input.metadata, + }, + insertItemRespList, + updateItemRespList, + deleteItemRespList, + ); + } catch (error) { + const errRespEvent = handleRtTfSingleEventError(input, error, {}); + batchErrorRespList.push(errRespEvent); + } + }); + + const batchSuccessfulRespList = batchResponseBuilder( + insertItemRespList, + updateItemRespList, + deleteItemRespList, + destination, + ); + return [...batchSuccessfulRespList, ...batchErrorRespList]; +}; diff --git a/src/cdk/v2/destinations/bloomreach_catalog/utils.ts b/src/cdk/v2/destinations/bloomreach_catalog/utils.ts new file mode 100644 index 0000000000..0e74ce9379 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/utils.ts @@ -0,0 +1,147 @@ +import { BatchUtils } from '@rudderstack/workflow-engine'; +import { base64Convertor } from '@rudderstack/integrations-lib'; +import { + getCreateBulkCatalogItemEndpoint, + getDeleteBulkCatalogItemEndpoint, + getUpdateBulkCatalogItemEndpoint, + MAX_ITEMS, + MAX_PAYLOAD_SIZE, +} from './config'; + +const buildBatchedRequest = ( + payload: string, + method: string, + endpoint: string, + headers: any, + metadata: any, + destination: any, +) => ({ + batchedRequest: { + body: { + JSON: {}, + JSON_ARRAY: { batch: payload }, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method, + endpoint, + headers, + params: {}, + files: {}, + }, + metadata, + batched: true, + statusCode: 200, + destination, +}); + +const getHeaders = (destination: any) => ({ + 'Content-Type': 'application/json', + Authorization: `Basic ${base64Convertor(`${destination.Config.apiKey}:${destination.Config.apiSecret}`)}`, +}); + +// returns merged metadata for a batch +const getMergedMetadata = (batch: any[]) => batch.map((input) => input.metadata); + +// returns merged payload for a batch +const getMergedEvents = (batch: any[]) => batch.map((input) => input.payload); + +// builds final batched response for insert action records +const insertItemBatchResponseBuilder = (insertItemRespList: any[], destination: any) => { + const insertItemBatchedResponse: any[] = []; + + const method = 'PUT'; + const endpoint = getCreateBulkCatalogItemEndpoint( + destination.Config.apiBaseUrl, + destination.Config.projectToken, + destination.Config.catalogID, + ); + const headers = getHeaders(destination); + + const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(insertItemRespList, { + maxSizeInBytes: MAX_PAYLOAD_SIZE, + maxItems: MAX_ITEMS, + }); + batchesOfEvents.items.forEach((batch: any) => { + const mergedPayload = JSON.stringify(getMergedEvents(batch)); + const mergedMetadata = getMergedMetadata(batch); + insertItemBatchedResponse.push( + buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination), + ); + }); + return insertItemBatchedResponse; +}; + +// builds final batched response for update action records +const updateItemBatchResponseBuilder = (updateItemRespList: any[], destination: any) => { + const updateItemBatchedResponse: any[] = []; + + const method = 'POST'; + const endpoint = getUpdateBulkCatalogItemEndpoint( + destination.Config.apiBaseUrl, + destination.Config.projectToken, + destination.Config.catalogID, + ); + const headers = getHeaders(destination); + + const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(updateItemRespList, { + maxSizeInBytes: MAX_PAYLOAD_SIZE, + maxItems: MAX_ITEMS, + }); + batchesOfEvents.items.forEach((batch: any) => { + const mergedPayload = JSON.stringify(getMergedEvents(batch)); + const mergedMetadata = getMergedMetadata(batch); + updateItemBatchedResponse.push( + buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination), + ); + }); + return updateItemBatchedResponse; +}; + +// builds final batched response for delete action records +const deleteItemBatchResponseBuilder = (deleteItemRespList: any[], destination: any) => { + const deleteItemBatchedResponse: any[] = []; + + const method = 'DELETE'; + const endpoint = getDeleteBulkCatalogItemEndpoint( + destination.Config.apiBaseUrl, + destination.Config.projectToken, + destination.Config.catalogID, + ); + const headers = getHeaders(destination); + + const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(deleteItemRespList, { + maxSizeInBytes: MAX_PAYLOAD_SIZE, + maxItems: MAX_ITEMS, + }); + batchesOfEvents.items.forEach((batch: any) => { + const mergedPayload = JSON.stringify(getMergedEvents(batch)); + const mergedMetadata = getMergedMetadata(batch); + deleteItemBatchedResponse.push( + buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination), + ); + }); + return deleteItemBatchedResponse; +}; + +// returns final batched response +export const batchResponseBuilder = ( + insertItemRespList: any, + updateItemRespList: any, + deleteItemRespList: any, + destination: any, +) => { + const response: any[] = []; + if (insertItemRespList.length > 0) { + response.push(...insertItemBatchResponseBuilder(insertItemRespList, destination)); + } + if (updateItemRespList.length > 0) { + response.push(...updateItemBatchResponseBuilder(updateItemRespList, destination)); + } + if (deleteItemRespList.length > 0) { + response.push(...deleteItemBatchResponseBuilder(deleteItemRespList, destination)); + } + return response; +}; diff --git a/src/cdk/v2/destinations/smartly/config.js b/src/cdk/v2/destinations/smartly/config.js new file mode 100644 index 0000000000..5083fde5fe --- /dev/null +++ b/src/cdk/v2/destinations/smartly/config.js @@ -0,0 +1,21 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +const ConfigCategories = { + TRACK: { + type: 'track', + name: 'trackMapping', + }, +}; + +const mappingConfig = getMappingConfig(ConfigCategories, __dirname); +const singleEventEndpoint = 'https://s2s.smartly.io/events'; +const batchEndpoint = 'https://s2s.smartly.io/events/batch'; + +module.exports = { + ConfigCategories, + mappingConfig, + singleEventEndpoint, + batchEndpoint, + TRACK_CONFIG: mappingConfig[ConfigCategories.TRACK.name], + MAX_BATCH_SIZE: 1000, +}; diff --git a/src/cdk/v2/destinations/smartly/data/trackMapping.json b/src/cdk/v2/destinations/smartly/data/trackMapping.json new file mode 100644 index 0000000000..55ba437f12 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/data/trackMapping.json @@ -0,0 +1,76 @@ +[ + { + "destKey": "value", + "sourceKeys": [ + "properties.total", + "properties.value", + "properties.revenue", + { + "operation": "multiplication", + "args": [ + { + "sourceKeys": "properties.price" + }, + { + "sourceKeys": "properties.quantity", + "default": 1 + } + ] + } + ], + "metadata": { + "type": "toNumber" + }, + "required": false + }, + { + "sourceKeys": ["properties.conversions", "properties.products.length"], + "required": false, + "metadata": { + "defaultValue": "1" + }, + "destKey": "conversions" + }, + { + "sourceKeys": ["properties.adUnitId", "properties.ad_unit_id"], + "required": true, + "destKey": "ad_unit_id", + "metadata": { + "type": "toString" + } + }, + { + "sourceKeys": ["properties.platform"], + "required": true, + "destKey": "platform" + }, + { + "sourceKeys": ["properties.adInteractionTime", "properties.ad_interaction_time"], + "required": true, + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "ad_interaction_time" + }, + { + "sourceKeys": ["properties.installTime"], + "required": false, + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "installTime" + }, + { + "sourceKeys": ["originalTimestamp", "timestamp"], + "required": false, + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "event_time" + }, + { + "sourceKeys": ["properties.currency"], + "required": false, + "destKey": "value_currency" + } +] diff --git a/src/cdk/v2/destinations/smartly/procWorkflow.yaml b/src/cdk/v2/destinations/smartly/procWorkflow.yaml new file mode 100644 index 0000000000..b69df0dd09 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/procWorkflow.yaml @@ -0,0 +1,31 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: constructPayload + path: ../../../../v0/util + - path: ./config + - path: ./utils +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.apiToken, "API Token is not present. Aborting"); + - name: preparePayload + template: | + const payload = $.removeUndefinedAndNullValues($.constructPayload(.message, $.TRACK_CONFIG)); + $.verifyAdInteractionTime(payload.ad_interaction_time); + $.context.payloadList = $.getPayloads(.message.event, .destination.Config, payload) + - name: buildResponse + template: | + const response = $.buildResponseList($.context.payloadList) + response diff --git a/src/cdk/v2/destinations/smartly/rtWorkflow.yaml b/src/cdk/v2/destinations/smartly/rtWorkflow.yaml new file mode 100644 index 0000000000..4d3afdb6d0 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/rtWorkflow.yaml @@ -0,0 +1,35 @@ +bindings: + - path: ./config + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "output": .body.JSON, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + $.batchResponseBuilder($.outputs.successfulEvents); + + - name: finalPayload + template: | + [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] diff --git a/src/cdk/v2/destinations/smartly/utils.js b/src/cdk/v2/destinations/smartly/utils.js new file mode 100644 index 0000000000..7d53ed0d27 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/utils.js @@ -0,0 +1,108 @@ +const { BatchUtils } = require('@rudderstack/workflow-engine'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const moment = require('moment'); +const config = require('./config'); +const { + getHashFromArrayWithDuplicate, + defaultRequestConfig, + isDefinedAndNotNull, +} = require('../../../../v0/util'); + +// docs reference : https://support.smartly.io/hc/en-us/articles/4406049685788-S2S-integration-API-description#01H8HBXZF6WSKSYBW1C6NY8A88 + +/** + * This function generates an array of payload objects, each with the event property set + * to different values associated with the given event name according to eventsMapping + * @param {*} event + * @param {*} eventsMapping + * @param {*} payload + * @returns + */ +const getPayloads = (event, Config, payload) => { + if (!isDefinedAndNotNull(event) || typeof event !== 'string') { + throw new InstrumentationError('Event is not defined or is not String'); + } + const eventsMap = getHashFromArrayWithDuplicate(Config.eventsMapping); + // eventsMap = hashmap {"prop1":["val1","val2"],"prop2":["val2"]} + const eventList = Array.isArray(eventsMap[event.toLowerCase()]) + ? eventsMap[event.toLowerCase()] + : Array.from(eventsMap[event.toLowerCase()] || [event]); + + const payloadLists = eventList.map((ev) => ({ ...payload, event_name: ev })); + return payloadLists; +}; + +// ad_interaction_time must be within one year in the future and three years in the past from the current date +// Example : "1735680000" +const verifyAdInteractionTime = (adInteractionTime) => { + if (isDefinedAndNotNull(adInteractionTime)) { + const now = moment(); + const threeYearAgo = now.clone().subtract(3, 'year'); + const oneYearFromNow = now.clone().add(1, 'year'); + const inputMoment = moment(adInteractionTime * 1000); // Convert to milliseconds + if (!inputMoment.isAfter(threeYearAgo) || !inputMoment.isBefore(oneYearFromNow)) { + throw new InstrumentationError( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + } + } +}; + +const buildResponseList = (payloadList) => + payloadList.map((payload) => { + const response = defaultRequestConfig(); + response.body.JSON = payload; + response.endpoint = config.singleEventEndpoint; + response.method = 'POST'; + return response; + }); + +const batchBuilder = (batch, destination) => ({ + batchedRequest: { + body: { + JSON: { events: batch.map((event) => event.output) }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: config.batchEndpoint, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${destination.Config.apiToken}`, + }, + params: {}, + files: {}, + }, + metadata: batch + .map((event) => event.metadata) + .filter((metadata, index, self) => self.findIndex((m) => m.jobId === metadata.jobId) === index), // handling jobId duplication for multiplexed events + batched: true, + statusCode: 200, + destination: batch[0].destination, +}); + +/** + * This fucntions make chunk of successful events based on MAX_BATCH_SIZE + * and then build the response for each chunk to be returned as object of an array + * @param {*} events + * @returns + */ +const batchResponseBuilder = (events) => { + if (events.length === 0) { + return []; + } + const { destination } = events[0]; + const batches = BatchUtils.chunkArrayBySizeAndLength(events, { maxItems: config.MAX_BATCH_SIZE }); + + const response = []; + batches.items.forEach((batch) => { + const batchedResponse = batchBuilder(batch, destination); + response.push(batchedResponse); + }); + return response; +}; + +module.exports = { batchResponseBuilder, getPayloads, buildResponseList, verifyAdInteractionTime }; diff --git a/src/cdk/v2/destinations/smartly/utils.test.js b/src/cdk/v2/destinations/smartly/utils.test.js new file mode 100644 index 0000000000..0ad73f5369 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/utils.test.js @@ -0,0 +1,59 @@ +const moment = require('moment'); +const { verifyAdInteractionTime } = require('./utils'); + +describe('verifyAdInteractionTime', () => { + it('should pass when adInteractionTime is 2 years in the past (UNIX timestamp)', () => { + // 2 years ago from now + const adInteractionTime = moment().subtract(2, 'years').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).not.toThrow(); + }); + + it('should pass when adInteractionTime is 10 months in the future (UNIX timestamp)', () => { + // 10 months in the future from now + const adInteractionTime = moment().add(10, 'months').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).not.toThrow(); + }); + + it('should fail when adInteractionTime is 4 years in the past (UNIX timestamp)', () => { + // 4 years ago from now + const adInteractionTime = moment().subtract(4, 'years').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should fail when adInteractionTime is 2 years in the future (UNIX timestamp)', () => { + // 2 years in the future from now + const adInteractionTime = moment().add(2, 'years').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should pass when adInteractionTime is exactly 1 year in the future (UTC date string)', () => { + // Exactly 1 year in the future from now + const adInteractionTime = moment.utc().add(1, 'year').toISOString(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow(); + }); + + it('should fail when adInteractionTime is 4 years in the past (UTC date string)', () => { + // 4 years ago from now + const adInteractionTime = moment.utc().subtract(4, 'years').toISOString(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should fail when adInteractionTime is 2 years in the future (UTC date string)', () => { + // 2 years in the future from now + const adInteractionTime = moment.utc().add(2, 'years').toISOString(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should not throw an error if adInteractionTime is null or undefined', () => { + expect(() => verifyAdInteractionTime(null)).not.toThrow(); + expect(() => verifyAdInteractionTime(undefined)).not.toThrow(); + }); +}); diff --git a/src/features.json b/src/features.json index 94e36a2416..a0261683a8 100644 --- a/src/features.json +++ b/src/features.json @@ -76,7 +76,9 @@ "WUNDERKIND": true, "CLICKSEND": true, "ZOHO": true, - "CORDIAL": true + "CORDIAL": true, + "BLOOMREACH_CATALOG": true, + "SMARTLY": true }, "regulations": [ "BRAZE", diff --git a/src/v0/destinations/reddit/networkHandler.js b/src/v0/destinations/reddit/networkHandler.js index 55087b52ac..e691255a26 100644 --- a/src/v0/destinations/reddit/networkHandler.js +++ b/src/v0/destinations/reddit/networkHandler.js @@ -1,4 +1,5 @@ const { RetryableError } = require('@rudderstack/integrations-lib'); +const isString = require('lodash/isString'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess } = require('../../util/index'); const { REFRESH_TOKEN } = require('../../../adapters/networkhandler/authConstants'); @@ -9,12 +10,23 @@ const redditRespHandler = (destResponse) => { const { status, response } = destResponse; // to handle the case when authorization-token is invalid - if (status === 401 && response.includes('Authorization Required')) { + if (status === 401) { + let errorMessage = 'Authorization failed'; + let authErrorCategory = ''; + + if (isString(response) && response.includes('Authorization Required')) { + errorMessage = `Request failed due to ${response}`; + authErrorCategory = REFRESH_TOKEN; + } else if (response?.error?.reason === 'UNAUTHORIZED') { + errorMessage = response.error.explanation || errorMessage; + authErrorCategory = REFRESH_TOKEN; + } + throw new RetryableError( - `Request failed due to ${response} 'during reddit response transformation'`, - 500, + `${errorMessage} during reddit response transformation`, + status, destResponse, - REFRESH_TOKEN, + authErrorCategory, ); } }; diff --git a/src/v1/destinations/bloomreach_catalog/networkHandler.js b/src/v1/destinations/bloomreach_catalog/networkHandler.js new file mode 100644 index 0000000000..1fb987b840 --- /dev/null +++ b/src/v1/destinations/bloomreach_catalog/networkHandler.js @@ -0,0 +1,85 @@ +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { proxyRequest, prepareProxyRequest } = require('../../../adapters/network'); +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); +const tags = require('../../../v0/util/tags'); + +// Catalog response +// [ +// { +// "errors": { +// "properties": [ +// "Fields [field1, field2] are not properly defined." +// ] +// }, +// "queued": false, +// "success": false +// }, +// { +// "success" : "True", +// "queued" : "True", +// }, +// ] +const checkIfEventIsAbortableAndExtractErrorMessage = (element) => { + if (element.success) { + return { isAbortable: false, errorMsg: '' }; + } + + const errorMsg = Object.values(element.errors || {}) + .flat() + .join(', '); + + return { isAbortable: true, errorMsg }; +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + + const message = '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully'; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + response.forEach((event, idx) => { + const proxyOutput = { + statusCode: 200, + error: 'success', + metadata: rudderJobMetadata[idx], + }; + // update status of partial event if abortable + const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage(event); + if (isAbortable) { + proxyOutput.error = errorMsg; + proxyOutput.statusCode = 400; + } + responseWithIndividualEvents.push(proxyOutput); + }); + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + throw new TransformerProxyError( + `BLOOMREACH_CATALOG: Error encountered in transformer proxy V1`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); +}; +function networkHandler() { + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.prepareProxy = prepareProxyRequest; + this.responseHandler = responseHandler; +} +module.exports = { networkHandler }; diff --git a/test/integrations/destinations/bloomreach_catalog/common.ts b/test/integrations/destinations/bloomreach_catalog/common.ts new file mode 100644 index 0000000000..2b4266837b --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/common.ts @@ -0,0 +1,81 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'bloomreach_catalog'; +const destTypeInUpperCase = 'BLOOMREACH_CATALOG'; +const displayName = 'bloomreach catalog'; +const channel = 'web'; +const destination: Destination = { + Config: { + apiBaseUrl: 'https://demoapp-api.bloomreach.com', + apiKey: 'test-api-key', + apiSecret: 'test-api-secret', + projectToken: 'test-project-token', + catalogID: 'test-catalog-id', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const insertEndpoint = + 'https://demoapp-api.bloomreach.com/data/v2/projects/test-project-token/catalogs/test-catalog-id/items'; +const updateEndpoint = + 'https://demoapp-api.bloomreach.com/data/v2/projects/test-project-token/catalogs/test-catalog-id/items/partial-update'; +const deleteEndpoint = + 'https://demoapp-api.bloomreach.com/data/v2/projects/test-project-token/catalogs/test-catalog-id/items/bulk-delete'; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const proxyV1RetryableErrorStatTags = { + ...RouterInstrumentationErrorStatTags, + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hcGkta2V5OnRlc3QtYXBpLXNlY3JldA==', +}; + +const sampleContext = { + destinationFields: 'item_id, title, status, unprinted', + mappedToDestination: 'true', +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + headers, + proxyV1RetryableErrorStatTags, + insertEndpoint, + updateEndpoint, + deleteEndpoint, + sampleContext, +}; diff --git a/test/integrations/destinations/bloomreach_catalog/dataDelivery/data.ts b/test/integrations/destinations/bloomreach_catalog/dataDelivery/data.ts new file mode 100644 index 0000000000..f8cccd04ed --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/dataDelivery/data.ts @@ -0,0 +1,197 @@ +import { ProxyV1TestData } from '../../../testTypes'; +import { destType, headers, updateEndpoint } from '../common'; +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; + +export const data: ProxyV1TestData[] = [ + { + id: 'bloomreach_catalog_v1_business_scenario_1', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid record request - where the destination responds with 200 with error for request 2 in a batch', + successCriteria: 'Should return 200 with partial failures within the response payload', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id","properties":{"unprinted":"1"}},{"item_id":"test-item-id-faulty","properties":{"unprinted1":"1"}}]', + }, + endpoint: updateEndpoint, + }, + [generateMetadata(1), generateMetadata(2)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: [ + { + success: true, + queued: true, + }, + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: 'Fields [unprinted1] are not properly defined.', + }, + ], + }, + }, + }, + }, + }, + { + id: 'bloomreach_catalog_v1_business_scenario_2', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid rETL request - where the destination responds with 200 without any error', + successCriteria: 'Should return 200 with no error with destination response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id-1","properties":{"unprinted":"1"}},{"item_id":"test-item-id-2","properties":{"unprinted":"2"}}]', + }, + endpoint: updateEndpoint, + }, + [generateMetadata(3), generateMetadata(4)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: [ + { + success: true, + queued: true, + }, + { + success: true, + queued: true, + }, + ], + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(3), + error: 'success', + }, + { + statusCode: 200, + metadata: generateMetadata(4), + error: 'success', + }, + ], + }, + }, + }, + }, + }, + { + id: 'bloomreach_catalog_v1_business_scenario_3', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid rETL request - where the destination responds with 200 with error', + successCriteria: 'Should return 400 with error message', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: {}, + JSON_ARRAY: { + batch: '[{"item_id":"test-item-id-faulty","properties":{"unprinted1":"1"}}]', + }, + endpoint: updateEndpoint, + }, + [generateMetadata(5)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: [ + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + }, + response: [ + { + statusCode: 400, + metadata: generateMetadata(5), + error: 'Fields [unprinted1] are not properly defined.', + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach_catalog/mocks.ts b/test/integrations/destinations/bloomreach_catalog/mocks.ts new file mode 100644 index 0000000000..bb07c0ea72 --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/mocks.ts @@ -0,0 +1,5 @@ +import * as config from '../../../../src/cdk/v2/destinations/bloomreach_catalog/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_ITEMS', 2 as typeof config.MAX_ITEMS); +}; diff --git a/test/integrations/destinations/bloomreach_catalog/network.ts b/test/integrations/destinations/bloomreach_catalog/network.ts new file mode 100644 index 0000000000..b8ae078498 --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/network.ts @@ -0,0 +1,108 @@ +import { destType, headers, updateEndpoint } from './common'; + +export const networkCallsData = [ + { + httpReq: { + url: updateEndpoint, + data: [ + { + item_id: 'test-item-id', + properties: { + unprinted: '1', + }, + }, + { + item_id: 'test-item-id-faulty', + properties: { + unprinted1: '1', + }, + }, + ], + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: [ + { + success: true, + queued: true, + }, + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: updateEndpoint, + data: [ + { + item_id: 'test-item-id-1', + properties: { + unprinted: '1', + }, + }, + { + item_id: 'test-item-id-2', + properties: { + unprinted: '2', + }, + }, + ], + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: [ + { + success: true, + queued: true, + }, + { + success: true, + queued: true, + }, + ], + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: updateEndpoint, + data: [ + { + item_id: 'test-item-id-faulty', + properties: { + unprinted1: '1', + }, + }, + ], + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: [ + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + statusText: 'Ok', + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach_catalog/router/data.ts b/test/integrations/destinations/bloomreach_catalog/router/data.ts new file mode 100644 index 0000000000..68ab422444 --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/router/data.ts @@ -0,0 +1,328 @@ +import { generateMetadata } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; +import { + destType, + destination, + headers, + RouterInstrumentationErrorStatTags, + insertEndpoint, + updateEndpoint, + deleteEndpoint, + sampleContext, +} from '../common'; + +const routerRequest = { + input: [ + { + message: { + type: 'record', + action: 'insert', + fields: { + item_id: 'test-item-id', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '1', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'record', + action: 'insert', + fields: { + item_id: 'test-item-id-7', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + test_empty: '', + test_null: null, + test_empty_array: [], + }, + channel: 'sources', + context: sampleContext, + recordId: '2', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'record', + action: 'update', + fields: { + item_id: 'test-item-id', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 3, + }, + channel: 'sources', + context: sampleContext, + recordId: '3', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'record', + action: 'update', + fields: { + item_id: 'test-item-id', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 2, + }, + channel: 'sources', + context: sampleContext, + recordId: '4', + }, + metadata: generateMetadata(4), + destination, + }, + { + message: { + type: 'record', + action: 'delete', + fields: { + item_id: 'test-item-id-1', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '5', + }, + metadata: generateMetadata(5), + destination, + }, + { + message: { + type: 'record', + action: 'delete', + fields: { + item_id: 'test-item-id-2', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '6', + }, + metadata: generateMetadata(6), + destination, + }, + { + message: { + type: 'record', + action: 'delete', + fields: { + item_id: 'test-item-id-3', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '7', + }, + metadata: generateMetadata(7), + destination, + }, + { + message: { + type: 'record', + action: 'insert', + fields: {}, + channel: 'sources', + context: sampleContext, + recordId: '8', + }, + metadata: generateMetadata(8), + destination, + }, + { + message: { + type: 'record', + action: 'dummy-action', + fields: { + item_id: 'test-item-id', + }, + channel: 'sources', + context: sampleContext, + recordId: '9', + }, + metadata: generateMetadata(9), + destination, + }, + { + message: { + type: 'record', + action: 'insert', + fields: { + item_id: null, + }, + channel: 'sources', + context: sampleContext, + recordId: '10', + }, + metadata: generateMetadata(10), + destination, + }, + ], + destType, +}; + +export const data = [ + { + id: 'bloomreach-catalog-router-test-1', + name: destType, + description: 'Basic Router Test to test record payloads', + scenario: 'Framework', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'PUT', + endpoint: insertEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":1}},{"item_id":"test-item-id-7","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":1,"test_empty":"","test_null":null,"test_empty_array":[]}}]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: updateEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":3}},{"item_id":"test-item-id","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":2}}]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(3), generateMetadata(4)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: deleteEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: '["test-item-id-1","test-item-id-2"]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(5), generateMetadata(6)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: deleteEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: '["test-item-id-3"]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(7)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(8)], + batched: false, + statusCode: 400, + error: '`fields` cannot be empty', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(9)], + batched: false, + statusCode: 400, + error: + 'Invalid action type dummy-action. You can only add, update or remove items from the catalog', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(10)], + batched: false, + statusCode: 400, + error: '`item_id` cannot be empty', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/reddit/dataDelivery/oauth.ts b/test/integrations/destinations/reddit/dataDelivery/oauth.ts index 90368cd60b..8c0d486a7a 100644 --- a/test/integrations/destinations/reddit/dataDelivery/oauth.ts +++ b/test/integrations/destinations/reddit/dataDelivery/oauth.ts @@ -81,7 +81,7 @@ export const v0oauthScenarios = [ }, output: { response: { - status: 500, + status: 401, body: { output: { authErrorCategory: 'REFRESH_TOKEN', @@ -90,9 +90,53 @@ export const v0oauthScenarios = [ status: 401, }, message: - "Request failed due to Authorization Required 'during reddit response transformation'", + 'Request failed due to Authorization Required during reddit response transformation', statTags: expectedStatTags, - status: 500, + status: 401, + }, + }, + }, + }, + }, + { + id: 'reddit_v0_oauth_scenario_2', + name: 'reddit', + description: '[Proxy v0 API] :: Oauth where error response is an object from destination', + successCriteria: 'Should return 401 with authErrorCategory as REFRESH_TOKEN', + scenario: 'Oauth', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: generateProxyV0Payload({ + ...commonRequestParameters, + endpoint: 'https://ads-api.reddit.com/api/v2.0/conversions/events/a2_objResp_401', + }), + method: 'POST', + }, + }, + output: { + response: { + status: 401, + body: { + output: { + authErrorCategory: 'REFRESH_TOKEN', + destinationResponse: { + response: { + success: false, + error: { + reason: 'UNAUTHORIZED', + explanation: + 'This server could not verify that you are authorized to access the document you requested.', + }, + }, + status: 401, + }, + message: + 'This server could not verify that you are authorized to access the document you requested. during reddit response transformation', + statTags: expectedStatTags, + status: 401, }, }, }, @@ -124,21 +168,65 @@ export const v1oauthScenarios = [ }, output: { response: { - status: 500, + status: 401, body: { output: { authErrorCategory: 'REFRESH_TOKEN', message: - "Request failed due to Authorization Required 'during reddit response transformation'", + 'Request failed due to Authorization Required during reddit response transformation', response: [ { error: '"Authorization Required"', metadata: generateMetadata(1), - statusCode: 500, + statusCode: 401, + }, + ], + statTags: expectedStatTags, + status: 401, + }, + }, + }, + }, + }, + { + id: 'reddit_v1_oauth_scenario_2', + name: 'reddit', + description: '[Proxy v1 API] :: Oauth where error response is an object from destination', + successCriteria: 'Should return 401 with authErrorCategory as REFRESH_TOKEN', + scenario: 'Oauth', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + ...commonRequestParameters, + endpoint: 'https://ads-api.reddit.com/api/v2.0/conversions/events/a2_objResp_401', + }, + [generateMetadata(1)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 401, + body: { + output: { + authErrorCategory: 'REFRESH_TOKEN', + message: + 'This server could not verify that you are authorized to access the document you requested. during reddit response transformation', + response: [ + { + error: + '{"success":false,"error":{"reason":"UNAUTHORIZED","explanation":"This server could not verify that you are authorized to access the document you requested."}}', + metadata: generateMetadata(1), + statusCode: 401, }, ], statTags: expectedStatTags, - status: 500, + status: 401, }, }, }, diff --git a/test/integrations/destinations/reddit/network.ts b/test/integrations/destinations/reddit/network.ts index 7c436e8fb8..562c8e95ad 100644 --- a/test/integrations/destinations/reddit/network.ts +++ b/test/integrations/destinations/reddit/network.ts @@ -97,4 +97,61 @@ export const networkCallsData = [ }, httpRes: { data: 'Authorization Required', status: 401, statusText: 'Unauthorized' }, }, + { + httpReq: { + url: 'https://ads-api.reddit.com/api/v2.0/conversions/events/a2_objResp_401', + data: { + events: [ + { + event_at: '2019-10-14T09:03:17.562Z', + event_type: { + tracking_type: 'ViewContent', + }, + user: { + aaid: 'c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a', + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 3, + products: [ + { + id: '123', + name: 'Monopoly', + category: 'Games', + }, + { + id: '345', + name: 'UNO', + category: 'Games', + }, + ], + }, + }, + ], + }, + params: { destination: 'reddit' }, + headers: { + Authorization: 'Bearer dummyAccessToken', + 'Content-Type': 'application/json', + }, + method: 'POST', + }, + httpRes: { + data: { + success: false, + error: { + reason: 'UNAUTHORIZED', + explanation: + 'This server could not verify that you are authorized to access the document you requested.', + }, + }, + status: 401, + statusText: 'Unauthorized', + }, + }, ]; diff --git a/test/integrations/destinations/smartly/commonConfig.ts b/test/integrations/destinations/smartly/commonConfig.ts new file mode 100644 index 0000000000..f5b0a6f4d4 --- /dev/null +++ b/test/integrations/destinations/smartly/commonConfig.ts @@ -0,0 +1,40 @@ +export const destination = { + ID: 'random_id', + Name: 'smartly', + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiToken: 'testAuthToken', + eventsMapping: [ + { + from: 'product list viewed', + to: 'event1', + }, + { + from: 'product list viewed', + to: 'event2', + }, + ], + }, +}; + +export const routerInstrumentationErrorStatTags = { + destType: 'SMARTLY', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', +}; +export const processInstrumentationErrorStatTags = { + destType: 'SMARTLY', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'dummyDestId', +}; diff --git a/test/integrations/destinations/smartly/mocks.ts b/test/integrations/destinations/smartly/mocks.ts new file mode 100644 index 0000000000..78773d8853 --- /dev/null +++ b/test/integrations/destinations/smartly/mocks.ts @@ -0,0 +1,6 @@ +import config from '../../../../src/cdk/v2/destinations/smartly/config'; + +export const defaultMockFns = () => { + jest.useFakeTimers().setSystemTime(new Date('2024-02-01')); + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 2); +}; diff --git a/test/integrations/destinations/smartly/processor/data.ts b/test/integrations/destinations/smartly/processor/data.ts new file mode 100644 index 0000000000..a94f6b220f --- /dev/null +++ b/test/integrations/destinations/smartly/processor/data.ts @@ -0,0 +1,9 @@ +import { trackTestData } from './track'; +import { validationFailures } from './validation'; + +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2024-02-01')); +}; + +export const data = [...trackTestData, ...validationFailures].map((d) => ({ ...d, mockFns })); diff --git a/test/integrations/destinations/smartly/processor/track.ts b/test/integrations/destinations/smartly/processor/track.ts new file mode 100644 index 0000000000..944327fce3 --- /dev/null +++ b/test/integrations/destinations/smartly/processor/track.ts @@ -0,0 +1,71 @@ +import { destination } from '../commonConfig'; + +export const trackTestData = [ + { + name: 'smartly', + description: 'Test 0', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + event: 'Add to cart', + properties: { + platform: 'meta', + ad_unit_id: '228287', + ad_interaction_time: 1735680000, + email: 'eventIdn01@sample.com', + }, + type: 'track', + userId: 'eventIdn01', + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events', + headers: {}, + params: {}, + body: { + JSON: { + platform: 'meta', + ad_unit_id: '228287', + ad_interaction_time: 1735680000, + conversions: '1', + event_name: 'Add to cart', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/smartly/processor/validation.ts b/test/integrations/destinations/smartly/processor/validation.ts new file mode 100644 index 0000000000..996afc34b3 --- /dev/null +++ b/test/integrations/destinations/smartly/processor/validation.ts @@ -0,0 +1,174 @@ +import { processInstrumentationErrorStatTags, destination } from '../commonConfig'; + +export const validationFailures = [ + { + id: 'Smartly-validation-test-1', + name: 'smartly', + description: 'Required field anonymousId not present', + scenario: 'Framework', + successCriteria: 'Transformationn Error for anonymousId not present', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + event: 'product purchased', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'john123', + properties: { + products: [{}], + ad_unit_id: '22123387', + ad_interaction_time: '1690867200', + }, + integrations: { + All: true, + }, + originalTimestamp: '2021-01-25T15:32:56.409Z', + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Missing required value from ["properties.platform"]: Workflow: procWorkflow, Step: preparePayload, ChildStep: undefined, OriginalError: Missing required value from ["properties.platform"]', + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + statTags: processInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'Smartly-test-2', + name: 'smartly', + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Transformationn Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + event_name: 'purchase', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'john67', + channel: 'mobile', + rudderId: 'b7b24f86-cccx-46d8-b2b4-ccaxxx80239c', + messageId: 'dummy_msg_id', + properties: { + platform: 'snapchat', + ad_unit_id: '2653387', + ad_interaction_time: '1690867200', + }, + anonymousId: 'anon_123', + integrations: { + All: true, + }, + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + statTags: processInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'Smartly-test-3', + name: 'smartly', + description: 'Event name not defined', + scenario: 'Framework', + successCriteria: 'Transformationn Error for Undefined Event', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'john67', + channel: 'mobile', + rudderId: 'b7b24f86-cccx-46d8-b2b4-ccaxxx80239c', + messageId: 'dummy_msg_id', + properties: { + platform: 'snapchat', + ad_unit_id: '2653387', + ad_interaction_time: 1675094400, + }, + anonymousId: 'anon_123', + integrations: { + All: true, + }, + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Event is not defined or is not String: Workflow: procWorkflow, Step: preparePayload, ChildStep: undefined, OriginalError: Event is not defined or is not String', + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + statTags: processInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/smartly/router/data.ts b/test/integrations/destinations/smartly/router/data.ts new file mode 100644 index 0000000000..7c2d74e6f0 --- /dev/null +++ b/test/integrations/destinations/smartly/router/data.ts @@ -0,0 +1,329 @@ +import { destination, routerInstrumentationErrorStatTags } from '../commonConfig'; +import { defaultMockFns } from '../mocks'; + +export const data = [ + { + name: 'smartly', + id: 'Test 0 - router', + description: 'Track call with multiplexing and batching', + scenario: 'Framework+Buisness', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'track', + event: 'product list viewed', + properties: { + platform: 'meta', + conversions: 1, + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 2, userId: 'u2' }, + destination, + }, + { + message: { + type: 'track', + event: 'add to cart', + properties: { + conversions: 3, + platform: 'snapchat', + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 3, userId: 'u3' }, + destination, + }, + ], + destType: 'smartly', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + body: { + JSON: { + events: [ + { + conversions: 1, + ad_unit_id: '221187', + platform: 'meta', + ad_interaction_time: 1690867200, + event_name: 'event1', + }, + { + conversions: 1, + ad_unit_id: '221187', + platform: 'meta', + ad_interaction_time: 1690867200, + event_name: 'event2', + }, + { + conversions: 3, + ad_unit_id: '77187', + platform: 'snapchat', + ad_interaction_time: 1690867200, + event_name: 'add to cart', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events/batch', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer testAuthToken`, + }, + params: {}, + files: {}, + }, + metadata: [ + { + jobId: 2, + userId: 'u2', + }, + { + jobId: 3, + userId: 'u3', + }, + ], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + mockFns: () => { + jest.useFakeTimers().setSystemTime(new Date('2024-02-01')); + }, + }, + { + name: 'smartly', + id: 'Test 1 - router', + description: 'Batch calls with 4 succesfull events including multiplexing and 2 failed events', + scenario: 'Framework+Buisness', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'track', + event: 'product list viewed', + properties: { + platform: 'meta', + conversions: 1, + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 11, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + event: 'purchase', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + platform: 'snapchat', + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 13, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + platform: 'snapchat', + ad_unit_id: '12387', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 14, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + event: 'random event', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 15, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + event: 'add to cart', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + platform: 'tiktok', + ad_unit_id: '789187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 16, userId: 'u1' }, + destination, + }, + ], + destType: 'smartly', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: false, + destination, + error: 'Event is not defined or is not String', + metadata: [{ jobId: 14, userId: 'u1' }], + statTags: routerInstrumentationErrorStatTags, + statusCode: 400, + }, + { + batched: false, + destination, + error: 'Missing required value from ["properties.platform"]', + metadata: [{ jobId: 15, userId: 'u1' }], + statTags: routerInstrumentationErrorStatTags, + statusCode: 400, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events/batch', + params: {}, + body: { + FORM: {}, + JSON: { + events: [ + { + platform: 'meta', + conversions: 1, + event_name: 'event1', + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + { + platform: 'meta', + conversions: 1, + event_name: 'event2', + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer testAuthToken`, + }, + files: {}, + }, + metadata: [{ jobId: 11, userId: 'u1' }], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events/batch', + params: {}, + body: { + FORM: {}, + JSON: { + events: [ + { + conversions: 3, + event_name: 'purchase', + platform: 'snapchat', + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + { + conversions: 3, + event_name: 'add to cart', + platform: 'tiktok', + ad_unit_id: '789187', + ad_interaction_time: 1690867200, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer testAuthToken`, + }, + files: {}, + }, + metadata: [ + { jobId: 13, userId: 'u1' }, + { jobId: 16, userId: 'u1' }, + ], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +];