From 53d2bef38ad5d7bc5504111ec797b3c3973546dd Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Wed, 20 Nov 2024 03:28:06 +0530 Subject: [PATCH 01/21] feat: add support for getting anonymousId by note attributes array --- .../serverSideTransform.js | 19 +++---------- .../webhookTransformations/serverSideUtlis.js | 27 ++++++++++++++++++- .../integrations/sources/shopify/constants.ts | 15 +++++++++++ .../CheckoutEventsTests.ts | 27 ++++++++++--------- .../webhookTestScenarios/GenericTrackTests.ts | 12 +++++---- 5 files changed, 66 insertions(+), 34 deletions(-) diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js index c31bc74bf1..4f91b0e6c1 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -1,18 +1,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ const lodash = require('lodash'); const get = require('get-value'); -// const { RedisError } = require('@rudderstack/integrations-lib'); const stats = require('../../../../util/stats'); -const { - getShopifyTopic, - // createPropertiesForEcomEvent, - extractEmailFromPayload, - getAnonymousIdAndSessionId, - // getHashLineItems, -} = require('../../../../v0/sources/shopify/util'); -// const logger = require('../../../logger'); +const { getShopifyTopic, extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util'); -// const { RedisDB } = require('../../../util/redis/redisConnector'); const Message = require('../../../../v0/sources/message'); const { EventType } = require('../../../../constants'); const { @@ -28,6 +19,7 @@ const { const { createPropertiesForEcomEventFromWebhook, getProductsFromLineItems, + getAnonymousIdFromAttributes, } = require('./serverSideUtlis'); const NO_OPERATION_SUCCESS = { @@ -128,12 +120,9 @@ const processEvent = async (inputEvent, metricMetadata) => { message.setProperty('traits.email', email); } } + // attach anonymousId if the event is track event using note_attributes if (message.type !== EventType.IDENTIFY) { - const { anonymousId } = await getAnonymousIdAndSessionId( - message, - { shopifyTopic, ...metricMetadata }, - null, - ); + const anonymousId = await getAnonymousIdFromAttributes(event); if (isDefinedAndNotNull(anonymousId)) { message.setProperty('anonymousId', anonymousId); } diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index eed03de71f..0c38dd1d10 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -1,5 +1,6 @@ +const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); +const { v5 } = require('uuid'); const { constructPayload } = require('../../../../v0/util'); - const { lineItemsMappingJSON, productMappingJSON, @@ -39,7 +40,31 @@ const createPropertiesForEcomEventFromWebhook = (message) => { return mappedPayload; }; +/** + * Returns the anonymousId from the noteAttributes array in the webhook event + * @param {Object} event + * @returns {String} anonymousId + */ +const getAnonymousIdFromAttributes = async (event) => { + let anonymousId = null; + let cartToken = null; + const noteAttributes = event.note_attributes; + if (isDefinedAndNotNull(event) && isDefinedAndNotNull(noteAttributes)) { + const rudderAnonymousIdObject = noteAttributes.find( + (attr) => attr.name === 'rudderAnonymousId', + ); + anonymousId = rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; + const cartTokenObject = noteAttributes.find((attr) => attr.name === 'cartToken'); + cartToken = cartTokenObject ? cartTokenObject.value : null; + if (!isDefinedAndNotNull(anonymousId) && isDefinedAndNotNull(cartToken)) { + anonymousId = v5(cartToken, v5.URL); + } + } + return anonymousId; +}; + module.exports = { createPropertiesForEcomEventFromWebhook, getProductsFromLineItems, + getAnonymousIdFromAttributes, }; diff --git a/test/integrations/sources/shopify/constants.ts b/test/integrations/sources/shopify/constants.ts index f9df305841..af53a3180e 100644 --- a/test/integrations/sources/shopify/constants.ts +++ b/test/integrations/sources/shopify/constants.ts @@ -83,6 +83,21 @@ export const dummyContext = { }, }; +export const note_attributes = [ + { + name: 'cartId', + value: '9c623f099fc8819aa4d6a958b65dfe7d', + }, + { + name: 'cartToken', + value: 'Z2NwLXVzLWVhc3QxOjAxSkQzNUFXVEI4VkVUNUpTTk1LSzBCMzlF', + }, + { + name: 'rudderAnonymousId', + value: '50ead33e-d763-4854-b0ab-765859ef05cb', + }, +]; + export const responseDummyContext = { document: { location: { diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts index ade496efb7..a154ccb890 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -1,7 +1,6 @@ // This file contains the test scenarios for the server-side events from the Shopify GraphQL API for // the v1 transformation flow -import { mockFns } from '../mocks'; -import { dummySourceConfig } from '../constants'; +import { dummySourceConfig, note_attributes } from '../constants'; export const checkoutEventsTestScenarios = [ { @@ -27,7 +26,7 @@ export const checkoutEventsTestScenarios = [ updated_at: '2024-11-05T21:22:02-05:00', landing_site: '/', note: '', - note_attributes: [], + note_attributes, referring_site: '', shipping_lines: [], shipping_address: [], @@ -145,7 +144,7 @@ export const checkoutEventsTestScenarios = [ updated_at: '2024-11-05T21:22:02-05:00', landing_site: '/', note: '', - note_attributes: [], + note_attributes, referring_site: '', shipping_lines: [], shipping_address: [], @@ -237,7 +236,7 @@ export const checkoutEventsTestScenarios = [ traits: { shippingAddress: [], }, - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, @@ -269,7 +268,7 @@ export const checkoutEventsTestScenarios = [ created_at: '2024-09-16T03:50:1500:00', updated_at: '2024-09-17T03:29:02-04:00', note: '', - note_attributes: [], + note_attributes, shipping_address: { first_name: 'testuser', address1: 'oakwood bridge', @@ -396,7 +395,7 @@ export const checkoutEventsTestScenarios = [ output: { batch: [ { - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', context: { cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', integration: { @@ -415,7 +414,7 @@ export const checkoutEventsTestScenarios = [ created_at: '2024-09-16T03:50:1500:00', updated_at: '2024-09-17T03:29:02-04:00', note: '', - note_attributes: [], + note_attributes, shipping_address: { first_name: 'testuser', address1: 'oakwood bridge', @@ -680,7 +679,7 @@ export const checkoutEventsTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, number: 17, order_number: 1017, order_status_url: @@ -988,7 +987,7 @@ export const checkoutEventsTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, number: 17, order_number: 1017, order_status_url: @@ -1289,7 +1288,7 @@ export const checkoutEventsTestScenarios = [ }, }, timestamp: '2024-11-06T02:54:50.000Z', - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, @@ -1326,6 +1325,7 @@ export const checkoutEventsTestScenarios = [ current_total_tax: '0.00', email: 'henry@wfls.com', name: '#1017', + note_attributes, order_number: 1017, order_status_url: 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', @@ -1486,6 +1486,7 @@ export const checkoutEventsTestScenarios = [ current_total_tax: '0.00', email: 'henry@wfls.com', name: '#1017', + note_attributes, order_number: 1017, order_status_url: 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', @@ -1675,7 +1676,7 @@ export const checkoutEventsTestScenarios = [ }, }, timestamp: '2024-11-06T02:54:50.000Z', - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, @@ -1684,4 +1685,4 @@ export const checkoutEventsTestScenarios = [ }, }, }, -].map((d1) => ({ ...d1, mockFns })); +]; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts index f04fd7e08e..d68d0a8f59 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -1,7 +1,7 @@ // This file contains the test scenarios for the server-side events from the Shopify GraphQL API for // the v1 transformation flow import { mockFns } from '../mocks'; -import { dummySourceConfig } from '../constants'; +import { dummySourceConfig, note_attributes } from '../constants'; export const genericTrackTestScenarios = [ { @@ -24,6 +24,7 @@ export const genericTrackTestScenarios = [ token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', line_items: [], note: '', + note_attributes, updated_at: '2024-09-17T08:15:13.280Z', created_at: '2024-09-16T03:50:15.478Z', }, @@ -43,7 +44,7 @@ export const genericTrackTestScenarios = [ output: { batch: [ { - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', context: { integration: { name: 'SHOPIFY', @@ -58,6 +59,7 @@ export const genericTrackTestScenarios = [ id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', line_items: [], note: '', + note_attributes, token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', updated_at: '2024-09-17T08:15:13.280Z', }, @@ -149,7 +151,7 @@ export const genericTrackTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, order_number: 1017, original_total_additional_fees_set: null, original_total_duties_set: null, @@ -363,7 +365,7 @@ export const genericTrackTestScenarios = [ merchant_of_record_app_id: null, name: '#1017', note: null, - note_attributes: [], + note_attributes, order_number: 1017, original_total_additional_fees_set: null, original_total_duties_set: null, @@ -545,7 +547,7 @@ export const genericTrackTestScenarios = [ traits: { email: 'henry@wfls.com', }, - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, ], }, From a0d4efcb165e61c27a96a2d436cd3af625522842 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 22 Nov 2024 01:09:32 +0530 Subject: [PATCH 02/21] chore: allow only anonymousId set from note attributes --- .../shopify/webhookTransformations/serverSideUtlis.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index 0c38dd1d10..1ac60f967d 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -1,5 +1,4 @@ const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); -const { v5 } = require('uuid'); const { constructPayload } = require('../../../../v0/util'); const { lineItemsMappingJSON, @@ -47,18 +46,12 @@ const createPropertiesForEcomEventFromWebhook = (message) => { */ const getAnonymousIdFromAttributes = async (event) => { let anonymousId = null; - let cartToken = null; const noteAttributes = event.note_attributes; if (isDefinedAndNotNull(event) && isDefinedAndNotNull(noteAttributes)) { const rudderAnonymousIdObject = noteAttributes.find( (attr) => attr.name === 'rudderAnonymousId', ); anonymousId = rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; - const cartTokenObject = noteAttributes.find((attr) => attr.name === 'cartToken'); - cartToken = cartTokenObject ? cartTokenObject.value : null; - if (!isDefinedAndNotNull(anonymousId) && isDefinedAndNotNull(cartToken)) { - anonymousId = v5(cartToken, v5.URL); - } } return anonymousId; }; From 83c8f52d76739b46f558c2d8682f4b831f9c701a Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:23:00 +0530 Subject: [PATCH 03/21] chore: add unit tests --- .../serverSideUtils.test.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js index a611d1d8dc..4a2839103b 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -1,6 +1,7 @@ const { getProductsFromLineItems, createPropertiesForEcomEventFromWebhook, + getAnonymousIdFromAttributes, } = require('./serverSideUtlis'); const { constructPayload } = require('../../../../v0/util'); @@ -109,4 +110,21 @@ describe('serverSideUtils.js', () => { }); }); }); + + describe('getAnonymousIdFromAttributes', () => { + // Handles empty note_attributes array gracefully + it('should return null when note_attributes is an empty array', async () => { + const event = { note_attributes: [] }; + const result = await getAnonymousIdFromAttributes(event); + expect(result).toBeNull(); + }); + + it('get anonymousId from noteAttributes', async () => { + const event = { + note_attributes: [{ name: 'rudderAnonymousId', value: '123456' }], + }; + const result = await getAnonymousIdFromAttributes(event); + expect(result).toEqual('123456'); + }); + }); }); From 70fa123a771914d6fc1387e7e6ebc0b2e4db4d5d Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 2 Dec 2024 12:32:08 +0530 Subject: [PATCH 04/21] chore: address initialization comment --- .../sources/shopify/webhookTransformations/serverSideUtlis.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index 1ac60f967d..0afbf30277 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -46,8 +46,8 @@ const createPropertiesForEcomEventFromWebhook = (message) => { */ const getAnonymousIdFromAttributes = async (event) => { let anonymousId = null; - const noteAttributes = event.note_attributes; - if (isDefinedAndNotNull(event) && isDefinedAndNotNull(noteAttributes)) { + if (isDefinedAndNotNull(event) && isDefinedAndNotNull(event.note_attributes)) { + const noteAttributes = event.note_attributes; const rudderAnonymousIdObject = noteAttributes.find( (attr) => attr.name === 'rudderAnonymousId', ); From 481d149c9b2786dd2360f3ee0ab220660d461178 Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Tue, 3 Dec 2024 09:12:00 +0530 Subject: [PATCH 05/21] chore: update default case handling in facebook destinations (#3908) * chore: update default case handling in facebook destinations * fix: update default handling for some error codes - used error_user_msg field as error message - when no error is seen, handled error by stringifying - update test-cases --------- Co-authored-by: Sai Sankeerth --- src/v0/util/facebookUtils/networkHandler.js | 38 +++++----- .../facebook_pixel/dataDelivery/business.ts | 4 +- .../facebook_pixel/dataDelivery/data.ts | 70 +++++++++++++++++-- .../facebook_pixel/dataDelivery/oauth.ts | 4 +- .../facebook_pixel/dataDelivery/other.ts | 10 +-- .../destinations/facebook_pixel/network.ts | 26 +++++++ .../destinations/fb/dataDelivery/data.ts | 3 +- .../destinations/fb/dataDelivery/other.ts | 4 +- test/integrations/destinations/fb/network.ts | 1 + .../fb_custom_audience/dataDelivery/data.ts | 2 + .../fb_custom_audience/network.ts | 2 + 11 files changed, 132 insertions(+), 32 deletions(-) diff --git a/src/v0/util/facebookUtils/networkHandler.js b/src/v0/util/facebookUtils/networkHandler.js index fbb7899efe..46ac59a07a 100644 --- a/src/v0/util/facebookUtils/networkHandler.js +++ b/src/v0/util/facebookUtils/networkHandler.js @@ -85,7 +85,10 @@ const errorDetailsMap = { "Object with ID 'PIXEL_ID' / 'DATASET_ID' / 'AUDIENCE_ID' does not exist, cannot be loaded due to missing permissions, or does not support this operation", ) .build(), - default: new ErrorDetailsExtractorBuilder().setStatus(400).setMessageField('message').build(), + default: new ErrorDetailsExtractorBuilder() + .setStatus(400) + .setMessageField('error_user_msg') + .build(), }, 1: { // An unknown error occurred. @@ -107,9 +110,7 @@ const errorDetailsMap = { .setStat({ [TAG_NAMES.ERROR_TYPE]: ERROR_TYPES.AUTH, }) - .setMessage( - 'The session has been invalidated because the user changed their password or Facebook has changed the session for security reasons', - ) + .setMessageField('error_user_msg') .build(), 463: new ErrorDetailsExtractorBuilder() @@ -117,56 +118,56 @@ const errorDetailsMap = { .setStat({ [TAG_NAMES.ERROR_TYPE]: ERROR_TYPES.AUTH, }) - .setMessageField('message') + .setMessageField('error_user_msg') .build(), default: new ErrorDetailsExtractorBuilder() .setStatus(400) .setStat({ [TAG_NAMES.ERROR_TYPE]: ERROR_TYPES.AUTH, }) - .setMessage('Invalid OAuth 2.0 access token') + .setMessageField('error_user_msg') .build(), }, 3: { default: new ErrorDetailsExtractorBuilder() .setStatus(400) - .setMessage('Capability or permissions issue.') + .setMessageField('error_user_msg') .build(), }, 2: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Temporary issue due to downtime.') + .setMessageField('error_user_msg') .build(), }, 341: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Application limit reached: Temporary issue due to downtime or throttling') + .setMessageField('error_user_msg') .build(), }, 368: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Temporarily blocked for policies violations.') + .setMessageField('error_user_msg') .build(), }, 5000: { default: new ErrorDetailsExtractorBuilder() .setStatus(500) - .setMessage('Unknown Error Code') + .setMessageField('error_user_msg') .build(), }, 4: { default: new ErrorDetailsExtractorBuilder() .setStatus(429) - .setMessage('API Too Many Calls') + .setMessageField('error_user_msg') .build(), }, 17: { default: new ErrorDetailsExtractorBuilder() .setStatus(429) - .setMessage('API User Too Many Calls') + .setMessageField('error_user_msg') .build(), }, // facebook custom audience related error codes @@ -176,9 +177,7 @@ const errorDetailsMap = { 294: { default: new ErrorDetailsExtractorBuilder() .setStatus(400) - .setMessage( - 'Missing permission. Please make sure you have ads_management permission and the application is included in the allowlist', - ) + .setMessageField('error_user_msg') .build(), }, 1487301: { @@ -214,7 +213,10 @@ const errorDetailsMap = { .build(), }, 200: { - default: new ErrorDetailsExtractorBuilder().setStatus(403).setMessageField('message').build(), + default: new ErrorDetailsExtractorBuilder() + .setStatus(403) + .setMessageField('error_user_msg') + .build(), }, 21009: { default: new ErrorDetailsExtractorBuilder().setStatus(500).setMessageField('message').build(), @@ -236,9 +238,11 @@ const getStatus = (error) => { const isErrorDetailEmpty = isEmpty(errorDetail); if (isErrorDetailEmpty) { // Unhandled error response + const errorMessage = get(error?.error || error, 'error_user_msg'); return { status: errorStatus, stats: { [TAG_NAMES.META]: METADATA.UNHANDLED_STATUS_CODE }, + errorMessage: errorMessage || JSON.stringify(error), }; } errorStatus = errorDetail.status; diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts index 1b425ac5fa..2b4af61ac3 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/business.ts @@ -42,7 +42,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ body: { output: { status: 400, - message: 'Invalid OAuth 2.0 access token', + message: 'The access token could not be decrypted', statTags: { ...statTags, errorCategory: 'dataValidation', @@ -51,7 +51,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ }, response: [ { - error: 'Invalid OAuth 2.0 access token', + error: 'The access token could not be decrypted', statusCode: 400, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts index 3366e62c5a..0e004c1183 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/data.ts @@ -45,12 +45,13 @@ export const v0TestData = [ body: { output: { status: 400, - message: 'Invalid OAuth 2.0 access token', + message: 'The access token could not be decrypted', destinationResponse: { error: { code: 190, fbtrace_id: 'fbpixel_trace_id', message: 'The access token could not be decrypted', + error_user_msg: 'The access token could not be decrypted', type: 'OAuthException', }, status: 500, @@ -202,7 +203,7 @@ export const v0TestData = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', destinationResponse: { error: { message: 'User request limit reached', @@ -414,7 +415,7 @@ export const v0TestData = [ body: { output: { status: 400, - message: 'Capability or permissions issue.', + message: 'Some error in permission', destinationResponse: { error: { message: 'Some error in permission', @@ -466,7 +467,8 @@ export const v0TestData = [ body: { output: { status: 500, - message: 'Unhandled random error', + message: + '{"message":"Unhandled random error","type":"RandomException","code":5,"error_subcode":12,"fbtrace_id":"facebook_px_trace_id_10"}', destinationResponse: { error: { message: 'Unhandled random error', @@ -488,6 +490,66 @@ export const v0TestData = [ }, }, + { + name: 'facebook_pixel', + description: 'Test 10: should handle error with code: 100 & subcode: 2804009', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + body: { + XML: {}, + FORM: testFormData, + JSON: {}, + JSON_ARRAY: {}, + }, + type: 'REST', + files: {}, + method: 'POST', + userId: '', + headers: {}, + version: '1', + endpoint: `https://graph.facebook.com/${VERSION}/12345678912804009/events?access_token=2804009_valid_access_token`, + params: { + destination: 'facebook_pixel', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 400, + body: { + output: { + status: 400, + message: + 'Your purchase event doesn’t include a value parameter. Enter a value. For example: 1.99', + destinationResponse: { + error: { + message: 'Invalid parameter', + type: 'OAuthException', + code: 100, + error_user_msg: + 'Your purchase event doesn’t include a value parameter. Enter a value. For example: 1.99', + error_user_title: 'Missing Value for Purchase Event', + error_subcode: 2804009, + is_transient: false, + fbtrace_id: 'AP4G-xxxxxxxx', + }, + status: 400, + }, + statTags: { + ...statTags, + errorType: 'aborted', + }, + }, + }, + }, + }, + }, { name: 'facebook_pixel', description: 'Test 9: should handle error with code: 21009', diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts index c6d938c627..bb4fd582f0 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/oauth.ts @@ -28,11 +28,11 @@ export const oauthScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 400, - message: 'Capability or permissions issue.', + message: 'Some error in permission', statTags, response: [ { - error: 'Capability or permissions issue.', + error: 'Some error in permission', statusCode: 400, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts b/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts index e25cc8e07c..154c6f75ad 100644 --- a/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts +++ b/test/integrations/destinations/facebook_pixel/dataDelivery/other.ts @@ -31,14 +31,14 @@ export const otherScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', statTags: { ...statTags, errorType: 'throttled', }, response: [ { - error: 'API User Too Many Calls', + error: 'User request limit reached', statusCode: 429, metadata: generateMetadata(1), }, @@ -72,7 +72,8 @@ export const otherScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 500, - message: 'Unhandled random error', + message: + '{"message":"Unhandled random error","type":"RandomException","code":5,"error_subcode":12,"fbtrace_id":"facebook_px_trace_id_10"}', statTags: { ...statTags, errorType: 'retryable', @@ -80,7 +81,8 @@ export const otherScenariosV1: ProxyV1TestData[] = [ }, response: [ { - error: 'Unhandled random error', + error: + '{"message":"Unhandled random error","type":"RandomException","code":5,"error_subcode":12,"fbtrace_id":"facebook_px_trace_id_10"}', statusCode: 500, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/facebook_pixel/network.ts b/test/integrations/destinations/facebook_pixel/network.ts index 411d36cf19..a6ff50623b 100644 --- a/test/integrations/destinations/facebook_pixel/network.ts +++ b/test/integrations/destinations/facebook_pixel/network.ts @@ -15,6 +15,7 @@ export const networkCallsData = [ data: { error: { message: 'The access token could not be decrypted', + error_user_msg: 'The access token could not be decrypted', type: 'OAuthException', code: 190, fbtrace_id: 'fbpixel_trace_id', @@ -208,4 +209,29 @@ export const networkCallsData = [ statusText: 'OK', }, }, + { + httpReq: { + url: `https://graph.facebook.com/${VERSION}/12345678912804009/events?access_token=2804009_valid_access_token`, + data: getFormData(testFormData).toString(), + params: { destination: 'facebook_pixel' }, + headers: { 'User-Agent': 'RudderLabs' }, + method: 'POST', + }, + httpRes: { + status: 400, + data: { + error: { + code: 100, + error_subcode: 2804009, + error_user_msg: + 'Your purchase event doesn’t include a value parameter. Enter a value. For example: 1.99', + error_user_title: 'Missing Value for Purchase Event', + fbtrace_id: 'AP4G-xxxxxxxx', + is_transient: false, + message: 'Invalid parameter', + type: 'OAuthException', + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/fb/dataDelivery/data.ts b/test/integrations/destinations/fb/dataDelivery/data.ts index 9ee19af265..dfa5dbc65e 100644 --- a/test/integrations/destinations/fb/dataDelivery/data.ts +++ b/test/integrations/destinations/fb/dataDelivery/data.ts @@ -59,6 +59,7 @@ export const existingTestData = [ message: 'Invalid OAuth 2.0 access token', destinationResponse: { error: { + error_user_msg: 'Invalid OAuth 2.0 access token', code: 190, fbtrace_id: 'fbpixel_trace_id', message: 'The access token could not be decrypted', @@ -274,7 +275,7 @@ export const existingTestData = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', destinationResponse: { error: { message: 'User request limit reached', diff --git a/test/integrations/destinations/fb/dataDelivery/other.ts b/test/integrations/destinations/fb/dataDelivery/other.ts index 9ac3f14fb5..36d3e45f53 100644 --- a/test/integrations/destinations/fb/dataDelivery/other.ts +++ b/test/integrations/destinations/fb/dataDelivery/other.ts @@ -31,14 +31,14 @@ export const otherScenariosV1: ProxyV1TestData[] = [ body: { output: { status: 429, - message: 'API User Too Many Calls', + message: 'User request limit reached', statTags: { ...statTags, errorType: 'throttled', }, response: [ { - error: 'API User Too Many Calls', + error: 'User request limit reached', statusCode: 429, metadata: generateMetadata(1), }, diff --git a/test/integrations/destinations/fb/network.ts b/test/integrations/destinations/fb/network.ts index 31bbaf0b6e..1aed48136c 100644 --- a/test/integrations/destinations/fb/network.ts +++ b/test/integrations/destinations/fb/network.ts @@ -30,6 +30,7 @@ export const networkCallsData = [ data: { error: { message: 'The access token could not be decrypted', + error_user_msg: 'Invalid OAuth 2.0 access token', type: 'OAuthException', code: 190, fbtrace_id: 'fbpixel_trace_id', diff --git a/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts b/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts index b41c656d9f..cd440aaa37 100644 --- a/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts +++ b/test/integrations/destinations/fb_custom_audience/dataDelivery/data.ts @@ -623,6 +623,8 @@ export const existingTestData = [ fbtrace_id: 'A3b8C6PpI-kdIOwPwV4PANi', message: 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', + error_user_msg: + 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', type: 'OAuthException', }, status: 400, diff --git a/test/integrations/destinations/fb_custom_audience/network.ts b/test/integrations/destinations/fb_custom_audience/network.ts index 369c27afa9..ba14d537ad 100644 --- a/test/integrations/destinations/fb_custom_audience/network.ts +++ b/test/integrations/destinations/fb_custom_audience/network.ts @@ -514,6 +514,8 @@ export const networkCallsData = [ error: { message: 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', + error_user_msg: + 'Error validating access token: Session has expired on Tuesday, 01-Aug-23 10:12:14 PDT. The current time is Sunday, 28-Jan-24 16:01:17 PST.', type: 'OAuthException', code: 190, error_subcode: 463, From 3eb5c551f6551e8da9451787159d5616114aa1c8 Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:50:06 +0530 Subject: [PATCH 06/21] chore: update function as per review suggestion --- .../webhookTransformations/serverSideUtlis.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index 0afbf30277..59a1ddc04f 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -45,15 +45,14 @@ const createPropertiesForEcomEventFromWebhook = (message) => { * @returns {String} anonymousId */ const getAnonymousIdFromAttributes = async (event) => { - let anonymousId = null; - if (isDefinedAndNotNull(event) && isDefinedAndNotNull(event.note_attributes)) { - const noteAttributes = event.note_attributes; - const rudderAnonymousIdObject = noteAttributes.find( - (attr) => attr.name === 'rudderAnonymousId', - ); - anonymousId = rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; + if (!isDefinedAndNotNull(event) || !isDefinedAndNotNull(event.note_attributes)) { + return null; // Return early if event or note_attributes is invalid } - return anonymousId; + + const noteAttributes = event.note_attributes; + const rudderAnonymousIdObject = noteAttributes.find((attr) => attr.name === 'rudderAnonymousId'); + + return rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; }; module.exports = { From b3f7140cd9444e74955bf91200dc7c3150ce335e Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 6 Dec 2024 00:32:04 +0530 Subject: [PATCH 07/21] chore: fix sonar issues --- .../shopify/webhookTransformations/serverSideTransform.js | 2 +- .../sources/shopify/webhookTransformations/serverSideUtlis.js | 3 +-- src/v1/sources/shopify/webpixelTransformations/pixelUtils.js | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js index 4f91b0e6c1..292a980a93 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -122,7 +122,7 @@ const processEvent = async (inputEvent, metricMetadata) => { } // attach anonymousId if the event is track event using note_attributes if (message.type !== EventType.IDENTIFY) { - const anonymousId = await getAnonymousIdFromAttributes(event); + const anonymousId = getAnonymousIdFromAttributes(event); if (isDefinedAndNotNull(anonymousId)) { message.setProperty('anonymousId', anonymousId); } diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index 59a1ddc04f..951fa479e4 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -17,7 +17,6 @@ const getProductsFromLineItems = (lineItems, mapping) => { } const products = []; lineItems.forEach((lineItem) => { - // const product = constructPayload(lineItem, lineItemsMappingJSON); const product = constructPayload(lineItem, mapping); products.push(product); }); @@ -44,7 +43,7 @@ const createPropertiesForEcomEventFromWebhook = (message) => { * @param {Object} event * @returns {String} anonymousId */ -const getAnonymousIdFromAttributes = async (event) => { +const getAnonymousIdFromAttributes = (event) => { if (!isDefinedAndNotNull(event) || !isDefinedAndNotNull(event.note_attributes)) { return null; // Return early if event or note_attributes is invalid } diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js index 0c1007f311..46ae59e0cf 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js @@ -14,7 +14,7 @@ const { function getNestedValue(object, path) { const keys = path.split('.'); - return keys.reduce((nestedObject, key) => nestedObject && nestedObject[key], object); + return keys.reduce((nestedObject, key) => nestedObject?.[key], object); } function setNestedValue(object, path, value) { From 4a77eda2caf133526a911a239e84e2764a88448d Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:15:35 +0530 Subject: [PATCH 08/21] chore: update repo name for staging deployment also (#3916) --- .github/workflows/prepare-for-staging-deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/prepare-for-staging-deploy.yml b/.github/workflows/prepare-for-staging-deploy.yml index 8eeafc376e..cdf27f7832 100644 --- a/.github/workflows/prepare-for-staging-deploy.yml +++ b/.github/workflows/prepare-for-staging-deploy.yml @@ -78,6 +78,7 @@ jobs: runs-on: ubuntu-latest needs: [generate-tag-names, build-transformer-image, build-user-transformer-image] env: + TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }} UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }} steps: @@ -113,6 +114,7 @@ jobs: git checkout -b $BRANCH_NAME cd helm-charts/shared-services/per-az/environment/staging + yq eval -i ".rudder-transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" staging.yaml yq eval -i ".rudder-transformer.image.tag=\"$TAG_NAME\"" staging.yaml yq eval -i ".user-transformer.image.tag=\"$TAG_NAME\"" staging.yaml git add staging.yaml From f44f1488acb05bb5096e5f9d7bade06f76f55d13 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Fri, 6 Dec 2024 14:05:11 +0530 Subject: [PATCH 09/21] chore: add metrics for mailjet (#3917) --- src/util/prometheus.js | 6 ++++++ src/v0/destinations/mailjet/transform.js | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/util/prometheus.js b/src/util/prometheus.js index c8dd55068b..a4c12e4ea8 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -520,6 +520,12 @@ class Prometheus { type: 'counter', labelNames: ['destination_id'], }, + { + name: 'mailjet_packing_size', + help: 'mailjet_packing_size', + type: 'gauge', + labelNames: ['group'], + }, { name: 'hs_batch_size', help: 'hs_batch_size', diff --git a/src/v0/destinations/mailjet/transform.js b/src/v0/destinations/mailjet/transform.js index 78b4f766d1..0a742b4cf7 100644 --- a/src/v0/destinations/mailjet/transform.js +++ b/src/v0/destinations/mailjet/transform.js @@ -1,5 +1,6 @@ const lodash = require('lodash'); const { TransformationError, InstrumentationError } = require('@rudderstack/integrations-lib'); +const stats = require('../../../util/stats'); const { getSuccessRespEvents, defaultRequestConfig, @@ -105,6 +106,9 @@ const batchEvents = (successRespList) => { const eventChunks = lodash.chunk(eventGroups[combination], MAX_BATCH_SIZE); // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] eventChunks.forEach((chunk) => { + stats.gauge('mailjet_packing_size', chunk.length, { + group: combination, + }); const batchEventResponse = generateBatchedPaylaodForArray(chunk, combination); batchedResponseList.push( getSuccessRespEvents( From 313710ca725538e5ffe357216d9c88e444f995c8 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:04:06 +0530 Subject: [PATCH 10/21] fix: remove redundant ids and userIdentifier when gbraid or wbraid are there (#3910) * fix: remove redundant ids and userIdentifier when gbraid or wbraid are there * chore: address comment * chore: fix sonar cloud issue * chore: fix sonar cloud issue * chore: decrease Cognitive Complexity from 17 to the 15 * chore: fix lint error * chore: specify head commit hash for checkout temporarily * chore: fix fetch depth * chore: fix fetch depth to 0 * chore: reset commit sha --------- Co-authored-by: Sai Kumar Battinoju --- .../dt-test-and-report-code-coverage.yml | 2 +- .../transform.js | 4 + .../utils.js | 62 +++- .../utils.test.js | 4 +- .../processor/data.ts | 287 ++++++++++++++++-- .../router/data.ts | 2 - 6 files changed, 315 insertions(+), 46 deletions(-) diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 2c76898882..12bd9ac78b 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout uses: actions/checkout@v4.2.1 with: - fetch-depth: 1 + fetch-depth: 0 - name: Setup Node uses: actions/setup-node@v4.0.4 diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index 76b12587cd..2648f03e8a 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -18,6 +18,7 @@ const { getClickConversionPayloadAndEndpoint, getConsentsDataFromIntegrationObj, getCallConversionPayload, + updateConversion, } = require('./utils'); const helper = require('./helper'); @@ -48,6 +49,9 @@ const getConversions = (message, metadata, { Config }, event, conversionType) => filteredCustomerId, eventLevelConsentsData, ); + convertedPayload.payload.conversions[0] = updateConversion( + convertedPayload.payload.conversions[0], + ); payload = convertedPayload.payload; endpoint = convertedPayload.endpoint; } else if (conversionType === 'store') { diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 89fe609df9..2d47095eea 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -317,6 +317,35 @@ const getStoreConversionPayload = (message, Config, event) => { return payload; }; +const hasClickId = (conversion) => { + const { gbraid, wbraid, gclid } = conversion; + return gclid || wbraid || gbraid; +}; +const populateUserIdentifier = ({ email, phone, properties, payload, UserIdentifierSource }) => { + const copiedPayload = cloneDeep(payload); + // userIdentifierSource + // if userIdentifierSource doesn't exist in properties + // then it is taken from the webapp config + if (!properties.userIdentifierSource && UserIdentifierSource !== 'none') { + set( + copiedPayload, + 'conversions[0].userIdentifiers[0].userIdentifierSource', + UserIdentifierSource, + ); + // one of email or phone must be provided when none of gclid, wbraid and gbraid provided + } + if (!email && !phone) { + if (!hasClickId(copiedPayload.conversions[0])) { + throw new InstrumentationError( + `Either an email address or a phone number is required for user identification when none of gclid, wbraid, or gbraid is provided.`, + ); + } else { + // we are deleting userIdentifiers if any one of gclid, wbraid and gbraid is there but email or phone is not present + delete copiedPayload.conversions[0].userIdentifiers; + } + } + return copiedPayload; +}; const getClickConversionPayloadAndEndpoint = ( message, Config, @@ -335,7 +364,7 @@ const getClickConversionPayloadAndEndpoint = ( updatedClickMapping = removeHashToSha256TypeFromMappingJson(updatedClickMapping); } - const payload = constructPayload(message, updatedClickMapping); + let payload = constructPayload(message, updatedClickMapping); const endpoint = CLICK_CONVERSION.replace(':customerId', filteredCustomerId); @@ -353,17 +382,8 @@ const getClickConversionPayloadAndEndpoint = ( set(payload, 'conversions[0].cartData.items', itemList); } - // userIdentifierSource - // if userIdentifierSource doesn't exist in properties - // then it is taken from the webapp config - if (!properties.userIdentifierSource && UserIdentifierSource !== 'none') { - set(payload, 'conversions[0].userIdentifiers[0].userIdentifierSource', UserIdentifierSource); + payload = populateUserIdentifier({ email, phone, properties, payload, UserIdentifierSource }); - // one of email or phone must be provided - if (!email && !phone) { - throw new InstrumentationError(`Either of email or phone is required for user identifier`); - } - } // either of email or phone should be passed // defaultUserIdentifier depends on the webapp configuration // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion @@ -411,6 +431,25 @@ const getConsentsDataFromIntegrationObj = (message) => { return integrationObj?.consents || {}; }; +/** + * remove redundant ids + * @param {*} conversionCopy + */ +const updateConversion = (conversion) => { + const conversionCopy = cloneDeep(conversion); + if (conversionCopy.gclid) { + delete conversionCopy.wbraid; + delete conversionCopy.gbraid; + } + if (conversionCopy.wbraid && conversionCopy.gbraid) { + throw new InstrumentationError(`You can't use both wbraid and gbraid.`); + } + if (conversionCopy.wbraid || conversionCopy.gbraid) { + delete conversionCopy.userIdentifiers; + } + return conversionCopy; +}; + module.exports = { validateDestinationConfig, generateItemListFromProducts, @@ -423,4 +462,5 @@ module.exports = { getExisitingUserIdentifier, getConsentsDataFromIntegrationObj, getCallConversionPayload, + updateConversion, }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js index 2d1863413c..b6c6653782 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js @@ -244,7 +244,9 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { }; expect(() => getClickConversionPayloadAndEndpoint(fittingPayload, config, '9625812972'), - ).toThrow('Either of email or phone is required for user identifier'); + ).toThrow( + 'Either an email address or a phone number is required for user identification when none of gclid, wbraid, or gbraid is provided.', + ); }); it('finaliseConsent', () => { diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts index ab3e19dc2f..3ecae2b92d 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts @@ -231,8 +231,6 @@ export const data = [ JSON: { conversions: [ { - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -528,8 +526,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -821,8 +817,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -1911,7 +1905,6 @@ export const data = [ userId: '12345', properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -1935,7 +1928,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -2052,7 +2044,6 @@ export const data = [ ], properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -2076,7 +2067,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -2093,7 +2083,6 @@ export const data = [ adUserData: 'UNSPECIFIED', }, gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', @@ -2110,15 +2099,7 @@ export const data = [ }, ], }, - userIdentifiers: [ - { - userIdentifierSource: 'FIRST_PARTY', - hashedPhoneNumber: - '04e1dabb7c1348b72bfa87da179c9697c69af74827649266a5da8cdbb367abcd', - }, - ], conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: 1, currencyCode: 'GBP', @@ -2169,7 +2150,6 @@ export const data = [ ], properties: { gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', conversionCustomVariable: 'conversionCustomVariable', @@ -2193,7 +2173,6 @@ export const data = [ ], userIdentifierSource: 'FIRST_PARTY', conversionEnvironment: 'WEB', - gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: '1', currency: 'GBP', @@ -3751,20 +3730,73 @@ export const data = [ metadata: { secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, }, - statusCode: 400, - error: 'Either of email or phone is required for user identifier', - statTags: { - errorCategory: 'dataValidation', - errorType: 'instrumentation', - destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', - module: 'destination', - implementation: 'native', - feature: 'processor', + output: { + body: { + FORM: {}, + JSON: { + conversions: [ + { + cartData: { + items: [ + { + productId: '123445', + quantity: 123, + }, + ], + }, + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + conversionDateTime: '2022-01-01 12:32:45-08:00', + conversionEnvironment: 'WEB', + gclid: 'gclid', + }, + ], + partialFailure: true, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + files: {}, + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': '8617859087', + }, + method: 'POST', + params: { + customVariables: [ + { + from: 'value', + to: 'revenue', + }, + { + from: 'total', + to: 'cost', + }, + ], + customerId: '9625812972', + event: 'Sign-up - click', + properties: { + conversionDateTime: '2022-01-01 12:32:45-08:00', + gclid: 'gclid', + product_id: '123445', + quantity: 123, + }, + }, + type: 'REST', + userId: '', + version: '1', }, + statusCode: 200, }, ], }, @@ -5704,4 +5736,197 @@ export const data = [ }, mockFns: timestampMock, }, + { + name: 'google_adwords_offline_conversions', + description: 'Test 28 : when both gbraid and wbraid are available', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + id: '0572f78fa49c648e', + name: 'generic_x86_arm', + type: 'Android', + model: 'AOSP on IA Emulator', + manufacturer: 'Google', + adTrackingEnabled: true, + advertisingId: '44c97318-9040-4361-8bc7-4eb30f665ca8', + }, + traits: { + phone: 'alex@example.com', + firstName: 'John', + lastName: 'Gomes', + city: 'London', + state: 'England', + countryCode: 'GB', + postalCode: 'EC3M', + streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + name: '', + version: '', + }, + screen: { + density: 2, + }, + }, + event: 'Promotion Clicked', + type: 'track', + messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', + originalTimestamp: '2019-10-14T11:15:18.299Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + properties: { + gbraid: 'gbraid', + wbraid: 'wbraid', + externalAttributionCredit: 10, + externalAttributionModel: 'externalAttributionModel', + conversionCustomVariable: 'conversionCustomVariable', + value: 'value', + merchantId: '9876merchantId', + feedCountryCode: 'feedCountryCode', + feedLanguageCode: 'feedLanguageCode', + localTransactionCost: 20, + products: [ + { + product_id: '507f1f77bcf86cd799439011', + quantity: '2', + price: '50', + sku: '45790-32', + name: 'Monopoly: 3rd Edition', + position: '1', + category: 'cars', + url: 'https://www.example.com/product/path', + image_url: 'https://www.example.com/product/path.jpg', + }, + ], + userIdentifierSource: 'FIRST_PARTY', + conversionEnvironment: 'WEB', + conversionValue: '1', + currency: 'GBP', + orderId: 'PL-123QR', + }, + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + customerId: '962-581-2972', + subAccount: false, + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Sign up completed', + to: 'click', + }, + { + from: 'Download', + to: 'call', + }, + { + from: 'Promotion Clicked', + to: 'click', + }, + { + from: 'Product Searched', + to: 'call', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Sign up completed', + to: 'Sign-up - click', + }, + { + from: 'Download', + to: 'Page view', + }, + { + from: 'Promotion Clicked', + to: 'Sign-up - click', + }, + { + from: 'Product Searched', + to: 'search', + }, + ], + customVariables: [ + { + from: 'value', + to: 'revenue', + }, + { + from: 'total', + to: 'cost', + }, + ], + UserIdentifierSource: 'THIRD_PARTY', + conversionEnvironment: 'WEB', + hashUserIdentifier: true, + defaultUserIdentifier: 'email', + validateOnly: false, + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: "You can't use both wbraid and gbraid.", + metadata: { + secret: { + access_token: 'abcd1234', + developer_token: 'ijkl91011', + refresh_token: 'efgh5678', + }, + }, + statTags: { + destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + mockFns: timestampMock, + }, ]; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index 9d1ba220c8..82c8e9b3ff 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -738,8 +738,6 @@ export const data = [ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', }, - gbraid: 'gbraid', - wbraid: 'wbraid', externalAttributionData: { externalAttributionCredit: 10, externalAttributionModel: 'externalAttributionModel', From 12621c8eee641f5a03a997e95ed016cff0eefde7 Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Mon, 9 Dec 2024 10:36:42 +0530 Subject: [PATCH 11/21] fix: skipping users events for snowpipe streaming (#3836) --- .../snowpipe_streaming/transform.js | 27 +- src/warehouse/config/ReservedKeywords.json | 93 ++++ src/warehouse/index.js | 18 +- src/warehouse/v1/util.js | 4 +- test/__tests__/data/warehouse/events.js | 501 ++++++++++++++++++ .../warehouse/integration_options_events.js | 166 +++++- .../integrations/jsonpaths/legacy/aliases.js | 73 +++ .../integrations/jsonpaths/legacy/extract.js | 45 ++ .../integrations/jsonpaths/legacy/groups.js | 73 +++ .../jsonpaths/legacy/identifies.js | 89 ++++ .../integrations/jsonpaths/legacy/pages.js | 69 +++ .../integrations/jsonpaths/legacy/screens.js | 69 +++ .../integrations/jsonpaths/legacy/tracks.js | 151 ++++++ .../integrations/jsonpaths/new/aliases.js | 70 +++ .../integrations/jsonpaths/new/extract.js | 45 ++ .../integrations/jsonpaths/new/groups.js | 70 +++ .../integrations/jsonpaths/new/identifies.js | 89 ++++ .../integrations/jsonpaths/new/pages.js | 66 +++ .../integrations/jsonpaths/new/screens.js | 66 +++ .../integrations/jsonpaths/new/tracks.js | 151 ++++++ test/__tests__/data/warehouse/names.js | 39 ++ test/__tests__/warehouse.test.js | 285 +++++----- 22 files changed, 2120 insertions(+), 139 deletions(-) diff --git a/src/v0/destinations/snowpipe_streaming/transform.js b/src/v0/destinations/snowpipe_streaming/transform.js index 3b0a6e1a93..36fe19ceff 100644 --- a/src/v0/destinations/snowpipe_streaming/transform.js +++ b/src/v0/destinations/snowpipe_streaming/transform.js @@ -1 +1,26 @@ -module.exports = require('../snowflake/transform'); +const transform = require('../snowflake/transform'); +const { processWarehouseMessage } = require('../../../warehouse'); + +const provider = 'snowpipe_streaming'; + +function process(event) { + const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; + const whStoreEvent = event.destination.Config.storeFullEvent === true; + const destJsonPaths = event.destination?.Config?.jsonPaths || ''; + return processWarehouseMessage(event.message, { + metadata: event.metadata, + whSchemaVersion, + whStoreEvent, + getDataTypeOverride: transform.getDataTypeOverride, + provider, + sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destJsonPaths, + destConfig: event.destination?.Config, + }); +} + +module.exports = { + provider, + process, + getDataTypeOverride: transform.getDataTypeOverride, +}; diff --git a/src/warehouse/config/ReservedKeywords.json b/src/warehouse/config/ReservedKeywords.json index 3c383d24bf..53b8738e1d 100644 --- a/src/warehouse/config/ReservedKeywords.json +++ b/src/warehouse/config/ReservedKeywords.json @@ -1519,6 +1519,99 @@ "WHERE": true, "WITH": true }, + "SNOWPIPE_STREAMING": { + "ACCOUNT": true, + "ALL": true, + "ALTER": true, + "AND": true, + "ANY": true, + "AS": true, + "BETWEEN": true, + "BY": true, + "CASE": true, + "CAST": true, + "CHECK": true, + "COLUMN": true, + "CONNECT": true, + "CONNECTION": true, + "CONSTRAINT": true, + "CREATE": true, + "CROSS": true, + "CURRENT": true, + "CURRENT_DATE": true, + "CURRENT_TIME": true, + "CURRENT_TIMESTAMP": true, + "CURRENT_USER": true, + "DATABASE": true, + "DELETE": true, + "DISTINCT": true, + "DROP": true, + "ELSE": true, + "EXISTS": true, + "FALSE": true, + "FOLLOWING": true, + "FOR": true, + "FROM": true, + "FULL": true, + "GRANT": true, + "GROUP": true, + "GSCLUSTER": true, + "HAVING": true, + "ILIKE": true, + "IN": true, + "INCREMENT": true, + "INNER": true, + "INSERT": true, + "INTERSECT": true, + "INTO": true, + "IS": true, + "ISSUE": true, + "JOIN": true, + "LATERAL": true, + "LEFT": true, + "LIKE": true, + "LOCALTIME": true, + "LOCALTIMESTAMP": true, + "MINUS": true, + "NATURAL": true, + "NOT": true, + "NULL": true, + "OF": true, + "ON": true, + "OR": true, + "ORDER": true, + "ORGANIZATION": true, + "QUALIFY": true, + "REGEXP": true, + "REVOKE": true, + "RIGHT": true, + "RLIKE": true, + "ROW": true, + "ROWS": true, + "SAMPLE": true, + "SCHEMA": true, + "SELECT": true, + "SET": true, + "SOME": true, + "START": true, + "TABLE": true, + "TABLESAMPLE": true, + "THEN": true, + "TO": true, + "TRIGGER": true, + "TRUE": true, + "TRY_CAST": true, + "UNION": true, + "UNIQUE": true, + "UPDATE": true, + "USING": true, + "VALUES": true, + "VIEW": true, + "WHEN": true, + "WHENEVER": true, + "WHERE": true, + "WITH": true + }, "CLICKHOUSE": {}, "S3_DATALAKE": { "ALL": true, diff --git a/src/warehouse/index.js b/src/warehouse/index.js index 3491a257da..ea663c9b2f 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -349,7 +349,10 @@ function stringLikeObjectToString(obj) { */ function getColumns(options, event, columnTypes) { const columns = {}; - const uuidTS = options.provider === 'snowflake' ? 'UUID_TS' : 'uuid_ts'; + const uuidTS = + options.provider === 'snowflake' || options.provider === 'snowpipe_streaming' + ? 'UUID_TS' + : 'uuid_ts'; columns[uuidTS] = 'datetime'; // add loaded_at for bq to be segment compatible if (options.provider === 'bq') { @@ -377,6 +380,7 @@ function getColumns(options, event, columnTypes) { const fullEventColumnTypeByProvider = { snowflake: 'json', + snowpipe_streaming: 'json', rs: 'text', bq: 'string', postgres: 'json', @@ -613,6 +617,15 @@ function enhanceContextWithSourceDestInfo(message, metadata) { message.context = context; } +function shouldSkipUsersTable(options) { + return ( + options.provider === 'snowpipe_streaming' || + options.destConfig?.skipUsersTable || + options.integrationOptions?.skipUsersTable || + false + ); +} + function processWarehouseMessage(message, options) { const utils = getVersionedUtils(options.whSchemaVersion); options.utils = utils; @@ -638,8 +651,7 @@ function processWarehouseMessage(message, options) { const eventType = message.type?.toLowerCase(); const skipTracksTable = options.destConfig?.skipTracksTable || options.integrationOptions.skipTracksTable || false; - const skipUsersTable = - options.destConfig?.skipUsersTable || options.integrationOptions.skipUsersTable || false; + const skipUsersTable = shouldSkipUsersTable(options); const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; diff --git a/src/warehouse/v1/util.js b/src/warehouse/v1/util.js index d1289bc674..95dd59b030 100644 --- a/src/warehouse/v1/util.js +++ b/src/warehouse/v1/util.js @@ -11,7 +11,7 @@ function safeTableName(options, name = '') { if (tableName === '') { throw new TransformationError('Table name cannot be empty.'); } - if (provider === 'snowflake') { + if (provider === 'snowflake' || provider === 'snowpipe_streaming') { tableName = tableName.toUpperCase(); } else if (provider === 'postgres') { tableName = tableName.substr(0, 63); @@ -41,7 +41,7 @@ function safeColumnName(options, name = '') { if (columnName === '') { throw new TransformationError('Column name cannot be empty.'); } - if (provider === 'snowflake') { + if (provider === 'snowflake' || provider === 'snowpipe_streaming') { columnName = columnName.toUpperCase(); } else if (provider === 'postgres') { columnName = columnName.substr(0, 63); diff --git a/test/__tests__/data/warehouse/events.js b/test/__tests__/data/warehouse/events.js index bca6f776be..99343e43e1 100644 --- a/test/__tests__/data/warehouse/events.js +++ b/test/__tests__/data/warehouse/events.js @@ -368,6 +368,146 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "TRACKS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked" + } + }, + { + metadata: { + table: "BUTTON_CLICKED", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + STACK_HISTORY_ERROR_DETAILS: [ + { + "message": "Cannot set headers after they are sent to the client", + "toString": "[function]" + } + ], + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "button clicked", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + EVENT: "button_clicked", + } + } + ], s3_datalake: [ { metadata: { @@ -845,6 +985,78 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "IDENTIFIES", + columns: { + UUID_TS: "datetime", + CITY: "string", + COUNTRY: "string", + EMAIL: "string", + FIRSTNAME: "string", + LASTNAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CITY: "Disney", + COUNTRY: "UK", + EMAIL: "mickey@disney.com", + FIRSTNAME: "Mickey", + LASTNAME: "Mouse", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "2536eda4-d638-4c93-8014-8ffe3f083214", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.363Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.362Z", + CHANNEL: "web" + } + }, + ], s3_datalake: [ { metadata: { @@ -1170,6 +1382,66 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "PAGES", + columns: { + UUID_TS: "datetime", + PATH: "string", + URL: "string", + NAME: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_USER_AGENT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + PATH: "/tests/html/index2.html", + URL: "http://localhost/tests/html/index2.html", + NAME: "sample title", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "dd266c67-9199-4a52-ba32-f46ddde67312", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.359Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.358Z", + CHANNEL: "web" + } + } + ], s3_datalake: [ { metadata: { @@ -1424,6 +1696,67 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "SCREENS", + columns: { + UUID_TS: "datetime", + PATH: "string", + CATEGORY: "string", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string", + NAME: "string" + } + }, + data: { + PATH: "/abc", + CATEGORY: "test-category", + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + ID: "5e10d13a-bf9a-44bf-b884-43a9e591ea71", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2019-10-14T11:15:53.296Z", + ORIGINAL_TIMESTAMP: "2019-10-14T11:15:18.299Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + TIMESTAMP: "2020-01-24T06:29:02.402Z", + CHANNEL: "web", + NAME: "ApplicationLoaded" + } + } + ], s3_datalake: [ { metadata: { @@ -1687,6 +2020,73 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "GROUPS", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_LOCALE: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + CONTEXT_OS_ID: "string", + CONTEXT_OS_MANUFACTURER: "string", + CONTEXT_OS_MODEL: "string", + CONTEXT_OS_TYPE: "string", + CONTEXT_SCREEN_DENSITY: "int", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + GROUP_ID: "string", + SENT_AT: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + TIMESTAMP: "datetime", + CHANNEL: "string" + } + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.0", + CONTEXT_TRAITS_EMAIL: "test@rudderstack.com", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.0", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + CONTEXT_LOCALE: "en-US", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_OS_ID: "72e528f869711c3d", + CONTEXT_OS_MANUFACTURER: "Google", + CONTEXT_OS_MODEL: "sdk_gphone_x86", + CONTEXT_OS_NAME: "", + CONTEXT_OS_TOKEN: "", + CONTEXT_OS_TYPE: "android", + CONTEXT_SCREEN_DENSITY: 2, + ID: "84e26acc-56a5-4835-8233-591137fca468", + ANONYMOUS_ID: "00000000000000000000000000", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + GROUP_ID: "g1", + SENT_AT: "2019-10-14T09:03:22.563Z", + ORIGINAL_TIMESTAMP: "2019-10-14T09:03:17.562Z", + TIMESTAMP: "2020-01-24T11:59:02.403Z", + RECEIVED_AT: "2020-01-24T11:59:02.403Z", + CHANNEL: "web" + } + } + ], s3_datalake: [ { metadata: { @@ -1956,6 +2356,70 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "ALIASES", + columns: { + UUID_TS: "datetime", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USER_AGENT: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + PREVIOUS_ID: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USER_AGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + ID: "79313729-7fe5-4204-963a-dc46f4205e4e", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "1234abc", + SENT_AT: "2020-01-24T06:29:02.366Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.366Z", + CHANNEL: "web", + PREVIOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca" + } + } + ], s3_datalake: [ { metadata: { @@ -2147,6 +2611,40 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "EXTRACT_EVENT", + columns: { + DATE_PROPERTY: "datetime", + DATE_PROPERTY_2: "datetime", + CONTEXT_SOURCES_JOB_ID: "string", + CONTEXT_SOURCES_JOB_RUN_ID: "string", + CONTEXT_SOURCES_TASK_RUN_ID: "string", + CONTEXT_SOURCES_VERSION: "string", + EVENT: "string", + ID: "string", + USER_ID: "string", + RECEIVED_AT: "datetime", + BOOLEAN_PROPERTY: "boolean", + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + DATE_PROPERTY: "2023-02-01T07:53:31.430Z", + DATE_PROPERTY_2: "2023-01-31T07:39:10.002Z", + CONTEXT_SOURCES_JOB_ID: "2JABSy1nq89H7xeJimBL2pCtOOp", + CONTEXT_SOURCES_JOB_RUN_ID: "cfd6705nsevh5p2l77ag", + CONTEXT_SOURCES_TASK_RUN_ID: "cfd6705nsevh5p2l77b0", + CONTEXT_SOURCES_VERSION: "version", + ID: "some-uuid", + USER_ID: "dummy-user-id-inside-properties", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + EVENT: "extract_event", + BOOLEAN_PROPERTY: true, + } + } + ], s3_datalake: [ { metadata: { @@ -2193,6 +2691,9 @@ function output(eventType, provider) { if (provider === "snowflake") { return _.cloneDeep(sampleEvents[eventType].output.snowflake); } + if (provider === "snowpipe_streaming") { + return _.cloneDeep(sampleEvents[eventType].output.snowpipe_streaming); + } if (provider === "s3_datalake") { return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); } diff --git a/test/__tests__/data/warehouse/integration_options_events.js b/test/__tests__/data/warehouse/integration_options_events.js index 35eafb6a9b..b3f076851b 100644 --- a/test/__tests__/data/warehouse/integration_options_events.js +++ b/test/__tests__/data/warehouse/integration_options_events.js @@ -74,6 +74,13 @@ const sampleEvents = { jsonPaths: ["tMap"] } }, + SNOWPIPE_STREAMING: { + options: { + skipTracksTable: true, + useBlendoCasing: true, + jsonPaths: ["tMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -432,6 +439,88 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + metadata: { + table: "_GROUPS", + columns: { + UUID_TS: "datetime", + CURRENCY: "string", + REVENUE: "int", + PATH_TO_$1_000_000: "string", + _9OMEGA: "boolean", + CAMELCASE123KEY: "string", + TESTMAP_NESTEDMAP: "json", + TMAP: "json", + TESTARRAY: "json", + CONTEXT_APP_BUILD: "string", + CONTEXT_APP_NAME: "string", + CONTEXT_APP_NAMESPACE: "string", + CONTEXT_APP_VERSION: "string", + CONTEXT_LIBRARY_NAME: "string", + CONTEXT_LIBRARY_VERSION: "string", + CONTEXT_LOCALE: "string", + CONTEXT_SCREEN_DENSITY: "int", + CONTEXT_TRAITS_CITY: "string", + CONTEXT_TRAITS_COUNTRY: "string", + CONTEXT_TRAITS_EMAIL: "string", + CONTEXT_TRAITS_FIRSTNAME: "string", + CONTEXT_USERAGENT: "string", + EVENT_TEXT: "string", + ID: "string", + ANONYMOUS_ID: "string", + USER_ID: "string", + SENT_AT: "datetime", + TIMESTAMP: "datetime", + RECEIVED_AT: "datetime", + ORIGINAL_TIMESTAMP: "datetime", + CHANNEL: "string", + CONTEXT_IP: "string", + CONTEXT_REQUEST_IP: "string", + CONTEXT_PASSED_IP: "string", + EVENT: "string" + }, + receivedAt: "2020-01-24T11:59:02.403+05:30" + }, + data: { + CURRENCY: "USD", + REVENUE: 50, + PATH_TO_$1_000_000: "None", + _9OMEGA: true, + CAMELCASE123KEY: "camel case", + TESTMAP_NESTEDMAP: '{"n1":"nested prop 1"}', + TMAP: '{"t1":10,"t2":20}', + TESTARRAY: '["This is","an","array"]', + CONTEXT_APP_BUILD: "1.0.0", + CONTEXT_APP_NAME: "RudderLabs JavaScript SDK", + CONTEXT_APP_NAMESPACE: "com.rudderlabs.javascript", + CONTEXT_APP_VERSION: "1.0.5", + CONTEXT_LIBRARY_NAME: "RudderLabs JavaScript SDK", + CONTEXT_LIBRARY_VERSION: "1.0.5", + CONTEXT_LOCALE: "en-GB", + CONTEXT_SCREEN_DENSITY: 2, + CONTEXT_TRAITS_CITY: "Disney", + CONTEXT_TRAITS_COUNTRY: "USA", + CONTEXT_TRAITS_EMAIL: "mickey@disney.com", + CONTEXT_TRAITS_FIRSTNAME: "Mickey", + CONTEXT_USERAGENT: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + EVENT_TEXT: "groups", + ID: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + ANONYMOUS_ID: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + USER_ID: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + SENT_AT: "2020-01-24T06:29:02.364Z", + TIMESTAMP: "2020-01-24T06:29:02.403Z", + RECEIVED_AT: "2020-01-24T06:29:02.403Z", + ORIGINAL_TIMESTAMP: "2020-01-24T06:29:02.364Z", + CHANNEL: "web", + CONTEXT_IP: "0.0.0.0", + CONTEXT_REQUEST_IP: "[::1]:53708", + CONTEXT_PASSED_IP: "0.0.0.0", + EVENT: "groups" + } + } + ], s3_datalake: [ { metadata: { @@ -730,7 +819,7 @@ const sampleEvents = { ] } }, - users: { + identify: { input: { destination: { Config: { @@ -1255,6 +1344,75 @@ const sampleEvents = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], s3_datalake: [ { "data": { @@ -1639,6 +1797,8 @@ function opOutput(eventType, provider) { switch (provider) { case "snowflake": return _.cloneDeep(sampleEvents[eventType].output.snowflake); + case "snowpipe_streaming": + return _.cloneDeep(sampleEvents[eventType].output.snowpipe_streaming); case "s3_datalake": return _.cloneDeep(sampleEvents[eventType].output.s3_datalake); case "rs": @@ -1646,13 +1806,13 @@ function opOutput(eventType, provider) { case "bq": return _.cloneDeep(sampleEvents[eventType].output.bq); case "gcs_datalake": - if (eventType === 'users') { + if (eventType === 'identify') { return _.cloneDeep(sampleEvents[eventType].output.gcs_datalake); } else { return _.cloneDeep(sampleEvents[eventType].output.default); } case "azure_datalake": - if (eventType === 'users') { + if (eventType === 'identify') { return _.cloneDeep(sampleEvents[eventType].output.azure_datalake); } else { return _.cloneDeep(sampleEvents[eventType].output.default); diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js index 30cce51fb7..7615abe5e1 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -78,6 +78,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -426,5 +434,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js index 9d38ecc292..b28b3a161e 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -59,6 +59,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -276,6 +284,43 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP_N_1": "nested prop 1", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP_N_1": "string", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js index bbae993b27..457d251a74 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -78,6 +78,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, GCS_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -431,5 +439,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js index 42f5cccf49..d91ef5f44b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -53,6 +53,16 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -879,6 +889,85 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "UP_MAP_NESTED_MAP_N_1": "nested prop 1", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP_N_1": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP_N_1": "string", + "UP_MAP_NESTED_MAP_N_1": "string", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], datalake: [ { "data": { diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js index dc2d3a3318..322ee8f3e6 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -67,6 +67,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -404,5 +412,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js index 9b789d2747..27d7b9a69b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -67,6 +67,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -404,5 +412,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js index 691ce57ca1..aabfce149b 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -78,6 +78,15 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -828,6 +837,148 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js index d5d57f85b9..4b74fd3768 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -66,6 +66,11 @@ module.exports = { jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -414,5 +419,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js index a950b7f526..03745c0502 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -59,6 +59,14 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -276,6 +284,43 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js index 6979aa1100..1e7c8110da 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -66,6 +66,11 @@ module.exports = { jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, GCS_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -419,5 +424,70 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js index 89fcc23cd5..d9c3b80df3 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -53,6 +53,16 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -879,6 +889,85 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "json", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP": "json", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP": "json", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + ], datalake: [ { "data": { diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js index d5282c1cfd..1f72ec31cf 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -55,6 +55,11 @@ module.exports = { jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -392,5 +397,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js index 3eec47b89d..68b5aafcb5 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -55,6 +55,11 @@ module.exports = { jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -392,5 +397,66 @@ module.exports = { } } ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], } } diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js index 6b411835db..70c20b8b8e 100644 --- a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -78,6 +78,15 @@ module.exports = { ] } }, + SNOWPIPE_STREAMING: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, S3_DATALAKE: { options: { skipReservedKeywordsEscaping: true @@ -828,6 +837,148 @@ module.exports = { "table": "PRODUCT_ADDED" } } + ], + snowpipe_streaming: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } ] } } diff --git a/test/__tests__/data/warehouse/names.js b/test/__tests__/data/warehouse/names.js index b0380dc06d..e6015cef2c 100644 --- a/test/__tests__/data/warehouse/names.js +++ b/test/__tests__/data/warehouse/names.js @@ -41,6 +41,19 @@ const names = { C_Z: "string", CAMEL_CASE_123_KEY: "string", _1_C_COMEGA: "string" + }, + snowpipe_streaming: { + OMEGA: "string", + OMEGA_V_2: "string", + _9_MEGA: "string", + MEGA: "string", + OME_GA: "string", + ALPHA: "string", + OME_GA: "string", + _9_MEGA_90: "string", + C_Z: "string", + CAMEL_CASE_123_KEY: "string", + _1_C_COMEGA: "string" } }, data: { @@ -69,6 +82,19 @@ const names = { C_Z: "test", CAMEL_CASE_123_KEY: "test", _1_C_COMEGA: "test" + }, + snowpipe_streaming: { + OMEGA: "test", + OMEGA_V_2: "test", + _9_MEGA: "test", + MEGA: "test", + OME_GA: "test", + ALPHA: "test", + OME_GA: "test", + _9_MEGA_90: "test", + C_Z: "test", + CAMEL_CASE_123_KEY: "test", + _1_C_COMEGA: "test" } }, namesMap: { @@ -97,6 +123,19 @@ const names = { Cízǔ: "C_Z", CamelCase123Key: "CAMEL_CASE_123_KEY", "1CComega": "_1_C_COMEGA" + }, + snowpipe_streaming: { + omega: "OMEGA", + "omega v2": "OMEGA_V_2", + "9mega": "_9_MEGA", + "mega&": "MEGA", + ome$ga: "OME_GA", + alpha$: "ALPHA", + "ome_ ga": "OME_GA", + "9mega________-________90": "_9_MEGA_90", + Cízǔ: "C_Z", + CamelCase123Key: "CAMEL_CASE_123_KEY", + "1CComega": "_1_C_COMEGA" } } } diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 7fdecbd7cf..44d1d33ff8 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -37,14 +37,6 @@ const integrations = [ "s3_datalake", "gcs_datalake", ]; - -const integration = (index ) => { - const it = integrations[index]; - if (it === "snowflake" || it === "snowpipe_streaming") { - return "snowflake"; - } - return it; -} const transformers = integrations.map(integration => require(`../../src/${version}/destinations/${integration}/transform`) ); @@ -62,7 +54,7 @@ const propsKeyMap = { }; const integrationCasedString = (integration, str) => { - if (integration === "snowflake") { + if (integration === "snowflake" || integration === "snowpipe_streaming") { return str.toUpperCase(); } return str; @@ -74,7 +66,7 @@ describe("event types", () => { const i = input("track"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("track", integration(index))); + expect(received).toMatchObject(output("track", integrations[index])); }); }); }); @@ -85,7 +77,7 @@ describe("event types", () => { // also verfies priority order between traits and context.traits transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("identify", integration(index))); + expect(received).toMatchObject(output("identify", integrations[index])); }); }); }); @@ -95,7 +87,7 @@ describe("event types", () => { const i = input("page"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -104,7 +96,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); }); @@ -114,7 +106,7 @@ describe("event types", () => { const i = input("screen"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -123,7 +115,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); }); @@ -133,7 +125,7 @@ describe("event types", () => { const i = input("alias"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("alias", integration(index))); + expect(received).toMatchObject(output("alias", integrations[index])); }); }); }); @@ -143,7 +135,7 @@ describe("event types", () => { const i = input("extract"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("extract", integration(index))); + expect(received).toMatchObject(output("extract", integrations[index])); }); }); }); @@ -161,8 +153,9 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - const provider = - (integration(index) === "snowflake" || integration(index) == "snowpipe_streaming") ? "snowflake" : "default"; + const provider = ["snowflake", "snowpipe_streaming"].includes(integrations[index]) + ? integrations[index] + : "default"; expect(received[1].metadata.columns).toMatchObject( names.output.columns[provider] @@ -195,7 +188,7 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" @@ -219,7 +212,7 @@ describe("column & table names", () => { //KEY should be trimmed to 63 return; } - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(received[1].metadata).toHaveProperty( "table", "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" @@ -242,7 +235,7 @@ describe("column & table names", () => { ); return; } - if (integration(index) === "s3_datalake" || integration(index) === "gcs_datalake" || integration(index) === "azure_datalake") { + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" @@ -325,7 +318,7 @@ describe("conflict between rudder set props and user set props", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { let sampleRudderPropKey = "id"; - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { sampleRudderPropKey = "ID"; } @@ -363,7 +356,7 @@ describe("handle reserved words", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { const reserverdKeywordsMap = - reservedANSIKeywordsMap[integration(index).toUpperCase()]; + reservedANSIKeywordsMap[integrations[index].toUpperCase()]; i.message[propsKey] = Object.assign( i.message[propsKey] || {}, @@ -373,7 +366,7 @@ describe("handle reserved words", () => { const received = transformer.process(i); const out = - evType === "track" || evType === "identify" + evType === "track" || (evType === "identify" && integrations[index] !== 'snowpipe_streaming') ? received[1] : received[0]; @@ -386,7 +379,7 @@ describe("handle reserved words", () => { } else { k = snakeCasedKey; } - if (integration(index) === "snowflake") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(out.metadata.columns).toHaveProperty(k); } else { expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); @@ -470,24 +463,24 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); } @@ -505,23 +498,23 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); } @@ -542,10 +535,10 @@ describe("remove rudder property if rudder property is null", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); }); }); @@ -560,29 +553,29 @@ describe("remove any property if event is object ", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); }); i.message.channel = { channel: "android" }; transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); }); @@ -598,13 +591,13 @@ describe("store full rudder event", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); const columnName = integrationCasedString( - integration(index), + integrations[index], "rudder_event" ); expect(received[0].metadata.columns).toHaveProperty(columnName); expect(received[0].metadata.columns[columnName]).toEqual( - fullEventColumnTypeByProvider[integration(index)] + fullEventColumnTypeByProvider[integrations[index]] ); expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); @@ -644,7 +637,7 @@ describe("rudder reserved columns", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); checkProps.forEach(k => { - k = integrationCasedString(integration(index), k); + k = integrationCasedString(integrations[index], k); expect(received[0].metadata.columns).not.toHaveProperty(k); expect(received[0].data).not.toHaveProperty(k); if (received[1]) { @@ -664,14 +657,25 @@ describe("id column datatype for users table", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); + if (integrations[index] === 'snowpipe_streaming') { + expect(received).toHaveLength(1); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("int"); + return; + } + + expect(received).toHaveLength(2); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("int"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); }); @@ -681,14 +685,25 @@ describe("id column datatype for users table", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); + if (integrations[index] === 'snowpipe_streaming') { + expect(received).toHaveLength(1); + expect( + received[0].metadata.columns[ + integrationCasedString(integrations[index], "user_id") + ] + ).toEqual("float"); + return; + } + + expect(received).toHaveLength(2); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("float"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("float"); }); @@ -707,22 +722,22 @@ describe("handle leading underscores in properties", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); }); }); @@ -736,22 +751,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -764,22 +779,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -792,17 +807,17 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("string"); }); @@ -818,28 +833,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -854,28 +869,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -915,10 +930,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -943,10 +958,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -956,7 +971,7 @@ describe("Handle no of columns in an event", () => { it("should throw an error if no of columns are more than 200", () => { const i = input("track"); transformers - .filter((transformer, index) => integration(index) !== "s3_datalake" && integration(index) !== "gcs_datalake" && integration(index) !== "azure_datalake") + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") .forEach((transformer, index) => { i.message.properties = largeNoOfColumnsevent; expect(() => transformer.process(i)).toThrow( @@ -984,13 +999,13 @@ describe("Add auto generated messageId for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toMatch(/auto-.*/); }); }); @@ -1007,10 +1022,10 @@ describe("Add receivedAt for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); }); }); @@ -1023,22 +1038,22 @@ describe("Integration options", () => { const i = opInput("track"); transformers.forEach((transformer, index) => { const {jsonPaths} = i.destination.Config; - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { delete i.destination.Config.jsonPaths; } const received = transformer.process(i); i.destination.Config.jsonPaths = jsonPaths; - expect(received).toEqual(opOutput("track", integration(index))); + expect(received).toEqual(opOutput("track", integrations[index])); }); }); }); describe("users", () => { it("should skip users when skipUsersTable is set", () => { - const i = opInput("users"); + const i = opInput("identify"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toEqual(opOutput("users", integration(index))); + expect(received).toEqual(opOutput("identify", integrations[index])); }); }); }); @@ -1054,6 +1069,8 @@ describe("Integration options", () => { return _.cloneDeep(config.output.postgres); case "snowflake": return _.cloneDeep(config.output.snowflake); + case "snowpipe_streaming": + return _.cloneDeep(config.output.snowpipe_streaming); case "s3_datalake": case "gcs_datalake": case "azure_datalake": @@ -1093,18 +1110,18 @@ describe("Integration options", () => { for (const testCase of testCases) { transformers.forEach((transformer, index) => { - it(`new ${testCase.eventType} for ${integration(index)}`, () => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) - it(`legacy ${testCase.eventType} for ${integration(index)}`, () => { + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); } @@ -1242,7 +1259,10 @@ describe("Destination config", () => { transformers.forEach((transformer, index) => { const received = transformer.process(scenario.event); - expect(received).toHaveLength(scenario.expected.length); + const expectedLength = integrations[index] === "snowpipe_streaming" && scenario.event.message.type === "identify" + ? 1 + : scenario.expected.length; + expect(received).toHaveLength(expectedLength); for (const i in received) { const evt = received[i]; expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); @@ -1285,7 +1305,9 @@ describe("Destination config", () => { } } const output = transformer.process(event); - const events = [output[0], output[1]]; // identifies and users event + const events = integrations[index] === "snowpipe_streaming" + ? [output[0]] + : [output[0], output[1]]; const traitsToCheck = { 'city': 'Disney', 'country': 'USA', @@ -1294,10 +1316,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data[integrationCasedString(integration(index), trait)]).toEqual(value); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1325,13 +1347,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v_2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); }); }); }); @@ -1364,7 +1386,10 @@ describe("Destination config", () => { } } const received = transformer.process(event); - const events = [received[0], received[1]]; // identifies and users event + const events = integrations[index] === "snowpipe_streaming" + ? [received[0]] + : [received[0], received[1]]; + // identifies and users event const traitsToCheck = { 'city': 'Disney', 'country': 'USA', @@ -1373,10 +1398,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1402,13 +1427,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); }); }); }); @@ -1615,8 +1640,8 @@ describe("context traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/context_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); } @@ -1708,8 +1733,8 @@ describe("group traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/group_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); }); From ebcf84e07bf121d882c99df973af265a915a1ce1 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 9 Dec 2024 13:10:01 +0530 Subject: [PATCH 12/21] fix: braze subscription group fixes (#3901) --- src/util/prometheus.js | 12 ++ src/v0/destinations/braze/braze.util.test.js | 151 ++++++++++++++++-- src/v0/destinations/braze/config.js | 2 +- src/v0/destinations/braze/transform.js | 2 +- src/v0/destinations/braze/util.js | 54 ++++++- .../destinations/braze/processor/data.ts | 4 +- .../destinations/braze/router/data.ts | 33 +++- 7 files changed, 240 insertions(+), 18 deletions(-) diff --git a/src/util/prometheus.js b/src/util/prometheus.js index a4c12e4ea8..2a3a1fb22a 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -520,6 +520,18 @@ class Prometheus { type: 'counter', labelNames: ['destination_id'], }, + { + name: 'braze_batch_subscription_size', + help: 'braze_batch_subscription_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'braze_batch_subscription_combined_size', + help: 'braze_batch_subscription_combined_size', + type: 'gauge', + labelNames: ['destination_id'], + }, { name: 'mailjet_packing_size', help: 'mailjet_packing_size', diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 985f2434d5..7d10035233 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -6,6 +6,7 @@ const { getPurchaseObjs, setAliasObject, handleReservedProperties, + combineSubscriptionGroups, } = require('./util'); const { processBatch } = require('./util'); const { @@ -958,7 +959,9 @@ describe('processBatch', () => { attributes: [{ id: i, name: 'test', xyz: 'abc' }], events: [{ id: i, event: 'test', xyz: 'abc' }], purchases: [{ id: i, purchase: 'test', xyz: 'abc' }], - subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }], + subscription_groups: [ + { subscription_group_id: i, group: 'test', subscription_state: 'abc' }, + ], merge_updates: [{ id: i, alias: 'test', xyz: 'abc' }], }, }, @@ -972,7 +975,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(8); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -981,10 +984,12 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases - expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); - expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates + expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Third batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // Fourth batch contains 25 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates }); test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => { @@ -1055,7 +1060,9 @@ describe('processBatch', () => { batchedRequest: { body: { JSON: { - subscription_groups: [{ id: i, group: 'test', xyz: 'abc' }], + subscription_groups: [ + { subscription_group_id: i, group: 'test', subscription_state: 'abc' }, + ], }, }, }, @@ -1093,7 +1100,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40) - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(7); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -1103,9 +1110,10 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates }); test('check success and failure scenarios both for processBatch', () => { @@ -1751,3 +1759,124 @@ describe('handleReservedProperties', () => { }); }); }); + +describe('combineSubscriptionGroups', () => { + it('should merge external_ids, emails, and phones for the same subscription_group_id and subscription_state', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1', 'id2'], + emails: ['email1@example.com', 'email2@example.com'], + phones: ['+1234567890', '+0987654321'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id2', 'id3'], + emails: ['email2@example.com', 'email3@example.com'], + phones: ['+1234567890', '+1122334455'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1', 'id2', 'id3'], + emails: ['email1@example.com', 'email2@example.com', 'email3@example.com'], + phones: ['+1234567890', '+0987654321', '+1122334455'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should handle groups with missing external_ids, emails, or phones', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + emails: ['email1@example.com'], + }, + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + phones: ['+1234567890'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + emails: ['email1@example.com'], + phones: ['+1234567890'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should handle multiple unique subscription groups', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group2', + subscription_state: 'Unsubscribed', + external_ids: ['id2'], + emails: ['email2@example.com'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + { + subscription_group_id: 'group2', + subscription_state: 'Unsubscribed', + external_ids: ['id2'], + emails: ['email2@example.com'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); + + it('should not include undefined fields in the output', () => { + const input = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + ]; + + const expectedOutput = [ + { + subscription_group_id: 'group1', + subscription_state: 'Subscribed', + external_ids: ['id1'], + }, + ]; + + const result = combineSubscriptionGroups(input); + expect(result).toEqual(expectedOutput); + }); +}); diff --git a/src/v0/destinations/braze/config.js b/src/v0/destinations/braze/config.js index 2bbade2754..8ccabbdb0a 100644 --- a/src/v0/destinations/braze/config.js +++ b/src/v0/destinations/braze/config.js @@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50; // https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/ const ALIAS_BRAZE_MAX_REQ_COUNT = 50; -const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50; +const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25; const DEL_MAX_BATCH_SIZE = 50; const DESTINATION = 'braze'; diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js index 09fb0205c5..2a9cb8e7dc 100644 --- a/src/v0/destinations/braze/transform.js +++ b/src/v0/destinations/braze/transform.js @@ -383,7 +383,7 @@ function processGroup(message, destination) { ); } subscriptionGroup.subscription_state = message.traits.subscriptionState; - subscriptionGroup.external_id = [message.userId]; + subscriptionGroup.external_ids = [message.userId]; const phone = getFieldValueFromMessage(message, 'phone'); const email = getFieldValueFromMessage(message, 'email'); if (phone) { diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 6c8cf64265..3778c34c43 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -45,6 +45,42 @@ const getEndpointFromConfig = (destination) => { return endpoint; }; +// Merges external_ids, emails, and phones for entries with the same subscription_group_id and subscription_state +const combineSubscriptionGroups = (subscriptionGroups) => { + const uniqueGroups = {}; + + subscriptionGroups.forEach((group) => { + const key = `${group.subscription_group_id}-${group.subscription_state}`; + if (!uniqueGroups[key]) { + uniqueGroups[key] = { + ...group, + external_ids: [...(group.external_ids || [])], + emails: [...(group.emails || [])], + phones: [...(group.phones || [])], + }; + } else { + uniqueGroups[key].external_ids.push(...(group.external_ids || [])); + uniqueGroups[key].emails.push(...(group.emails || [])); + uniqueGroups[key].phones.push(...(group.phones || [])); + } + }); + + return Object.values(uniqueGroups).map((group) => { + const result = { + subscription_group_id: group.subscription_group_id, + subscription_state: group.subscription_state, + external_ids: [...new Set(group.external_ids)], + }; + if (group.emails?.length) { + result.emails = [...new Set(group.emails)]; + } + if (group.phones?.length) { + result.phones = [...new Set(group.phones)]; + } + return result; + }); +}; + const CustomAttributeOperationUtil = { customAttributeUpdateOperation(key, data, traits, mergeObjectsUpdateOperation) { data[key] = {}; @@ -381,8 +417,22 @@ function prepareGroupAndAliasBatch(arrayChunks, responseArray, destination, type } else if (type === 'subscription') { response.endpoint = getSubscriptionGroupEndPoint(getEndpointFromConfig(destination)); const subscription_groups = chunk; + // maketool transformed event + logger.info(`braze subscription chunk ${JSON.stringify(subscription_groups)}`); + + stats.gauge('braze_batch_subscription_size', subscription_groups.length, { + destination_id: destination.ID, + }); + + // Deduplicate the subscription groups before constructing the response body + const deduplicatedSubscriptionGroups = combineSubscriptionGroups(subscription_groups); + + stats.gauge('braze_batch_subscription_combined_size', deduplicatedSubscriptionGroups.length, { + destination_id: destination.ID, + }); + response.body.JSON = removeUndefinedAndNullValues({ - subscription_groups, + subscription_groups: deduplicatedSubscriptionGroups, }); } responseArray.push({ @@ -490,6 +540,7 @@ const processBatch = (transformedEvents) => { prepareGroupAndAliasBatch(mergeUsersArrayChunks, responseArray, destination, 'merge'); if (successMetadata.length > 0) { + console.log(`Response 1 batchRequest ${JSON.stringify(responseArray)}`); finalResponse.push({ batchedRequest: responseArray, metadata: successMetadata, @@ -756,4 +807,5 @@ module.exports = { collectStatsForAliasFailure, collectStatsForAliasMissConfigurations, handleReservedProperties, + combineSubscriptionGroups, }; diff --git a/test/integrations/destinations/braze/processor/data.ts b/test/integrations/destinations/braze/processor/data.ts index 240206791e..9b6f6dac65 100644 --- a/test/integrations/destinations/braze/processor/data.ts +++ b/test/integrations/destinations/braze/processor/data.ts @@ -3160,7 +3160,7 @@ export const data = [ { subscription_group_id: '1234', subscription_state: 'subscribed', - external_id: ['Randomuser2222'], + external_ids: ['Randomuser2222'], phones: ['5055077683'], }, ], @@ -3281,7 +3281,7 @@ export const data = [ { subscription_group_id: '1234', subscription_state: 'unsubscribed', - external_id: ['Randomuser2222'], + external_ids: ['Randomuser2222'], emails: ['abc@test.com'], }, ], diff --git a/test/integrations/destinations/braze/router/data.ts b/test/integrations/destinations/braze/router/data.ts index 6803742e86..b788e22741 100644 --- a/test/integrations/destinations/braze/router/data.ts +++ b/test/integrations/destinations/braze/router/data.ts @@ -219,6 +219,34 @@ export const data = [ metadata: { jobId: 6, userId: 'u1' }, message: { type: 'alias', previousId: 'adsfsaf2', userId: 'dsafsdf2' }, }, + { + destination: { + Config: { + restApiKey: 'dummyApiKey', + prefixProperties: true, + useNativeSDK: false, + dataCenter: 'us-01', + enableSubscriptionGroupInGroupCall: true, + }, + DestinationDefinition: { + DisplayName: 'Braze', + ID: '1WhbSZ6uA3H5ChVifHpfL2H6sie', + Name: 'BRAZE', + }, + Enabled: true, + ID: '1WhcOCGgj9asZu850HvugU2C3Aq', + Name: 'Braze', + Transformations: [], + }, + metadata: { jobId: 7, userId: 'u1' }, + message: { + anonymousId: '56yrtsdfgbgxcb-22b4-401d-aae5-1b994be9afdf', + groupId: 'c90f0fd2-2a02-4f2f-bf07-7e7d2c2ed2b1', + traits: { phone: '5055077683', subscriptionState: 'subscribed' }, + userId: 'user12345', + type: 'group', + }, + }, ], destType: 'braze', }, @@ -299,13 +327,13 @@ export const data = [ JSON: { subscription_groups: [ { - external_id: ['user123'], + external_ids: ['user123', 'user12345'], phones: ['5055077683'], subscription_group_id: 'c90f0fd2-2a02-4f2f-bf07-7e7d2c2ed2b1', subscription_state: 'subscribed', }, { - external_id: ['user877'], + external_ids: ['user877'], phones: ['5055077683'], subscription_group_id: '58d0a278-b55b-4f10-b7d2-98d1c5dd4c30', subscription_state: 'subscribed', @@ -356,6 +384,7 @@ export const data = [ { jobId: 4, userId: 'u1' }, { jobId: 5, userId: 'u1' }, { jobId: 6, userId: 'u1' }, + { jobId: 7, userId: 'u1' }, ], batched: true, statusCode: 200, From a2462934b289fd8d43c5586640945c47de6f21fd Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 9 Dec 2024 10:18:19 +0000 Subject: [PATCH 13/21] chore(release): 1.86.0 --- CHANGELOG.md | 15 +++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb1779fd3a..e742ea1f55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.86.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.1...v1.86.0) (2024-12-09) + + +### Features + +* add support for getting anonymousId by note attributes array ([53d2bef](https://github.com/rudderlabs/rudder-transformer/commit/53d2bef38ad5d7bc5504111ec797b3c3973546dd)) +* add support for getting anonymousId by note attributes array ([#3893](https://github.com/rudderlabs/rudder-transformer/issues/3893)) ([d7f390c](https://github.com/rudderlabs/rudder-transformer/commit/d7f390cb471e44afb276484b8b804d1f257c539c)) + + +### Bug Fixes + +* braze subscription group fixes ([#3901](https://github.com/rudderlabs/rudder-transformer/issues/3901)) ([ebcf84e](https://github.com/rudderlabs/rudder-transformer/commit/ebcf84e07bf121d882c99df973af265a915a1ce1)) +* remove redundant ids and userIdentifier when gbraid or wbraid are there ([#3910](https://github.com/rudderlabs/rudder-transformer/issues/3910)) ([313710c](https://github.com/rudderlabs/rudder-transformer/commit/313710ca725538e5ffe357216d9c88e444f995c8)) +* skipping users events for snowpipe streaming ([#3836](https://github.com/rudderlabs/rudder-transformer/issues/3836)) ([12621c8](https://github.com/rudderlabs/rudder-transformer/commit/12621c8eee641f5a03a997e95ed016cff0eefde7)) + ### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21) diff --git a/package-lock.json b/package-lock.json index 40033e278b..991d341e9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.85.1", + "version": "1.86.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.85.1", + "version": "1.86.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index e0b17a4f47..407ee1d1ab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.85.1", + "version": "1.86.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 5f299de8c899e6924760c4537a9213a44d6aaf10 Mon Sep 17 00:00:00 2001 From: Dilip Kola <33080863+koladilip@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:16:50 +0530 Subject: [PATCH 14/21] refactor: code to address sonar issues (#3926) * refactor: code to address sonar issues * refactor: garl record transform --- package-lock.json | 4 +- .../v2/destinations/pinterest_tag/utils.js | 62 +++--- src/v0/destinations/amazon_audience/utils.js | 3 +- .../recordTransform.js | 207 ++++++------------ 4 files changed, 94 insertions(+), 182 deletions(-) diff --git a/package-lock.json b/package-lock.json index 40033e278b..14832566c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10621,7 +10621,9 @@ "license": "MIT" }, "node_modules/cross-spawn": { - "version": "7.0.3", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "license": "MIT", "dependencies": { "path-key": "^3.1.0", diff --git a/src/cdk/v2/destinations/pinterest_tag/utils.js b/src/cdk/v2/destinations/pinterest_tag/utils.js index 31a897f133..dbaeea9501 100644 --- a/src/cdk/v2/destinations/pinterest_tag/utils.js +++ b/src/cdk/v2/destinations/pinterest_tag/utils.js @@ -1,8 +1,8 @@ -/* eslint-disable no-param-reassign */ const sha256 = require('sha256'); const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { API_VERSION } = require('./config'); +const { CommonUtils } = require('../../../../util/common'); const VALID_ACTION_SOURCES = ['app_android', 'app_ios', 'web', 'offline']; @@ -21,9 +21,15 @@ const ecomEventMaps = [ }, ]; -const USER_NON_ARRAY_PROPERTIES = ['client_user_agent', 'client_ip_address']; +const USER_NON_ARRAY_PROPERTIES = [ + 'client_user_agent', + 'client_ip_address', + 'click_id', + 'partner_id', +]; -const getHashedValue = (key, value) => { +const transformValue = (key, value) => { + const arrayValue = CommonUtils.toArray(value); switch (key) { case 'em': case 'ct': @@ -32,33 +38,18 @@ const getHashedValue = (key, value) => { case 'ln': case 'fn': case 'ge': - value = Array.isArray(value) - ? value.map((val) => val.toString().trim().toLowerCase()) - : value.toString().trim().toLowerCase(); - break; + return arrayValue.map((val) => val.toString().trim().toLowerCase()); case 'ph': - // phone numbers should only contain digits & should not contain leading zeros - value = Array.isArray(value) - ? value.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, '')) - : value.toString().replace(/\D/g, '').replace(/^0+/, ''); - break; + return arrayValue.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, '')); case 'zp': - // zip fields should only contain digits - value = Array.isArray(value) - ? value.map((val) => val.toString().trim().replace(/\D/g, '')) - : value.toString().replace(/\D/g, ''); - break; - case 'hashed_maids': - case 'external_id': - case 'db': - // no action needed on value - break; + return arrayValue.map((val) => val.toString().trim().replace(/\D/g, '')); default: - return String(value); + return arrayValue; } - return Array.isArray(value) ? value.map((val) => sha256(val)) : [sha256(value)]; }; +const getHashedValue = (key, value) => transformValue(key, value).map((val) => sha256(val)); + /** * * @param {*} userPayload Payload mapped from user fields @@ -67,10 +58,15 @@ const getHashedValue = (key, value) => { * Ref: https://s.pinimg.com/ct/docs/conversions_api/dist/v3.html */ const processUserPayload = (userPayload) => { - Object.keys(userPayload).forEach((key) => { - userPayload[key] = getHashedValue(key, userPayload[key]); + const newPayload = { ...userPayload }; + Object.keys(newPayload).forEach((key) => { + if (USER_NON_ARRAY_PROPERTIES.includes(key)) { + newPayload[key] = String(newPayload[key]); + } else { + newPayload[key] = getHashedValue(key, newPayload[key]); + } }); - return userPayload; + return newPayload; }; /** @@ -99,11 +95,7 @@ const processHashedUserPayload = (userPayload, message) => { const processedHashedUserPayload = {}; Object.keys(userPayload).forEach((key) => { if (!USER_NON_ARRAY_PROPERTIES.includes(key)) { - if (Array.isArray(userPayload[key])) { - processedHashedUserPayload[key] = [...userPayload[key]]; - } else { - processedHashedUserPayload[key] = [userPayload[key]]; - } + processedHashedUserPayload[key] = CommonUtils.toArray(userPayload[key]); } else { processedHashedUserPayload[key] = userPayload[key]; } @@ -111,10 +103,8 @@ const processHashedUserPayload = (userPayload, message) => { // multiKeyMap will works on only specific values like m, male, MALE, f, F, Female // if hashed data is sent from the user, it is directly set over here const gender = message.traits?.gender || message.context?.traits?.gender; - if (gender && Array.isArray(gender)) { - processedHashedUserPayload.ge = [...gender]; - } else if (gender) { - processedHashedUserPayload.ge = [gender]; + if (gender) { + processedHashedUserPayload.ge = CommonUtils.toArray(gender); } return processedHashedUserPayload; }; diff --git a/src/v0/destinations/amazon_audience/utils.js b/src/v0/destinations/amazon_audience/utils.js index c25f301378..350d071a47 100644 --- a/src/v0/destinations/amazon_audience/utils.js +++ b/src/v0/destinations/amazon_audience/utils.js @@ -21,7 +21,8 @@ const buildResponseWithUsers = (users, action, config, jobIdList, secret) => { if (!secret?.clientId) { throw new OAuthSecretError('OAuth - Client Id not found'); } - const externalId = `Rudderstack_${sha256(`${jobIdList}`)}`; + const jobIdHash = sha256(String(jobIdList)); + const externalId = `Rudderstack_${jobIdHash}`; const response = defaultRequestConfig(); response.endpoint = ''; response.method = defaultPostRequestConfig.requestMethod; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index 5866b66538..c3bf27a75d 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -1,4 +1,3 @@ -/* eslint-disable no-const-assign */ const lodash = require('lodash'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); const { @@ -19,29 +18,22 @@ const { const { getErrorResponse, createFinalResponse } = require('../../util/recordUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); -const processRecordEventArray = ( - records, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - operationType, -) => { - let outputPayloads = {}; - // ** only send it if identifier > 0 - - const fieldsArray = []; - const metadata = []; - records.forEach((record) => { - fieldsArray.push(record.message.fields); - metadata.push(record.metadata); - }); +const processRecordEventArray = (records, context, operationType) => { + const { + message, + destination, + accessToken, + developerToken, + audienceId, + typeOfList, + userSchema, + isHashRequired, + userDataConsent, + personalizationConsent, + } = context; + + const fieldsArray = records.map((record) => record.message.fields); + const metadata = records.map((record) => record.metadata); const userIdentifiersList = populateIdentifiersForRecordEvent( fieldsArray, @@ -51,131 +43,60 @@ const processRecordEventArray = ( ); const outputPayload = constructPayload(message, offlineDataJobsMapping); - outputPayload.operations = []; - // breaking the userIdentiFier array in chunks of 20 + const userIdentifierChunks = returnArrayOfSubarrays(userIdentifiersList, 20); - // putting each chunk in different create/remove operations - switch (operationType) { - case 'add': - // for add operation - userIdentifierChunks.forEach((element) => { - const operations = { - create: {}, - }; - operations.create.userIdentifiers = element; - outputPayload.operations.push(operations); - }); - outputPayloads = { ...outputPayloads, create: outputPayload }; - break; - case 'remove': - // for remove operation - userIdentifierChunks.forEach((element) => { - const operations = { - remove: {}, - }; - operations.remove.userIdentifiers = element; - outputPayload.operations.push(operations); - }); - outputPayloads = { ...outputPayloads, remove: outputPayload }; - break; - default: - } + outputPayload.operations = userIdentifierChunks.map((chunk) => ({ + [operationType]: { userIdentifiers: chunk }, + })); - const toSendEvents = []; - Object.values(outputPayloads).forEach((data) => { - const consentObj = populateConsentFromConfig( - { userDataConsent, personalizationConsent }, - consentConfigMap, - ); - toSendEvents.push( - responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), - ); - }); + const consentObj = populateConsentFromConfig( + { userDataConsent, personalizationConsent }, + consentConfigMap, + ); - const successResponse = getSuccessRespEvents(toSendEvents, metadata, destination, true); + const toSendEvents = [outputPayload].map((data) => + responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), + ); - return successResponse; + return getSuccessRespEvents(toSendEvents, metadata, destination, true); }; -function preparepayload(events, config) { +function preparePayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { - audienceId, - typeOfList, - isHashRequired, - userSchema, - userDataConsent, - personalizationConsent, - } = config; + + const context = { + message, + destination, + accessToken, + developerToken, + ...config, + }; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), ); - let insertResponse; - let deleteResponse; - let updateResponse; - - if (groupedRecordsByAction.delete) { - deleteResponse = processRecordEventArray( - groupedRecordsByAction.delete, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - 'remove', - ); - } - - if (groupedRecordsByAction.insert) { - insertResponse = processRecordEventArray( - groupedRecordsByAction.insert, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - 'add', - ); - } - - if (groupedRecordsByAction.update) { - updateResponse = processRecordEventArray( - groupedRecordsByAction.update, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - 'add', - ); - } + const actionResponses = ['delete', 'insert', 'update'].reduce((responses, action) => { + const operationType = action === 'delete' ? 'remove' : 'create'; + if (groupedRecordsByAction[action]) { + return { + ...responses, + [action]: processRecordEventArray(groupedRecordsByAction[action], context, operationType), + }; + } + return responses; + }, {}); const errorResponse = getErrorResponse(groupedRecordsByAction); const finalResponse = createFinalResponse( - deleteResponse, - insertResponse, - updateResponse, + actionResponses.delete, + actionResponses.insert, + actionResponses.update, errorResponse, ); + if (finalResponse.length === 0) { throw new InstrumentationError( 'Missing valid parameters, unable to generate transformed payload', @@ -196,14 +117,16 @@ function processRecordInputsV0(groupedRecordInputs) { personalizationConsent, } = destination.Config; - return preparepayload(groupedRecordInputs, { + const config = { audienceId: getOperationAudienceId(audienceId, message), typeOfList, userSchema, isHashRequired, userDataConsent, personalizationConsent, - }); + }; + + return preparePayload(groupedRecordInputs, config); } function processRecordInputsV1(groupedRecordInputs) { @@ -211,11 +134,7 @@ function processRecordInputsV1(groupedRecordInputs) { const { audienceId, typeOfList, isHashRequired, userDataConsent, personalizationConsent } = connection.config.destination; - const identifiers = message?.identifiers; - let userSchema; - if (identifiers) { - userSchema = Object.keys(identifiers); - } + const userSchema = message?.identifiers ? Object.keys(message.identifiers) : undefined; const events = groupedRecordInputs.map((record) => ({ ...record, @@ -225,23 +144,23 @@ function processRecordInputsV1(groupedRecordInputs) { }, })); - return preparepayload(events, { + const config = { audienceId, typeOfList, userSchema, isHashRequired, userDataConsent, personalizationConsent, - }); + }; + + return preparePayload(events, config); } function processRecordInputs(groupedRecordInputs) { const event = groupedRecordInputs[0]; - // First check for rETL flow and second check for ES flow - if (isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event)) { - return processRecordInputsV0(groupedRecordInputs); - } - return processRecordInputsV1(groupedRecordInputs); + return isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event) + ? processRecordInputsV0(groupedRecordInputs) + : processRecordInputsV1(groupedRecordInputs); } module.exports = { From aa352ab50bcb93df692774c8f2ee49531486a387 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Tue, 10 Dec 2024 14:44:21 +0530 Subject: [PATCH 15/21] fix: removed console statement (#3927) --- src/v0/destinations/braze/util.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 3778c34c43..74cb7fb953 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -540,7 +540,6 @@ const processBatch = (transformedEvents) => { prepareGroupAndAliasBatch(mergeUsersArrayChunks, responseArray, destination, 'merge'); if (successMetadata.length > 0) { - console.log(`Response 1 batchRequest ${JSON.stringify(responseArray)}`); finalResponse.push({ batchedRequest: responseArray, metadata: successMetadata, From 31e422c2654326126bb3eda79b2520935af3f785 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Tue, 10 Dec 2024 16:06:36 +0530 Subject: [PATCH 16/21] chore: fix build workflow (#3928) --- .github/workflows/build-push-docker-image.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index c7e59dce86..49162e433b 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -96,15 +96,15 @@ jobs: with: ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 - + - name: Login to DockerHub uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 - name: Build Docker Image uses: docker/build-push-action@v6.9.0 @@ -149,16 +149,16 @@ jobs: with: ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 - + - name: Login to DockerHub uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 + - name: Build Docker Image uses: docker/build-push-action@v6.9.0 with: From 6a0f47511f1bb0a57406bfbac4aa0c2f64bfb548 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Tue, 10 Dec 2024 16:30:34 +0530 Subject: [PATCH 17/21] chore: fix build workflow (#3929) --- .github/workflows/build-push-docker-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 49162e433b..ba62e5d8d1 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -198,15 +198,15 @@ jobs: needs: [build-transformer-image-amd64, build-transformer-image-arm64] steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 - - name: Login to DockerHub uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 + - name: Create multi-arch manifest run: | docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64 From 325dabe1070b5a2bf3a84b12ce10f19abf40a6a8 Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:45:08 +0530 Subject: [PATCH 18/21] chore: remove references to legacy consent management fields (#3931) --- src/v0/destinations/braze/braze.util.test.js | 1 - .../data_scenarios/cdk_v2/failure.json | 9 +-- .../data_scenarios/cdk_v2/success.json | 9 +-- .../destination/batch/failure_batch.json | 20 ++---- .../destination/batch/successful_batch.json | 16 ++--- .../proc/batch_input_multiplex.json | 6 +- .../destination/proc/multiplex_failure.json | 3 +- .../proc/multiplex_partial_failure.json | 6 +- .../destination/proc/multiplex_success.json | 3 +- .../destination/router/failure_test.json | 15 ++--- .../destinations/algolia/router/data.ts | 20 ++---- .../destinations/braze/router/data.ts | 7 --- .../destinations/candu/processor/data.ts | 63 +++---------------- .../destinations/candu/router/data.ts | 14 +---- .../destinations/clicksend/commonConfig.ts | 5 -- .../destinations/clicksend/router/data.ts | 5 -- .../destinations/emarsys/deleteUsers/data.ts | 20 ------ .../destinations/emarsys/processor/data.ts | 55 ---------------- .../destinations/emarsys/router/data.ts | 5 -- .../fb_custom_audience/router/data.ts | 4 -- .../fb_custom_audience/router/rETL.ts | 2 - .../destinations/gainsight_px/router/data.ts | 2 +- .../router/data.ts | 7 --- .../destinations/hs/router/data.ts | 60 ------------------ .../processor/configLevelFeaturesTestData.ts | 5 -- .../linkedin_ads/processor/trackTestData.ts | 5 -- .../processor/validationTestData.ts | 5 -- .../destinations/linkedin_ads/router/data.ts | 5 -- .../destinations/rockerbox/processor/data.ts | 28 --------- .../destinations/rockerbox/router/data.ts | 12 ---- .../tune/processor/trackTestData.ts | 2 - .../destinations/tune/router/data.ts | 2 - .../destinations/webhook/processor/data.ts | 4 +- test/integrations/destinations/zoho/common.ts | 25 -------- 34 files changed, 44 insertions(+), 406 deletions(-) diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 7d10035233..6fe4dbbb44 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -255,7 +255,6 @@ describe('dedup utility tests', () => { enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'test-rest-api-key', supportDedup: true, trackAnonymousUser: true, diff --git a/test/apitests/data_scenarios/cdk_v2/failure.json b/test/apitests/data_scenarios/cdk_v2/failure.json index 154d24481d..c252761e88 100644 --- a/test/apitests/data_scenarios/cdk_v2/failure.json +++ b/test/apitests/data_scenarios/cdk_v2/failure.json @@ -94,7 +94,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -108,7 +108,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -271,7 +270,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -285,7 +284,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -448,7 +446,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -462,7 +460,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/apitests/data_scenarios/cdk_v2/success.json b/test/apitests/data_scenarios/cdk_v2/success.json index 88f430dd7c..ce819c3f80 100644 --- a/test/apitests/data_scenarios/cdk_v2/success.json +++ b/test/apitests/data_scenarios/cdk_v2/success.json @@ -94,7 +94,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -108,7 +108,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -271,7 +270,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -285,7 +284,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -448,7 +446,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -462,7 +460,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/apitests/data_scenarios/destination/batch/failure_batch.json b/test/apitests/data_scenarios/destination/batch/failure_batch.json index 6352ca1a11..80595e44f4 100644 --- a/test/apitests/data_scenarios/destination/batch/failure_batch.json +++ b/test/apitests/data_scenarios/destination/batch/failure_batch.json @@ -221,8 +221,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -257,7 +256,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -576,8 +574,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -612,7 +609,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -933,8 +929,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -969,7 +964,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -1247,8 +1241,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -1283,7 +1276,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -1722,8 +1714,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -1758,7 +1749,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], diff --git a/test/apitests/data_scenarios/destination/batch/successful_batch.json b/test/apitests/data_scenarios/destination/batch/successful_batch.json index 32745f49d5..45ceb1a545 100644 --- a/test/apitests/data_scenarios/destination/batch/successful_batch.json +++ b/test/apitests/data_scenarios/destination/batch/successful_batch.json @@ -221,8 +221,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -257,7 +256,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -578,8 +576,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -614,7 +611,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -935,8 +931,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -971,7 +966,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], @@ -1557,8 +1551,7 @@ "unsetParamsReferrerOnNewSession", "batchEvents", "eventUploadPeriodMillis", - "eventUploadThreshold", - "oneTrustCookieCategories" + "eventUploadThreshold" ] }, "excludeKeys": [], @@ -1593,7 +1586,6 @@ "useIdfaAsDeviceId", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption", "mapDeviceBrand" ], diff --git a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json index 3deb7d4b8b..c24dd08e8d 100644 --- a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json +++ b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json @@ -94,7 +94,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -108,7 +108,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -271,7 +270,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -285,7 +284,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_failure.json b/test/apitests/data_scenarios/destination/proc/multiplex_failure.json index 68c7fc3baa..d08949ffb6 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_failure.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_failure.json @@ -93,7 +93,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -107,7 +107,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json index a2652855d5..78e067e84b 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json @@ -94,7 +94,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -108,7 +108,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -271,7 +270,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -285,7 +284,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_success.json b/test/apitests/data_scenarios/destination/proc/multiplex_success.json index ba4d5266f3..2b05a2fad8 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_success.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_success.json @@ -94,7 +94,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -108,7 +108,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/apitests/data_scenarios/destination/router/failure_test.json b/test/apitests/data_scenarios/destination/router/failure_test.json index 197456f66a..4b341142e9 100644 --- a/test/apitests/data_scenarios/destination/router/failure_test.json +++ b/test/apitests/data_scenarios/destination/router/failure_test.json @@ -157,7 +157,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -171,7 +171,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -395,7 +394,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -409,7 +408,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -633,7 +631,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -647,7 +645,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -1018,7 +1015,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -1032,7 +1029,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, @@ -1234,7 +1230,7 @@ "whitelistedEvents", "eventFilteringOption" ], - "web": ["useNativeSDK", "oneTrustCookieCategories"] + "web": ["useNativeSDK"] }, "excludeKeys": [], "includeKeys": [ @@ -1248,7 +1244,6 @@ "deduplicationKey", "blacklistedEvents", "whitelistedEvents", - "oneTrustCookieCategories", "eventFilteringOption" ], "saveDestinationResponse": false, diff --git a/test/integrations/destinations/algolia/router/data.ts b/test/integrations/destinations/algolia/router/data.ts index dca899693e..adc7be1596 100644 --- a/test/integrations/destinations/algolia/router/data.ts +++ b/test/integrations/destinations/algolia/router/data.ts @@ -2007,7 +2007,6 @@ export const data = [ { from: 'Order Completed', to: 'conversion' }, { from: 'Product Added', to: 'click' }, ], - oneTrustCookieCategories: [], eventDelivery: false, eventDeliveryTS: 1687213909459, }, @@ -2017,16 +2016,11 @@ export const data = [ DestinationDefinition: { Config: { destConfig: { - defaultConfig: [ - 'apiKey', - 'applicationId', - 'eventTypeSettings', - 'oneTrustCookieCategories', - ], + defaultConfig: ['apiKey', 'applicationId', 'eventTypeSettings'], }, secretKeys: ['apiKey', 'applicationId'], excludeKeys: [], - includeKeys: ['oneTrustCookieCategories'], + includeKeys: [], transformAt: 'router', cdkV2Enabled: true, transformAtV1: 'router', @@ -2146,21 +2140,15 @@ export const data = [ { from: 'Order Completed', to: 'conversion' }, { from: 'Product Added', to: 'click' }, ], - oneTrustCookieCategories: [], }, DestinationDefinition: { Config: { cdkV2Enabled: true, destConfig: { - defaultConfig: [ - 'apiKey', - 'applicationId', - 'eventTypeSettings', - 'oneTrustCookieCategories', - ], + defaultConfig: ['apiKey', 'applicationId', 'eventTypeSettings'], }, excludeKeys: [], - includeKeys: ['oneTrustCookieCategories'], + includeKeys: [], saveDestinationResponse: true, secretKeys: ['apiKey', 'applicationId'], supportedMessageTypes: ['track'], diff --git a/test/integrations/destinations/braze/router/data.ts b/test/integrations/destinations/braze/router/data.ts index b788e22741..8ab04c5d04 100644 --- a/test/integrations/destinations/braze/router/data.ts +++ b/test/integrations/destinations/braze/router/data.ts @@ -432,7 +432,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, @@ -502,7 +501,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, @@ -572,7 +570,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, @@ -626,7 +623,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, @@ -680,7 +676,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, @@ -833,7 +828,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, @@ -869,7 +863,6 @@ export const data = [ enableNestedArrayOperations: false, enableSubscriptionGroupInGroupCall: false, eventFilteringOption: 'disable', - oneTrustCookieCategories: [], restApiKey: 'dummyApiKey', supportDedup: true, trackAnonymousUser: true, diff --git a/test/integrations/destinations/candu/processor/data.ts b/test/integrations/destinations/candu/processor/data.ts index 6e5ae636bd..22bd0b3567 100644 --- a/test/integrations/destinations/candu/processor/data.ts +++ b/test/integrations/destinations/candu/processor/data.ts @@ -21,12 +21,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -182,12 +177,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -347,12 +337,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -478,12 +463,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -619,12 +599,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -757,12 +732,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -894,12 +864,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -1056,12 +1021,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -1221,12 +1181,7 @@ export const data = [ defaultConfig: ['apiKey'], }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], diff --git a/test/integrations/destinations/candu/router/data.ts b/test/integrations/destinations/candu/router/data.ts index e6c7d96a15..02e1caa3e9 100644 --- a/test/integrations/destinations/candu/router/data.ts +++ b/test/integrations/destinations/candu/router/data.ts @@ -20,12 +20,7 @@ export const data = [ Config: { destConfig: { defaultConfig: ['apiKey'] }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], @@ -152,12 +147,7 @@ export const data = [ Config: { destConfig: { defaultConfig: ['apiKey'] }, excludeKeys: [], - includeKeys: [ - 'apiKey', - 'blackListedEvents', - 'whiteListedEvents', - 'oneTrustCookieCategories', - ], + includeKeys: ['apiKey', 'blackListedEvents', 'whiteListedEvents'], saveDestinationResponse: true, secretKeys: ['apiKey'], supportedMessageTypes: ['identify', 'track'], diff --git a/test/integrations/destinations/clicksend/commonConfig.ts b/test/integrations/destinations/clicksend/commonConfig.ts index 815973b8d9..c5c49e2b92 100644 --- a/test/integrations/destinations/clicksend/commonConfig.ts +++ b/test/integrations/destinations/clicksend/commonConfig.ts @@ -14,11 +14,6 @@ export const destination = { defaultSource: 'php', defaultSenderId: 'abc@gmail.com', defaultSenderPhoneNumber: '+919XXXXXXXX8', - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }; diff --git a/test/integrations/destinations/clicksend/router/data.ts b/test/integrations/destinations/clicksend/router/data.ts index 3aa3c0abc4..54018787b0 100644 --- a/test/integrations/destinations/clicksend/router/data.ts +++ b/test/integrations/destinations/clicksend/router/data.ts @@ -18,11 +18,6 @@ const commonDestination = { defaultSenderId: 'abc@gmail.com', defaultSenderPhoneNumber: '+919XXXXXXXX8', defaultSource: 'php', - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, DestinationDefinition: { Config: { diff --git a/test/integrations/destinations/emarsys/deleteUsers/data.ts b/test/integrations/destinations/emarsys/deleteUsers/data.ts index 96b27cad0d..2596e9648c 100644 --- a/test/integrations/destinations/emarsys/deleteUsers/data.ts +++ b/test/integrations/destinations/emarsys/deleteUsers/data.ts @@ -64,11 +64,6 @@ export const data = [ defaultContactList: 'dummy', eventsMapping: commonEventMap, fieldMapping: commonFieldMap, - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, ], @@ -113,11 +108,6 @@ export const data = [ defaultContactList: undefined, eventsMapping: commonEventMap, fieldMapping: commonFieldMap, - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, ], @@ -161,11 +151,6 @@ export const data = [ defaultContactList: 'dummy', eventsMapping: commonEventMap, fieldMapping: commonFieldMap, - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, ], @@ -210,11 +195,6 @@ export const data = [ defaultContactList: 'dummy', eventsMapping: commonEventMap, fieldMapping: commonFieldMap, - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, ], diff --git a/test/integrations/destinations/emarsys/processor/data.ts b/test/integrations/destinations/emarsys/processor/data.ts index badd14e7cc..ddb61b83c7 100644 --- a/test/integrations/destinations/emarsys/processor/data.ts +++ b/test/integrations/destinations/emarsys/processor/data.ts @@ -112,11 +112,6 @@ export const data = [ emersysProperty: '3', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -285,11 +280,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -456,11 +446,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -599,11 +584,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -722,11 +702,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -842,11 +817,6 @@ export const data = [ emersysProperty: '2', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -964,11 +934,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -1093,11 +1058,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -1220,11 +1180,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -1346,11 +1301,6 @@ export const data = [ emersysProperty: 'custom_id', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, @@ -1463,11 +1413,6 @@ export const data = [ emersysProperty: '3', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }, }, diff --git a/test/integrations/destinations/emarsys/router/data.ts b/test/integrations/destinations/emarsys/router/data.ts index 5f6dad1077..4a7bcf9332 100644 --- a/test/integrations/destinations/emarsys/router/data.ts +++ b/test/integrations/destinations/emarsys/router/data.ts @@ -29,11 +29,6 @@ const config = { emersysProperty: '2', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }; const commonDestination = { diff --git a/test/integrations/destinations/fb_custom_audience/router/data.ts b/test/integrations/destinations/fb_custom_audience/router/data.ts index cfc24968a8..834b6315f6 100644 --- a/test/integrations/destinations/fb_custom_audience/router/data.ts +++ b/test/integrations/destinations/fb_custom_audience/router/data.ts @@ -611,7 +611,6 @@ export const data = [ disableFormat: false, isHashRequired: true, isRaw: false, - oneTrustCookieCategories: [], skipVerify: false, subType: 'NA', type: 'NA', @@ -687,7 +686,6 @@ export const data = [ disableFormat: false, isHashRequired: true, isRaw: false, - oneTrustCookieCategories: [], skipVerify: false, subType: 'NA', type: 'NA', @@ -795,7 +793,6 @@ export const data = [ disableFormat: false, isHashRequired: true, isRaw: false, - oneTrustCookieCategories: [], skipVerify: false, subType: 'NA', type: 'NA', @@ -951,7 +948,6 @@ export const data = [ disableFormat: false, isHashRequired: true, isRaw: false, - oneTrustCookieCategories: [], skipVerify: false, subType: 'NA', type: 'NA', diff --git a/test/integrations/destinations/fb_custom_audience/router/rETL.ts b/test/integrations/destinations/fb_custom_audience/router/rETL.ts index 0ba7f8b531..f8d5fc89a0 100644 --- a/test/integrations/destinations/fb_custom_audience/router/rETL.ts +++ b/test/integrations/destinations/fb_custom_audience/router/rETL.ts @@ -8,7 +8,6 @@ const destinationV2: Destination = { disableFormat: false, isHashRequired: true, isRaw: false, - oneTrustCookieCategories: [], skipVerify: false, subType: 'NA', type: 'NA', @@ -167,7 +166,6 @@ export const destinationV1: Destination = { disableFormat: false, isHashRequired: true, isRaw: false, - oneTrustCookieCategories: [], skipVerify: false, subType: 'NA', type: 'NA', diff --git a/test/integrations/destinations/gainsight_px/router/data.ts b/test/integrations/destinations/gainsight_px/router/data.ts index 1b3d5be875..3a5255f7d2 100644 --- a/test/integrations/destinations/gainsight_px/router/data.ts +++ b/test/integrations/destinations/gainsight_px/router/data.ts @@ -57,7 +57,7 @@ const destination2 = { { from: 'inboxready_signup_date', to: 'inboxready_signup_date' }, { from: 'gpt_setup', to: 'gpt_setup' }, ], - oneTrustCookieCategories: [], + apiKey: 'sample-api-key', eventDelivery: false, eventDeliveryTS: 1624472902670, diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index 82c8e9b3ff..bcc718485b 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -314,7 +314,6 @@ export const data = [ { from: 'Product Clicked', to: 'Store sales' }, ], authStatus: 'active', - oneTrustCookieCategories: [], customVariables: [{ from: '', to: '' }], }, }, @@ -390,7 +389,6 @@ export const data = [ { from: 'Product Clicked', to: 'Store sales' }, ], authStatus: 'active', - oneTrustCookieCategories: [], customVariables: [{ from: '', to: '' }], }, }, @@ -464,7 +462,6 @@ export const data = [ { from: 'Product Clicked', to: 'Store sales' }, ], authStatus: 'active', - oneTrustCookieCategories: [], customVariables: [{ from: '', to: '' }], }, }, @@ -674,7 +671,6 @@ export const data = [ { from: 'Product Clicked', to: 'Store sales' }, ], authStatus: 'active', - oneTrustCookieCategories: [], customVariables: [{ from: '', to: '' }], }, }, @@ -986,7 +982,6 @@ export const data = [ { from: 'Product Clicked', to: 'Store sales' }, ], authStatus: 'active', - oneTrustCookieCategories: [], customVariables: [{ from: '', to: '' }], }, }, @@ -1152,7 +1147,6 @@ export const data = [ defaultUserIdentifier: 'email', hashUserIdentifier: true, validateOnly: false, - oneTrustCookieCategories: [], eventDelivery: false, eventDeliveryTS: 1715104236592, rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', @@ -1265,7 +1259,6 @@ export const data = [ defaultUserIdentifier: 'email', hashUserIdentifier: true, validateOnly: false, - oneTrustCookieCategories: [], eventDelivery: false, eventDeliveryTS: 1715104236592, rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', diff --git a/test/integrations/destinations/hs/router/data.ts b/test/integrations/destinations/hs/router/data.ts index b47d6b7f07..2f0879528b 100644 --- a/test/integrations/destinations/hs/router/data.ts +++ b/test/integrations/destinations/hs/router/data.ts @@ -1988,13 +1988,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2070,13 +2064,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2150,13 +2138,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2231,13 +2213,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2330,13 +2306,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2425,13 +2395,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2511,13 +2475,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2621,13 +2579,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2702,13 +2654,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, @@ -2801,13 +2747,7 @@ export const data = [ rsEventName: 'Order Complete', }, ], - ketchConsentPurposes: [ - { - purpose: '', - }, - ], lookupField: 'email', - oneTrustCookieCategories: [], useNativeSDK: false, whitelistedEvents: [], }, diff --git a/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts b/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts index 287e35e5a7..29ff1f10a8 100644 --- a/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts +++ b/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts @@ -36,11 +36,6 @@ const commonDestination: Destination = { to: '34567', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, Enabled: true, }; diff --git a/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts b/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts index f9dfc528db..53272e73bf 100644 --- a/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts +++ b/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts @@ -32,11 +32,6 @@ const commonDestination: Destination = { to: '34567', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, Enabled: true, }; diff --git a/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts b/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts index 5cb6ff8cf2..653ad32056 100644 --- a/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts +++ b/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts @@ -36,11 +36,6 @@ const commonDestination: Destination = { to: '34567', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, Enabled: true, }; diff --git a/test/integrations/destinations/linkedin_ads/router/data.ts b/test/integrations/destinations/linkedin_ads/router/data.ts index cf7defe6af..16abc0cd06 100644 --- a/test/integrations/destinations/linkedin_ads/router/data.ts +++ b/test/integrations/destinations/linkedin_ads/router/data.ts @@ -20,11 +20,6 @@ const config = { to: '34567', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }; const commonDestination = { diff --git a/test/integrations/destinations/rockerbox/processor/data.ts b/test/integrations/destinations/rockerbox/processor/data.ts index 76dd8ef11b..4a64ff09f6 100644 --- a/test/integrations/destinations/rockerbox/processor/data.ts +++ b/test/integrations/destinations/rockerbox/processor/data.ts @@ -85,13 +85,6 @@ export const data = [ clientAuthId: { web: 'test-client-auth-id', }, - oneTrustCookieCategories: { - web: [ - { - oneTrustCookieCategory: 'Marketing Sample', - }, - ], - }, customDomain: { web: 'https://cookiedomain.com', }, @@ -206,13 +199,6 @@ export const data = [ clientAuthId: { web: 'test-client-auth-id', }, - oneTrustCookieCategories: { - web: [ - { - oneTrustCookieCategory: 'Marketing Sample', - }, - ], - }, customDomain: { web: 'https://cookiedomain.com', }, @@ -339,13 +325,6 @@ export const data = [ clientAuthId: { web: 'test-client-auth-id', }, - oneTrustCookieCategories: { - web: [ - { - oneTrustCookieCategory: 'Marketing Sample', - }, - ], - }, customDomain: { web: 'https://cookiedomain.com', }, @@ -501,13 +480,6 @@ export const data = [ clientAuthId: { web: '', }, - oneTrustCookieCategories: { - web: [ - { - oneTrustCookieCategory: '', - }, - ], - }, customDomain: { web: '', }, diff --git a/test/integrations/destinations/rockerbox/router/data.ts b/test/integrations/destinations/rockerbox/router/data.ts index af943d7e6b..535b4f7da5 100644 --- a/test/integrations/destinations/rockerbox/router/data.ts +++ b/test/integrations/destinations/rockerbox/router/data.ts @@ -19,9 +19,6 @@ export const data = [ eventsMap: [{ from: 'Product Added', to: 'conv.add_to_cart' }], useNativeSDK: { web: false }, clientAuthId: { web: 'test-client-auth-id' }, - oneTrustCookieCategories: { - web: [{ oneTrustCookieCategory: 'Marketing Sample' }], - }, customDomain: { web: 'https://cookiedomain.com' }, enableCookieSync: { web: true }, }, @@ -108,9 +105,6 @@ export const data = [ enableCookieSync: { web: true }, eventFilteringOption: 'disable', eventsMap: [{ from: 'Product Added', to: 'conv.add_to_cart' }], - oneTrustCookieCategories: { - web: [{ oneTrustCookieCategory: 'Marketing Sample' }], - }, useNativeSDK: { web: false }, whitelistedEvents: [{ eventName: '' }], }, @@ -143,9 +137,6 @@ export const data = [ eventsMap: [{ from: 'Product Viewed', to: 'conv.add_to_cart' }], useNativeSDK: { web: false }, clientAuthId: { web: 'test-client-auth-id' }, - oneTrustCookieCategories: { - web: [{ oneTrustCookieCategory: 'Marketing Sample' }], - }, customDomain: { web: 'https://cookiedomain.com' }, enableCookieSync: { web: true }, }, @@ -232,9 +223,6 @@ export const data = [ enableCookieSync: { web: true }, eventFilteringOption: 'disable', eventsMap: [{ from: 'Product Viewed', to: 'conv.add_to_cart' }], - oneTrustCookieCategories: { - web: [{ oneTrustCookieCategory: 'Marketing Sample' }], - }, useNativeSDK: { web: false }, whitelistedEvents: [{ eventName: '' }], }, diff --git a/test/integrations/destinations/tune/processor/trackTestData.ts b/test/integrations/destinations/tune/processor/trackTestData.ts index 63178d0e33..d9bfab54e3 100644 --- a/test/integrations/destinations/tune/processor/trackTestData.ts +++ b/test/integrations/destinations/tune/processor/trackTestData.ts @@ -22,8 +22,6 @@ const destination: Destination = { }, subdomain: 'demo', consentManagement: {}, - oneTrustCookieCategories: {}, - ketchConsentPurposes: {}, tuneEvents: [ { eventName: 'Product added', diff --git a/test/integrations/destinations/tune/router/data.ts b/test/integrations/destinations/tune/router/data.ts index f21714d058..4aee5b8967 100644 --- a/test/integrations/destinations/tune/router/data.ts +++ b/test/integrations/destinations/tune/router/data.ts @@ -17,8 +17,6 @@ const destination: Destination = { }, subdomain: 'demo', consentManagement: {}, - oneTrustCookieCategories: {}, - ketchConsentPurposes: {}, tuneEvents: [ { eventName: 'Product added', diff --git a/test/integrations/destinations/webhook/processor/data.ts b/test/integrations/destinations/webhook/processor/data.ts index 92fe8f50d9..7720bc683c 100644 --- a/test/integrations/destinations/webhook/processor/data.ts +++ b/test/integrations/destinations/webhook/processor/data.ts @@ -3285,7 +3285,7 @@ export const data = [ Config: { webhookUrl: 'https://webhook.site/81dc2730-807f-4bbc-8914-5b37d21c8a55', webhookMethod: 'POST', - oneTrustCookieCategories: [], + connectionMode: 'cloud', eventDelivery: false, eventDeliveryTS: 1720497286192, @@ -3294,7 +3294,7 @@ export const data = [ Config: { secretKeys: ['headers.to'], excludeKeys: [], - includeKeys: ['oneTrustCookieCategories', 'consentManagement'], + includeKeys: ['consentManagement'], cdkV2Enabled: true, transformAtV1: 'processor', isAudienceSupported: true, diff --git a/test/integrations/destinations/zoho/common.ts b/test/integrations/destinations/zoho/common.ts index bea4437e6f..1d89dbce6d 100644 --- a/test/integrations/destinations/zoho/common.ts +++ b/test/integrations/destinations/zoho/common.ts @@ -65,11 +65,6 @@ const commonDeletionDestConfig: Destination = { to: 'false', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }; @@ -186,11 +181,6 @@ const commonUpsertDestConfig: Destination = { to: 'false', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }; @@ -225,11 +215,6 @@ const commonUpsertDestConfig2: Destination = { to: 'false', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }; @@ -264,11 +249,6 @@ const commonUpsertDestConfig2CustomModule: Destination = { to: 'false', }, ], - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }; @@ -293,11 +273,6 @@ const commonUpsertDestConfig3: Destination = { module: 'Leads', trigger: 'workflow', addDefaultDuplicateCheck: true, - oneTrustCookieCategories: [ - { - oneTrustCookieCategory: 'Marketing', - }, - ], }, }; From b7692c85adfb4c495ef12e1b31c6764b859ffc67 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Date: Wed, 11 Dec 2024 11:13:38 +0530 Subject: [PATCH 19/21] chore: silent testcase logs in local (#3932) --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index e0b17a4f47..cb7379bb0f 100644 --- a/package.json +++ b/package.json @@ -36,11 +36,11 @@ "test": "jest -c jest.config.js --detectOpenHandles", "test:ci": "npm run test -- --coverage --expand --maxWorkers=50%", "test:js": "jest -c jest.default.config.js --detectOpenHandles", - "test:js:silent": "npm run test:js -- --silent", + "test:js:silent": "export LOG_LEVEL=silent && npm run test:js -- --silent", "test:js:ci": "npm run test:js -- --coverage --expand --maxWorkers=50%", "test:ts": "jest -c jest.config.typescript.js --detectOpenHandles", "test:ts:component:generateNwMocks": "npm run test:ts -- component --generate=true", - "test:ts:silent": "npm run test:ts -- --silent", + "test:ts:silent": "export LOG_LEVEL=silent && npm run test:ts -- --silent", "test:ts:ci": "npm run test:ts -- --coverage --expand --maxWorkers=50%", "test:ut:integration": "jest \"user_transformation.integration.test.js\" --detectOpenHandles --notify", "test:ut:integration:silent": "npm run test:ut:integration -- --silent", From d40db6c1e7f71c5ab5fc3a3659d1fc51b6d527fa Mon Sep 17 00:00:00 2001 From: Manish Kumar <144022547+manish339k@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:43:17 +0530 Subject: [PATCH 20/21] fix: handling partial error in a batch for reddit destination (#3935) * fix: handling partial error in a batch for reddit destination * fix: minor change * fix: moving networkHandler to v1 from v0 * fix: sonarcloud code analysis --- .../v2/destinations/reddit/rtWorkflow.yaml | 6 +- src/cdk/v2/destinations/reddit/utils.js | 1 + .../destinations/reddit/networkHandler.js | 65 +++-- .../reddit/dataDelivery/business.ts | 136 ++++++++- .../destinations/reddit/dataDelivery/oauth.ts | 10 +- .../destinations/reddit/network.ts | 107 +++++++ .../destinations/reddit/router/data.ts | 266 ++++++++++++++++++ 7 files changed, 566 insertions(+), 25 deletions(-) rename src/{v0 => v1}/destinations/reddit/networkHandler.js (52%) diff --git a/src/cdk/v2/destinations/reddit/rtWorkflow.yaml b/src/cdk/v2/destinations/reddit/rtWorkflow.yaml index 937ff021f4..fd315b381a 100644 --- a/src/cdk/v2/destinations/reddit/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/reddit/rtWorkflow.yaml @@ -36,7 +36,11 @@ steps: description: Batches the successfulEvents using endpoint condition: $.outputs.successfulEvents.length template: | - let batches = $.batchEvents($.outputs.successfulEvents); + const dontBatchTrueEvents = $.outputs.successfulEvents{.metadata.dontBatch}[]; + const dontBatchFalseEvents = $.outputs.successfulEvents{!.metadata.dontBatch}[]; + const dontBatchTrueEventsChunks = $.chunk(dontBatchTrueEvents, 1); + + let batches = [...$.batchEvents(dontBatchFalseEvents), ...$.batchEventChunks(dontBatchTrueEventsChunks)]; batches@batch.({ "batchedRequest": { "body": { diff --git a/src/cdk/v2/destinations/reddit/utils.js b/src/cdk/v2/destinations/reddit/utils.js index f562d31313..e51a72142a 100644 --- a/src/cdk/v2/destinations/reddit/utils.js +++ b/src/cdk/v2/destinations/reddit/utils.js @@ -81,6 +81,7 @@ const populateRevenueField = (eventType, properties) => { }; module.exports = { batchEvents, + batchEventChunks, populateRevenueField, calculateDefaultRevenue, }; diff --git a/src/v0/destinations/reddit/networkHandler.js b/src/v1/destinations/reddit/networkHandler.js similarity index 52% rename from src/v0/destinations/reddit/networkHandler.js rename to src/v1/destinations/reddit/networkHandler.js index 7c9b32eaa4..608408a7fb 100644 --- a/src/v0/destinations/reddit/networkHandler.js +++ b/src/v1/destinations/reddit/networkHandler.js @@ -1,16 +1,31 @@ const { RetryableError, TAG_NAMES, NetworkError } = require('@rudderstack/integrations-lib'); const isString = require('lodash/isString'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); -const { isHttpStatusSuccess } = require('../../util/index'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); const { REFRESH_TOKEN } = require('../../../adapters/networkhandler/authConstants'); const { processAxiosResponse, getDynamicErrorType, } = require('../../../adapters/utils/networkUtils'); +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); -const redditRespHandler = (destResponse) => { - const { status, response } = destResponse; +const populateResponseWithDontBatch = (rudderJobMetadata, response) => { + const errorMessage = JSON.stringify(response); + return rudderJobMetadata.map((metadata) => { + // eslint-disable-next-line no-param-reassign + metadata.dontBatch = true; + return { + statusCode: 500, + metadata, + error: errorMessage, + }; + }); +}; + +const redditRespHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + const { status, response } = destinationResponse; // to handle the case when authorization-token is invalid if (status === 401) { @@ -28,26 +43,41 @@ const redditRespHandler = (destResponse) => { throw new RetryableError( `${errorMessage} during reddit response transformation`, status, - destResponse, + destinationResponse, authErrorCategory, ); } + if (status === 400 && Array.isArray(rudderJobMetadata) && rudderJobMetadata.length > 1) { + // sending back 500 for retry only when events came in a batch + const responseWithDontBatch = populateResponseWithDontBatch(rudderJobMetadata, response); + throw new TransformerProxyError( + `REDDIT: Error transformer proxy during REDDIT response transformation`, + 500, + { + [TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(500), + }, + destinationResponse, + '', + responseWithDontBatch, + ); + } + throw new NetworkError( `${JSON.stringify(response)} during reddit response transformation`, status, { [TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), }, - destResponse, + destinationResponse, ); }; const responseHandler = (responseParams) => { - const { destinationResponse } = responseParams; + const { destinationResponse, rudderJobMetadata } = responseParams; const message = `Request Processed Successfully`; const { status } = destinationResponse; if (!isHttpStatusSuccess(status)) { // if error, successfully return status, message and original destination response - redditRespHandler(destinationResponse); + redditRespHandler(responseParams); } const { response } = destinationResponse; const errorMessage = @@ -55,20 +85,23 @@ const responseHandler = (responseParams) => { ? response?.invalid_events[0]?.error_message : null; const destResp = errorMessage || destinationResponse; + const responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ + statusCode: 200, + metadata, + error: 'success', + })); // Mostly any error will not have a status of 2xx return { status, message, - destResp, + destinationResponse: destResp, + response: responseWithIndividualEvents, }; }; -// eslint-disable-next-line @typescript-eslint/naming-convention -class networkHandler { - constructor() { - this.responseHandler = responseHandler; - this.proxy = proxyRequest; - this.prepareProxy = prepareProxyRequest; - this.processAxiosResponse = processAxiosResponse; - } +function networkHandler() { + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.prepareProxy = prepareProxyRequest; + this.responseHandler = responseHandler; } module.exports = { networkHandler }; diff --git a/test/integrations/destinations/reddit/dataDelivery/business.ts b/test/integrations/destinations/reddit/dataDelivery/business.ts index b48004a2ed..cc2feccffa 100644 --- a/test/integrations/destinations/reddit/dataDelivery/business.ts +++ b/test/integrations/destinations/reddit/dataDelivery/business.ts @@ -39,6 +39,71 @@ const validRequestPayload = { ], }; +const validRequestMultipleEventsInPayload = { + events: [ + { + event_at: '2019-10-14T09:03:17.562Z', + event_type: { + tracking_type: 'Purchase', + }, + user: { + aaid: 'c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a', + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 3, + products: [ + { + id: '123', + name: 'Monopoly', + category: 'Games', + }, + { + id: '345', + name: 'UNO', + category: 'Games', + }, + ], + }, + }, + { + event_at: '2018-10-14T09:03:17.562Z', + event_type: { + tracking_type: 'Purchase', + }, + user: { + aaid: 'c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a', + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 3, + products: [ + { + id: '123', + name: 'Monopoly', + category: 'Games', + }, + { + id: '345', + name: 'UNO', + category: 'Games', + }, + ], + }, + }, + ], +}; + const commonHeaders = { Authorization: 'Bearer dummyAccessToken', 'Content-Type': 'application/json', @@ -73,13 +138,14 @@ export const testScenariosForV0API = [ status: 200, body: { output: { - destResp: { + destinationResponse: { response: { message: 'Successfully processed 1 conversion events.', }, status: 200, }, - message: 'Request Processed Successfully', + message: + '[Generic Response Handler] Request for destination: reddit Processed Successfully', status: 200, }, }, @@ -116,8 +182,15 @@ export const testScenariosForV1API = [ body: { output: { message: 'Request Processed Successfully', + destinationResponse: { + response: { + message: 'Successfully processed 1 conversion events.', + }, + status: 200, + }, response: [ { + error: 'success', metadata: generateMetadata(1), statusCode: 200, }, @@ -180,4 +253,63 @@ export const testScenariosForV1API = [ }, }, }, + { + id: 'reddit_v1_scenario_3', + name: 'reddit', + description: + '[Proxy v1 API] :: Test for a valid request with a partial event failure from the destination', + scenario: 'PartialFailure', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers: commonHeaders, + JSON: validRequestMultipleEventsInPayload, + endpoint: + 'https://ads-api.reddit.com/api/v2.0/conversions/events/partial_failed_events', + }, + [generateMetadata(1), generateMetadata(2)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: 'REDDIT: Error transformer proxy during REDDIT response transformation', + response: [ + { + metadata: { ...generateMetadata(1), dontBatch: true }, + statusCode: 500, + error: + '{"message":"There were 1 invalid conversion events. None were processed.","invalid_events":[{"error_message":"event_at timestamp must be less than 168h0m0s old","event":{"event_at":"2018-10-14T09:03:17.562Z","event_type":{"tracking_type":"Purchase"},"event_metadata":{"item_count":0,"products":[{}],"conversion_id":"c054005afd85a4de74638a776eb8348d44ee875184d7a401830705b7a06e7df1"},"user":{"aaid":"c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a","email":"ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2","external_id":"7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d","ip_address":"e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36","screen_dimensions":{}}}}]}', + }, + { + metadata: { ...generateMetadata(2), dontBatch: true }, + statusCode: 500, + error: + '{"message":"There were 1 invalid conversion events. None were processed.","invalid_events":[{"error_message":"event_at timestamp must be less than 168h0m0s old","event":{"event_at":"2018-10-14T09:03:17.562Z","event_type":{"tracking_type":"Purchase"},"event_metadata":{"item_count":0,"products":[{}],"conversion_id":"c054005afd85a4de74638a776eb8348d44ee875184d7a401830705b7a06e7df1"},"user":{"aaid":"c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a","email":"ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2","external_id":"7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d","ip_address":"e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36","screen_dimensions":{}}}}]}', + }, + ], + statTags: { + destType: 'REDDIT', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + status: 500, + }, + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/reddit/dataDelivery/oauth.ts b/test/integrations/destinations/reddit/dataDelivery/oauth.ts index 8c0d486a7a..e3c5875a06 100644 --- a/test/integrations/destinations/reddit/dataDelivery/oauth.ts +++ b/test/integrations/destinations/reddit/dataDelivery/oauth.ts @@ -84,14 +84,13 @@ export const v0oauthScenarios = [ status: 401, body: { output: { - authErrorCategory: 'REFRESH_TOKEN', destinationResponse: { response: 'Authorization Required', status: 401, }, message: - 'Request failed due to Authorization Required during reddit response transformation', - statTags: expectedStatTags, + '[Generic Response Handler] Request failed for destination reddit with status: 401', + statTags: { ...expectedStatTags, errorType: 'aborted' }, status: 401, }, }, @@ -121,7 +120,6 @@ export const v0oauthScenarios = [ status: 401, body: { output: { - authErrorCategory: 'REFRESH_TOKEN', destinationResponse: { response: { success: false, @@ -134,8 +132,8 @@ export const v0oauthScenarios = [ status: 401, }, message: - 'This server could not verify that you are authorized to access the document you requested. during reddit response transformation', - statTags: expectedStatTags, + '[Generic Response Handler] Request failed for destination reddit with status: 401', + statTags: { ...expectedStatTags, errorType: 'aborted' }, status: 401, }, }, diff --git a/test/integrations/destinations/reddit/network.ts b/test/integrations/destinations/reddit/network.ts index 80c18c00a0..54d7a95b73 100644 --- a/test/integrations/destinations/reddit/network.ts +++ b/test/integrations/destinations/reddit/network.ts @@ -210,4 +210,111 @@ export const networkCallsData = [ statusText: 'Unauthorized', }, }, + { + httpReq: { + url: 'https://ads-api.reddit.com/api/v2.0/conversions/events/partial_failed_events', + data: { + events: [ + { + event_at: '2019-10-14T09:03:17.562Z', + event_type: { + tracking_type: 'Purchase', + }, + user: { + aaid: 'c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a', + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 3, + products: [ + { + id: '123', + name: 'Monopoly', + category: 'Games', + }, + { + id: '345', + name: 'UNO', + category: 'Games', + }, + ], + }, + }, + { + event_at: '2018-10-14T09:03:17.562Z', + event_type: { + tracking_type: 'Purchase', + }, + user: { + aaid: 'c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a', + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 3, + products: [ + { + id: '123', + name: 'Monopoly', + category: 'Games', + }, + { + id: '345', + name: 'UNO', + category: 'Games', + }, + ], + }, + }, + ], + }, + params: { destination: 'reddit' }, + headers: { + Authorization: 'Bearer dummyAccessToken', + 'Content-Type': 'application/json', + }, + method: 'POST', + }, + httpRes: { + data: { + message: 'There were 1 invalid conversion events. None were processed.', + invalid_events: [ + { + error_message: 'event_at timestamp must be less than 168h0m0s old', + event: { + event_at: '2018-10-14T09:03:17.562Z', + event_type: { + tracking_type: 'Purchase', + }, + event_metadata: { + item_count: 0, + products: [{}], + conversion_id: 'c054005afd85a4de74638a776eb8348d44ee875184d7a401830705b7a06e7df1', + }, + user: { + aaid: 'c12d34889302d3c656b5699fa9190b51c50d6f62fce57e13bd56b503d66c487a', + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + }, + }, + ], + }, + status: 400, + statusText: 'Bad Request', + }, + }, ]; diff --git a/test/integrations/destinations/reddit/router/data.ts b/test/integrations/destinations/reddit/router/data.ts index b5bed48ae7..f2dd887b84 100644 --- a/test/integrations/destinations/reddit/router/data.ts +++ b/test/integrations/destinations/reddit/router/data.ts @@ -178,6 +178,131 @@ export const data = [ userId: 'u1', }, }, + { + message: { + context: { + traits: { email: 'testone@gmail.com' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + ip: '54.100.200.255', + }, + type: 'track', + session_id: '16733896350494', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'Order Completed', + userId: 'testuserId1', + properties: { + checkout_id: '12345', + order_id: '1234', + affiliation: 'Apple Store', + total: 20, + revenue: 15, + shipping: 4, + tax: 1, + discount: 1.5, + coupon: 'ImagePro', + currency: 'USD', + products: [ + { + product_id: '123', + sku: 'G-32', + name: 'Monopoly', + price: 14, + quantity: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '345', + sku: 'F-32', + name: 'UNO', + price: 3.45, + quantity: 2, + category: 'Games', + }, + ], + }, + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { + accountId: 'a2_fsddXXXfsfd', + hashData: true, + eventsMapping: [{ from: 'Order Completed', to: 'Purchase' }], + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + secret: { accessToken: 'dummyAccessToken' }, + jobId: 4, + userId: 'u1', + dontBatch: true, + }, + }, + { + message: { + context: { + traits: { email: 'testone@gmail.com' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + ip: '54.100.200.255', + }, + type: 'track', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + event: 'Product List Viewed', + userId: 'testuserId1', + properties: { + list_id: 'list1', + category: "What's New", + value: 2600, + value_decimal: 26, + products: [ + { + product_id: '017c6f5d5cf86a4b22432066', + sku: '8732-98', + name: 'Just Another Game', + price: 22, + position: 2, + category: 'Games and Entertainment', + url: 'https://www.myecommercewebsite.com/product', + image_url: 'https://www.myecommercewebsite.com/product/path.jpg', + }, + { + product_id: '89ac6f5d5cf86a4b64eac145', + sku: '1267-01', + name: 'Wrestling Trump Cards', + price: 4, + position: 21, + category: 'Card Games', + }, + ], + }, + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { + accountId: 'a2_fsddXXXfsfd', + hashData: true, + eventsMapping: [{ from: 'Order Completed', to: 'Purchase' }], + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + secret: { accessToken: 'dummyAccessToken' }, + jobId: 5, + userId: 'u1', + dontBatch: true, + }, + }, ], destType: 'reddit', }, @@ -325,6 +450,147 @@ export const data = [ DestinationDefinition: { Config: { cdkV2Enabled: true } }, }, }, + { + batchedRequest: { + body: { + JSON: { + events: [ + { + event_at: '2019-10-14T09:03:17.562Z', + event_type: { tracking_type: 'Purchase' }, + user: { + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: + '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: + 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 2, + currency: 'USD', + value: 1500, + value_decimal: 15, + products: [ + { id: '123', name: 'Monopoly', category: 'Games' }, + { id: '345', name: 'UNO', category: 'Games' }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://ads-api.reddit.com/api/v2.0/conversions/events/a2_fsddXXXfsfd', + headers: { + Authorization: 'Bearer dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + files: {}, + }, + metadata: [ + { + destinationId: 'destId', + workspaceId: 'wspId', + secret: { accessToken: 'dummyAccessToken' }, + jobId: 4, + userId: 'u1', + dontBatch: true, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + accountId: 'a2_fsddXXXfsfd', + hashData: true, + eventsMapping: [{ from: 'Order Completed', to: 'Purchase' }], + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + { + batchedRequest: { + body: { + JSON: { + events: [ + { + event_at: '2019-10-14T09:03:17.562Z', + event_type: { tracking_type: 'ViewContent' }, + user: { + email: 'ac144532d9e4efeab19475d9253a879173ea12a3d2238d1cb8a332a7b3a105f2', + external_id: + '7b023241a3132b792a5a33915a5afb3133cbb1e13d72879689bf6504de3b036d', + ip_address: + 'e80bd55a3834b7c2a34ade23c7ecb54d2a49838227080f50716151e765a619db', + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + screen_dimensions: {}, + }, + event_metadata: { + item_count: 0, + value: 2600, + value_decimal: 26, + products: [ + { + id: '017c6f5d5cf86a4b22432066', + name: 'Just Another Game', + category: 'Games and Entertainment', + }, + { + id: '89ac6f5d5cf86a4b64eac145', + name: 'Wrestling Trump Cards', + category: 'Card Games', + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://ads-api.reddit.com/api/v2.0/conversions/events/a2_fsddXXXfsfd', + headers: { + Authorization: 'Bearer dummyAccessToken', + 'Content-Type': 'application/json', + }, + params: {}, + files: {}, + }, + metadata: [ + { + destinationId: 'destId', + workspaceId: 'wspId', + secret: { accessToken: 'dummyAccessToken' }, + jobId: 5, + userId: 'u1', + dontBatch: true, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + accountId: 'a2_fsddXXXfsfd', + hashData: true, + eventsMapping: [{ from: 'Order Completed', to: 'Purchase' }], + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, ], }, }, From 227419f1ff618f96aafa849862828e2315e4ac55 Mon Sep 17 00:00:00 2001 From: Aanshi Lahoti <110057617+aanshi07@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:42:46 +0530 Subject: [PATCH 21/21] feat: onboard topsort destination (#3913) * chore: topsort changes * chore: boilerplates * chore: topsort changes * chore: function updates * chore: product array changes * chore: purchase event implementation * chore: purchase function updated * chore: build finalPayload * chore: few updations * chore: mockfn changes * chore: restructured files - moved impressions and clicks logic into its own file - moved purchase related logic into its own file * chore: update formatting * chore: test cases added * chore: destType updated --------- Co-authored-by: Utsab Chowdhury Co-authored-by: Sai Sankeerth --- src/features.ts | 1 + src/v0/destinations/topsort/config.js | 31 + .../data/TopSortPurchaseProductConfig.json | 24 + .../topsort/data/TopsortItemConfig.json | 15 + .../topsort/data/TopsortPlacementConfig.json | 36 ++ .../topsort/data/TopsortTrackConfig.json | 27 + .../topsort/impressions-and-clicks.js | 95 +++ src/v0/destinations/topsort/purchase.js | 56 ++ src/v0/destinations/topsort/transform.js | 158 +++++ src/v0/destinations/topsort/utils.js | 53 ++ src/v0/util/index.js | 8 + .../destinations/topsort/mocks.ts | 5 + .../destinations/topsort/processor/data.ts | 9 + .../topsort/processor/trackClicksTestData.ts | 571 ++++++++++++++++++ .../processor/trackImpressionsTestData.ts | 381 ++++++++++++ .../processor/trackPurchasesTestData.ts | 457 ++++++++++++++ .../destinations/topsort/router/data.ts | 418 +++++++++++++ test/integrations/testTypes.ts | 2 +- test/integrations/testUtils.ts | 1 + 19 files changed, 2347 insertions(+), 1 deletion(-) create mode 100644 src/v0/destinations/topsort/config.js create mode 100644 src/v0/destinations/topsort/data/TopSortPurchaseProductConfig.json create mode 100644 src/v0/destinations/topsort/data/TopsortItemConfig.json create mode 100644 src/v0/destinations/topsort/data/TopsortPlacementConfig.json create mode 100644 src/v0/destinations/topsort/data/TopsortTrackConfig.json create mode 100644 src/v0/destinations/topsort/impressions-and-clicks.js create mode 100644 src/v0/destinations/topsort/purchase.js create mode 100644 src/v0/destinations/topsort/transform.js create mode 100644 src/v0/destinations/topsort/utils.js create mode 100644 test/integrations/destinations/topsort/mocks.ts create mode 100644 test/integrations/destinations/topsort/processor/data.ts create mode 100644 test/integrations/destinations/topsort/processor/trackClicksTestData.ts create mode 100644 test/integrations/destinations/topsort/processor/trackImpressionsTestData.ts create mode 100644 test/integrations/destinations/topsort/processor/trackPurchasesTestData.ts create mode 100644 test/integrations/destinations/topsort/router/data.ts diff --git a/src/features.ts b/src/features.ts index 4ff419a7fe..fb91ae2883 100644 --- a/src/features.ts +++ b/src/features.ts @@ -93,6 +93,7 @@ const defaultFeaturesConfig: FeaturesConfig = { AMAZON_AUDIENCE: true, INTERCOM_V2: true, LINKEDIN_AUDIENCE: true, + TOPSORT: true, }, regulations: [ 'BRAZE', diff --git a/src/v0/destinations/topsort/config.js b/src/v0/destinations/topsort/config.js new file mode 100644 index 0000000000..5d4060f526 --- /dev/null +++ b/src/v0/destinations/topsort/config.js @@ -0,0 +1,31 @@ +const { getMappingConfig } = require('../../util'); + +const ENDPOINT = 'https://api.topsort.com/v2/events'; + +const ConfigCategory = { + TRACK: { + type: 'track', + name: 'TopsortTrackConfig', + }, + PLACEMENT: { name: 'TopsortPlacementConfig' }, + ITEM: { name: 'TopsortItemConfig' }, + PURCHASE_ITEM: { name: 'TopSortPurchaseProductConfig' }, +}; + +const ECOMM_EVENTS_WITH_PRODUCT_ARRAY = [ + 'Cart Viewed', + 'Checkout Started', + 'Order Updated', + 'Order Completed', + 'Order Refunded', + 'Order Cancelled', +]; + +const mappingConfig = getMappingConfig(ConfigCategory, __dirname); + +module.exports = { + mappingConfig, + ConfigCategory, + ENDPOINT, + ECOMM_EVENTS_WITH_PRODUCT_ARRAY, +}; diff --git a/src/v0/destinations/topsort/data/TopSortPurchaseProductConfig.json b/src/v0/destinations/topsort/data/TopSortPurchaseProductConfig.json new file mode 100644 index 0000000000..f3f9d877a9 --- /dev/null +++ b/src/v0/destinations/topsort/data/TopSortPurchaseProductConfig.json @@ -0,0 +1,24 @@ +[ + { + "destKey": "productId", + "sourceKeys": ["product_id", "properties.product_id"] + }, + { + "destKey": "unitPrice", + "sourceKeys": ["price", "properties.price"], + "metadata": { + "type": "toNumber" + } + }, + { + "destKey": "quantity", + "sourceKeys": ["quantity", "properties.quantity"], + "metadata": { + "toInt": true + } + }, + { + "destKey": "vendorId", + "sourceKeys": "properties.vendorId" + } +] diff --git a/src/v0/destinations/topsort/data/TopsortItemConfig.json b/src/v0/destinations/topsort/data/TopsortItemConfig.json new file mode 100644 index 0000000000..ff8c77a7ac --- /dev/null +++ b/src/v0/destinations/topsort/data/TopsortItemConfig.json @@ -0,0 +1,15 @@ +[ + { + "destKey": "position", + "sourceKeys": ["properties.position", "position"], + "metadata": { + "toInt": true + }, + "required": false + }, + { + "destKey": "productId", + "sourceKeys": ["properties.product_id", "product_id"], + "required": false + } +] diff --git a/src/v0/destinations/topsort/data/TopsortPlacementConfig.json b/src/v0/destinations/topsort/data/TopsortPlacementConfig.json new file mode 100644 index 0000000000..7c67bb183d --- /dev/null +++ b/src/v0/destinations/topsort/data/TopsortPlacementConfig.json @@ -0,0 +1,36 @@ +[ + { + "destKey": "path", + "sourceKeys": "context.page.path", + "required": false + }, + { + "destKey": "searchQuery", + "sourceKeys": "properties.query", + "required": false + }, + { + "destKey": "page", + "sourceKeys": "properties.pageNumber", + "metadata": { + "toInt": true + }, + "required": false + }, + { + "destKey": "pageSize", + "sourceKeys": "properties.pageSize", + "metadata": { + "toInt": true + }, + "required": false + }, + { + "destKey": "categoryIds", + "sourceKeys": "properties.category_id", + "required": false, + "metadata": { + "toArray": true + } + } +] diff --git a/src/v0/destinations/topsort/data/TopsortTrackConfig.json b/src/v0/destinations/topsort/data/TopsortTrackConfig.json new file mode 100644 index 0000000000..fcc9c57d1b --- /dev/null +++ b/src/v0/destinations/topsort/data/TopsortTrackConfig.json @@ -0,0 +1,27 @@ +[ + { + "destKey": "occurredAt", + "sourceKeys": ["timestamp", "originalTimestamp"], + "required": true + }, + { + "destKey": "opaqueUserId", + "sourceKeys": "anonymousId", + "required": true + }, + { + "destKey": "resolvedBidId", + "sourceKeys": "properties.resolvedBidId", + "required": false + }, + { + "destKey": "entity", + "sourceKeys": "properties.entity", + "required": false + }, + { + "destKey": "additionalAttribution", + "sourceKeys": "properties.additionalAttribution", + "required": false + } +] diff --git a/src/v0/destinations/topsort/impressions-and-clicks.js b/src/v0/destinations/topsort/impressions-and-clicks.js new file mode 100644 index 0000000000..ca3c6ad860 --- /dev/null +++ b/src/v0/destinations/topsort/impressions-and-clicks.js @@ -0,0 +1,95 @@ +const { ConfigCategory, mappingConfig } = require('./config'); +const { getItemPayloads, addFinalPayload } = require('./utils'); +const { constructPayload, generateUUID } = require('../../util'); + +const processImpressionsAndClicksUtility = { + // Create event data object + createEventData(basePayload, placementPayload, itemPayload, event) { + return { + topsortPayload: { + ...basePayload, + placement: { + ...placementPayload, + ...itemPayload, + }, + id: generateUUID(), + }, + event, + }; + }, + + // Process events with a product array + processProductArray({ + products, + basePayload, + placementPayload, + topsortEventName, + finalPayloads, + }) { + const itemPayloads = getItemPayloads(products, mappingConfig[ConfigCategory.ITEM.name]); + itemPayloads.forEach((itemPayload) => { + const eventData = this.createEventData( + basePayload, + placementPayload, + itemPayload, + topsortEventName, + ); + addFinalPayload(eventData, finalPayloads); + }); + }, + + // Process events with a single product + processSingleProduct({ + basePayload, + placementPayload, + message, + topsortEventName, + finalPayloads, + }) { + const itemPayload = constructPayload(message, mappingConfig[ConfigCategory.ITEM.name]); + const eventData = this.createEventData( + basePayload, + placementPayload, + itemPayload, + topsortEventName, + ); + + // Ensure messageId is used instead of generating a UUID for single product events + eventData.topsortPayload.id = message.messageId; + + // Add final payload with appropriate ID and other headers + addFinalPayload(eventData, finalPayloads); + }, + + processImpressionsAndClicks({ + isProductArrayAvailable, + basePayload, + topsortEventName, + finalPayloads, + products, + message, + placementPayload, + }) { + if (isProductArrayAvailable) { + // If product array is available, process the event with multiple products + this.processProductArray({ + basePayload, + topsortEventName, + finalPayloads, + products, + placementPayload, + }); + } else { + // Otherwise, process the event with a single product + this.processSingleProduct({ + basePayload, + topsortEventName, + finalPayloads, + message, + placementPayload, + }); + } + }, +}; + +module.exports = { processImpressionsAndClicksUtility }; diff --git a/src/v0/destinations/topsort/purchase.js b/src/v0/destinations/topsort/purchase.js new file mode 100644 index 0000000000..497188cb10 --- /dev/null +++ b/src/v0/destinations/topsort/purchase.js @@ -0,0 +1,56 @@ +const { ConfigCategory, mappingConfig } = require('./config'); +const { getItemPayloads, addFinalPayload } = require('./utils'); +const { constructPayload, generateUUID } = require('../../util'); + +const processPurchaseEventUtility = { + // Create event data object for purchase events + createEventData(basePayload, items, event) { + return { + topsortPayload: { + ...basePayload, + items, + id: generateUUID(), + }, + event, + }; + }, + + // Function to process events with a product array for purchase events + processProductArray(args) { + const { products, basePayload, topsortEventName, finalPayloads } = args; + const itemPayloads = getItemPayloads( + products, + mappingConfig[ConfigCategory.PURCHASE_ITEM.name], + ); + const eventData = this.createEventData(basePayload, itemPayloads, topsortEventName); + addFinalPayload(eventData, finalPayloads); + }, + + // Function to process events with a single product for purchase events + processSingleProduct(args) { + const { basePayload, message, topsortEventName, finalPayloads } = args; + const itemPayload = constructPayload(message, mappingConfig[ConfigCategory.PURCHASE_ITEM.name]); + const eventData = this.createEventData(basePayload, [itemPayload], topsortEventName); + + // Ensure messageId is used instead of generating a UUID for single product events + eventData.topsortPayload.id = message.messageId; + + // Add final payload with appropriate ID and other headers + addFinalPayload(eventData, finalPayloads); + }, + + // Function to process purchase events (either with a product array or single product) + processPurchaseEvent(args) { + if (args.isProductArrayAvailable) { + // Process the event with multiple products (product array) + this.processProductArray(args); + } else { + // Process the event with a single product + this.processSingleProduct(args); + } + }, +}; + +module.exports = { + processPurchaseEventUtility, +}; diff --git a/src/v0/destinations/topsort/transform.js b/src/v0/destinations/topsort/transform.js new file mode 100644 index 0000000000..e5997ab37d --- /dev/null +++ b/src/v0/destinations/topsort/transform.js @@ -0,0 +1,158 @@ +const { + InstrumentationError, + ConfigurationError, + getHashFromArray, +} = require('@rudderstack/integrations-lib'); +const { + mappingConfig, + ECOMM_EVENTS_WITH_PRODUCT_ARRAY, + ConfigCategory, + ENDPOINT, +} = require('./config'); +const { + constructPayload, + handleRtTfSingleEventError, + defaultRequestConfig, + defaultPostRequestConfig, + getSuccessRespEvents, +} = require('../../util'); +const { isProductArrayValid, getMappedEventName } = require('./utils'); +const { JSON_MIME_TYPE } = require('../../util/constant'); +const { processPurchaseEventUtility } = require('./purchase'); +const { processImpressionsAndClicksUtility } = require('./impressions-and-clicks'); + +const processTopsortEvents = (message, { Config }, finalPayloads) => { + const { topsortEvents } = Config; + const { event, properties } = message; + const { products } = properties; + + // Parse Topsort event mappings + const mappedEventName = getMappedEventName(getHashFromArray(topsortEvents), event); + + if (!mappedEventName) { + throw new InstrumentationError("Event not mapped in 'topsortEvents'. Dropping the event."); + } + + const topsortEventName = mappedEventName; + + // Construct base and placement payloads + const basePayload = constructPayload(message, mappingConfig[ConfigCategory.TRACK.name]); + + const commonArgs = { + basePayload, + topsortEventName, + finalPayloads, + products, + message, + isProductArrayAvailable: + ECOMM_EVENTS_WITH_PRODUCT_ARRAY.includes(event) && isProductArrayValid(event, properties), + }; + + // Process events based on type and construct payload within each logic block + if (topsortEventName === 'impressions' || topsortEventName === 'clicks') { + const placementPayload = constructPayload( + message, + mappingConfig[ConfigCategory.PLACEMENT.name], + ); + processImpressionsAndClicksUtility.processImpressionsAndClicks({ + ...commonArgs, + placementPayload, // Only pass placementPayload for impressions and clicks + }); + } else if (topsortEventName === 'purchases') { + processPurchaseEventUtility.processPurchaseEvent({ + ...commonArgs, + }); + } else { + throw new InstrumentationError(`Event not mapped: ${topsortEventName}`); + } + + return finalPayloads; +}; + +const processEvent = (message, destination, finalPayloads) => { + // Check for missing API Key or missing Advertiser ID + if (!destination.Config.apiKey) { + throw new ConfigurationError('API Key is missing. Aborting message.', 400); + } + if (!message.type) { + throw new InstrumentationError('Message Type is missing. Aborting message.', 400); + } + + const messageType = message.type.toLowerCase(); + + // Handle 'track' event type + if (messageType !== 'track') { + throw new InstrumentationError('Only "track" events are supported. Dropping event.', 400); + } + + processTopsortEvents(message, destination, finalPayloads); +}; + +// Process function that is called per event +const process = (event) => { + const finalPayloads = { + impressions: [], + clicks: [], + purchases: [], + }; + + processEvent(event.message, event.destination, finalPayloads); + + const response = defaultRequestConfig(); + const { apiKey } = event.destination.Config; + + response.method = defaultPostRequestConfig.requestMethod; + response.body.JSON = finalPayloads; + response.headers = { + 'content-type': JSON_MIME_TYPE, + Authorization: `Bearer ${apiKey}`, + }; + + response.endpoint = ENDPOINT; + + return response; +}; + +// Router destination handler to process a batch of events +const processRouterDest = async (inputs, reqMetadata) => { + const finalPayloads = { + impressions: [], + clicks: [], + purchases: [], + }; + + const failureResponses = []; + const successMetadatas = []; + + inputs.forEach((input) => { + try { + // Process the event + processEvent(input.message, input.destination, finalPayloads); + // Add to successMetadatas array + successMetadatas.push(input.metadata); + } catch (error) { + // Handle error and store the error details + const failureResponse = handleRtTfSingleEventError(input, error, reqMetadata); + failureResponses.push(failureResponse); + } + }); + + const response = defaultRequestConfig(); + const { destination } = inputs[0]; + const { apiKey } = destination.Config; + + response.method = defaultPostRequestConfig.requestMethod; + response.body.JSON = finalPayloads; + response.headers = { + 'content-type': JSON_MIME_TYPE, + Authorization: `Bearer ${apiKey}`, + }; + + response.endpoint = ENDPOINT; + + const successResponses = getSuccessRespEvents(response, successMetadatas, destination, true); + + return [successResponses, ...failureResponses]; +}; + +module.exports = { process, processRouterDest }; diff --git a/src/v0/destinations/topsort/utils.js b/src/v0/destinations/topsort/utils.js new file mode 100644 index 0000000000..ca9edfbcea --- /dev/null +++ b/src/v0/destinations/topsort/utils.js @@ -0,0 +1,53 @@ +const { ConfigurationError } = require('@rudderstack/integrations-lib'); +const { constructPayload } = require('../../util'); + +// Function to check if a product array is valid +const isProductArrayValid = (event, properties) => + Array.isArray(properties?.products) && properties?.products.length > 0; + +// Function to construct item payloads for each product +const getItemPayloads = (products, mappingConfigs) => + products.map((product) => constructPayload(product, mappingConfigs)); + +// Function to add the structured event data to the final payloads array +const addFinalPayload = (eventData, finalPayloads) => { + switch (eventData.event) { + case 'impressions': + finalPayloads.impressions.push(eventData.topsortPayload); + break; + case 'clicks': + finalPayloads.clicks.push(eventData.topsortPayload); + break; + case 'purchases': + finalPayloads.purchases.push(eventData.topsortPayload); + break; + default: + throw new ConfigurationError('Invalid event mapping'); + } +}; + +// Function to retrieve mapped event name from Topsort event mappings. +const getMappedEventName = (parsedTopsortEventMappings, event) => { + const eventName = event.toLowerCase(); + + const mappedEventNames = parsedTopsortEventMappings[eventName]; + + // Check if mapping exists + if (!mappedEventNames) { + throw new ConfigurationError(`Event '${eventName}' not found in Topsort event mappings`); + } + + // If there are multiple mappings, pick the first one or apply your logic + if (Array.isArray(mappedEventNames)) { + return mappedEventNames[0]; // Return the first mapping + } + + return mappedEventNames; // Return the single mapping if not an array +}; + +module.exports = { + isProductArrayValid, + getItemPayloads, + addFinalPayload, + getMappedEventName, +}; diff --git a/src/v0/util/index.js b/src/v0/util/index.js index ad08be448e..0f9abfb040 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -970,6 +970,7 @@ const handleMetadataForValue = (value, metadata, destKey, integrationsObj = null strictMultiMap, validateTimestamp, allowedKeyCheck, + toArray, } = metadata; // if value is null and defaultValue is supplied - use that @@ -1037,6 +1038,13 @@ const handleMetadataForValue = (value, metadata, destKey, integrationsObj = null } } + if (toArray) { + if (Array.isArray(formattedVal)) { + return formattedVal; + } + return [formattedVal]; + } + return formattedVal; }; diff --git a/test/integrations/destinations/topsort/mocks.ts b/test/integrations/destinations/topsort/mocks.ts new file mode 100644 index 0000000000..0a3e24b30f --- /dev/null +++ b/test/integrations/destinations/topsort/mocks.ts @@ -0,0 +1,5 @@ +import utils from '../../../../src/v0/util'; + +export const defaultMockFns = () => { + jest.spyOn(utils, 'generateUUID').mockReturnValue('test-id-123-123-123'); +}; diff --git a/test/integrations/destinations/topsort/processor/data.ts b/test/integrations/destinations/topsort/processor/data.ts new file mode 100644 index 0000000000..73f838f140 --- /dev/null +++ b/test/integrations/destinations/topsort/processor/data.ts @@ -0,0 +1,9 @@ +import { trackClicksTestData } from './trackClicksTestData'; +import { trackImpressionsTestData } from './trackImpressionsTestData'; +import { trackPurchasesTestData } from './trackPurchasesTestData'; + +export const data = [ + ...trackClicksTestData, + ...trackImpressionsTestData, + ...trackPurchasesTestData, +]; diff --git a/test/integrations/destinations/topsort/processor/trackClicksTestData.ts b/test/integrations/destinations/topsort/processor/trackClicksTestData.ts new file mode 100644 index 0000000000..2cfd458905 --- /dev/null +++ b/test/integrations/destinations/topsort/processor/trackClicksTestData.ts @@ -0,0 +1,571 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { + generateMetadata, + generateSimplifiedTrackPayload, + transformResultBuilder, +} from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; + +const destination: Destination = { + ID: '123', + Name: 'topsort', + DestinationDefinition: { + ID: '123', + Name: 'topsort', + DisplayName: 'topsort', + Config: { + endpoint: 'https://api.topsort.com/v2/events', // Base URL for Topsort API + }, + }, + Config: { + apiKey: 'test-api', + connectionMode: { + web: 'cloud', + }, + consentManagement: {}, + oneTrustCookieCategories: {}, + ketchConsentPurposes: {}, + topsortEvents: [ + { + from: 'Product Clicked', + to: 'clicks', + }, + { + from: 'Order Completed', + to: 'clicks', + }, + { + from: 'Order Refunded', + to: 'clicks', + }, + ], + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +export const trackClicksTestData: ProcessorTestData[] = [ + { + id: 'Test 0', + name: 'topsort', + description: + 'Verifies that a Product Clicked event with all necessary properties is successfully processed and mapped correctly by Topsort.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Product Clicked', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + impressions: [], + clicks: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-msg-id', + }, + ], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 1', + name: 'topsort', + description: + 'Validates the correct processing and mapping of an Order Completed event with multiple products.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Order Completed', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + impressions: [], + clicks: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-id-123-123-123', + }, + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 2, + productId: '577c6f5d5cf86a4c7735ba03', + }, + id: 'test-id-123-123-123', + }, + ], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 2', + name: 'topsort', + description: + 'Tests the handling of an invalid event type (abc) and ensures that Topsort correctly drops the event with a 400 error indicating unsupported event type.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'abc', + event: 'Order Refunded', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }, + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Only "track" events are supported. Dropping event.', + statTags: { + destType: 'TOPSORT', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + metadata: generateMetadata(1), + statusCode: 400, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 3', + name: 'topsort', + description: + 'Verifies the correct processing of an Order Completed event with multiple products and handles an error for an unrecognized Order Updated2 event in Topsort.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Order Completed', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Order Updated2', + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + impressions: [], + clicks: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-id-123-123-123', + }, + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 2, + productId: '577c6f5d5cf86a4c7735ba03', + }, + id: 'test-id-123-123-123', + }, + ], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: "Event 'order updated2' not found in Topsort event mappings", + statTags: { + destType: 'TOPSORT', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'processor', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + metadata: generateMetadata(1), + statusCode: 400, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/topsort/processor/trackImpressionsTestData.ts b/test/integrations/destinations/topsort/processor/trackImpressionsTestData.ts new file mode 100644 index 0000000000..2f1fc60571 --- /dev/null +++ b/test/integrations/destinations/topsort/processor/trackImpressionsTestData.ts @@ -0,0 +1,381 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { + generateMetadata, + generateSimplifiedTrackPayload, + transformResultBuilder, +} from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; + +const destination: Destination = { + ID: '123', + Name: 'topsort', + DestinationDefinition: { + ID: '123', + Name: 'topsort', + DisplayName: 'topsort', + Config: { + endpoint: 'https://api.topsort.com/v2/events', // Base URL for Topsort API + }, + }, + Config: { + apiKey: 'test-api', + connectionMode: { + web: 'cloud', + }, + consentManagement: {}, + oneTrustCookieCategories: {}, + ketchConsentPurposes: {}, + topsortEvents: [ + { + from: 'Product Viewed', + to: 'impressions', + }, + { + from: 'Checkout Started', + to: 'impressions', + }, + ], + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +export const trackImpressionsTestData: ProcessorTestData[] = [ + { + id: 'Test 0', + name: 'topsort', + description: + 'Track call with impressions event, verifies that a Product Viewed event is correctly mapped', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Product Viewed', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + impressions: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-msg-id', + }, + ], + clicks: [], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 1', + name: 'topsort', + description: + 'Verifies that a Checkout Started event with multiple products is correctly mapped and ingested as impressions in Topsort.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Checkout Started', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + impressions: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-id-123-123-123', + }, + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/categories/dairy', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 2, + productId: '577c6f5d5cf86a4c7735ba03', + }, + id: 'test-id-123-123-123', + }, + ], + clicks: [], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 2', + name: 'topsort', + description: + 'Verifies that an invalid event (Checkout done) that is not found in Topsort’s event mappings is handled and returns an error with a status code 400.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Checkout done', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: "Event 'checkout done' not found in Topsort event mappings", + statTags: { + destType: 'TOPSORT', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'processor', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + metadata: generateMetadata(1), + statusCode: 400, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/topsort/processor/trackPurchasesTestData.ts b/test/integrations/destinations/topsort/processor/trackPurchasesTestData.ts new file mode 100644 index 0000000000..a4736fb471 --- /dev/null +++ b/test/integrations/destinations/topsort/processor/trackPurchasesTestData.ts @@ -0,0 +1,457 @@ +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; +import { + generateMetadata, + generateSimplifiedTrackPayload, + transformResultBuilder, +} from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; + +const destination: Destination = { + ID: '123', + Name: 'topsort', + DestinationDefinition: { + ID: '123', + Name: 'topsort', + DisplayName: 'topsort', + Config: { + endpoint: 'https://api.topsort.com/v2/events', // Base URL for Topsort API + }, + }, + Config: { + apiKey: 'test-api', + connectionMode: { + web: 'cloud', + }, + consentManagement: {}, + oneTrustCookieCategories: {}, + ketchConsentPurposes: {}, + topsortEvents: [ + { + from: 'Order Completed', + to: 'purchases', + }, + { + from: 'Product Added', + to: 'purchases', + }, + ], + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +export const trackPurchasesTestData: ProcessorTestData[] = [ + { + id: 'Test 0', + name: 'topsort', + description: + 'Verifies that a Product Added event is correctly mapped and ingested as a purchase event in Topsort with the appropriate product details.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Product Added', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + purchases: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + items: [ + { + productId: '622c6f5d5cf86a4c77358033', + quantity: 5, + unitPrice: 49.99, + }, + ], + id: 'test-msg-id', + }, + ], + clicks: [], + impressions: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 1', + name: 'topsort', + description: + 'Verifies that a Checkout Started event with multiple products is correctly mapped with items.', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Order Completed', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + purchases: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + items: [ + { + productId: '622c6f5d5cf86a4c77358033', + unitPrice: 40, + }, + { + productId: '577c6f5d5cf86a4c7735ba03', + unitPrice: 5, + }, + ], + id: 'test-id-123-123-123', + }, + ], + clicks: [], + impressions: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'Test 2', + name: 'topsort', + description: + 'Verifies that both a Product Added and an Order Completed event are correctly mapped and ingested into Topsort as purchase events', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200 and correctly map the properties to the specified parameters.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Product Added', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + { + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'Order Completed', // The RudderStack event + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + }, + context: { + page: { + path: '/categories/dairy', + }, + }, + anonymousId: 'david_bowie_anonId', + }), + metadata: generateMetadata(1), + destination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + purchases: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + items: [ + { + productId: '622c6f5d5cf86a4c77358033', + quantity: 5, + unitPrice: 49.99, + }, + ], + id: 'test-msg-id', + }, + ], + clicks: [], + impressions: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + { + output: transformResultBuilder({ + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + userId: '', + JSON: { + purchases: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'david_bowie_anonId', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + items: [ + { + productId: '622c6f5d5cf86a4c77358033', + quantity: 5, + unitPrice: 49.99, + }, + ], + id: 'test-msg-id', + }, + ], + clicks: [], + impressions: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }), + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/topsort/router/data.ts b/test/integrations/destinations/topsort/router/data.ts new file mode 100644 index 0000000000..0cabdcbac8 --- /dev/null +++ b/test/integrations/destinations/topsort/router/data.ts @@ -0,0 +1,418 @@ +import { Destination } from '../../../../../src/types'; +import { RouterTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; + +const destination: Destination = { + ID: '123', + Name: 'topsort', + DestinationDefinition: { + ID: '123', + Name: 'topsort', + DisplayName: 'topsort', + Config: { + endpoint: 'https://api.topsort.com/v2/events', + }, + }, + Config: { + apiKey: 'test-api', + connectionMode: { + web: 'cloud', + }, + consentManagement: {}, + oneTrustCookieCategories: {}, + ketchConsentPurposes: {}, + topsortEvents: [ + { + from: 'Product Clicked', + to: 'clicks', + }, + { + from: 'Product Added', + to: 'purchases', + }, + { + from: 'Product Removed', + to: 'impressions', + }, + ], + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], +}; + +export const data: RouterTestData[] = [ + { + id: 'topsort-router-test-1', + name: 'topsort', + description: 'Basic Router Test for track call having clicks event', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200, and the output should correctly map the properties.', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + destination, + metadata: generateMetadata(1), + message: { + type: 'track', + event: 'Product Clicked', + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + anonymousId: 'sampath', + channel: 'web', + context: { + page: { + path: '/category/123', + }, + ip: '0.0.0.0', + }, + integrations: { All: true }, + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + }, + }, + }, + ], + destType: 'topsort', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + body: { + JSON: { + impressions: [], + clicks: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'sampath', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/category/123', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-msg-id', + }, + ], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, + { + id: 'topsort-router-test-2', + name: 'topsort', + description: 'Basic Router Test for track call having impressions event', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200, and the output should correctly map the properties.', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + destination, + metadata: generateMetadata(1), + message: { + type: 'track', + event: 'Product Removed', + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + anonymousId: 'sampath', + channel: 'web', + context: { + page: { + path: '/category/123', + }, + ip: '0.0.0.0', + }, + integrations: { All: true }, + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + price: 5, + position: 2, + }, + ], + }, + }, + }, + ], + destType: 'topsort', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + body: { + JSON: { + impressions: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'sampath', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + placement: { + path: '/category/123', + pageSize: 15, + categoryIds: ['9BLIe'], + position: 1, + productId: '622c6f5d5cf86a4c77358033', + }, + id: 'test-msg-id', + }, + ], + clicks: [], + purchases: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, + { + id: 'topsort-router-test-3', + name: 'topsort', + description: 'Basic Router Test for track call having purchases event', + scenario: 'Business', + successCriteria: + 'The response should have a status code of 200, and the output should correctly map the properties.', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + destination, + metadata: generateMetadata(1), + message: { + type: 'track', + event: 'Product Added', + originalTimestamp: '2024-11-05T15:19:08+00:00', + messageId: 'test-msg-id', + anonymousId: 'sampath', + channel: 'web', + context: { + page: { + path: '/category/123', + }, + ip: '0.0.0.0', + }, + integrations: { All: true }, + properties: { + product_id: '622c6f5d5cf86a4c77358033', + price: 49.99, + quantity: 5, + position: 1, + resolvedBidId: '13841873482r7903r823', + page: 1, + pageSize: 15, + category_id: '9BLIe', + url: 'https://www.website.com/product/path', + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + entity: { + id: '235', + type: 'product', + }, + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + price: 40, + position: 1, + }, + ], + }, + }, + }, + ], + destType: 'topsort', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.topsort.com/v2/events', + headers: { + 'content-type': 'application/json', + Authorization: 'Bearer test-api', + }, + params: {}, + body: { + JSON: { + purchases: [ + { + occurredAt: '2024-11-05T15:19:08+00:00', + opaqueUserId: 'sampath', + resolvedBidId: '13841873482r7903r823', + entity: { + id: '235', + type: 'product', + }, + additionalAttribution: { + id: 'a13362', + type: 'product', + }, + items: [ + { + productId: '622c6f5d5cf86a4c77358033', + quantity: 5, + unitPrice: 49.99, + }, + ], + id: 'test-msg-id', + }, + ], + clicks: [], + impressions: [], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, +]; diff --git a/test/integrations/testTypes.ts b/test/integrations/testTypes.ts index 3c5cf60600..b11403f50a 100644 --- a/test/integrations/testTypes.ts +++ b/test/integrations/testTypes.ts @@ -101,7 +101,7 @@ export type ProcessorTestData = { body: ProcessorTransformationResponse[]; }; }; - mockFns?: (mockAdapter: MockAdapter) => {}; + mockFns?: (mockAdapter: MockAdapter) => void; }; export type RouterTestData = { id: string; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 7e6e6b9acb..73d08e452c 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -277,6 +277,7 @@ export const generateSimplifiedTrackPayload: any = (parametersOverride: any) => device: parametersOverride.context.device, os: parametersOverride.context.os, app: parametersOverride.context.app, + page: parametersOverride.context.page, }), anonymousId: parametersOverride.anonymousId || 'default-anonymousId', originalTimestamp: parametersOverride.originalTimestamp || '2021-01-03T17:02:53.193Z',