From 932792561c98833baf9881a83ee36ae5000e37b4 Mon Sep 17 00:00:00 2001 From: Gauravudia <60897972+Gauravudia@users.noreply.github.com> Date: Mon, 12 Aug 2024 11:23:22 +0530 Subject: [PATCH 01/12] fix: handle attentive tag null, undefined properties (#3647) --- src/v0/destinations/attentive_tag/util.js | 12 ++- .../attentive_tag/processor/data.ts | 91 +++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/v0/destinations/attentive_tag/util.js b/src/v0/destinations/attentive_tag/util.js index c96d03028f..abecf76542 100644 --- a/src/v0/destinations/attentive_tag/util.js +++ b/src/v0/destinations/attentive_tag/util.js @@ -19,11 +19,13 @@ const { mappingConfig, ConfigCategory } = require('./config'); */ const getPropertiesKeyValidation = (payload) => { const validationArray = [`'`, `"`, `{`, `}`, `[`, `]`, ',', `,`]; - const keys = Object.keys(payload.properties); - for (const key of keys) { - for (const validationChar of validationArray) { - if (key.includes(validationChar)) { - return false; + if (payload.properties) { + const keys = Object.keys(payload.properties); + for (const key of keys) { + for (const validationChar of validationArray) { + if (key.includes(validationChar)) { + return false; + } } } } diff --git a/test/integrations/destinations/attentive_tag/processor/data.ts b/test/integrations/destinations/attentive_tag/processor/data.ts index 12e0f4bea1..2e602610e0 100644 --- a/test/integrations/destinations/attentive_tag/processor/data.ts +++ b/test/integrations/destinations/attentive_tag/processor/data.ts @@ -1678,4 +1678,95 @@ export const data = [ }, }, }, + { + name: 'attentive_tag', + description: 'Test 16', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: {}, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { + id: '72e528f869711c3d', + manufacturer: 'Google', + model: 'sdk_gphone_x86', + name: 'generic_x86_arm', + token: 'some_device_token', + type: 'android', + }, + screen: { + density: 2, + }, + }, + type: 'track', + event: 'Application Backgrounded', + anonymousId: '00000000000000000000000000', + userId: '123456', + integrations: { + All: true, + }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + signUpSourceId: '240654', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + userId: '', + endpoint: 'https://api.attentivemobile.com/v1/events/custom', + headers: { + Authorization: 'Bearer dummyApiKey', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + user: {}, + type: 'Application Backgrounded', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + statusCode: 200, + }, + ], + }, + }, + }, ].map((d) => ({ ...d, mockFns })); From 0a9b68118241158623d45ee28896377296bafd91 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Mon, 12 Aug 2024 11:49:41 +0530 Subject: [PATCH 02/12] fix: handle null values for braze dedupe (#3638) --- src/v0/destinations/braze/braze.util.test.js | 130 +++++++++++++++++++ src/v0/destinations/braze/util.js | 9 +- 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index f2726c3283..8199aae70b 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -785,6 +785,136 @@ describe('dedup utility tests', () => { const result = BrazeDedupUtility.deduplicate(userData, store); expect(result).toBeNull(); }); + + test('deduplicates user data correctly when user data is null and it doesnt exist in stored data', () => { + const userData = { + external_id: '123', + nullProperty: null, + color: 'green', + age: 30, + }; + const storeData = { + external_id: '123', + custom_attributes: { + color: 'green', + age: 30, + }, + }; + store.set('123', storeData); + const result = BrazeDedupUtility.deduplicate(userData, store); + expect(store.size).toBe(1); + expect(result).toEqual(null); + }); + + test('deduplicates user data correctly when user data is null and it is same in stored data', () => { + const userData = { + external_id: '123', + nullProperty: null, + color: 'green', + age: 30, + }; + const storeData = { + external_id: '123', + custom_attributes: { + color: 'green', + age: 30, + nullProperty: null, + }, + }; + store.set('123', storeData); + const result = BrazeDedupUtility.deduplicate(userData, store); + expect(store.size).toBe(1); + expect(result).toEqual(null); + }); + + test('deduplicates user data correctly when user data is null and it is different in stored data', () => { + const userData = { + external_id: '123', + nullProperty: null, + color: 'green', + age: 30, + }; + const storeData = { + external_id: '123', + custom_attributes: { + color: 'green', + age: 30, + nullProperty: 'valid data', + }, + }; + store.set('123', storeData); + const result = BrazeDedupUtility.deduplicate(userData, store); + expect(store.size).toBe(1); + expect(result).toEqual({ + external_id: '123', + nullProperty: null, + }); + }); + + test('deduplicates user data correctly when user data is undefined and it doesnt exist in stored data', () => { + const userData = { + external_id: '123', + undefinedProperty: undefined, + color: 'green', + age: 30, + }; + const storeData = { + external_id: '123', + custom_attributes: { + color: 'green', + age: 30, + }, + }; + store.set('123', storeData); + const result = BrazeDedupUtility.deduplicate(userData, store); + expect(store.size).toBe(1); + expect(result).toEqual(null); + }); + + test('deduplicates user data correctly when user data is undefined and it is same in stored data', () => { + const userData = { + external_id: '123', + undefinedProperty: undefined, + color: 'green', + age: 30, + }; + const storeData = { + external_id: '123', + custom_attributes: { + color: 'green', + undefinedProperty: undefined, + age: 30, + }, + }; + store.set('123', storeData); + const result = BrazeDedupUtility.deduplicate(userData, store); + expect(store.size).toBe(1); + expect(result).toEqual(null); + }); + + test('deduplicates user data correctly when user data is undefined and it is defined in stored data', () => { + const userData = { + external_id: '123', + undefinedProperty: undefined, + color: 'green', + age: 30, + }; + const storeData = { + external_id: '123', + custom_attributes: { + color: 'green', + undefinedProperty: 'defined data', + age: 30, + }, + }; + store.set('123', storeData); + const result = BrazeDedupUtility.deduplicate(userData, store); + expect(store.size).toBe(1); + expect(result).toEqual({ + external_id: '123', + undefinedProperty: undefined, + }); + }); }); }); diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 00ef308fe9..7f4afaf6c1 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -288,7 +288,14 @@ const BrazeDedupUtility = { if (keys.length > 0) { keys.forEach((key) => { - if (!_.isEqual(userData[key], storedUserData[key])) { + // ref: https://www.braze.com/docs/user_guide/data_and_analytics/custom_data/custom_attributes/#adding-descriptions + // null is a valid value in braze for unsetting, so we need to compare the values only if the key is present in the stored user data + // in case of keys having null values only compare if the key is present in the stored user data + if (userData[key] === null) { + if (isDefinedAndNotNull(storedUserData[key])) { + deduplicatedUserData[key] = userData[key]; + } + } else if (!_.isEqual(userData[key], storedUserData[key])) { deduplicatedUserData[key] = userData[key]; } }); From 8450021672c51ac798ec0aeab422f5fceea5e53e Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:35:16 +0530 Subject: [PATCH 03/12] feat: move hubspot to transformer proxy to enable partial batch handling (#3308) * feat: move hubspot to transformer proxy to enable partial batch handling * chore: refactor code * chore: add unit tests * chore: refactor code * chore: update error message * fix: update network handler for legacy api * chore: address comment * chore: update failed tests * fix: update network handler * feat: add verification of single event * chore: add test cases --- src/v0/destinations/hs/transform.js | 31 +- src/v0/destinations/hs/util.js | 32 + src/v1/destinations/hs/networkHandler.ts | 147 +++ .../destinations/hs/dataDelivery/business.ts | 593 +++++++++++ .../destinations/hs/dataDelivery/data.ts | 4 + .../destinations/hs/dataDelivery/other.ts | 245 +++++ test/integrations/destinations/hs/network.ts | 376 +++++++ .../destinations/hs/router/data.ts | 931 ++++++++++++++++++ 8 files changed, 2354 insertions(+), 5 deletions(-) create mode 100644 src/v1/destinations/hs/networkHandler.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/business.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/data.ts create mode 100644 test/integrations/destinations/hs/dataDelivery/other.ts diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 6cf69e3c3b..1d1a81b6b0 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -19,6 +19,7 @@ const { fetchFinalSetOfTraits, getProperties, validateDestinationConfig, + convertToResponseFormat, } = require('./util'); const processSingleMessage = async ({ message, destination, metadata }, propertyMap) => { @@ -147,20 +148,38 @@ const processBatchRouter = async (inputs, reqMetadata) => { }), ); - if (successRespList.length > 0) { + const dontBatchTrueResponses = []; + const dontBatchFalseOrUndefinedResponses = []; + // segregating successRepList depending on dontbatch value + successRespList.forEach((successResp) => { + if (successResp.metadata?.dontBatch) { + dontBatchTrueResponses.push(successResp); + } else { + dontBatchFalseOrUndefinedResponses.push(successResp); + } + }); + + // batch implementation + if (dontBatchFalseOrUndefinedResponses.length > 0) { if (destination.Config.apiVersion === API_VERSION.v3) { - batchedResponseList = batchEvents(successRespList); + batchedResponseList = batchEvents(dontBatchFalseOrUndefinedResponses); } else { - batchedResponseList = legacyBatchEvents(successRespList); + batchedResponseList = legacyBatchEvents(dontBatchFalseOrUndefinedResponses); } } - return { batchedResponseList, errorRespList }; + return { + batchedResponseList, + errorRespList, + // if there are any events where dontbatch set to true we need to update them according to the response format + dontBatchEvents: convertToResponseFormat(dontBatchTrueResponses), + }; }; // we are batching by default at routerTransform const processRouterDest = async (inputs, reqMetadata) => { const tempNewInputs = batchEventsInOrder(inputs); const batchedResponseList = []; const errorRespList = []; + const dontBatchEvents = []; const promises = tempNewInputs.map(async (inputEvents) => { const response = await processBatchRouter(inputEvents, reqMetadata); return response; @@ -171,8 +190,10 @@ const processRouterDest = async (inputs, reqMetadata) => { results.forEach((response) => { errorRespList.push(...response.errorRespList); batchedResponseList.push(...response.batchedResponseList); + dontBatchEvents.push(...response.dontBatchEvents); }); - return [...batchedResponseList, ...errorRespList]; + console.log(JSON.stringify([...batchedResponseList, ...errorRespList, ...dontBatchEvents])); + return [...batchedResponseList, ...errorRespList, ...dontBatchEvents]; }; module.exports = { process, processRouterDest }; diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index 38b2e636b9..fc22a89d69 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -22,6 +22,9 @@ const { getValueFromMessage, isNull, validateEventName, + defaultBatchRequestConfig, + defaultPostRequestConfig, + getSuccessRespEvents, } = require('../../util'); const { CONTACT_PROPERTY_MAP_ENDPOINT, @@ -844,6 +847,34 @@ const addExternalIdToHSTraits = (message) => { set(getFieldValueFromMessage(message, 'traits'), externalIdObj.identifierType, externalIdObj.id); }; +const convertToResponseFormat = (successRespListWithDontBatchTrue) => { + const response = []; + if (Array.isArray(successRespListWithDontBatchTrue)) { + successRespListWithDontBatchTrue.forEach((event) => { + const { message, metadata, destination } = event; + const endpoint = get(message, 'endpoint'); + + const batchedResponse = defaultBatchRequestConfig(); + batchedResponse.batchedRequest.headers = message.headers; + batchedResponse.batchedRequest.endpoint = endpoint; + batchedResponse.batchedRequest.body = message.body; + batchedResponse.batchedRequest.params = message.params; + batchedResponse.batchedRequest.method = defaultPostRequestConfig.requestMethod; + batchedResponse.metadata = [metadata]; + batchedResponse.destination = destination; + + response.push( + getSuccessRespEvents( + batchedResponse.batchedRequest, + batchedResponse.metadata, + batchedResponse.destination, + ), + ); + }); + } + return response; +}; + module.exports = { validateDestinationConfig, addExternalIdToHSTraits, @@ -863,4 +894,5 @@ module.exports = { getObjectAndIdentifierType, extractIDsForSearchAPI, getRequestData, + convertToResponseFormat, }; diff --git a/src/v1/destinations/hs/networkHandler.ts b/src/v1/destinations/hs/networkHandler.ts new file mode 100644 index 0000000000..8293a7240c --- /dev/null +++ b/src/v1/destinations/hs/networkHandler.ts @@ -0,0 +1,147 @@ +import { TransformerProxyError } from '../../../v0/util/errorTypes'; +import { prepareProxyRequest, proxyRequest } from '../../../adapters/network'; +import { isHttpStatusSuccess, getAuthErrCategoryFromStCode } from '../../../v0/util/index'; +import { DeliveryV1Response, DeliveryJobState } from '../../../types/index'; + +import { processAxiosResponse, getDynamicErrorType } from '../../../adapters/utils/networkUtils'; + +const tags = require('../../../v0/util/tags'); + +/** + * + * @param results + * @param rudderJobMetadata + * @param destinationConfig + * @returns boolean + */ + +const findFeatureandVersion = (response, rudderJobMetadata, destinationConfig) => { + const { results, errors } = response; + if (Array.isArray(rudderJobMetadata) && rudderJobMetadata.length === 1) { + return 'singleEvent'; + } + if (destinationConfig?.apiVersion === 'legacyApi') { + return 'legacyApiWithMultipleEvents'; + } + if (destinationConfig?.apiVersion === 'newApi') { + if (Array.isArray(results) && results.length === rudderJobMetadata.length) + return 'newApiWithMultipleEvents'; + + if ( + Array.isArray(results) && + results.length !== rudderJobMetadata.length && + results.length + errors.length === rudderJobMetadata.length + ) + return 'newApiWithMultipleEventsAndErrors'; + } + return 'unableToFindVersionWithMultipleEvents'; +}; + +const populateResponseWithDontBatch = (rudderJobMetadata, response) => { + const errorMessage = JSON.stringify(response); + const responseWithIndividualEvents: DeliveryJobState[] = []; + + rudderJobMetadata.forEach((metadata) => { + responseWithIndividualEvents.push({ + statusCode: 500, + metadata: { ...metadata, dontBatch: true }, + error: errorMessage, + }); + }); + return responseWithIndividualEvents; +}; + +type Response = { + status?: string; + results?: Array; + errors?: Array; + startedAt?: Date; + completedAt?: Date; + message?: string; + correlationId?: string; + failureMessages?: Array; +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata, destinationRequest } = responseParams; + const successMessage = `[HUBSPOT Response V1 Handler] - Request Processed Successfully`; + const failureMessage = + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation'; + const responseWithIndividualEvents: DeliveryJobState[] = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // populate different response for each event + const destResponse = response as Response; + let proxyOutputObj: DeliveryJobState; + const featureAndVersion = findFeatureandVersion( + destResponse, + rudderJobMetadata, + destinationRequest?.destinationConfig, + ); + switch (featureAndVersion) { + case 'singleEvent': + proxyOutputObj = { + statusCode: status, + metadata: rudderJobMetadata[0], + error: JSON.stringify(destResponse), + }; + responseWithIndividualEvents.push(proxyOutputObj); + break; + case 'newApiWithMultipleEvents': + rudderJobMetadata.forEach((metadata: any, index: string | number) => { + proxyOutputObj = { + statusCode: 200, + metadata, + error: JSON.stringify(destResponse.results?.[index]), + }; + responseWithIndividualEvents.push(proxyOutputObj); + }); + break; + default: + rudderJobMetadata.forEach((metadata) => { + proxyOutputObj = { + statusCode: 200, + metadata, + error: 'success', + }; + responseWithIndividualEvents.push(proxyOutputObj); + }); + break; + } + return { + status, + message: successMessage, + response: responseWithIndividualEvents, + } as DeliveryV1Response; + } + + // At least one event in the batch is invalid. + if (status === 400 && Array.isArray(rudderJobMetadata) && rudderJobMetadata.length > 1) { + // sending back 500 for retry only when events came in a batch + return { + status: 500, + message: failureMessage, + response: populateResponseWithDontBatch(rudderJobMetadata, response), + } as DeliveryV1Response; + } + throw new TransformerProxyError( + failureMessage, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + response, + ); +}; + +function networkHandler(this: any) { + this.prepareProxy = prepareProxyRequest; + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.responseHandler = responseHandler; +} + +module.exports = { networkHandler }; diff --git a/test/integrations/destinations/hs/dataDelivery/business.ts b/test/integrations/destinations/hs/dataDelivery/business.ts new file mode 100644 index 0000000000..2239abfb95 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/business.ts @@ -0,0 +1,593 @@ +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; + +const commonStatTags = { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', +}; +export const businessData = [ + { + name: 'hs', + description: 'successfully creating users from a batch with legacy api', + feature: 'dataDelivery', + module: 'destination', + id: 'successWithLegacyApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', + JSON_ARRAY: { + batch: + '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.com","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', + }, + headers: { + Authorization: 'Bearer validApiKey', + 'Content-Type': 'application/json', + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: 'success', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed to create users from a batch with legacy api', + feature: 'dataDelivery', + module: 'destination', + id: 'failureWithLegacyApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/contacts/v1/contact/batch/', + JSON_ARRAY: { + batch: + '[{"email":"identify111051@test.com","properties":[{"property":"firstname","value":"John1051"},{"property":"lastname","value":"Sparrow1051"}]},{"email":"identify111052@test.con","properties":[{"property":"firstname","value":"John1052"},{"property":"lastname","value":"Sparrow1052"}]},{"email":"identify111053@test.com","properties":[{"property":"firstname","value":"John1053"},{"property":"lastname","value":"Sparrow1053"}]}]', + }, + headers: { + Authorization: 'Bearer inValidApiKey', + 'Content-Type': 'application/json', + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', + metadata: { ...generateMetadata(1), dontBatch: true }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Errors found processing batch update","correlationId":"a716ef20-79df-44d4-98bd-9136af7bdefc","invalidEmails":["identify111052@test.con"],"failureMessages":[{"index":1,"error":{"status":"error","message":"Email address identify111052@test.con is invalid"}}]}', + metadata: { ...generateMetadata(2), dontBatch: true }, + statusCode: 500, + }, + ], + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'successfully deliver events with legacy api', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + id: 'successEventsWithLegacyApi', + input: { + request: { + body: generateProxyV1Payload( + { + headers: { + 'Content-Type': 'application/json', + }, + method: 'GET', + params: { + _a: 'dummy-hubId', + _n: 'test track event HS', + _m: 4.99, + email: 'testhubspot2@email.com', + firstname: 'Test Hubspot', + }, + endpoint: 'https://track.hubspot.com/v1/event', + }, + [generateMetadata(1)], + { + apiVersion: 'legacyApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: '{}', + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'successfully creating users from a batch with new api', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + id: 'successCreatingBatchOfUsersWithNewApi', + input: { + request: { + body: generateProxyV1Payload( + { + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer validAccessToken', + }, + method: 'POST', + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + JSON: { + inputs: [ + { + properties: { + email: 'testuser31848@testmail.com', + }, + }, + { + properties: { + email: 'testuser31847@testmail.com', + }, + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'newApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: + '{"id":"44188066992","properties":{"createdate":"2024-07-31T03:21:03.176Z","email":"testuser31848@testmail.com","hs_all_contact_vids":"44188066992","hs_email_domain":"testmail.com","hs_is_contact":"true","hs_is_unworked":"true","hs_lifecyclestage_lead_date":"2024-07-31T03:21:03.176Z","hs_membership_has_accessed_private_content":"0","hs_object_id":"44188066992","hs_object_source":"INTEGRATION","hs_object_source_id":"3209723","hs_object_source_label":"INTEGRATION","hs_pipeline":"contacts-lifecycle-pipeline","hs_registered_member":"0","lastmodifieddate":"2024-07-31T03:21:03.176Z","lifecyclestage":"lead"},"createdAt":"2024-07-31T03:21:03.176Z","updatedAt":"2024-07-31T03:21:03.176Z","archived":false}', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: + '{"id":"44188066993","properties":{"createdate":"2024-07-31T03:21:03.176Z","email":"testuser31847@testmail.com","hs_all_contact_vids":"44188066993","hs_email_domain":"testmail.com","hs_is_contact":"true","hs_is_unworked":"true","hs_lifecyclestage_lead_date":"2024-07-31T03:21:03.176Z","hs_membership_has_accessed_private_content":"0","hs_object_id":"44188066993","hs_object_source":"INTEGRATION","hs_object_source_id":"3209723","hs_object_source_label":"INTEGRATION","hs_pipeline":"contacts-lifecycle-pipeline","hs_registered_member":"0","lastmodifieddate":"2024-07-31T03:21:03.176Z","lifecyclestage":"lead"},"createdAt":"2024-07-31T03:21:03.176Z","updatedAt":"2024-07-31T03:21:03.176Z","archived":false}', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'successfully updating users from a batch with new api', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + { + apiVersion: 'newApi', + }, + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: + '{"id":"12877907025","properties":{"createdate":"2024-04-16T09:50:16.034Z","email":"test1@mail.com","firstname":"test1","hs_is_unworked":"true","hs_object_id":"12877907025","hs_pipeline":"contacts-lifecycle-pipeline","lastmodifieddate":"2024-04-23T11:52:03.723Z","lifecyclestage":"lead"},"createdAt":"2024-04-16T09:50:16.034Z","updatedAt":"2024-04-23T11:52:03.723Z","archived":false}', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: + '{"id":"12877907024","properties":{"createdate":"2024-04-16T09:50:16.034Z","firstname":"testmail1217","hs_is_unworked":"true","hs_object_id":"12877907024","hs_pipeline":"contacts-lifecycle-pipeline","lastmodifieddate":"2024-04-23T11:52:03.723Z","lifecyclestage":"lead"},"createdAt":"2024-04-16T09:50:16.034Z","updatedAt":"2024-04-23T11:52:03.723Z","archived":false}', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to duplicate in a batch', + id: 'hs_datadelivery_01', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + { + properties: { + firstname: 'test5', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + { + properties: { + firstname: 'testmail1217', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + [generateMetadata(1), generateMetadata(2)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Duplicate IDs found in batch input: [12877907025]. IDs must be unique","correlationId":"d24ec5cd-8998-4674-a928-59603ae6b0eb","context":{"ids":["12877907025"]},"category":"VALIDATION_ERROR"}', + metadata: { + ...generateMetadata(1), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Duplicate IDs found in batch input: [12877907025]. IDs must be unique","correlationId":"d24ec5cd-8998-4674-a928-59603ae6b0eb","context":{"ids":["12877907025"]},"category":"VALIDATION_ERROR"}', + metadata: { + ...generateMetadata(2), + dontBatch: true, + }, + statusCode: 500, + }, + ], + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'failed due to wrong email format in a batch', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + JSON: { + inputs: [ + [ + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + }, + { + properties: { + firstname: 'testmail1217', + email: 'testmail1217@testmail.com', + }, + }, + { + properties: { + firstname: 'test5', + email: 'test5@xmail.con', + }, + }, + ], + ], + }, + }, + [generateMetadata(1), generateMetadata(2), generateMetadata(3)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(1), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(2), + dontBatch: true, + }, + statusCode: 500, + }, + { + error: + '{"status":"error","message":"Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)","correlationId":"99df04b9-da11-4504-bd97-2c15f58d0943"}', + metadata: { + ...generateMetadata(3), + dontBatch: true, + }, + statusCode: 500, + }, + ], + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'succeed to send a custom event', + feature: 'dataDelivery', + module: 'destination', + id: 'succeedEventWithNewApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/events/v3/send', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + JSON: { + email: 'osvaldocostaferreira98@gmail.com', + eventName: 'pe22315509_rs_hub_test', + properties: { + value: 'name1', + }, + }, + }, + [generateMetadata(1)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: '{}', + metadata: generateMetadata(1), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'succeed to send a custom event', + feature: 'dataDelivery', + module: 'destination', + id: 'failedEventWithNewApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/events/v3/send', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer invalid-dummy-access-token', + }, + JSON: { + email: 'osvaldocostaferreira98@gmail.com', + eventName: 'pe22315509_rs_hub_test', + properties: { + value: 'name1', + }, + }, + }, + [generateMetadata(1)], + ), + }, + }, + output: { + response: { + status: 401, + body: { + output: { + authErrorCategory: 'REFRESH_TOKEN', + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"status":"error","message":"Authentication credentials not found. This API supports OAuth 2.0 authentication and you can find more details at https://developers.hubspot.com/docs/methods/auth/oauth-overview","correlationId":"501651f6-bb90-40f1-b0db-349f62916993","category":"INVALID_AUTHENTICATION"}', + metadata: generateMetadata(1), + statusCode: 401, + }, + ], + statTags: { ...commonStatTags, errorType: 'aborted' }, + status: 401, + }, + }, + }, + }, + }, + { + name: 'hs', + description: 'succeed to send an event with association', + feature: 'dataDelivery', + module: 'destination', + id: 'succeedAssociationWithNewApi', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://api.hubapi.com/crm/v3/associations/companies/contacts/batch/create', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + JSON: { + inputs: [{ to: { id: 1 }, from: { id: 9405415215 }, type: 'contact_to_company' }], + }, + }, + [generateMetadata(1)], + ), + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: + '{"completedAt":"2024-07-31T04:46:34.391Z","requestedAt":"2024-07-31T04:46:34.391Z","startedAt":"2024-07-31T04:46:34.391Z","results":[{"from":{"id":"9405415215"},"to":{"id":"1"},"type":"contact_to_company"}],"status":"PENDING"}', + metadata: generateMetadata(1), + statusCode: 201, + }, + ], + status: 201, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/hs/dataDelivery/data.ts b/test/integrations/destinations/hs/dataDelivery/data.ts new file mode 100644 index 0000000000..5b2060d001 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/data.ts @@ -0,0 +1,4 @@ +import { businessData } from './business'; +import { otherData } from './other'; + +export const data = [...businessData, ...otherData]; diff --git a/test/integrations/destinations/hs/dataDelivery/other.ts b/test/integrations/destinations/hs/dataDelivery/other.ts new file mode 100644 index 0000000000..202b665a51 --- /dev/null +++ b/test/integrations/destinations/hs/dataDelivery/other.ts @@ -0,0 +1,245 @@ +import { generateMetadata } from '../../../testUtils'; + +const commonStatTags = { + destType: 'HS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', +}; + +const commonBody = { + version: '1', + type: 'REST', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummyAccessToken', + }, + params: {}, + files: {}, + metadata: [generateMetadata(1), generateMetadata(2)], + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, +}; + +export const otherData = [ + { + name: 'hs', + id: 'hs_datadelivery_other_00', + description: 'failed due to gateway timeout from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_gateway_time_out', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '"Gateway Timeout"', + metadata: generateMetadata(1), + statusCode: 504, + }, + { + error: '"Gateway Timeout"', + metadata: generateMetadata(2), + statusCode: 504, + }, + ], + statTags: commonStatTags, + status: 504, + }, + }, + }, + }, + }, + { + name: 'hs', + id: 'hs_datadelivery_other_01', + description: 'failed due to internal server error from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_internal_server_error', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: '"Internal Server Error"', + metadata: generateMetadata(1), + statusCode: 500, + }, + { + error: '"Internal Server Error"', + metadata: generateMetadata(2), + statusCode: 500, + }, + ], + statTags: commonStatTags, + status: 500, + }, + }, + }, + }, + }, + { + name: 'hs', + id: 'hs_datadelivery_other_02', + description: 'failed due to service unavailable error from hubspot', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://random_test_url/test_for_service_not_available', + ...commonBody, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + 'HUBSPOT: Error in transformer proxy v1 during HUBSPOT response transformation', + response: [ + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + metadata: generateMetadata(1), + statusCode: 503, + }, + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + metadata: generateMetadata(2), + statusCode: 503, + }, + ], + statTags: commonStatTags, + status: 503, + }, + }, + }, + }, + }, + { + name: 'hs', + id: 'hs_datadelivery_other_03', + description: 'getting success response but not in the expected format', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + version: '1', + type: 'REST', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummyAccessToken', + }, + params: {}, + files: {}, + metadata: [generateMetadata(1), generateMetadata(2)], + body: { + JSON: { + inputs: [ + { + properties: { + firstname: 'testmail12178', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: '[HUBSPOT Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: 'success', + metadata: generateMetadata(1), + statusCode: 200, + }, + { + error: 'success', + metadata: generateMetadata(2), + statusCode: 200, + }, + ], + status: 200, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/hs/network.ts b/test/integrations/destinations/hs/network.ts index 3d3b8fd83f..9d8658ff6b 100644 --- a/test/integrations/destinations/hs/network.ts +++ b/test/integrations/destinations/hs/network.ts @@ -692,4 +692,380 @@ export const networkCallsData = [ status: 200, }, }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'testmail1217', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 200, + data: { + status: 'COMPLETE', + results: [ + { + id: '12877907025', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + email: 'test1@mail.com', + firstname: 'test1', + hs_is_unworked: 'true', + hs_object_id: '12877907025', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + { + id: '12877907024', + properties: { + createdate: '2024-04-16T09:50:16.034Z', + firstname: 'testmail1217', + hs_is_unworked: 'true', + hs_object_id: '12877907024', + hs_pipeline: 'contacts-lifecycle-pipeline', + lastmodifieddate: '2024-04-23T11:52:03.723Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-04-16T09:50:16.034Z', + updatedAt: '2024-04-23T11:52:03.723Z', + archived: false, + }, + ], + startedAt: '2024-04-24T05:11:51.090Z', + completedAt: '2024-04-24T05:11:51.190Z', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'test5', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + { + properties: { + firstname: 'testmail1217', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: 'Duplicate IDs found in batch input: [12877907025]. IDs must be unique', + correlationId: 'd24ec5cd-8998-4674-a928-59603ae6b0eb', + context: { + ids: ['12877907025'], + }, + category: 'VALIDATION_ERROR', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + [ + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + }, + { + properties: { + firstname: 'testmail1217', + email: 'testmail1217@testmail.com', + }, + }, + { + properties: { + firstname: 'test5', + email: 'test5@xmail.con', + }, + }, + ], + ], + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: + 'Invalid input JSON on line 3, column 9: Cannot deserialize value of type `com.hubspot.inbounddb.publicobject.core.v2.SimplePublicObjectBatchInput$Json` from Array value (token `JsonToken.START_ARRAY`)', + correlationId: '99df04b9-da11-4504-bd97-2c15f58d0943', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/update', + method: 'POST', + data: { + inputs: [ + { + properties: { + firstname: 'testmail12178', + }, + id: '12877907024', + }, + { + properties: { + firstname: 'test1', + email: 'test1@mail.com', + }, + id: '12877907025', + }, + ], + }, + }, + httpRes: { + status: 200, + data: { + message: 'unknown response', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/search', + method: 'POST', + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + }, + }, + httpRes: { + data: { + total: 0, + results: [], + }, + status: 200, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/contacts/v1/contact/batch/', + method: 'POST', + headers: { + 'User-Agent': 'RudderLabs', + 'Content-Type': 'application/json', + Authorization: 'Bearer validApiKey', + }, + }, + httpRes: { + status: 200, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/contacts/v1/contact/batch/', + method: 'POST', + headers: { + 'User-Agent': 'RudderLabs', + 'Content-Type': 'application/json', + Authorization: 'Bearer inValidApiKey', + }, + }, + httpRes: { + status: 400, + data: { + status: 'error', + message: 'Errors found processing batch update', + correlationId: 'a716ef20-79df-44d4-98bd-9136af7bdefc', + invalidEmails: ['identify111052@test.con'], + failureMessages: [ + { + index: 1, + error: { + status: 'error', + message: 'Email address identify111052@test.con is invalid', + }, + }, + ], + }, + }, + }, + { + httpReq: { + url: 'https://track.hubspot.com/v1/event', + method: 'GET', + params: { + _a: 'dummy-hubId', + _n: 'test track event HS', + _m: 4.99, + email: 'testhubspot2@email.com', + firstname: 'Test Hubspot', + }, + }, + httpRes: { + status: 200, + data: {}, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer validAccessToken', + }, + }, + httpRes: { + status: 200, + data: { + status: 'COMPLETE', + results: [ + { + id: '44188066992', + properties: { + createdate: '2024-07-31T03:21:03.176Z', + email: 'testuser31848@testmail.com', + hs_all_contact_vids: '44188066992', + hs_email_domain: 'testmail.com', + hs_is_contact: 'true', + hs_is_unworked: 'true', + hs_lifecyclestage_lead_date: '2024-07-31T03:21:03.176Z', + hs_membership_has_accessed_private_content: '0', + hs_object_id: '44188066992', + hs_object_source: 'INTEGRATION', + hs_object_source_id: '3209723', + hs_object_source_label: 'INTEGRATION', + hs_pipeline: 'contacts-lifecycle-pipeline', + hs_registered_member: '0', + lastmodifieddate: '2024-07-31T03:21:03.176Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-07-31T03:21:03.176Z', + updatedAt: '2024-07-31T03:21:03.176Z', + archived: false, + }, + { + id: '44188066993', + properties: { + createdate: '2024-07-31T03:21:03.176Z', + email: 'testuser31847@testmail.com', + hs_all_contact_vids: '44188066993', + hs_email_domain: 'testmail.com', + hs_is_contact: 'true', + hs_is_unworked: 'true', + hs_lifecyclestage_lead_date: '2024-07-31T03:21:03.176Z', + hs_membership_has_accessed_private_content: '0', + hs_object_id: '44188066993', + hs_object_source: 'INTEGRATION', + hs_object_source_id: '3209723', + hs_object_source_label: 'INTEGRATION', + hs_pipeline: 'contacts-lifecycle-pipeline', + hs_registered_member: '0', + lastmodifieddate: '2024-07-31T03:21:03.176Z', + lifecyclestage: 'lead', + }, + createdAt: '2024-07-31T03:21:03.176Z', + updatedAt: '2024-07-31T03:21:03.176Z', + archived: false, + }, + ], + startedAt: '2024-07-31T03:21:03.133Z', + completedAt: '2024-07-31T03:21:03.412Z', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/events/v3/send', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + }, + httpRes: { + status: 200, + data: {}, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/events/v3/send', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer invalid-dummy-access-token', + }, + }, + httpRes: { + status: 401, + data: { + status: 'error', + message: + 'Authentication credentials not found. This API supports OAuth 2.0 authentication and you can find more details at https://developers.hubspot.com/docs/methods/auth/oauth-overview', + correlationId: '501651f6-bb90-40f1-b0db-349f62916993', + category: 'INVALID_AUTHENTICATION', + }, + }, + }, + { + httpReq: { + url: 'https://api.hubapi.com/crm/v3/associations/companies/contacts/batch/create', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer dummy-access-token', + }, + }, + httpRes: { + status: 201, + data: { + completedAt: '2024-07-31T04:46:34.391Z', + requestedAt: '2024-07-31T04:46:34.391Z', + startedAt: '2024-07-31T04:46:34.391Z', + results: [ + { + from: { + id: '9405415215', + }, + to: { + id: '1', + }, + type: 'contact_to_company', + }, + ], + status: 'PENDING', + }, + }, + }, ]; diff --git a/test/integrations/destinations/hs/router/data.ts b/test/integrations/destinations/hs/router/data.ts index e12efef4d0..989a581e55 100644 --- a/test/integrations/destinations/hs/router/data.ts +++ b/test/integrations/destinations/hs/router/data.ts @@ -1913,6 +1913,937 @@ export const data = [ }, }, }, + { + name: 'hs', + description: 'if dontBatch is true we are not going to create a batch out of those events', + feature: 'router', + module: 'destination', + version: 'v0', + scenario: 'buisness', + id: 'dontbatchtrue', + successCriteria: + 'should not create a batch with the events if that events contains dontBatch true', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id425', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow', + email: 'identify425@test.com', + phone: '9112340425', + lastname: 'Sparrow', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.070+05:30', + receivedAt: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe', + }, + metadata: { + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + dontBatch: true, + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id738', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow738', + email: 'identify425@test.con', + phone: '9112340738', + lastname: 'Sparrow738', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe738', + }, + metadata: { + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + dontBatch: true, + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.462+05:30', + userId: 'sample_user_id803', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow803', + email: 'identify803@test.com', + phone: '9112340803', + lastname: 'Sparrow803', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + anonymousId: '8d872292709c6fbe803', + originalTimestamp: '2024-05-23T16:49:57.462+05:30', + }, + metadata: { + userId: '<<>>8d872292709c6fbe803<<>>sample_user_id803', + jobId: 3, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.462+05:30', + userId: 'sample_user_id804', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow804', + email: 'identify804@test.con', + phone: '9112340804', + lastname: 'Sparrow804', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + anonymousId: '8d872292709c6fbe804', + originalTimestamp: '2024-05-23T16:49:57.462+05:30', + }, + metadata: { + userId: '<<>>8d872292709c6fbe804<<>>sample_user_id804', + jobId: 4, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + dontBatch: false, + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: true, + batchedRequest: { + body: { + FORM: {}, + JSON: { + inputs: [ + { + properties: { + email: 'identify803@test.com', + firstname: 'John', + lastname: 'Sparrow803', + phone: '9112340803', + }, + }, + { + properties: { + email: 'identify804@test.con', + firstname: 'John', + lastname: 'Sparrow804', + phone: '9112340804', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + jobId: 3, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe803<<>>sample_user_id803', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + dontBatch: false, + jobId: 4, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe804<<>>sample_user_id804', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + FORM: {}, + JSON: { + properties: { + email: 'identify425@test.com', + firstname: 'John', + lastname: 'Sparrow', + phone: '9112340425', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + dontBatch: true, + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + FORM: {}, + JSON: { + properties: { + email: 'identify425@test.con', + firstname: 'John', + lastname: 'Sparrow738', + phone: '9112340738', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + dontBatch: true, + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + ], + }, + }, + }, + }, + { + name: 'hs', + description: 'if dontBatch is not available we are considering those as dontbatch false', + feature: 'router', + module: 'destination', + version: 'v0', + scenario: 'buisness', + id: 'dontbatchundefined', + successCriteria: 'all events should be batched', + input: { + request: { + body: { + input: [ + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id425', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow', + email: 'identify425@test.com', + phone: '9112340425', + lastname: 'Sparrow', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.070+05:30', + receivedAt: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe', + }, + metadata: { + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + { + message: { + type: 'identify', + sentAt: '2024-05-23T16:49:57.461+05:30', + userId: 'sample_user_id738', + channel: 'mobile', + context: { + traits: { + age: '30', + name: 'John Sparrow738', + email: 'identify425@test.con', + phone: '9112340738', + lastname: 'Sparrow738', + firstname: 'John', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + timestamp: '2024-05-23T16:49:57.071+05:30', + anonymousId: '8d872292709c6fbe738', + }, + metadata: { + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + }, + destination: { + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + Name: 'hs-1', + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + Transformations: [], + IsProcessorEnabled: true, + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + }, + request: { + query: {}, + }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: true, + batchedRequest: { + body: { + FORM: {}, + JSON: { + inputs: [ + { + properties: { + email: 'identify425@test.com', + firstname: 'John', + lastname: 'Sparrow', + phone: '9112340425', + }, + }, + { + properties: { + email: 'identify425@test.con', + firstname: 'John', + lastname: 'Sparrow738', + phone: '9112340738', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.hubapi.com/crm/v3/objects/contacts/batch/create', + files: {}, + headers: { + Authorization: 'Bearer dontbatchtrueaccesstoken', + 'Content-Type': 'application/json', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + accessToken: 'dontbatchtrueaccesstoken', + apiKey: '', + apiVersion: 'newApi', + authorizationType: 'newPrivateAppApi', + blacklistedEvents: [], + connectionMode: 'cloud', + doAssociation: false, + eventDelivery: false, + eventDeliveryTS: 1687884567403, + eventFilteringOption: 'disable', + hubID: '25092171', + hubspotEvents: [ + { + eventProperties: [ + { + from: 'first_name', + to: 'first_name', + }, + { + from: 'last_name', + to: 'last_name', + }, + ], + hubspotEventName: 'pedummy-hubId_rs_hub_chair', + rsEventName: 'Order Complete', + }, + ], + ketchConsentPurposes: [ + { + purpose: '', + }, + ], + lookupField: 'email', + oneTrustCookieCategories: [], + useNativeSDK: false, + whitelistedEvents: [], + }, + Enabled: true, + ID: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + IsProcessorEnabled: true, + Name: 'hs-1', + RevisionID: '2gqf7Mc7WEwqQtQy3G105O22s3D', + Transformations: [], + WorkspaceID: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + metadata: [ + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + jobId: 1, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + { + destinationId: '2RnSBhn4zPTOF8NdqAIrnVPPnfr', + jobId: 2, + sourceId: '2RnN36pc7p5lzoApxZnDfRnYFx0', + transformAt: 'router', + userId: '<<>>8d872292709c6fbe738<<>>sample_user_id738', + workerAssignedTime: '2024-05-23T16:49:58.569269+05:30', + workspaceId: '2QapBTEvZYwuf6O9KB5AEvvBt8j', + }, + ], + statusCode: 200, + }, + ], + }, + }, + }, + }, { name: 'hs', description: 'router job ordering ', From c4a01eb9589c2af50087e4628eb79e510dc01245 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 12 Aug 2024 08:20:29 +0000 Subject: [PATCH 04/12] chore(release): 1.75.0 --- CHANGELOG.md | 13 +++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5659f1fd9a..9305e68efa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.75.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.1...v1.75.0) (2024-08-12) + + +### Features + +* move hubspot to transformer proxy to enable partial batch handling ([#3308](https://github.com/rudderlabs/rudder-transformer/issues/3308)) ([8450021](https://github.com/rudderlabs/rudder-transformer/commit/8450021672c51ac798ec0aeab422f5fceea5e53e)) + + +### Bug Fixes + +* handle attentive tag null, undefined properties ([#3647](https://github.com/rudderlabs/rudder-transformer/issues/3647)) ([9327925](https://github.com/rudderlabs/rudder-transformer/commit/932792561c98833baf9881a83ee36ae5000e37b4)) +* handle null values for braze dedupe ([#3638](https://github.com/rudderlabs/rudder-transformer/issues/3638)) ([0a9b681](https://github.com/rudderlabs/rudder-transformer/commit/0a9b68118241158623d45ee28896377296bafd91)) + ### [1.74.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.0...v1.74.1) (2024-08-08) diff --git a/package-lock.json b/package-lock.json index b5db5aa72e..e765269ec7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.74.1", + "version": "1.75.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.74.1", + "version": "1.75.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index dce50ee36d..2dbaeda368 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.74.1", + "version": "1.75.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 62cdc4641d44d79e21949722660173df4c749f24 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:22:57 +0530 Subject: [PATCH 05/12] fix: add validation for concurrent_modification error (#3654) --- .../networkHandler.js | 14 ++-- .../dataDelivery/business.ts | 67 +++++++++++++++++++ .../network.ts | 48 +++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) diff --git a/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js b/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js index 6491215a93..8fa4867b78 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/networkHandler.js @@ -1,7 +1,7 @@ const { NetworkError } = require('@rudderstack/integrations-lib'); +const get = require('get-value'); const { prepareProxyRequest, handleHttpRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess } = require('../../util/index'); - const { processAxiosResponse, getDynamicErrorType, @@ -148,9 +148,15 @@ const gaAudienceProxyRequest = async (request) => { }; const gaAudienceRespHandler = (destResponse, stageMsg) => { - const { status, response } = destResponse; - // const respAttributes = response["@attributes"] || null; - // const { stat, err_code: errorCode } = respAttributes; + let { status } = destResponse; + const { response } = destResponse; + + if ( + status === 400 && + get(response, 'error.details.0.errors.0.errorCode.databaseError') === 'CONCURRENT_MODIFICATION' + ) { + status = 500; + } throw new NetworkError( `${JSON.stringify(response)} ${stageMsg}`, diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts index c0d8454548..ee9f7bbae2 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/dataDelivery/business.ts @@ -373,4 +373,71 @@ export const testScenariosForV1API = [ }, }, }, + { + id: 'garl_v1_scenario_4', + name: 'google_adwords_remarketing_lists', + description: + '[Proxy v1 API] :: getting concurrent_modification error code while sending request to GA audience API', + successCriteria: 'Should return 500 with destination response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers: commonHeaders, + params: { ...commonParams, customerId: 'wrongCustomerId' }, + JSON: validRequestPayload2, + endpoint: + 'https://googleads.googleapis.com/v15/customers/wrongCustomerId/offlineUserDataJobs', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + '{"error":{"code":400,"message":"Request contains an invalid argument.","status":"INVALID_ARGUMENT","details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"databaseError":"CONCURRENT_MODIFICATION"},"message":"Multiple requests were attempting to modify the same resource at once. Retry the request."}],"requestId":"08X6xmM1WJPf_lW1ppYfsA"}]}} during ga_audience response transformation', + response: [ + { + error: + '{"error":{"code":400,"message":"Request contains an invalid argument.","status":"INVALID_ARGUMENT","details":[{"@type":"type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure","errors":[{"errorCode":{"databaseError":"CONCURRENT_MODIFICATION"},"message":"Multiple requests were attempting to modify the same resource at once. Retry the request."}],"requestId":"08X6xmM1WJPf_lW1ppYfsA"}]}} during ga_audience response transformation', + metadata: { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 1, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + statusCode: 500, + }, + ], + statTags: { + destType: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + destinationId: 'default-destinationId', + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + status: 500, + }, + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/network.ts b/test/integrations/destinations/google_adwords_remarketing_lists/network.ts index d17f518c46..9c2c7f1709 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/network.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/network.ts @@ -289,4 +289,52 @@ export const networkCallsData = [ }, }, }, + { + httpReq: { + url: 'https://googleads.googleapis.com/v15/customers/wrongCustomerId/offlineUserDataJobs:create', + data: { + job: { + type: 'CUSTOMER_MATCH_USER_LIST', + customerMatchUserListMetadata: { + userList: 'customers/wrongCustomerId/userLists/709078448', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + }, + }, + headers: { + Authorization: 'Bearer dummy-access', + 'Content-Type': 'application/json', + 'developer-token': 'dummy-dev-token', + }, + method: 'POST', + }, + httpRes: { + status: 400, + data: { + error: { + code: 400, + message: 'Request contains an invalid argument.', + status: 'INVALID_ARGUMENT', + details: [ + { + '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', + errors: [ + { + errorCode: { + databaseError: 'CONCURRENT_MODIFICATION', + }, + message: + 'Multiple requests were attempting to modify the same resource at once. Retry the request.', + }, + ], + requestId: '08X6xmM1WJPf_lW1ppYfsA', + }, + ], + }, + }, + }, + }, ]; From 6c51487183b6b0b755620aac6cd51c2ffc966102 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:27:58 +0530 Subject: [PATCH 06/12] fix: clevertap bugsnag issue (#3656) fix: fixing clevertap bugsnag issue --- src/v0/destinations/clevertap/transform.js | 2 +- .../destinations/clevertap/processor/data.ts | 78 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/v0/destinations/clevertap/transform.js b/src/v0/destinations/clevertap/transform.js index 214f9e6409..e558b119f1 100644 --- a/src/v0/destinations/clevertap/transform.js +++ b/src/v0/destinations/clevertap/transform.js @@ -316,7 +316,7 @@ const responseBuilderSimple = (message, category, destination) => { // For 'Order Completed' type of events we are mapping it as 'Charged' // Special event in Clevertap. // Source: https://developer.clevertap.com/docs/concepts-events#recording-customer-purchases - if (get(message.event) && get(message.event).toLowerCase() === 'order completed') { + if (get(message.event) && get(message.event).toString().toLowerCase() === 'order completed') { eventPayload = { evtName: 'Charged', evtData: constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.ECOM.name]), diff --git a/test/integrations/destinations/clevertap/processor/data.ts b/test/integrations/destinations/clevertap/processor/data.ts index 49647e1937..f15fc409fb 100644 --- a/test/integrations/destinations/clevertap/processor/data.ts +++ b/test/integrations/destinations/clevertap/processor/data.ts @@ -3354,4 +3354,82 @@ export const data = [ }, }, }, + { + name: 'clevertap', + description: 'Should not throw bugsnag error if event is not of type string', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: { + Config: { + passcode: 'sample_passcode', + accountId: '476550467', + trackAnonymous: true, + enableObjectIdMapping: false, + }, + }, + message: { + type: 'track', + userId: 'user123', + event: 'Product Purchased', + properties: { + name: "Rubik's Cube", + revenue: 4.99, + }, + context: { + ip: '14.5.67.21', + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.clevertap.com/1/upload', + headers: { + 'X-CleverTap-Account-Id': '476550467', + 'X-CleverTap-Passcode': 'sample_passcode', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + d: [ + { + evtName: 'Product Purchased', + evtData: { + name: "Rubik's Cube", + revenue: 4.99, + }, + type: 'event', + identity: 'user123', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; From 31b03fc022ace0cda8df798c50e4764a9703c23b Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:58:48 +0530 Subject: [PATCH 07/12] fix: snapchat conversion bugsnag issue (#3657) * fix: fixing snapchat bugsnag issue * fix: code review suggestion Co-authored-by: Sankeerth --------- Co-authored-by: Sankeerth --- .../snapchat_conversion/transform.js | 4 +- .../snapchat_conversion/processor/data.ts | 114 ++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/snapchat_conversion/transform.js b/src/v0/destinations/snapchat_conversion/transform.js index 6fec6313a4..dbb2865e4a 100644 --- a/src/v0/destinations/snapchat_conversion/transform.js +++ b/src/v0/destinations/snapchat_conversion/transform.js @@ -179,7 +179,7 @@ const getEventConversionType = (message) => { // Returns the response for the track event after constructing the payload and setting necessary fields const trackResponseBuilder = (message, { Config }, mappedEvent) => { let payload = {}; - const event = mappedEvent.trim().replace(/\s+/g, '_'); + const event = mappedEvent?.toString().trim().replace(/\s+/g, '_'); const eventConversionType = getEventConversionType(message); const { apiKey, pixelId, snapAppId, appId, deduplicationKey, enableDeduplication } = Config; validateEventConfiguration(eventConversionType, pixelId, snapAppId, appId); @@ -305,7 +305,7 @@ const eventMappingHandler = (message, destination) => { if (!event) { throw new InstrumentationError('Event name is required'); } - event = event.trim().replace(/\s+/g, '_'); + event = event.toString().trim().replace(/\s+/g, '_'); let { rudderEventsToSnapEvents } = destination.Config; const mappedEvents = new Set(); diff --git a/test/integrations/destinations/snapchat_conversion/processor/data.ts b/test/integrations/destinations/snapchat_conversion/processor/data.ts index 7de7ed9b8d..b7fde67c4e 100644 --- a/test/integrations/destinations/snapchat_conversion/processor/data.ts +++ b/test/integrations/destinations/snapchat_conversion/processor/data.ts @@ -4734,6 +4734,120 @@ export const data = [ }, }, }, + { + name: 'snapchat_conversion', + description: "Test case non string event doesn't match with snapchat events", + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + messageId: 'ec5481b6-a926-4d2e-b293-0b3a77c4d3be', + originalTimestamp: '2022-04-22T10:57:58Z', + anonymousId: 'ea5cfab2-3961-4d8a-8187-3d1858c99090', + context: { + traits: { + email: 'test@email.com', + phone: '+91 2111111 ', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + locale: 'en-US', + os: { + name: 'iOS', + version: '14.4.1', + }, + screen: { + density: 2, + }, + externalId: [ + { + type: 'ga4AppInstanceId', + id: 'f0dd99v4f979fb997ce453373900f891', + }, + ], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + }, + type: 'track', + event: 1234, + properties: { + query: 't-shirts', + event_conversion_type: 'web', + }, + integrations: { + All: true, + }, + sentAt: '2022-04-22T10:57:58Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: false, + }, + }, + Config: { + pixelId: 'dummyPixelId', + apiKey: 'dummyApiKey', + rudderEventsToSnapEvents: [], + }, + }, + metadata: { + jobId: 45, + destinationId: 'd2', + workspaceId: 'w2', + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 45, + destinationId: 'd2', + workspaceId: 'w2', + }, + error: "Event 1234 doesn't match with Snapchat Events!", + statTags: { + destType: 'SNAPCHAT_CONVERSION', + destinationId: 'd2', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + workspaceId: 'w2', + }, + statusCode: 400, + }, + ], + }, + }, + }, ].map((tc) => ({ ...tc, mockFns: (_) => { From 1cb3f86c894f30bdf04df870484c6df3a956f36e Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:12:00 +0530 Subject: [PATCH 08/12] fix: validation for iterable object of HS (#3653) --- src/v0/destinations/hs/transform.js | 2 +- src/v0/destinations/hs/util.js | 9 + src/v0/destinations/hs/util.test.js | 19 ++ src/v1/destinations/hs/networkHandler.ts | 1 + .../destinations/hs/router/data.ts | 213 ++++++++++++++++++ test/integrations/testUtils.ts | 2 +- 6 files changed, 244 insertions(+), 2 deletions(-) diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 1d1a81b6b0..b6ca213f03 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -107,6 +107,7 @@ const processBatchRouter = async (inputs, reqMetadata) => { errorRespList: tempInputs.map((input) => handleRtTfSingleEventError(input, error, reqMetadata), ), + dontBatchEvents: [], }; } @@ -192,7 +193,6 @@ const processRouterDest = async (inputs, reqMetadata) => { batchedResponseList.push(...response.batchedResponseList); dontBatchEvents.push(...response.dontBatchEvents); }); - console.log(JSON.stringify([...batchedResponseList, ...errorRespList, ...dontBatchEvents])); return [...batchedResponseList, ...errorRespList, ...dontBatchEvents]; }; diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index fc22a89d69..56ec6e4167 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -875,6 +875,14 @@ const convertToResponseFormat = (successRespListWithDontBatchTrue) => { return response; }; +const isIterable = (obj) => { + // checks for null and undefined + if (obj == null) { + return false; + } + return typeof obj[Symbol.iterator] === 'function'; +}; + module.exports = { validateDestinationConfig, addExternalIdToHSTraits, @@ -895,4 +903,5 @@ module.exports = { extractIDsForSearchAPI, getRequestData, convertToResponseFormat, + isIterable, }; diff --git a/src/v0/destinations/hs/util.test.js b/src/v0/destinations/hs/util.test.js index ea2e10dc3d..819baff83c 100644 --- a/src/v0/destinations/hs/util.test.js +++ b/src/v0/destinations/hs/util.test.js @@ -3,6 +3,7 @@ const { extractIDsForSearchAPI, validatePayloadDataTypes, getObjectAndIdentifierType, + isIterable, } = require('./util'); const { primaryToSecondaryFields } = require('./config'); @@ -239,3 +240,21 @@ describe('getRequestDataAndRequestOptions utility test cases', () => { expect(requestData).toEqual(expectedRequestData); }); }); + +describe('isIterable utility test cases', () => { + it('should return true when the input is an array', () => { + const input = [1, 2, 3]; + const result = isIterable(input); + expect(result).toBe(true); + }); + it('should return false when the input is null', () => { + const input = null; + const result = isIterable(input); + expect(result).toBe(false); + }); + it('should return false when the input is undefined', () => { + const input = undefined; + const result = isIterable(input); + expect(result).toBe(false); + }); +}); diff --git a/src/v1/destinations/hs/networkHandler.ts b/src/v1/destinations/hs/networkHandler.ts index 8293a7240c..8dd22b465f 100644 --- a/src/v1/destinations/hs/networkHandler.ts +++ b/src/v1/destinations/hs/networkHandler.ts @@ -30,6 +30,7 @@ const findFeatureandVersion = (response, rudderJobMetadata, destinationConfig) = if ( Array.isArray(results) && results.length !== rudderJobMetadata.length && + Array.isArray(errors) && results.length + errors.length === rudderJobMetadata.length ) return 'newApiWithMultipleEventsAndErrors'; diff --git a/test/integrations/destinations/hs/router/data.ts b/test/integrations/destinations/hs/router/data.ts index 989a581e55..b47d6b7f07 100644 --- a/test/integrations/destinations/hs/router/data.ts +++ b/test/integrations/destinations/hs/router/data.ts @@ -3394,4 +3394,217 @@ export const data = [ }, }, }, + { + name: 'hs', + description: 'test when get properties call failed', + feature: 'router', + module: 'destination', + id: 'routerGetPropertiesCallFailed', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + channel: 'web', + context: { + mappedToDestination: true, + externalId: [ + { identifierType: 'email', id: 'testhubspot2@email.com', type: 'HS-lead' }, + ], + sources: { + job_id: '24c5HJxHomh6YCngEOCgjS5r1KX/Syncher', + task_id: 'vw_rs_mailchimp_mocked_hg_data', + version: 'v1.8.1', + batch_id: 'f252c69d-c40d-450e-bcd2-2cf26cb62762', + job_run_id: 'c8el40l6e87v0c4hkbl0', + task_run_id: 'c8el40l6e87v0c4hkblg', + }, + }, + type: 'identify', + traits: { firstname: 'Test Hubspot', anonymousId: '12345', country: 'India' }, + messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', + originalTimestamp: '2019-10-15T09:35:31.288Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { apiKey: 'invalid-api-key', hubID: 'dummy-hubId' }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + metadata: { jobId: 2, userId: 'u1' }, + }, + { + message: { + channel: 'web', + context: { + mappedToDestination: true, + externalId: [ + { identifierType: 'email', id: 'testhubspot@email.com', type: 'HS-lead' }, + ], + sources: { + job_id: '24c5HJxHomh6YCngEOCgjS5r1KX/Syncher', + task_id: 'vw_rs_mailchimp_mocked_hg_data', + version: 'v1.8.1', + batch_id: 'f252c69d-c40d-450e-bcd2-2cf26cb62762', + job_run_id: 'c8el40l6e87v0c4hkbl0', + task_run_id: 'c8el40l6e87v0c4hkblg', + }, + }, + type: 'identify', + traits: { firstname: 'Test Hubspot 1', anonymousId: '123451', country: 'India 1' }, + messageId: '50360b9c-ea8d-409c-b672-c9230f91cce5', + originalTimestamp: '2019-10-15T09:35:31.288Z', + anonymousId: '00000000000000000000000000', + userId: '12345', + integrations: { All: true }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + destination: { + Config: { apiKey: 'invalid-api-key', hubID: 'dummy-hubId' }, + secretConfig: {}, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + name: 'Hubspot', + enabled: true, + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + deleted: false, + createdAt: '2020-12-30T08:39:32.005Z', + updatedAt: '2021-02-03T16:22:31.374Z', + destinationDefinition: { + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + displayName: 'Hubspot', + createdAt: '2020-04-09T09:24:31.794Z', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + transformations: [], + isConnectionEnabled: true, + isProcessorEnabled: true, + }, + metadata: { jobId: 3, userId: 'u1' }, + }, + ], + destType: 'hs', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: false, + destination: { + Config: { + apiKey: 'invalid-api-key', + hubID: 'dummy-hubId', + }, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + createdAt: '2020-12-30T08:39:32.005Z', + deleted: false, + destinationDefinition: { + createdAt: '2020-04-09T09:24:31.794Z', + displayName: 'Hubspot', + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + enabled: true, + isConnectionEnabled: true, + isProcessorEnabled: true, + name: 'Hubspot', + secretConfig: {}, + transformations: [], + updatedAt: '2021-02-03T16:22:31.374Z', + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + error: + '{"message":"Failed to get hubspot properties: {\\"status\\":\\"error\\",\\"message\\":\\"The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/\\",\\"correlationId\\":\\"4d39ff11-e121-4514-bcd8-132a9dd1ff50\\",\\"category\\":\\"INVALID_AUTHENTICATION\\",\\"links\\":{\\"api key\\":\\"https://app.hubspot.com/l/api-key/\\"}}","destinationResponse":{"response":{"status":"error","message":"The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/","correlationId":"4d39ff11-e121-4514-bcd8-132a9dd1ff50","category":"INVALID_AUTHENTICATION","links":{"api key":"https://app.hubspot.com/l/api-key/"}},"status":401}}', + metadata: [ + { + jobId: 2, + userId: 'u1', + }, + ], + statTags: { + destType: 'HS', + errorCategory: 'network', + errorType: 'aborted', + feature: 'router', + implementation: 'native', + module: 'destination', + }, + statusCode: 401, + }, + { + batched: false, + destination: { + Config: { + apiKey: 'invalid-api-key', + hubID: 'dummy-hubId', + }, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + createdAt: '2020-12-30T08:39:32.005Z', + deleted: false, + destinationDefinition: { + createdAt: '2020-04-09T09:24:31.794Z', + displayName: 'Hubspot', + id: '1aIXqM806xAVm92nx07YwKbRrO9', + name: 'HS', + updatedAt: '2021-01-11T11:03:28.103Z', + }, + enabled: true, + isConnectionEnabled: true, + isProcessorEnabled: true, + name: 'Hubspot', + secretConfig: {}, + transformations: [], + updatedAt: '2021-02-03T16:22:31.374Z', + workspaceId: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + error: + '{"message":"Failed to get hubspot properties: {\\"status\\":\\"error\\",\\"message\\":\\"The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/\\",\\"correlationId\\":\\"4d39ff11-e121-4514-bcd8-132a9dd1ff50\\",\\"category\\":\\"INVALID_AUTHENTICATION\\",\\"links\\":{\\"api key\\":\\"https://app.hubspot.com/l/api-key/\\"}}","destinationResponse":{"response":{"status":"error","message":"The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/","correlationId":"4d39ff11-e121-4514-bcd8-132a9dd1ff50","category":"INVALID_AUTHENTICATION","links":{"api key":"https://app.hubspot.com/l/api-key/"}},"status":401}}', + metadata: [ + { + jobId: 3, + userId: 'u1', + }, + ], + statTags: { + destType: 'HS', + errorCategory: 'network', + errorType: 'aborted', + feature: 'router', + implementation: 'native', + module: 'destination', + }, + statusCode: 401, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index bec79f8570..6eaf662c78 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -27,7 +27,7 @@ export const getTestDataFilePaths = (dirPath: string, opts: OptionValues): strin if (opts.destination || opts.source) { testFilePaths = testFilePaths.filter((testFile) => - testFile.includes(opts.destination || opts.source), + testFile.includes(`${opts.destination}/` || opts.source), ); } if (opts.feature) { From 932a2968e95a51febc27b5ee2389f2631715545e Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Wed, 14 Aug 2024 07:44:18 +0000 Subject: [PATCH 09/12] chore(release): 1.75.1 --- CHANGELOG.md | 10 ++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9305e68efa..2e4a636b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14) + + +### Bug Fixes + +* add validation for concurrent_modification error ([#3654](https://github.com/rudderlabs/rudder-transformer/issues/3654)) ([62cdc46](https://github.com/rudderlabs/rudder-transformer/commit/62cdc4641d44d79e21949722660173df4c749f24)) +* clevertap bugsnag issue ([#3656](https://github.com/rudderlabs/rudder-transformer/issues/3656)) ([6c51487](https://github.com/rudderlabs/rudder-transformer/commit/6c51487183b6b0b755620aac6cd51c2ffc966102)) +* snapchat conversion bugsnag issue ([#3657](https://github.com/rudderlabs/rudder-transformer/issues/3657)) ([31b03fc](https://github.com/rudderlabs/rudder-transformer/commit/31b03fc022ace0cda8df798c50e4764a9703c23b)) +* validation for iterable object of HS ([#3653](https://github.com/rudderlabs/rudder-transformer/issues/3653)) ([1cb3f86](https://github.com/rudderlabs/rudder-transformer/commit/1cb3f86c894f30bdf04df870484c6df3a956f36e)) + ## [1.75.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.74.1...v1.75.0) (2024-08-12) diff --git a/package-lock.json b/package-lock.json index e765269ec7..d16e52424a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.75.0", + "version": "1.75.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.75.0", + "version": "1.75.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 2dbaeda368..2dad49bba6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.75.0", + "version": "1.75.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 866dbf3e81754e71ff8ac08b258b359ec5cc6889 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:20:01 +0530 Subject: [PATCH 10/12] fix: attentive tag bugsnag issue (#3663) * fix: attentive tag bugsnag issue * fix: review comment addressed * fix: review comment addressed * fix: editing test caes * fix: editing test caes --- .../destinations/attentive_tag/transform.js | 6 +-- src/v0/destinations/attentive_tag/util.js | 13 +++-- .../destinations/attentive_tag/util.test.js | 51 +++++++++++++++++++ .../attentive_tag/processor/data.ts | 2 +- 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 src/v0/destinations/attentive_tag/util.test.js diff --git a/src/v0/destinations/attentive_tag/transform.js b/src/v0/destinations/attentive_tag/transform.js index fa05eb6c21..5522c078b0 100644 --- a/src/v0/destinations/attentive_tag/transform.js +++ b/src/v0/destinations/attentive_tag/transform.js @@ -16,7 +16,7 @@ const { const { getDestinationItemProperties, getExternalIdentifiersMapping, - getPropertiesKeyValidation, + arePropertiesValid, validateTimestamp, } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -137,9 +137,9 @@ const trackResponseBuilder = (message, { Config }) => { payload = constructPayload(message, mappingConfig[ConfigCategory.TRACK.name]); endpoint = ConfigCategory.TRACK.endpoint; payload.type = get(message, 'event'); - if (!getPropertiesKeyValidation(payload)) { + if (!arePropertiesValid(payload.properties)) { throw new InstrumentationError( - '[Attentive Tag]:The event name contains characters which is not allowed', + '[Attentive Tag]:The properties contains characters which is not allowed', ); } } diff --git a/src/v0/destinations/attentive_tag/util.js b/src/v0/destinations/attentive_tag/util.js index c96d03028f..8e73f572fc 100644 --- a/src/v0/destinations/attentive_tag/util.js +++ b/src/v0/destinations/attentive_tag/util.js @@ -14,12 +14,19 @@ const { mappingConfig, ConfigCategory } = require('./config'); * The keys should not contain any of the values inside the validationArray. * STEP 1: Storing keys in the array. * Checking for the non-valid characters inside the keys of properties. + * ref: https://docs.attentivemobile.com/openapi/reference/tag/Custom-Events/ * @param {*} payload * @returns */ -const getPropertiesKeyValidation = (payload) => { +const arePropertiesValid = (properties) => { + if (!isDefinedAndNotNullAndNotEmpty(properties)) { + return true; + } + if (typeof properties !== 'object') { + return false; + } const validationArray = [`'`, `"`, `{`, `}`, `[`, `]`, ',', `,`]; - const keys = Object.keys(payload.properties); + const keys = Object.keys(properties); for (const key of keys) { for (const validationChar of validationArray) { if (key.includes(validationChar)) { @@ -134,6 +141,6 @@ const getDestinationItemProperties = (message, isItemsRequired) => { module.exports = { getDestinationItemProperties, getExternalIdentifiersMapping, - getPropertiesKeyValidation, + arePropertiesValid, validateTimestamp, }; diff --git a/src/v0/destinations/attentive_tag/util.test.js b/src/v0/destinations/attentive_tag/util.test.js new file mode 100644 index 0000000000..59fdc9f361 --- /dev/null +++ b/src/v0/destinations/attentive_tag/util.test.js @@ -0,0 +1,51 @@ +const { arePropertiesValid } = require('./util'); + +describe('arePropertiesValid', () => { + // returns true for valid properties object with no special characters in keys + it('should return true when properties object has no special characters in keys', () => { + const properties = { key1: 'value1', key2: 'value2' }; + const result = arePropertiesValid(properties); + expect(result).toBe(true); + }); + + // returns true for null properties input + it('should return true when properties input is null', () => { + const properties = null; + const result = arePropertiesValid(properties); + expect(result).toBe(true); + }); + + // returns false for properties object with keys containing special characters + it('should return false for properties object with keys containing special characters', () => { + const properties = { + key1: 'value1', + 'key,2': 'value2', + key3: 'value3', + }; + expect(arePropertiesValid(properties)).toBe(false); + }); + + // returns true for empty properties object + it('should return true for empty properties object', () => { + const properties = {}; + expect(arePropertiesValid(properties)).toBe(true); + }); + + // returns true for undefined properties input + it('should return true for undefined properties input', () => { + const result = arePropertiesValid(undefined); + expect(result).toBe(true); + }); + + // returns true for empty string properties input + it('should return true for empty string properties input', () => { + const result = arePropertiesValid(''); + expect(result).toBe(true); + }); + + // returns false for empty string properties input + it('should return false for non object properties input', () => { + const result = arePropertiesValid('1234'); + expect(result).toBe(false); + }); +}); diff --git a/test/integrations/destinations/attentive_tag/processor/data.ts b/test/integrations/destinations/attentive_tag/processor/data.ts index 12e0f4bea1..c026569439 100644 --- a/test/integrations/destinations/attentive_tag/processor/data.ts +++ b/test/integrations/destinations/attentive_tag/processor/data.ts @@ -1458,7 +1458,7 @@ export const data = [ body: [ { statusCode: 400, - error: '[Attentive Tag]:The event name contains characters which is not allowed', + error: '[Attentive Tag]:The properties contains characters which is not allowed', statTags: { destType: 'ATTENTIVE_TAG', errorCategory: 'dataValidation', From bcdabe2a9be41dd18881420a0012414acbad1715 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:20:46 +0530 Subject: [PATCH 11/12] chore(deps): bump axios from 1.6.5 to 1.7.3 (#3632) * chore(deps): bump axios from 1.6.5 to 1.7.3 Bumps [axios](https://github.com/axios/axios) from 1.6.5 to 1.7.3. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.6.5...v1.7.3) --- updated-dependencies: - dependency-name: axios dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: include package files to eslintignore * chore: include eslintignore file to prettierignore --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Co-authored-by: ItsSudip --- .eslintignore | 2 ++ .prettierignore | 1 + package-lock.json | 10 +++++----- package.json | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.eslintignore b/.eslintignore index ce54730f4b..abeac6b164 100644 --- a/.eslintignore +++ b/.eslintignore @@ -21,3 +21,5 @@ src/v0/destinations/personalize/scripts/ test/integrations/destinations/testTypes.d.ts *.config*.js scripts/skipPrepareScript.js +package-lock.json +package.json diff --git a/.prettierignore b/.prettierignore index 99747b29bb..10ea30580f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,3 +8,4 @@ test/**/*.js src/util/lodash-es-core.js src/util/url-search-params.min.js dist +.eslintignore \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8c7377da3e..c29db44559 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", "ajv-formats": "^2.1.1", - "axios": "^1.6.4", + "axios": "^1.7.3", "btoa": "^1.2.1", "component-each": "^0.2.6", "crypto-js": "^4.2.0", @@ -7244,11 +7244,11 @@ "integrity": "sha512-dihteGhwbJpT89kVbacWiyKeAZr+En0YGK6pAKQJLR0En9ZxSH2H4TTvfG4bBjzFq9gDAma4y9BrpDns6j5UiQ==" }, "node_modules/axios": { - "version": "1.6.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.5.tgz", - "integrity": "sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.3.tgz", + "integrity": "sha512-Ar7ND9pU99eJ9GpoGQKhKf58GpUOgnzuaB7ueNQ5BMi0p+LZ5oaEnfF999fAArcTIBwXTCHAmGcHOZJaWPq9Nw==", "dependencies": { - "follow-redirects": "^1.15.4", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } diff --git a/package.json b/package.json index c14e6b9ef5..d13f7f14da 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", "ajv-formats": "^2.1.1", - "axios": "^1.6.4", + "axios": "^1.7.3", "btoa": "^1.2.1", "component-each": "^0.2.6", "crypto-js": "^4.2.0", From 78e7a95dd1fabdface2f14d402b2b63f0f47c082 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:52:22 +0530 Subject: [PATCH 12/12] chore(deps): bump docker/build-push-action from 6.5.0 to 6.7.0 (#3666) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.5.0 to 6.7.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.5.0...v6.7.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sankeerth --- .github/workflows/build-push-docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 69f68c1676..cffb5f91da 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -107,7 +107,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - name: Build Docker Image - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.7.0 with: context: . file: ${{ inputs.dockerfile }} @@ -124,7 +124,7 @@ jobs: docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.7.0 with: context: . file: ${{ inputs.dockerfile }} @@ -160,7 +160,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - name: Build Docker Image - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.7.0 with: context: . file: ${{ inputs.dockerfile }} @@ -177,7 +177,7 @@ jobs: docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.7.0 with: context: . file: ${{ inputs.dockerfile }}