From e21ebd0085aadfe61cb6442da6689e32be33f52f Mon Sep 17 00:00:00 2001
From: Gauravudia <60897972+Gauravudia@users.noreply.github.com>
Date: Thu, 29 Aug 2024 17:20:30 +0530
Subject: [PATCH] feat: webhook v2 (#3651)
* feat: webhook v2
* feat: webhook v2
* refactor: getAuthHeaders utility
* feat: add xml support
* feat: add batching
* feat: update batching logic
* chore: remove logs
* fix: excludeMappedFields utility
* test: add testcases
* fix: use sha256 for better hashing
---
package-lock.json | 19 +-
package.json | 3 +-
.../destinations/webhook_v2/procWorkflow.yaml | 67 ++++
.../destinations/webhook_v2/rtWorkflow.yaml | 60 +++
src/cdk/v2/destinations/webhook_v2/utils.js | 146 ++++++++
src/features.json | 3 +-
.../destinations/webhook_v2/common.ts | 321 ++++++++++++++++
.../webhook_v2/processor/configuration.ts | 206 +++++++++++
.../destinations/webhook_v2/processor/data.ts | 2 +
.../destinations/webhook_v2/router/data.ts | 350 ++++++++++++++++++
10 files changed, 1171 insertions(+), 6 deletions(-)
create mode 100644 src/cdk/v2/destinations/webhook_v2/procWorkflow.yaml
create mode 100644 src/cdk/v2/destinations/webhook_v2/rtWorkflow.yaml
create mode 100644 src/cdk/v2/destinations/webhook_v2/utils.js
create mode 100644 test/integrations/destinations/webhook_v2/common.ts
create mode 100644 test/integrations/destinations/webhook_v2/processor/configuration.ts
create mode 100644 test/integrations/destinations/webhook_v2/processor/data.ts
create mode 100644 test/integrations/destinations/webhook_v2/router/data.ts
diff --git a/package-lock.json b/package-lock.json
index c29db44559..fa6221bebe 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -20,7 +20,7 @@
"@ndhoule/extend": "^2.0.0",
"@pyroscope/nodejs": "^0.2.9",
"@rudderstack/integrations-lib": "^0.2.10",
- "@rudderstack/json-template-engine": "^0.17.1",
+ "@rudderstack/json-template-engine": "^0.18.0",
"@rudderstack/workflow-engine": "^0.8.13",
"@shopify/jest-koa-mocks": "^5.1.1",
"ajv": "^8.12.0",
@@ -45,6 +45,7 @@
"json-diff": "^1.0.3",
"json-size": "^1.0.0",
"jsontoxml": "^1.0.1",
+ "jstoxml": "^5.0.2",
"koa": "^2.14.1",
"koa-bodyparser": "^4.4.0",
"koa2-swagger-ui": "^5.7.0",
@@ -5583,9 +5584,9 @@
}
},
"node_modules/@rudderstack/json-template-engine": {
- "version": "0.17.1",
- "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.17.1.tgz",
- "integrity": "sha512-i8kSHSwkZenx2TX0rZtUcrxpebSrYSMQruW2YnxfoBk4ElAW6jGCopOPQlZ1+mqyv4J5h2mcdQyP/UzLGxvfDw=="
+ "version": "0.18.0",
+ "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.18.0.tgz",
+ "integrity": "sha512-yArdj5flPrYbH3lq8xLixBGjt74WOv+TY0rrTF8gB7v6hFFBn7IrcsNcLDbN2SoLT604ycgMTMgIYcsAqAWWDg=="
},
"node_modules/@rudderstack/workflow-engine": {
"version": "0.8.13",
@@ -5623,6 +5624,11 @@
"tslib": "^2.6.2"
}
},
+ "node_modules/@rudderstack/workflow-engine/node_modules/@rudderstack/json-template-engine": {
+ "version": "0.17.1",
+ "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.17.1.tgz",
+ "integrity": "sha512-i8kSHSwkZenx2TX0rZtUcrxpebSrYSMQruW2YnxfoBk4ElAW6jGCopOPQlZ1+mqyv4J5h2mcdQyP/UzLGxvfDw=="
+ },
"node_modules/@shopify/jest-koa-mocks": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/@shopify/jest-koa-mocks/-/jest-koa-mocks-5.1.1.tgz",
@@ -15638,6 +15644,11 @@
"node": ">=0.2.0"
}
},
+ "node_modules/jstoxml": {
+ "version": "5.0.2",
+ "resolved": "https://registry.npmjs.org/jstoxml/-/jstoxml-5.0.2.tgz",
+ "integrity": "sha512-p/Uyi1nSlAcOL+FbWCbTLAHtMbk/QlPMAE/wRLek7W8646jWII3GtLEKSBzf97UitieRWj1VZcbZxs8arq2nbg=="
+ },
"node_modules/keygrip": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz",
diff --git a/package.json b/package.json
index d13f7f14da..d8e34f55fe 100644
--- a/package.json
+++ b/package.json
@@ -65,7 +65,7 @@
"@ndhoule/extend": "^2.0.0",
"@pyroscope/nodejs": "^0.2.9",
"@rudderstack/integrations-lib": "^0.2.10",
- "@rudderstack/json-template-engine": "^0.17.1",
+ "@rudderstack/json-template-engine": "^0.18.0",
"@rudderstack/workflow-engine": "^0.8.13",
"@shopify/jest-koa-mocks": "^5.1.1",
"ajv": "^8.12.0",
@@ -90,6 +90,7 @@
"json-diff": "^1.0.3",
"json-size": "^1.0.0",
"jsontoxml": "^1.0.1",
+ "jstoxml": "^5.0.2",
"koa": "^2.14.1",
"koa-bodyparser": "^4.4.0",
"koa2-swagger-ui": "^5.7.0",
diff --git a/src/cdk/v2/destinations/webhook_v2/procWorkflow.yaml b/src/cdk/v2/destinations/webhook_v2/procWorkflow.yaml
new file mode 100644
index 0000000000..873a9807ce
--- /dev/null
+++ b/src/cdk/v2/destinations/webhook_v2/procWorkflow.yaml
@@ -0,0 +1,67 @@
+bindings:
+ - name: EventType
+ path: ../../../../constants
+ - path: ../../bindings/jsontemplate
+ exportAll: true
+ - path: ../../../../v0/destinations/webhook/utils
+ - name: getHashFromArray
+ path: ../../../../v0/util
+ - name: getIntegrationsObj
+ path: ../../../../v0/util
+ - name: removeUndefinedAndNullValues
+ path: ../../../../v0/util
+ - name: defaultRequestConfig
+ path: ../../../../v0/util
+ - name: isEmptyObject
+ path: ../../../../v0/util
+ - path: ./utils
+
+steps:
+ - name: validateInput
+ template: |
+ $.assertConfig(.destination.Config.webhookUrl, "Webhook URL required. Aborting");
+ $.assertConfig(!(.destination.Config.auth === "basicAuth" && !(.destination.Config.username)), "Username is required for Basic Authentication. Aborting");
+ $.assertConfig(!(.destination.Config.auth === "bearerTokenAuth" && !(.destination.Config.bearerToken)), "Token is required for Bearer Token Authentication. Aborting");
+ $.assertConfig(!(.destination.Config.auth === "apiKeyAuth" && !(.destination.Config.apiKeyName)), "API Key Name is required for API Key Authentication. Aborting");
+ $.assertConfig(!(.destination.Config.auth === "apiKeyAuth" && !(.destination.Config.apiKeyValue)), "API Key Value is required for API Key Authentication. Aborting");
+
+ - name: deduceMethod
+ template: |
+ $.context.method = .destination.Config.method ?? 'POST';
+
+ - name: deduceBodyFormat
+ template: |
+ $.context.format = .destination.Config.format ?? 'JSON';
+
+ - name: buildHeaders
+ template: |
+ const configAuthHeaders = $.getAuthHeaders(.destination.Config);
+ const additionalConfigHeaders = $.getCustomMappings(.message, .destination.Config.headers);
+ $.context.headers = {
+ ...configAuthHeaders,
+ ...additionalConfigHeaders
+ }
+
+ - name: prepareParams
+ template: |
+ $.context.params = $.getCustomMappings(.message, .destination.Config.queryParams)
+
+ - name: deduceEndPoint
+ template: |
+ $.context.endpoint = $.addPathParams(.message, .destination.Config.webhookUrl);
+
+ - name: prepareBody
+ template: |
+ const payload = $.getCustomMappings(.message, .destination.Config.propertiesMapping);
+ $.context.payload = $.removeUndefinedAndNullValues($.excludeMappedFields(payload, .destination.Config.propertiesMapping))
+ $.context.format === "XML" && !$.isEmptyObject($.context.payload) ? $.context.payload = {payload: $.getXMLPayload($.context.payload)};
+
+ - name: buildResponseForProcessTransformation
+ template: |
+ const response = $.defaultRequestConfig();
+ $.context.format === "JSON" ? response.body.JSON = $.context.payload: response.body.XML = $.context.payload;
+ response.endpoint = $.context.endpoint;
+ response.headers = $.context.headers;
+ response.method = $.context.method;
+ response.params = $.context.params ?? {};
+ response
diff --git a/src/cdk/v2/destinations/webhook_v2/rtWorkflow.yaml b/src/cdk/v2/destinations/webhook_v2/rtWorkflow.yaml
new file mode 100644
index 0000000000..edc31d003d
--- /dev/null
+++ b/src/cdk/v2/destinations/webhook_v2/rtWorkflow.yaml
@@ -0,0 +1,60 @@
+bindings:
+ - name: handleRtTfSingleEventError
+ path: ../../../../v0/util/index
+ - path: ./utils
+ exportAll: true
+ - name: BatchUtils
+ path: '@rudderstack/workflow-engine'
+
+steps:
+ - name: validateInput
+ template: |
+ $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+ - name: transform
+ externalWorkflow:
+ path: ./procWorkflow.yaml
+ loopOverInput: true
+
+ - name: successfulEvents
+ template: |
+ $.outputs.transform#idx.output.({
+ "batchedRequest": .,
+ "batched": false,
+ "destination": ^[idx].destination,
+ "metadata": ^[idx].metadata[],
+ "statusCode": 200
+ })[]
+
+ - name: failedEvents
+ template: |
+ $.outputs.transform#idx.error.(
+ $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+ )[]
+
+ - name: bodyFormat
+ template: |
+ $.outputs.successfulEvents[0].destination.Config.format ?? "JSON";
+
+ - name: batchingEnabled
+ template: |
+ $.outputs.successfulEvents[0].destination.Config.isBatchingEnabled;
+
+ - name: batchSize
+ template: |
+ $.outputs.successfulEvents[0].destination.Config.maxBatchSize;
+
+ - name: batchSuccessfulEvents
+ description: Batches the successfulEvents
+ condition: $.outputs.batchingEnabled && $.outputs.bodyFormat === "JSON"
+ template: |
+ $.batchSuccessfulEvents($.outputs.successfulEvents, $.outputs.batchSize);
+
+ - name: finalPayloadWithBatching
+ condition: $.outputs.batchingEnabled && $.outputs.bodyFormat === "JSON"
+ template: |
+ [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]
+ else:
+ name: finalPayloadWithoutBatching
+ template: |
+ [...$.outputs.successfulEvents, ...$.outputs.failedEvents]
diff --git a/src/cdk/v2/destinations/webhook_v2/utils.js b/src/cdk/v2/destinations/webhook_v2/utils.js
new file mode 100644
index 0000000000..bf0f2179f1
--- /dev/null
+++ b/src/cdk/v2/destinations/webhook_v2/utils.js
@@ -0,0 +1,146 @@
+const { toXML } = require('jstoxml');
+const { groupBy } = require('lodash');
+const { createHash } = require('crypto');
+const { ConfigurationError } = require('@rudderstack/integrations-lib');
+const { BatchUtils } = require('@rudderstack/workflow-engine');
+const { base64Convertor, applyCustomMappings, isEmptyObject } = require('../../../../v0/util');
+
+const getAuthHeaders = (config) => {
+ let headers;
+ switch (config.auth) {
+ case 'basicAuth': {
+ const credentials = `${config.username}:${config.password}`;
+ const encodedCredentials = base64Convertor(credentials);
+ headers = {
+ Authorization: `Basic ${encodedCredentials}`,
+ };
+ break;
+ }
+ case 'bearerTokenAuth':
+ headers = { Authorization: `Bearer ${config.bearerToken}` };
+ break;
+ case 'apiKeyAuth':
+ headers = { [config.apiKeyName]: `${config.apiKeyValue}` };
+ break;
+ default:
+ headers = {};
+ }
+ return headers;
+};
+
+const getCustomMappings = (message, mapping) => {
+ try {
+ return applyCustomMappings(message, mapping);
+ } catch (e) {
+ throw new ConfigurationError(`[Webhook]:: Error in custom mappings: ${e.message}`);
+ }
+};
+
+// TODO: write a func to evaluate json path template
+const addPathParams = (message, webhookUrl) => webhookUrl;
+
+const excludeMappedFields = (payload, mapping) => {
+ const rawPayload = { ...payload };
+ if (mapping) {
+ mapping.forEach(({ from, to }) => {
+ // continue when from === to
+ if (from === to) return;
+
+ // Remove the '$.' prefix and split the remaining string by '.'
+ const keys = from.replace(/^\$\./, '').split('.');
+ let current = rawPayload;
+
+ // Traverse to the parent of the key to be removed
+ keys.slice(0, -1).forEach((key) => {
+ if (current && current[key]) {
+ current = current[key];
+ } else {
+ current = null;
+ }
+ });
+
+ if (current) {
+ // Remove the 'from' field from input payload
+ delete current[keys[keys.length - 1]];
+ }
+ });
+ }
+
+ return rawPayload;
+};
+
+const getXMLPayload = (payload) =>
+ toXML(payload, {
+ header: true,
+ });
+
+const getMergedEvents = (batch) => {
+ const events = [];
+ batch.forEach((event) => {
+ if (!isEmptyObject(event.batchedRequest.body.JSON)) {
+ events.push(event.batchedRequest.body.JSON);
+ }
+ });
+ return events;
+};
+
+const mergeMetadata = (batch) => batch.map((event) => event.metadata[0]);
+
+const createHashKey = (endpoint, headers, params) => {
+ const hash = createHash('sha256');
+ hash.update(endpoint);
+ hash.update(JSON.stringify(headers));
+ hash.update(JSON.stringify(params));
+ return hash.digest('hex');
+};
+
+const buildBatchedRequest = (batch) => ({
+ batchedRequest: {
+ body: {
+ JSON: {},
+ JSON_ARRAY: { batch: JSON.stringify(getMergedEvents(batch)) },
+ XML: {},
+ FORM: {},
+ },
+ version: '1',
+ type: 'REST',
+ method: batch[0].batchedRequest.method,
+ endpoint: batch[0].batchedRequest.endpoint,
+ headers: batch[0].batchedRequest.headers,
+ params: batch[0].batchedRequest.params,
+ files: {},
+ },
+ metadata: mergeMetadata(batch),
+ batched: true,
+ statusCode: 200,
+ destination: batch[0].destination,
+});
+
+const batchSuccessfulEvents = (events, batchSize) => {
+ const response = [];
+ // group events by endpoint, headers and query params
+ const groupedEvents = groupBy(events, (event) => {
+ const { endpoint, headers, params } = event.batchedRequest;
+ return createHashKey(endpoint, headers, params);
+ });
+
+ // batch the each grouped event
+ Object.keys(groupedEvents).forEach((groupKey) => {
+ const batches = BatchUtils.chunkArrayBySizeAndLength(groupedEvents[groupKey], {
+ maxItems: batchSize,
+ }).items;
+ batches.forEach((batch) => {
+ response.push(buildBatchedRequest(batch));
+ });
+ });
+ return response;
+};
+
+module.exports = {
+ getAuthHeaders,
+ getCustomMappings,
+ addPathParams,
+ excludeMappedFields,
+ getXMLPayload,
+ batchSuccessfulEvents,
+};
diff --git a/src/features.json b/src/features.json
index 94e36a2416..5249703f8a 100644
--- a/src/features.json
+++ b/src/features.json
@@ -76,7 +76,8 @@
"WUNDERKIND": true,
"CLICKSEND": true,
"ZOHO": true,
- "CORDIAL": true
+ "CORDIAL": true,
+ "WEBHOOK_V2": true
},
"regulations": [
"BRAZE",
diff --git a/test/integrations/destinations/webhook_v2/common.ts b/test/integrations/destinations/webhook_v2/common.ts
new file mode 100644
index 0000000000..c31a7aabad
--- /dev/null
+++ b/test/integrations/destinations/webhook_v2/common.ts
@@ -0,0 +1,321 @@
+import { Destination } from '../../../../src/types';
+
+const destType = 'webhook_v2';
+const destTypeInUpperCase = 'WEBHOOK_V2';
+const displayName = 'Webhook V2';
+const destinations: Destination[] = [
+ {
+ Config: {
+ webhookUrl: 'http://abc.com/contacts',
+ auth: 'noAuth',
+ method: 'POST',
+ format: 'JSON',
+ isBatchingEnabled: true,
+ maxBatchSize: '2',
+ propertiesMapping: [
+ {
+ from: '$.traits.firstName',
+ to: '$.contacts.first_name',
+ },
+ {
+ from: '$.traits.email',
+ to: '$.contacts.email',
+ },
+ {
+ from: '$.traits.address.pinCode',
+ to: '$.contacts.address.pin_code',
+ },
+ ],
+ },
+ DestinationDefinition: {
+ DisplayName: displayName,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Config: { cdkV2Enabled: true },
+ },
+ Enabled: true,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Transformations: [],
+ WorkspaceID: 'test-workspace-id',
+ },
+ {
+ Config: {
+ webhookUrl: 'http://abc.com/contact/$traits.userId',
+ auth: 'basicAuth',
+ username: 'test-user',
+ password: '',
+ method: 'GET',
+ format: 'JSON',
+ isBatchingEnabled: true,
+ maxBatchSize: 2,
+ headers: [
+ {
+ to: '$.h1',
+ from: "'val1'",
+ },
+ {
+ to: '$.h2',
+ from: '2',
+ },
+ {
+ to: "$.'content-type'",
+ from: "'application/json'",
+ },
+ {
+ to: '$.h3',
+ from: '$.traits.firstName',
+ },
+ ],
+ queryParams: [
+ {
+ to: '$.q1',
+ from: "'val1'",
+ },
+ {
+ to: '$.q2',
+ from: '$.traits.email',
+ },
+ ],
+ },
+ DestinationDefinition: {
+ DisplayName: displayName,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Config: { cdkV2Enabled: true },
+ },
+ Enabled: true,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Transformations: [],
+ WorkspaceID: 'test-workspace-id',
+ },
+ {
+ Config: {
+ webhookUrl: 'http://abc.com/contacts/$.traits.userId/',
+ auth: 'apiKeyAuth',
+ apiKeyName: 'x-api-key',
+ apiKeyValue: 'test-api-key',
+ method: 'DELETE',
+ isBatchingEnabled: true,
+ maxBatchSize: 4,
+ },
+ DestinationDefinition: {
+ DisplayName: displayName,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Config: { cdkV2Enabled: true },
+ },
+ Enabled: true,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Transformations: [],
+ WorkspaceID: 'test-workspace-id',
+ },
+ {
+ Config: {
+ webhookUrl: 'http://abc.com/contacts/$.traits.userId/',
+ auth: 'apiKeyAuth',
+ apiKeyName: 'x-api-key',
+ apiKeyValue: 'test-api-key',
+ method: 'GET',
+ isBatchingEnabled: true,
+ maxBatchSize: 4,
+ },
+ DestinationDefinition: {
+ DisplayName: displayName,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Config: { cdkV2Enabled: true },
+ },
+ Enabled: true,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Transformations: [],
+ WorkspaceID: 'test-workspace-id',
+ },
+ {
+ Config: {
+ webhookUrl: 'http://abc.com/events',
+ auth: 'bearerTokenAuth',
+ bearerToken: 'test-token',
+ method: 'POST',
+ format: 'XML',
+ headers: [
+ {
+ to: '$.h1',
+ from: "'val1'",
+ },
+ {
+ to: '$.h2',
+ from: '$.key1',
+ },
+ {
+ to: "$.'content-type'",
+ from: "'application/json'",
+ },
+ ],
+ propertiesMapping: [
+ {
+ from: '$.event',
+ to: '$.event',
+ },
+ {
+ from: '$.properties.currency',
+ to: '$.currency',
+ },
+ {
+ from: '$.userId',
+ to: '$.userId',
+ },
+ {
+ from: '$.properties.products[*].product_id',
+ to: '$.properties.items[*].item_id',
+ },
+ {
+ from: '$.properties.products[*].name',
+ to: '$.properties.items[*].name',
+ },
+ {
+ from: '$.properties.products[*].price',
+ to: '$.properties.items[*].price',
+ },
+ ],
+ },
+ DestinationDefinition: {
+ DisplayName: displayName,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Config: { cdkV2Enabled: true },
+ },
+ Enabled: true,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Transformations: [],
+ WorkspaceID: 'test-workspace-id',
+ },
+ {
+ Config: {
+ webhookUrl: 'http://abc.com/events',
+ auth: 'noAuth',
+ method: 'POST',
+ format: 'JSON',
+ isBatchingEnabled: true,
+ maxBatchSize: '4',
+ headers: [
+ {
+ to: "$.'content-type'",
+ from: "'application/json'",
+ },
+ ],
+ propertiesMapping: [
+ {
+ from: '$.event',
+ to: '$.event',
+ },
+ {
+ from: '$.properties.currency',
+ to: '$.currency',
+ },
+ {
+ from: '$.userId',
+ to: '$.userId',
+ },
+ {
+ from: '$.properties.products[*].product_id',
+ to: '$.properties.items[*].item_id',
+ },
+ {
+ from: '$.properties.products[*].name',
+ to: '$.properties.items[*].name',
+ },
+ {
+ from: '$.properties.products[*].price',
+ to: '$.properties.items[*].price',
+ },
+ ],
+ },
+ DestinationDefinition: {
+ DisplayName: displayName,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Config: { cdkV2Enabled: true },
+ },
+ Enabled: true,
+ ID: '123',
+ Name: destTypeInUpperCase,
+ Transformations: [],
+ WorkspaceID: 'test-workspace-id',
+ },
+];
+
+const traits = {
+ email: 'john.doe@example.com',
+ firstName: 'John',
+ lastName: 'Doe',
+ phone: '1234567890',
+ address: {
+ city: 'New York',
+ country: 'USA',
+ pinCode: '123456',
+ },
+};
+
+const properties = {
+ checkout_id: '70324a1f0eaf000000000000',
+ order_id: '40684e8f0eaf000000000000',
+ affiliation: 'Vandelay Games',
+ total: 52.0,
+ subtotal: 45.0,
+ revenue: 50.0,
+ shipping: 4.0,
+ tax: 3.0,
+ discount: 5.0,
+ coupon: 'NEWCUST5',
+ currency: 'USD',
+ products: [
+ {
+ product_id: '622c6f5d5cf86a4c77358033',
+ sku: '8472-998-0112',
+ name: 'Cones of Dunshire',
+ price: 40,
+ position: 1,
+ category: 'Games',
+ url: 'https://www.website.com/product/path',
+ image_url: 'https://www.website.com/product/path.jpg',
+ },
+ {
+ product_id: '577c6f5d5cf86a4c7735ba03',
+ sku: '3309-483-2201',
+ name: 'Five Crowns',
+ price: 5,
+ position: 2,
+ category: 'Games',
+ },
+ ],
+};
+
+const processorInstrumentationErrorStatTags = {
+ destType: destTypeInUpperCase,
+ errorCategory: 'dataValidation',
+ errorType: 'instrumentation',
+ feature: 'processor',
+ implementation: 'cdkV2',
+ module: 'destination',
+ destinationId: 'default-destinationId',
+ workspaceId: 'default-workspaceId',
+};
+
+const RouterInstrumentationErrorStatTags = {
+ ...processorInstrumentationErrorStatTags,
+ feature: 'router',
+};
+
+export {
+ destType,
+ destinations,
+ processorInstrumentationErrorStatTags,
+ RouterInstrumentationErrorStatTags,
+ traits,
+ properties,
+};
diff --git a/test/integrations/destinations/webhook_v2/processor/configuration.ts b/test/integrations/destinations/webhook_v2/processor/configuration.ts
new file mode 100644
index 0000000000..7a1c105ed0
--- /dev/null
+++ b/test/integrations/destinations/webhook_v2/processor/configuration.ts
@@ -0,0 +1,206 @@
+import { ProcessorTestData } from '../../../testTypes';
+import { generateMetadata, transformResultBuilder } from '../../../testUtils';
+import { destType, destinations, properties, traits } from '../common';
+
+export const configuration: ProcessorTestData[] = [
+ {
+ id: 'webhook_v2-configuration-test-1',
+ name: destType,
+ description: 'Identify call with properties mapping',
+ scenario: 'Business',
+ successCriteria: 'Response should be in json format with properties mapping',
+ feature: 'processor',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: [
+ {
+ destination: destinations[0],
+ message: {
+ type: 'identify',
+ userId: 'userId123',
+ anonymousId: 'anonId123',
+ traits,
+ },
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: [
+ {
+ output: transformResultBuilder({
+ method: 'POST',
+ userId: '',
+ endpoint: destinations[0].Config.webhookUrl,
+ JSON: {
+ contacts: {
+ first_name: 'John',
+ email: 'john.doe@example.com',
+ address: {
+ pin_code: '123456',
+ },
+ },
+ },
+ }),
+ statusCode: 200,
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ },
+ {
+ id: 'webhook_v2-configuration-test-2',
+ name: destType,
+ description: 'Identify call with api key auth, delete method and path params',
+ scenario: 'Business',
+ successCriteria: 'Response should contain delete method and api key auth',
+ feature: 'processor',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: [
+ {
+ destination: destinations[2],
+ message: {
+ type: 'identify',
+ userId: 'userId123',
+ anonymousId: 'anonId123',
+ traits,
+ },
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: [
+ {
+ output: transformResultBuilder({
+ method: 'DELETE',
+ userId: '',
+ endpoint: 'http://abc.com/contacts/$.traits.userId/',
+ headers: {
+ 'x-api-key': 'test-api-key',
+ },
+ }),
+ statusCode: 200,
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ },
+ {
+ id: 'webhook_v2-configuration-test-3',
+ name: destType,
+ description: 'Track call with basic auth, get method, headers and query params mapping',
+ scenario: 'Business',
+ successCriteria: 'Response should contain get method, headers and query params mapping',
+ feature: 'processor',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: [
+ {
+ destination: destinations[1],
+ message: {
+ type: 'track',
+ userId: 'userId123',
+ event: 'Order Completed',
+ properties,
+ },
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: [
+ {
+ output: transformResultBuilder({
+ method: 'GET',
+ userId: '',
+ endpoint: destinations[1].Config.webhookUrl,
+ headers: {
+ Authorization: 'Basic dGVzdC11c2VyOg==',
+ h1: 'val1',
+ h2: 2,
+ 'content-type': 'application/json',
+ },
+ params: {
+ q1: 'val1',
+ },
+ }),
+ statusCode: 200,
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ },
+ {
+ id: 'webhook_v2-configuration-test-4',
+ name: destType,
+ description:
+ 'Track call with bearer token, xml format, post method, additional headers and properties mapping',
+ scenario: 'Business',
+ successCriteria:
+ 'Response should be in xml format with post method, headers and properties mapping',
+ feature: 'processor',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: [
+ {
+ destination: destinations[4],
+ message: {
+ type: 'track',
+ userId: 'userId123',
+ event: 'Order Completed',
+ properties,
+ },
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: [
+ {
+ output: transformResultBuilder({
+ method: 'POST',
+ userId: '',
+ endpoint: destinations[4].Config.webhookUrl,
+ headers: {
+ Authorization: 'Bearer test-token',
+ h1: 'val1',
+ 'content-type': 'application/json',
+ },
+ XML: {
+ payload:
+ 'Order CompletedUSDuserId123622c6f5d5cf86a4c77358033Cones of Dunshire40577c6f5d5cf86a4c7735ba03Five Crowns5',
+ },
+ }),
+ statusCode: 200,
+ metadata: generateMetadata(1),
+ },
+ ],
+ },
+ },
+ },
+];
diff --git a/test/integrations/destinations/webhook_v2/processor/data.ts b/test/integrations/destinations/webhook_v2/processor/data.ts
new file mode 100644
index 0000000000..bb4e7ffa0d
--- /dev/null
+++ b/test/integrations/destinations/webhook_v2/processor/data.ts
@@ -0,0 +1,2 @@
+import { configuration } from './configuration';
+export const data = [...configuration];
diff --git a/test/integrations/destinations/webhook_v2/router/data.ts b/test/integrations/destinations/webhook_v2/router/data.ts
new file mode 100644
index 0000000000..44c9f0e6fe
--- /dev/null
+++ b/test/integrations/destinations/webhook_v2/router/data.ts
@@ -0,0 +1,350 @@
+import { generateMetadata } from '../../../testUtils';
+import {
+ destType,
+ destinations,
+ traits,
+ properties,
+ RouterInstrumentationErrorStatTags,
+} from '../common';
+
+const routerRequest1 = {
+ input: [
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId1',
+ traits,
+ },
+ metadata: generateMetadata(1),
+ destination: destinations[3],
+ },
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId2',
+ traits,
+ },
+ metadata: generateMetadata(2),
+ destination: destinations[3],
+ },
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId1',
+ traits,
+ },
+ metadata: generateMetadata(3),
+ destination: destinations[3],
+ },
+ ],
+ destType,
+};
+
+const routerRequest2 = {
+ input: [
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId1',
+ traits,
+ },
+ metadata: generateMetadata(1, 'userId1'),
+ destination: destinations[1],
+ },
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId2',
+ traits: { ...traits, firstName: 'Alex', lastName: 'T', email: 'alex.t@example.com' },
+ },
+ metadata: generateMetadata(2, 'userId2'),
+ destination: destinations[1],
+ },
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId1',
+ traits: { ...traits, phone: '2234567890' },
+ },
+ metadata: generateMetadata(3, 'userId1'),
+ destination: destinations[1],
+ },
+ {
+ message: {
+ type: 'identify',
+ userId: 'userId1',
+ traits: { ...traits, phone: '3234567890' },
+ },
+ metadata: generateMetadata(4, 'userId1'),
+ destination: destinations[1],
+ },
+ ],
+ destType,
+};
+
+const routerRequest3 = {
+ input: [
+ {
+ message: {
+ type: 'track',
+ userId: 'userId1',
+ event: 'Product Viewed',
+ context: { traits },
+ },
+ metadata: generateMetadata(1, 'userId1'),
+ destination: destinations[5],
+ },
+ {
+ message: {
+ type: 'track',
+ userId: 'userId2',
+ event: 'Order Completed',
+ context: { traits },
+ properties,
+ },
+ metadata: generateMetadata(2, 'userId2'),
+ destination: destinations[5],
+ },
+ {
+ message: {
+ type: 'track',
+ userId: 'userId3',
+ event: 'Product Added',
+ context: { traits },
+ properties,
+ },
+ metadata: generateMetadata(3, 'userId3'),
+ destination: destinations[5],
+ },
+ ],
+ destType,
+};
+
+// TODO: add failure testcases
+export const data = [
+ {
+ id: 'webhook_v2-router-test-1',
+ name: destType,
+ description: 'Batch multiple GET requests in a single batch with given batch size',
+ scenario: 'Framework',
+ successCriteria: 'All events should be transformed successfully and status code should be 200',
+ feature: 'router',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: routerRequest1,
+ method: 'POST',
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: {
+ output: [
+ {
+ batchedRequest: {
+ version: '1',
+ type: 'REST',
+ method: 'GET',
+ endpoint: 'http://abc.com/contacts/$.traits.userId/',
+ headers: {
+ 'x-api-key': 'test-api-key',
+ },
+ params: {},
+ body: {
+ JSON: {},
+ JSON_ARRAY: { batch: '[]' },
+ XML: {},
+ FORM: {},
+ },
+ files: {},
+ },
+ metadata: [generateMetadata(1), generateMetadata(2), generateMetadata(3)],
+ batched: true,
+ statusCode: 200,
+ destination: destinations[3],
+ },
+ ],
+ },
+ },
+ },
+ },
+ {
+ id: 'webhook_v2-router-test-2',
+ name: destType,
+ description:
+ 'Batch multiple GET requests in multiple batches when number of requests are greater then given batch size',
+ scenario: 'Framework',
+ successCriteria: 'All events should be transformed successfully and status code should be 200',
+ feature: 'router',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: routerRequest2,
+ method: 'POST',
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: {
+ output: [
+ {
+ batchedRequest: {
+ version: '1',
+ type: 'REST',
+ method: 'GET',
+ endpoint: 'http://abc.com/contact/$traits.userId',
+ headers: {
+ Authorization: 'Basic dGVzdC11c2VyOg==',
+ 'content-type': 'application/json',
+ h1: 'val1',
+ h2: 2,
+ h3: 'John',
+ },
+ params: {
+ q1: 'val1',
+ q2: 'john.doe@example.com',
+ },
+ body: {
+ JSON: {},
+ JSON_ARRAY: {
+ batch: '[]',
+ },
+ XML: {},
+ FORM: {},
+ },
+ files: {},
+ },
+ metadata: [generateMetadata(1, 'userId1'), generateMetadata(3, 'userId1')],
+ batched: true,
+ statusCode: 200,
+ destination: destinations[1],
+ },
+ {
+ batchedRequest: {
+ version: '1',
+ type: 'REST',
+ method: 'GET',
+ endpoint: 'http://abc.com/contact/$traits.userId',
+ headers: {
+ Authorization: 'Basic dGVzdC11c2VyOg==',
+ 'content-type': 'application/json',
+ h1: 'val1',
+ h2: 2,
+ h3: 'John',
+ },
+ params: {
+ q1: 'val1',
+ q2: 'john.doe@example.com',
+ },
+ body: {
+ JSON: {},
+ JSON_ARRAY: {
+ batch: '[]',
+ },
+ XML: {},
+ FORM: {},
+ },
+ files: {},
+ },
+ metadata: [generateMetadata(4, 'userId1')],
+ batched: true,
+ statusCode: 200,
+ destination: destinations[1],
+ },
+ {
+ batchedRequest: {
+ version: '1',
+ type: 'REST',
+ method: 'GET',
+ endpoint: 'http://abc.com/contact/$traits.userId',
+ headers: {
+ Authorization: 'Basic dGVzdC11c2VyOg==',
+ 'content-type': 'application/json',
+ h1: 'val1',
+ h2: 2,
+ h3: 'Alex',
+ },
+ params: {
+ q1: 'val1',
+ q2: 'alex.t@example.com',
+ },
+ body: {
+ JSON: {},
+ JSON_ARRAY: {
+ batch: '[]',
+ },
+ XML: {},
+ FORM: {},
+ },
+ files: {},
+ },
+ metadata: [generateMetadata(2, 'userId2')],
+ batched: true,
+ statusCode: 200,
+ destination: destinations[1],
+ },
+ ],
+ },
+ },
+ },
+ },
+ {
+ id: 'webhook_v2-router-test-3',
+ name: destType,
+ description: 'Batch multiple POST requests with properties mappings',
+ scenario: 'Framework',
+ successCriteria: 'All events should be transformed successfully and status code should be 200',
+ feature: 'router',
+ module: 'destination',
+ version: 'v0',
+ input: {
+ request: {
+ body: routerRequest3,
+ method: 'POST',
+ },
+ },
+ output: {
+ response: {
+ status: 200,
+ body: {
+ output: [
+ {
+ batchedRequest: {
+ version: '1',
+ type: 'REST',
+ method: 'POST',
+ endpoint: 'http://abc.com/events',
+ params: {},
+ headers: {
+ 'content-type': 'application/json',
+ },
+ body: {
+ JSON: {},
+ JSON_ARRAY: {
+ batch:
+ '[{"event":"Product Viewed","userId":"userId1","properties":{"items":[]}},{"event":"Order Completed","currency":"USD","userId":"userId2","properties":{"items":[{"item_id":"622c6f5d5cf86a4c77358033","name":"Cones of Dunshire","price":40},{"item_id":"577c6f5d5cf86a4c7735ba03","name":"Five Crowns","price":5}]}},{"event":"Product Added","currency":"USD","userId":"userId3","properties":{"items":[{"item_id":"622c6f5d5cf86a4c77358033","name":"Cones of Dunshire","price":40},{"item_id":"577c6f5d5cf86a4c7735ba03","name":"Five Crowns","price":5}]}}]',
+ },
+ XML: {},
+ FORM: {},
+ },
+ files: {},
+ },
+ metadata: [
+ generateMetadata(1, 'userId1'),
+ generateMetadata(2, 'userId2'),
+ generateMetadata(3, 'userId3'),
+ ],
+ batched: true,
+ statusCode: 200,
+ destination: destinations[5],
+ },
+ ],
+ },
+ },
+ },
+ },
+];