From 35bec9ba0c6c467ceac9ab3db7d4ff28ae77b3e8 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Tue, 10 Dec 2024 15:56:07 +0530 Subject: [PATCH] feat: feature flag integration with webhook body format support --- .vscode/settings.json | 2 +- package-lock.json | 36 ++++ package.json | 2 + src/cdk/v2/destinations/http/utils.js | 14 ++ .../v2/destinations/webhook/procWorkflow.yaml | 30 +++- src/cdk/v2/handler.ts | 1 + src/v0/destinations/webhook/utils.js | 39 ++++ .../destinations/webhook/processor/data.ts | 168 +++++++++++++++++- 8 files changed, 288 insertions(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8d723306a91..c5c4e38b2df 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "editor.formatOnSave": true }, "[javascript]": { - "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.defaultFormatter": "vscode.typescript-language-features", "editor.formatOnSave": true }, "[jsonc]": { diff --git a/package-lock.json b/package-lock.json index 40033e278be..3cd1dfb817e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "component-each": "^0.2.6", "crypto-js": "^4.2.0", "dotenv": "^16.0.3", + "feature-flag-sdk": "file:../rudder-control-plane/libs/featureflag-sdk-node", "flat": "^5.0.2", "form-data": "^4.0.0", "get-value": "^3.0.1", @@ -64,6 +65,7 @@ "parse-static-imports": "^1.1.0", "prom-client": "^14.2.0", "qs": "^6.11.1", + "querystring": "^0.2.1", "rs-jsonpath": "^1.1.2", "set-value": "^4.1.0", "sha256": "^0.2.0", @@ -124,6 +126,27 @@ "typescript": "^5.0.4" } }, + "../rudder-control-plane/libs/featureflag-sdk-node": { + "name": "feature-flag-sdk", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "async-rwlock": "^1.1.1", + "axios": "^1.1.3", + "flagsmith-nodejs": "^4.0.0" + }, + "devDependencies": { + "@types/jest": "^29.2.0", + "@types/node": "^18.11.9", + "@typescript-eslint/eslint-plugin": "^5.42.0", + "@typescript-eslint/parser": "^5.42.0", + "eslint": "^8.26.0", + "jest": "^29.2.2", + "rimraf": "^3.0.2", + "ts-jest": "^29.0.3", + "typescript": "^4.8.4" + } + }, "node_modules/@aashutoshrathi/word-wrap": { "version": "1.2.6", "license": "MIT", @@ -12283,6 +12306,10 @@ "bser": "2.1.1" } }, + "node_modules/feature-flag-sdk": { + "resolved": "../rudder-control-plane/libs/featureflag-sdk-node", + "link": true + }, "node_modules/fecha": { "version": "4.2.3", "license": "MIT" @@ -19513,6 +19540,15 @@ "node": ">=0.10.0" } }, + "node_modules/querystring": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.1.tgz", + "integrity": "sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg==", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "funding": [ diff --git a/package.json b/package.json index e0b17a4f47f..b98b65d6aa0 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "component-each": "^0.2.6", "crypto-js": "^4.2.0", "dotenv": "^16.0.3", + "feature-flag-sdk": "file:../rudder-control-plane/libs/featureflag-sdk-node", "flat": "^5.0.2", "form-data": "^4.0.0", "get-value": "^3.0.1", @@ -109,6 +110,7 @@ "parse-static-imports": "^1.1.0", "prom-client": "^14.2.0", "qs": "^6.11.1", + "querystring": "^0.2.1", "rs-jsonpath": "^1.1.2", "set-value": "^4.1.0", "sha256": "^0.2.0", diff --git a/src/cdk/v2/destinations/http/utils.js b/src/cdk/v2/destinations/http/utils.js index 355eb034870..7b68645c760 100644 --- a/src/cdk/v2/destinations/http/utils.js +++ b/src/cdk/v2/destinations/http/utils.js @@ -1,6 +1,7 @@ const { toXML } = require('jstoxml'); const { groupBy } = require('lodash'); const { createHash } = require('crypto'); +const querystring = require('querystring'); const { ConfigurationError } = require('@rudderstack/integrations-lib'); const { BatchUtils } = require('@rudderstack/workflow-engine'); const { @@ -84,6 +85,18 @@ const getXMLPayload = (payload) => header: true, }); +/** + * Converts JSON payload to application/x-www-form-urlencoded format. + * @param {Object} payload - The JSON payload to be converted. + * @returns {string} - The payload in application/x-www-form-urlencoded format. + */ +const getFORMPayload = (payload) => { + if (!payload) { + throw new ConfigurationError('Invalid payload for FORM format'); + } + return querystring.stringify(payload); +} + const getMergedEvents = (batch) => { const events = []; batch.forEach((event) => { @@ -152,5 +165,6 @@ module.exports = { addPathParams, excludeMappedFields, getXMLPayload, + getFORMPayload, batchSuccessfulEvents, }; diff --git a/src/cdk/v2/destinations/webhook/procWorkflow.yaml b/src/cdk/v2/destinations/webhook/procWorkflow.yaml index bfe7bd4c133..cf5f4d71dca 100644 --- a/src/cdk/v2/destinations/webhook/procWorkflow.yaml +++ b/src/cdk/v2/destinations/webhook/procWorkflow.yaml @@ -4,6 +4,7 @@ bindings: - path: ../../bindings/jsontemplate exportAll: true - path: ../../../../v0/destinations/webhook/utils + exportAll: true - name: getHashFromArray path: ../../../../v0/util - name: getIntegrationsObj @@ -60,7 +61,32 @@ steps: payload.context.ip = ip; ) $.context.payload = $.removeUndefinedAndNullValues(payload) - - name: buildResponseForProcessTransformation + - name: buildResponseForProcessTransformationWithBodyFormatting + condition: await $.isFormattedBodyFeatureEnabled(.metadata.workspaceId) === true + template: | + const finalPayload = { + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + userId: ^.message.anonymousId, + type: 'REST', + method: $.context.method, + endpoint: $.context.finalEndpoint, + headers: $.context.finalHeaders, + params: $.context.params, + files: {}, + }; + + const formattedPayload = $.getFormatedPayload($.context.finalHeaders, $.context.payload); + finalPayload.body[formattedPayload.contentTypeSimplified] = formattedPayload.payload || {}; + $.context.payload = finalPayload; + $.context.payload + - name: buildResponseForProcessTransformationWithDefaultBodyFormat + condition: await $.isFormattedBodyFeatureEnabled(.metadata.workspaceId) !== true template: | $.context.payload.({ "body": { @@ -77,4 +103,4 @@ steps: "headers": $.context.finalHeaders, "params": $.context.params, "files": {} - }) + }) \ No newline at end of file diff --git a/src/cdk/v2/handler.ts b/src/cdk/v2/handler.ts index 74ebb716e60..512a9e2ade0 100644 --- a/src/cdk/v2/handler.ts +++ b/src/cdk/v2/handler.ts @@ -19,6 +19,7 @@ import { import logger from '../../logger'; + const defTags = { [tags.TAG_NAMES.IMPLEMENTATION]: tags.IMPLEMENTATIONS.CDK_V2, }; diff --git a/src/v0/destinations/webhook/utils.js b/src/v0/destinations/webhook/utils.js index b7d166ad7f2..61220f23fa7 100644 --- a/src/v0/destinations/webhook/utils.js +++ b/src/v0/destinations/webhook/utils.js @@ -1,5 +1,41 @@ +const { getXMLPayload, getFORMPayload } = require('../../../cdk/v2/destinations/http/utils'); const { EventType } = require('../../../constants'); const { getFieldValueFromMessage, flattenJson } = require('../../util'); +const { isFeatureEnabled, setDefaultTraits } = require('feature-flag-sdk') + + +const JSON = 'JSON', + XML = 'XML', + FORM = 'FORM'; + +const ContentTypeConstants = { + 'application/json': JSON, + 'application/xml': XML, + 'text/xml': XML, + 'application/x-www-form-urlencoded': FORM +} + +const isFormattedBodyFeatureEnabled = async (workspaceId) => { + return await isFeatureEnabled(workspaceId, 'dest_transformer_webhook_form_support'); +} + +const getFormatedPayload = (headers, payload) => { + const normalizedHeaders = Object.keys(headers).reduce((acc, key) => { + acc[key.toLowerCase()] = headers[key]; + return acc; + }, {}); + const contentType = normalizedHeaders['content-type']; + const contentTypeSimplified = ContentTypeConstants[contentType] || JSON; + + switch (contentTypeSimplified) { + case XML: + return { payload: getXMLPayload(payload), contentTypeSimplified }; + case FORM: + return { payload: getFORMPayload(payload), contentTypeSimplified }; + default: + return { payload, contentTypeSimplified }; + } +}; const getPropertyParams = (message) => { if (message.type === EventType.IDENTIFY) { @@ -10,4 +46,7 @@ const getPropertyParams = (message) => { module.exports = { getPropertyParams, + getFormatedPayload, + isFormattedBodyFeatureEnabled, + ContentTypeConstants }; diff --git a/test/integrations/destinations/webhook/processor/data.ts b/test/integrations/destinations/webhook/processor/data.ts index 92fe8f50d93..e3e7eee18d8 100644 --- a/test/integrations/destinations/webhook/processor/data.ts +++ b/test/integrations/destinations/webhook/processor/data.ts @@ -1,6 +1,172 @@ import { head } from 'lodash'; export const data = [ + { + name: 'webhook', + description: 'Test with different content-type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + context: { + device: { + id: 'df16bffa-5c3d-4fbb-9bce-3bab098129a7R', + manufacturer: 'Xiaomi', + model: 'Redmi 6', + name: 'xiaomi', + }, + network: { + carrier: 'Banglalink', + }, + os: { + name: 'android', + version: '8.1.0', + }, + traits: { + address: { + city: 'Dhaka', + country: 'Bangladesh', + }, + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + }, + }, + event: 'spin_result', + integrations: { + All: true, + }, + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + properties: { + additional_bet_index: 0, + battle_id: 'N/A', + bet_amount: 9, + bet_level: 1, + bet_multiplier: 1, + coin_balance: 9466052, + current_module_name: 'CasinoGameModule', + days_in_game: 0, + extra_param: 'N/A', + fb_profile: '0', + featureGameType: 'N/A', + game_fps: 30, + game_id: 'fireEagleBase', + game_name: 'FireEagleSlots', + gem_balance: 0, + graphicsQuality: 'HD', + idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', + internetReachability: 'ReachableViaLocalAreaNetwork', + isLowEndDevice: 'False', + is_auto_spin: 'False', + is_turbo: 'False', + isf: 'False', + ishighroller: 'False', + jackpot_win_amount: 90, + jackpot_win_type: 'Silver', + level: 6, + lifetime_gem_balance: 0, + no_of_spin: 1, + player_total_battles: 0, + player_total_shields: 0, + start_date: '2019-08-01', + total_payments: 0, + tournament_id: 'T1561970819', + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + versionSessionCount: 2, + win_amount: 0, + }, + timestamp: '2019-09-01T15:46:51.693229+05:30', + type: 'track', + user_properties: { + coin_balance: 9466052, + current_module_name: 'CasinoGameModule', + fb_profile: '0', + game_fps: 30, + game_name: 'FireEagleSlots', + gem_balance: 0, + graphicsQuality: 'HD', + idfa: '2bf99787-33d2-4ae2-a76a-c49672f97252', + internetReachability: 'ReachableViaLocalAreaNetwork', + isLowEndDevice: false, + level: 6, + lifetime_gem_balance: 0, + player_total_battles: 0, + player_total_shields: 0, + start_date: '2019-08-01', + total_payments: 0, + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + versionSessionCount: 2, + }, + }, + destination: { + Config: { + webhookUrl: 'http://6b0e6a60.ngrok.io', + headers: [ + { + from: '', + to: '', + }, + { + from: 'test2', + to: 'value2', + }, + { + from: 'content-type', + to: 'application/x-www-form-urlencoded' + } + ], + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: '1QH0ppCx6PD5kMmirglALJMTKax', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: 'anonymousId=c82cbdff-e5be-4009-ac78-cdeea09ab4b1&context=&event=spin_result&integrations=&message_id=a80f82be-9bdc-4a9f-b2a5-15621ee41df8&properties=×tamp=2019-09-01T15%3A46%3A51.693229%2B05%3A30&type=track&user_properties=', + }, + version: '1', + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + type: 'REST', + method: 'POST', + endpoint: 'http://6b0e6a60.ngrok.io', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + test2: 'value2', + }, + params: {}, + files: {}, + }, + metadata: { + destinationId: 'destId', + workspaceId: '1QH0ppCx6PD5kMmirglALJMTKax', + }, + statusCode: 200, + }, + ], + }, + }, + }, { name: 'webhook', description: 'Test 0', @@ -3373,5 +3539,5 @@ export const data = [ ], }, }, - }, + } ];