From ca8d6953f386a72f66c0bdd8ad926e6108c1eafb Mon Sep 17 00:00:00 2001 From: Gauravudia Date: Tue, 5 Mar 2024 12:57:06 +0530 Subject: [PATCH] feat: onboard destination movable ink --- .../movable_ink/procWorkflow.yaml | 65 ++++++ .../destinations/movable_ink/rtWorkflow.yaml | 33 +++ src/features.json | 3 +- .../destinations/movable_ink/common.ts | 121 +++++++++++ .../movable_ink/processor/data.ts | 4 + .../movable_ink/processor/identify.ts | 64 ++++++ .../movable_ink/processor/track.ts | 189 ++++++++++++++++++ .../movable_ink/processor/validation.ts | 131 ++++++++++++ .../destinations/movable_ink/router/data.ts | 178 +++++++++++++++++ 9 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 src/cdk/v2/destinations/movable_ink/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml create mode 100644 test/integrations/destinations/movable_ink/common.ts create mode 100644 test/integrations/destinations/movable_ink/processor/data.ts create mode 100644 test/integrations/destinations/movable_ink/processor/identify.ts create mode 100644 test/integrations/destinations/movable_ink/processor/track.ts create mode 100644 test/integrations/destinations/movable_ink/processor/validation.ts create mode 100644 test/integrations/destinations/movable_ink/router/data.ts diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml new file mode 100644 index 0000000000..93e0c8699d --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -0,0 +1,65 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: toUnixTimestampInMS + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - path: ./utils + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting"); + $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting"); + $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting"); + $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + + $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting"); + + - name: preparePayload + template: | + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + const email = .message.().( + {{{{$.getGenericPaths("email")}}}}; + ); + const timestampInUnix = $.toUnixTimestampInMS(.message.().( + {{{{$.getGenericPaths("timestamp")}}}}; + )); + $.context.payload = { + ...(.message), + userId: userId ?? email, + timestamp: timestampInUnix, + anonymousId: .message.anonymousId + } + + - name: buildResponse + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = .destination.Config.endpoint; + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret) + } + response; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml new file mode 100644 index 0000000000..fc5d474d60 --- /dev/null +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -0,0 +1,33 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils + exportAll: true + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata[], + "statusCode": 200 + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: finalPayload + template: | + [...$.outputs.successfulEvents, ...$.outputs.failedEvents] diff --git a/src/features.json b/src/features.json index 5460111a22..bbb92dd20d 100644 --- a/src/features.json +++ b/src/features.json @@ -66,7 +66,8 @@ "REDDIT": true, "THE_TRADE_DESK": true, "INTERCOM": true, - "NINETAILED": true + "NINETAILED": true, + "MOVABLE_INK": true }, "regulations": [ "BRAZE", diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts new file mode 100644 index 0000000000..6744bdb396 --- /dev/null +++ b/test/integrations/destinations/movable_ink/common.ts @@ -0,0 +1,121 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'movable_ink'; +const destTypeInUpperCase = 'MOVABLE_INK'; +const displayName = 'Movable Ink'; +const channel = 'web'; +const destination: Destination = { + Config: { + endpoint: 'https://collector.movableink-dmz.com/behavioral/abc123', + accessKey: 'test-access-key', + accessSecret: 'test_access_secret', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const traits = { + email: 'test@example.com', + firstName: 'John', + lastName: 'Doe', + phone: '1234567890', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hY2Nlc3Mta2V5OnRlc3RfYWNjZXNzX3NlY3JldA==', +}; + +const commonProperties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + categories: [ + { url: 'https://example1', id: '1' }, + { url: 'https://example2', id: '2' }, + ], + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', +}; + +const trackTestProperties = { + 'Product Added': commonProperties, + 'Product Viewed': commonProperties, + 'Order Completed': { + checkout_id: '70324a1f0eaf000000000000', + order_id: '40684e8f0eaf000000000000', + affiliation: 'Vandelay Games', + total: 52, + subtotal: 45, + revenue: 50, + shipping: 4, + tax: 3, + discount: 5, + coupon: 'NEWCUST5', + currency: 'USD', + products: [ + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + ], + }, + 'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' }, + 'Custom event': { ...commonProperties, key1: 'value1', key2: true }, +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + traits, + headers, + trackTestProperties, +}; diff --git a/test/integrations/destinations/movable_ink/processor/data.ts b/test/integrations/destinations/movable_ink/processor/data.ts new file mode 100644 index 0000000000..45453c74cd --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/data.ts @@ -0,0 +1,4 @@ +import { validation } from './validation'; +import { identify } from './identify'; +import { track } from './track'; +export const data = [...identify, ...track, ...validation]; diff --git a/test/integrations/destinations/movable_ink/processor/identify.ts b/test/integrations/destinations/movable_ink/processor/identify.ts new file mode 100644 index 0000000000..27186da05c --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/identify.ts @@ -0,0 +1,64 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, traits, headers } from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'MovableInk-identify-test-1', + name: destType, + description: 'Identify call with traits and anonymousId', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/track.ts b/test/integrations/destinations/movable_ink/processor/track.ts new file mode 100644 index 0000000000..5f30a3de83 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/track.ts @@ -0,0 +1,189 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, channel, destination, headers, trackTestProperties } from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'MovableInk-track-test-1', + name: destType, + description: 'Track call: Product Added event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-2', + name: destType, + description: 'Track call: Order Completed event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'MovableInk-track-test-3', + name: destType, + description: 'Track call: Custom event', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destination.Config.endpoint, + headers, + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts new file mode 100644 index 0000000000..f9f6c6a927 --- /dev/null +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -0,0 +1,131 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'MovableInk-validation-test-1', + name: destType, + description: 'All of the required fields — userId, email, and anonymousId — are missing.', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of userId or email or anonymousId is required. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Either one of userId or email or anonymousId is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'MovableInk-validation-test-3', + name: destType, + description: 'Missing required field -> timestamp', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Timestamp is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Timestamp is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts new file mode 100644 index 0000000000..fcb4324e7b --- /dev/null +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -0,0 +1,178 @@ +import { RouterTestData } from '../../../testTypes'; +import { RouterTransformationRequest } from '../../../../../src/types'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { + destType, + channel, + destination, + traits, + headers, + trackTestProperties, + RouterInstrumentationErrorStatTags, +} from '../common'; + +const routerRequest: RouterTransformationRequest = { + input: [ + { + message: { + type: 'identify', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + }, + metadata: generateMetadata(4), + destination, + }, + ], + destType, +}; + +export const data: RouterTestData[] = [ + { + id: 'MovableInk-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: + 'Some events should be transformed successfully and some should fail for missing fields and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + type: 'identify', + userId: traits.email, + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + properties: trackTestProperties['Product Added'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(3)], + batched: false, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(2)], + batched: false, + statusCode: 400, + error: 'Either one of userId or email or anonymousId is required. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(4)], + batched: false, + statusCode: 400, + error: 'Timestamp is not present. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + }, +];