diff --git a/src/cdk/v2/destinations/algolia/rtWorkflow.yaml b/src/cdk/v2/destinations/algolia/rtWorkflow.yaml index 4ededf3d24..545962e42b 100644 --- a/src/cdk/v2/destinations/algolia/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/algolia/rtWorkflow.yaml @@ -2,7 +2,6 @@ bindings: - path: ../../../../v0/destinations/algolia/config - name: handleRtTfSingleEventError path: ../../../../v0/util/index - - path: ./util steps: - name: validateInput template: | @@ -28,14 +27,47 @@ steps: $.outputs.transform#idx.error.( $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) )[] + - name: separateDontBatchEventForProxy template: | - const payload = $.constructFullPayload(^, $.outputs.successfulEvents); - payload - - name: batchSuccessfulEvents + { + dontBatchTrueEvents: $.outputs.successfulEvents{.metadata.dontBatch}[], + dontBatchFalseEvents: $.outputs.successfulEvents{!.metadata.dontBatch}[] + } + + - name: batchSuccessfulEventsDontBatchTrue + description: Batches the successfulEvents + template: | + let batches = $.chunk($.outputs.separateDontBatchEventForProxy.dontBatchTrueEvents, 1); + batches@batch.({ + "batchedRequest": { + "body": { + "JSON": {"events": ~r batch.output[]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": {{$.ENDPOINT}}, + "headers": batch[0].destination.Config.().({ + "X-Algolia-Application-Id": .applicationId, + "X-Algolia-API-Key": .apiKey + }), + "params": {}, + "files": {} + }, + "metadata": ~r batch.metadata[], + "batched": true, + "statusCode": 200, + "destination": batch[0].destination + })[]; + + - name: batchSuccessfulEventsDontBatchFalse description: Batches the successfulEvents template: | - let batches = $.chunk($.outputs.successfulEvents, $.MAX_BATCH_SIZE); + let batches = $.chunk($.outputs.separateDontBatchEventForProxy.dontBatchFalseEvents, $.MAX_BATCH_SIZE); batches@batch.({ "batchedRequest": { "body": { @@ -60,6 +92,12 @@ steps: "statusCode": 200, "destination": batch[0].destination })[]; + + - name: combineSuccessfulEvents + description: Combines the successfulEvents + template: | + [...$.outputs.batchSuccessfulEventsDontBatchFalse, ...$.outputs.batchSuccessfulEventsDontBatchTrue] + - name: finalPayload template: | - [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] + [...$.outputs.failedEvents, ...$.outputs.combineSuccessfulEvents] diff --git a/test/integrations/destinations/algolia/router/data.ts b/test/integrations/destinations/algolia/router/data.ts index 65c74342dc..49d90698c8 100644 --- a/test/integrations/destinations/algolia/router/data.ts +++ b/test/integrations/destinations/algolia/router/data.ts @@ -2220,4 +2220,874 @@ export const data = [ }, }, }, + { + name: 'algolia', + description: 'dontBatch true for all', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + }, + "message": { + "channel": "web", + "context": { + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + } + }, + "type": "track", + "anonymousId": "345345", + "event": "product clicked", + "userId": "test", + "properties": { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ] + }, + "integrations": { + "All": true + } + }, + "metadata": { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + }, + { + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + }, + "message": { + "channel": "web", + "context": { + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + } + }, + "type": "track", + "anonymousId": "345345", + "event": "product clicked", + "userId": "test", + "properties": { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ] + }, + "integrations": { + "All": true + } + }, + "metadata": { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + }, + { + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + }, + "message": { + "channel": "web", + "context": { + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + } + }, + "type": "track", + "anonymousId": "345345", + "event": "product clicked", + "userId": "test", + "properties": { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ] + }, + "integrations": { + "All": true + } + }, + "metadata": { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + }, + { + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + }, + "message": { + "channel": "web", + "context": { + "traits": { + "email": "testone@gmail.com", + "firstName": "test", + "lastName": "one" + } + }, + "type": "track", + "anonymousId": "345345", + "event": "product clicked", + "userId": "test", + "properties": { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ] + }, + "integrations": { + "All": true + } + }, + "metadata": { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + } + ], + destType: 'algolia', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + "batchedRequest": { + "body": { + "JSON": { + "events": [ + { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ], + "userToken": "test", + "eventName": "product clicked", + "eventType": "click" + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "dummyApiKey" + }, + "params": {}, + "files": {} + }, + "metadata": [ + { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + ], + "batched": true, + "statusCode": 200, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + } + }, + { + "batchedRequest": { + "body": { + "JSON": { + "events": [ + { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ], + "userToken": "test", + "eventName": "product clicked", + "eventType": "click" + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "dummyApiKey" + }, + "params": {}, + "files": {} + }, + "metadata": [ + { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + ], + "batched": true, + "statusCode": 200, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + } + }, + { + "batchedRequest": { + "body": { + "JSON": { + "events": [ + { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ], + "userToken": "test", + "eventName": "product clicked", + "eventType": "click" + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "dummyApiKey" + }, + "params": {}, + "files": {} + }, + "metadata": [ + { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + ], + "batched": true, + "statusCode": 200, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + } + }, + { + "batchedRequest": { + "body": { + "JSON": { + "events": [ + { + "index": "products", + "filters": [ + "field1:hello", + "val1:val2" + ], + "userToken": "test", + "eventName": "product clicked", + "eventType": "click" + } + ] + }, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://insights.algolia.io/1/events", + "headers": { + "X-Algolia-Application-Id": "O2YARRI15I", + "X-Algolia-API-Key": "dummyApiKey" + }, + "params": {}, + "files": {} + }, + "metadata": [ + { + "jobId": 1, + "attemptNum": 0, + "userId": "", + "sourceId": "2bAiFXtSLvENPDRAxVRxf0Udfaz", + "destinationId": "2dH3xYIQnTNqfgPCqfVs88gWQdQ", + "workspaceId": "2Csl0lSTbuM3qyHdaOQB2GcDH8o", + "secret": null, + "dontBatch": true + } + ], + "batched": true, + "statusCode": 200, + "destination": { + "DestinationDefinition": { + "Config": { + "cdkV2Enabled": true + } + }, + "Config": { + "cdkV2Enabled": true, + "apiKey": "dummyApiKey", + "applicationId": "O2YARRI15I", + "eventTypeSettings": [ + { + "from": "Product List Filtered", + "to": "click" + }, + { + "from": "Product List Viewed", + "to": "view" + }, + { + "from": "Order Completed", + "to": "view" + }, + { + "from": "Product Added", + "to": "conversion" + }, + { + "from": "Product Viewed", + "to": "view" + }, + { + "from": "Product Clicked", + "to": "click" + } + ], + "pixelId": "123456789", + "advertiserId": "429047995", + "eventId": "429047995", + "enhancedMatch": true, + "enableDeduplication": true, + "deduplicationKey": "messageId", + "sendingUnHashedData": true, + "customProperties": [ + { + "properties": "presentclass" + }, + { + "properties": "presentgrade" + } + ], + "eventsMapping": [ + { + "from": "ABC Searched", + "to": "WatchVideo" + } + ] + }, + "ID": "1pYpzzvcn7AQ2W9GGIAZSsN6Mfq", + "Name": "ALGOLIA", + "Enabled": true, + "cdkV2Enabled": true, + "Transformations": [] + } + } + ], + }, + }, + }, + }, ];