From 776d2c4abe23cd279195064684b9ccc807d83afc Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 11 Dec 2023 13:24:10 +0530 Subject: [PATCH 1/2] fix: removed retry logic from v1 cm360, added adapter for v1 to v0 conversion (#2860) * fix: removed retry logic from v1 cm360 proxy handler * feat: adapter to convert v1 payload to v0 transformer Proxy --- src/adapters/networkHandlerFactory.js | 10 +- src/adapters/networkHandlerFactory.test.js | 29 +- src/controllers/delivery.ts | 68 +- src/features.json | 2 +- src/interfaces/DestinationService.ts | 7 +- src/routes/delivery.ts | 9 +- src/services/comparator.ts | 8 +- src/services/delivertTest/deliveryTest.ts | 5 +- src/services/destination/cdkV1Integration.ts | 7 +- src/services/destination/cdkV2Integration.ts | 15 +- src/services/destination/nativeIntegration.ts | 63 +- .../destination/postTransformation.ts | 34 +- src/types/index.ts | 80 +- .../campaign_manager/networkHandler.js | 2 +- .../destinations/salesforce/networkHandler.js | 2 +- .../salesforce_oauth/networkHandler.js | 2 +- src/v0/util/errorTypes/index.js | 2 + .../util/errorTypes/transformerProxyError.js | 28 + .../campaign_manager/networkHandler.js | 47 +- test/__tests__/data/marketo_input.json | 16 +- .../destinations/braze/dataDelivery/data.ts | 999 +++++++++++------- .../destinations/braze/network.ts | 29 + 22 files changed, 942 insertions(+), 522 deletions(-) create mode 100644 src/v0/util/errorTypes/transformerProxyError.js diff --git a/src/adapters/networkHandlerFactory.js b/src/adapters/networkHandlerFactory.js index f4940553f59..e8c3748d15a 100644 --- a/src/adapters/networkHandlerFactory.js +++ b/src/adapters/networkHandlerFactory.js @@ -37,8 +37,14 @@ SUPPORTED_VERSIONS.forEach((version) => { }); const getNetworkHandler = (type, version) => { - const NetworkHandler = handlers[version][type] || handlers.generic; - return new NetworkHandler(); + let handlerVersion = version; + let NetworkHandler = handlers[version][type] || handlers.generic; + if (version === 'v1' && NetworkHandler === handlers.generic) { + NetworkHandler = handlers.v0[type] || handlers.generic; + handlerVersion = 'v0'; + } + const networkHandler = new NetworkHandler(); + return { networkHandler, handlerVersion }; }; module.exports = { diff --git a/src/adapters/networkHandlerFactory.test.js b/src/adapters/networkHandlerFactory.test.js index c4713e66a86..ff5f26a02d7 100644 --- a/src/adapters/networkHandlerFactory.test.js +++ b/src/adapters/networkHandlerFactory.test.js @@ -3,23 +3,34 @@ const { networkHandler: GenericNetworkHandler } = require('./networkhandler/gene describe(`Network Handler Tests`, () => { it('Should return v0 networkhandler', () => { - let proxyHandler = getNetworkHandler('campaign_manager', `v0`); + let { networkHandler, handlerVersion } = getNetworkHandler('campaign_manager', `v0`); const cmProxy = require(`../v0/destinations/campaign_manager/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new cmProxy()); + expect(networkHandler).toEqual(new cmProxy()); + }); - proxyHandler = getNetworkHandler('braze', `v0`); + it('Should return v0 networkhandler braze', () => { + let { networkHandler, handlerVersion } = getNetworkHandler('braze', `v0`); const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new brazeProxy()); + expect(networkHandler).toEqual(new brazeProxy()); }); it('Should return v1 networkhandler', () => { - let proxyHandler = getNetworkHandler('campaign_manager', `v1`); + let { networkHandler, handlerVersion } = getNetworkHandler('campaign_manager', `v1`); const cmProxy = require(`../v1/destinations/campaign_manager/networkHandler`).networkHandler; - expect(proxyHandler).toEqual(new cmProxy()); + expect(networkHandler).toEqual(new cmProxy()); }); - it('Should return genericHandler if v1 proxy and handler is not present for destination', () => { - let proxyHandler = getNetworkHandler('braze', `v1`); - expect(proxyHandler).toEqual(new GenericNetworkHandler()); + it('Should return v0 handler if v1 version and handler is present for destination in v0', () => { + const { networkHandler, handlerVersion } = getNetworkHandler('braze', `v1`); + const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; + console.log(networkHandler); + expect(networkHandler).toEqual(new brazeProxy()); + }); + + it('Should return generic handler', () => { + const { networkHandler, handlerVersion } = getNetworkHandler('abc', `v1`); + const brazeProxy = require(`../v0/destinations/braze/networkHandler`).networkHandler; + console.log(networkHandler); + expect(networkHandler).toEqual(new GenericNetworkHandler()); }); }); diff --git a/src/controllers/delivery.ts b/src/controllers/delivery.ts index 3ccc241b87c..0cc5e12d78d 100644 --- a/src/controllers/delivery.ts +++ b/src/controllers/delivery.ts @@ -1,6 +1,14 @@ +/* eslint-disable prefer-destructuring */ +/* eslint-disable sonarjs/no-duplicate-string */ import { Context } from 'koa'; import { MiscService } from '../services/misc'; -import { DeliveryResponse, ProcessorTransformationOutput } from '../types/index'; +import { + DeliveriesResponse, + DeliveryResponse, + ProcessorTransformationOutput, + ProxyDeliveriesRequest, + ProxyDeliveryRequest, +} from '../types/index'; import { ServiceSelector } from '../helpers/serviceSelector'; import { DeliveryTestService } from '../services/delivertTest/deliveryTest'; import { ControllerUtility } from './util'; @@ -9,30 +17,32 @@ import { DestinationPostTransformationService } from '../services/destination/po import tags from '../v0/util/tags'; import { FixMe } from '../util/types'; +const NON_DETERMINABLE = 'Non-determinable'; + export class DeliveryController { public static async deliverToDestination(ctx: Context) { logger.debug('Native(Delivery):: Request to transformer::', JSON.stringify(ctx.request.body)); let deliveryResponse: DeliveryResponse; const requestMetadata = MiscService.getRequestMetadata(ctx); - const event = ctx.request.body as ProcessorTransformationOutput; + const deliveryRequest = ctx.request.body as ProxyDeliveryRequest; const { destination }: { destination: string } = ctx.params; - const { version }: { version: string } = ctx.params; const integrationService = ServiceSelector.getNativeDestinationService(); try { - deliveryResponse = await integrationService.deliver( - event, + deliveryResponse = (await integrationService.deliver( + deliveryRequest, destination, requestMetadata, - version, - ); + 'v0', + )) as DeliveryResponse; } catch (error: any) { + const { metadata } = deliveryRequest; const metaTO = integrationService.getTags( destination, - event.metadata?.destinationId || 'Non-determininable', - event.metadata?.workspaceId || 'Non-determininable', + metadata?.destinationId || NON_DETERMINABLE, + metadata?.workspaceId || NON_DETERMINABLE, tags.FEATURES.DATA_DELIVERY, ); - metaTO.metadata = event.metadata; + metaTO.metadata = metadata; deliveryResponse = DestinationPostTransformationService.handleDeliveryFailureEvents( error, metaTO, @@ -40,6 +50,42 @@ export class DeliveryController { } ctx.body = { output: deliveryResponse }; ControllerUtility.deliveryPostProcess(ctx, deliveryResponse.status); + + logger.debug('Native(Delivery):: Response from transformer::', JSON.stringify(ctx.body)); + return ctx; + } + + public static async deliverToDestinationV1(ctx: Context) { + logger.debug('Native(Delivery):: Request to transformer::', JSON.stringify(ctx.request.body)); + let deliveryResponse: DeliveriesResponse; + const requestMetadata = MiscService.getRequestMetadata(ctx); + const deliveryRequest = ctx.request.body as ProxyDeliveriesRequest; + const { destination }: { destination: string } = ctx.params; + const integrationService = ServiceSelector.getNativeDestinationService(); + try { + deliveryResponse = (await integrationService.deliver( + deliveryRequest, + destination, + requestMetadata, + 'v1', + )) as DeliveriesResponse; + } catch (error: any) { + const { metadata } = deliveryRequest; + const metaTO = integrationService.getTags( + destination, + metadata[0].destinationId || NON_DETERMINABLE, + metadata[0].workspaceId || NON_DETERMINABLE, + tags.FEATURES.DATA_DELIVERY, + ); + metaTO.metadatas = metadata; + deliveryResponse = DestinationPostTransformationService.handlevV1DeliveriesFailureEvents( + error, + metaTO, + ); + } + ctx.body = { output: deliveryResponse }; + ControllerUtility.deliveryPostProcess(ctx); + logger.debug('Native(Delivery):: Response from transformer::', JSON.stringify(ctx.body)); return ctx; } @@ -50,6 +96,7 @@ export class DeliveryController { JSON.stringify(ctx.request.body), ); const { destination }: { destination: string } = ctx.params; + const { version }: { version: string } = ctx.params; const { deliveryPayload, destinationRequestPayload, @@ -61,6 +108,7 @@ export class DeliveryController { destination, destinationRequestPayload, deliveryPayload, + version, ); ctx.body = { output: response }; ControllerUtility.postProcess(ctx); diff --git a/src/features.json b/src/features.json index a3d29bb325d..e0745c22469 100644 --- a/src/features.json +++ b/src/features.json @@ -66,5 +66,5 @@ "REDDIT": true }, "supportSourceTransformV1": true, - "supportTransformerProxyV1": false + "supportTransformerProxyV1": true } diff --git a/src/interfaces/DestinationService.ts b/src/interfaces/DestinationService.ts index 16f6b9349cd..bf39024d85e 100644 --- a/src/interfaces/DestinationService.ts +++ b/src/interfaces/DestinationService.ts @@ -5,9 +5,10 @@ import { ProcessorTransformationResponse, RouterTransformationRequestData, RouterTransformationResponse, - ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, + DeliveriesResponse, } from '../types/index'; export interface DestinationService { @@ -44,11 +45,11 @@ export interface DestinationService { ): RouterTransformationResponse[]; deliver( - event: ProcessorTransformationOutput, + event: ProxyRequest, destinationType: string, requestMetadata: NonNullable, version: string, - ): Promise; + ): Promise; processUserDeletion( requests: UserDeletionRequest[], diff --git a/src/routes/delivery.ts b/src/routes/delivery.ts index 0591dc8b9eb..30f4e2fb788 100644 --- a/src/routes/delivery.ts +++ b/src/routes/delivery.ts @@ -5,12 +5,19 @@ import { RouteActivationMiddleware } from '../middlewares/routeActivation'; const router = new Router(); router.post( - '/:version/destinations/:destination/proxy', + '/v0/destinations/:destination/proxy', RouteActivationMiddleware.isDeliveryRouteActive, RouteActivationMiddleware.destinationDeliveryFilter, DeliveryController.deliverToDestination, ); +router.post( + '/v1/destinations/:destination/proxy', + RouteActivationMiddleware.isDeliveryRouteActive, + RouteActivationMiddleware.destinationDeliveryFilter, + DeliveryController.deliverToDestinationV1, +); + router.post( '/:version/destinations/:destination/proxyTest', RouteActivationMiddleware.isDeliveryTestRouteActive, diff --git a/src/services/comparator.ts b/src/services/comparator.ts index 58c96beabb3..d1e085b4bd6 100644 --- a/src/services/comparator.ts +++ b/src/services/comparator.ts @@ -1,13 +1,14 @@ /* eslint-disable class-methods-use-this */ import { DestinationService } from '../interfaces/DestinationService'; import { + DeliveriesResponse, DeliveryResponse, Destination, ErrorDetailer, MetaTransferObject, - ProcessorTransformationOutput, ProcessorTransformationRequest, ProcessorTransformationResponse, + ProxyRequest, RouterTransformationRequestData, RouterTransformationResponse, UserDeletionRequest, @@ -365,11 +366,11 @@ export class ComparatorService implements DestinationService { } public async deliver( - event: ProcessorTransformationOutput, + event: ProxyRequest, destinationType: string, requestMetadata: NonNullable, version: string, - ): Promise { + ): Promise { const primaryResplist = await this.primaryService.deliver( event, destinationType, @@ -377,7 +378,6 @@ export class ComparatorService implements DestinationService { version, ); logger.error('[LIVE_COMPARE_TEST] not implemented for delivery routine'); - return primaryResplist; } diff --git a/src/services/delivertTest/deliveryTest.ts b/src/services/delivertTest/deliveryTest.ts index 0d960ade172..2f5db183e56 100644 --- a/src/services/delivertTest/deliveryTest.ts +++ b/src/services/delivertTest/deliveryTest.ts @@ -12,12 +12,13 @@ export class DeliveryTestService { destination: string, routerDestReqPayload: any, routerDeliveryPayload: any, + version, ) { let response: any; try { - const destNetworkHandler = networkHandlerFactory.getNetworkHandler(destination); + const { networkHandler } = networkHandlerFactory.getNetworkHandler(destination, version); - const proxyDestReqPayload = destNetworkHandler.prepareProxy(routerDeliveryPayload); + const proxyDestReqPayload = networkHandler.prepareProxy(routerDeliveryPayload); response = { destinationRequestPayload: proxyDestReqPayload, }; diff --git a/src/services/destination/cdkV1Integration.ts b/src/services/destination/cdkV1Integration.ts index 8ccd3341e59..197e3162ea6 100644 --- a/src/services/destination/cdkV1Integration.ts +++ b/src/services/destination/cdkV1Integration.ts @@ -11,9 +11,10 @@ import { ProcessorTransformationResponse, RouterTransformationRequestData, RouterTransformationResponse, - ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, + DeliveriesResponse, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import tags from '../../v0/util/tags'; @@ -117,10 +118,10 @@ export class CDKV1DestinationService implements DestinationService { } public deliver( - _event: ProcessorTransformationOutput, + _event: ProxyRequest, _destinationType: string, _requestMetadata: NonNullable, - ): Promise { + ): Promise { throw new TransformationError('CDV1 Does not Implement Delivery Routine'); } diff --git a/src/services/destination/cdkV2Integration.ts b/src/services/destination/cdkV2Integration.ts index f3be2c01443..be7f0e51d5c 100644 --- a/src/services/destination/cdkV2Integration.ts +++ b/src/services/destination/cdkV2Integration.ts @@ -15,6 +15,8 @@ import { ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, + DeliveriesResponse, } from '../../types/index'; import tags from '../../v0/util/tags'; import { DestinationPostTransformationService } from './postTransformation'; @@ -64,7 +66,7 @@ export class CDKV2DestinationService implements DestinationService { destinationType, event, tags.FEATURES.PROCESSOR, - requestMetadata + requestMetadata, ); stats.increment('event_transform_success', { @@ -127,7 +129,12 @@ export class CDKV2DestinationService implements DestinationService { metaTo.metadata = destInputArray[0].metadata; try { const doRouterTransformationResponse: RouterTransformationResponse[] = - await processCdkV2Workflow(destinationType, destInputArray, tags.FEATURES.ROUTER, requestMetadata); + await processCdkV2Workflow( + destinationType, + destInputArray, + tags.FEATURES.ROUTER, + requestMetadata, + ); return DestinationPostTransformationService.handleRouterTransformSuccessEvents( doRouterTransformationResponse, undefined, @@ -160,10 +167,10 @@ export class CDKV2DestinationService implements DestinationService { } public deliver( - _event: ProcessorTransformationOutput, + _event: ProxyRequest, _destinationType: string, _requestMetadata: NonNullable, - ): Promise { + ): Promise { throw new TransformationError('CDKV2 Does not Implement Delivery Routine'); } diff --git a/src/services/destination/nativeIntegration.ts b/src/services/destination/nativeIntegration.ts index 6bb79ce27aa..c3b0237cdc6 100644 --- a/src/services/destination/nativeIntegration.ts +++ b/src/services/destination/nativeIntegration.ts @@ -1,3 +1,5 @@ +/* eslint-disable prefer-destructuring */ +/* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-unused-vars */ import groupBy from 'lodash/groupBy'; import cloneDeep from 'lodash/cloneDeep'; @@ -13,6 +15,11 @@ import { ProcessorTransformationOutput, UserDeletionRequest, UserDeletionResponse, + ProxyRequest, + ProxyDeliveriesRequest, + ProxyDeliveryRequest, + DeliveriesResponse, + DeliveryJobState, } from '../../types/index'; import { DestinationPostTransformationService } from './postTransformation'; import networkHandlerFactory from '../../adapters/networkHandlerFactory'; @@ -170,30 +177,66 @@ export class NativeIntegrationDestinationService implements DestinationService { } public async deliver( - destinationRequest: ProcessorTransformationOutput, + deliveryRequest: ProxyRequest, destinationType: string, _requestMetadata: NonNullable, version: string, - ): Promise { + ): Promise { try { - const networkHandler = networkHandlerFactory.getNetworkHandler(destinationType, version); - const rawProxyResponse = await networkHandler.proxy(destinationRequest, destinationType); + const { networkHandler, handlerVersion } = networkHandlerFactory.getNetworkHandler( + destinationType, + version, + ); + const rawProxyResponse = await networkHandler.proxy(deliveryRequest, destinationType); const processedProxyResponse = networkHandler.processAxiosResponse(rawProxyResponse); - return networkHandler.responseHandler( + let rudderJobMetadata = + version.toLowerCase() === 'v1' + ? (deliveryRequest as ProxyDeliveriesRequest).metadata + : (deliveryRequest as ProxyDeliveryRequest).metadata; + + if (version.toLowerCase() === 'v1' && handlerVersion.toLowerCase() === 'v0') { + rudderJobMetadata = rudderJobMetadata[0]; + } + + let responseProxy = networkHandler.responseHandler( { ...processedProxyResponse, - rudderJobMetadata: destinationRequest.metadata, + rudderJobMetadata, }, destinationType, - ) as DeliveryResponse; + ); + // Adaption Logic for V0 to V1 + if (handlerVersion.toLowerCase() === 'v0' && version.toLowerCase() === 'v1') { + const v0Response = responseProxy as DeliveryResponse; + const jobStates = (deliveryRequest as ProxyDeliveriesRequest).metadata.map( + (metadata) => + ({ + error: JSON.stringify(v0Response.destinationResponse?.response), + statusCode: v0Response.status, + metadata, + } as DeliveryJobState), + ); + responseProxy = { + response: jobStates, + } as DeliveriesResponse; + } + return responseProxy; } catch (err: any) { + const metadata = Array.isArray(deliveryRequest.metadata) + ? deliveryRequest.metadata[0] + : deliveryRequest.metadata; const metaTO = this.getTags( destinationType, - destinationRequest.metadata?.destinationId || 'Non-determininable', - destinationRequest.metadata?.workspaceId || 'Non-determininable', + metadata?.destinationId || 'Non-determininable', + metadata?.workspaceId || 'Non-determininable', tags.FEATURES.DATA_DELIVERY, ); - metaTO.metadata = destinationRequest.metadata; + + if (version.toLowerCase() === 'v1') { + metaTO.metadatas = (deliveryRequest as ProxyDeliveriesRequest).metadata; + return DestinationPostTransformationService.handlevV1DeliveriesFailureEvents(err, metaTO); + } + metaTO.metadata = (deliveryRequest as ProxyDeliveryRequest).metadata; return DestinationPostTransformationService.handleDeliveryFailureEvents(err, metaTO); } } diff --git a/src/services/destination/postTransformation.ts b/src/services/destination/postTransformation.ts index 076f5ba55e1..de2f5f1b42a 100644 --- a/src/services/destination/postTransformation.ts +++ b/src/services/destination/postTransformation.ts @@ -2,6 +2,7 @@ import cloneDeep from 'lodash/cloneDeep'; import isObject from 'lodash/isObject'; import isEmpty from 'lodash/isEmpty'; +import { PlatformError } from '@rudderstack/integrations-lib'; import { ProcessorTransformationRequest, ProcessorTransformationResponse, @@ -10,6 +11,8 @@ import { DeliveryResponse, MetaTransferObject, UserDeletionResponse, + DeliveriesResponse, + DeliveryJobState, } from '../../types/index'; import { generateErrorObject } from '../../v0/util'; import { ErrorReportingService } from '../errorReporting'; @@ -139,7 +142,7 @@ export class DestinationPostTransformationService { } public static handleDeliveryFailureEvents( - error: NonNullable, + error: any, metaTo: MetaTransferObject, ): DeliveryResponse { const errObj = generateErrorObject(error, metaTo.errorDetails, false); @@ -152,6 +155,35 @@ export class DestinationPostTransformationService { authErrorCategory: errObj.authErrorCategory, }), } as DeliveryResponse; + + ErrorReportingService.reportError(error, metaTo.errorContext, resp); + return resp; + } + + public static handlevV1DeliveriesFailureEvents( + error: NonNullable, + metaTo: MetaTransferObject, + ): DeliveriesResponse { + const errObj = generateErrorObject(error, metaTo.errorDetails, false); + const metadataArray = metaTo.metadatas; + if (!Array.isArray(metadataArray)) { + // Panic + throw new PlatformError('Proxy v1 endpoint error : metadataArray is not an array'); + } + const responses = metadataArray.map((metadata) => { + const resp = { + error: errObj.message || '[Delivery] Error occured while processing payload', + statusCode: errObj.status, + metadata, + } as DeliveryJobState; + return resp; + }); + + const resp = { + response: responses, + statTags: errObj.statTags, + } as DeliveriesResponse; + ErrorReportingService.reportError(error, metaTo.errorContext, resp); return resp; } diff --git a/src/types/index.ts b/src/types/index.ts index 7a231321730..f4432e5c2ac 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -16,9 +16,46 @@ type ProcessorTransformationOutput = { FORM?: Record; }; files?: Record; - metadata?: Metadata; }; +type ProxyDeliveryRequest = { + version: string; + type: string; + method: string; + endpoint: string; + userId: string; + headers?: Record; + params?: Record; + body?: { + JSON?: Record; + JSON_ARRAY?: Record; + XML?: Record; + FORM?: Record; + }; + files?: Record; + metadata: Metadata; +}; + +type ProxyDeliveriesRequest = { + version: string; + type: string; + method: string; + endpoint: string; + userId: string; + headers?: Record; + params?: Record; + body?: { + JSON?: Record; + JSON_ARRAY?: Record; + XML?: Record; + FORM?: Record; + }; + files?: Record; + metadata: Metadata[]; +}; + +type ProxyRequest = ProxyDeliveryRequest | ProxyDeliveriesRequest; + type Metadata = { sourceId: string; workspaceId: string; @@ -50,6 +87,7 @@ type Metadata = { sourceDefinitionId: string; destinationDefinitionId: string; transformationId: string; + dontBatch?: boolean; }; type MessageIdMetadataMap = { @@ -140,7 +178,20 @@ type DeliveryResponse = { destinationResponse: any; statTags: object; authErrorCategory?: string; - response?: object; +}; + +type DeliveryJobState = { + error: string; + statusCode: number; + metadata: Metadata; +}; + +type DeliveriesResponse = { + status?: number; + message?: string; + statTags?: object; + authErrorCategory?: string; + response: DeliveryJobState[]; }; enum MessageType { @@ -254,26 +305,31 @@ type SourceInput = { source?: Source; }; export { - Metadata, + ComparatorInput, + DeliveryJobState, + DeliveryResponse, + DeliveriesResponse, + Destination, + ErrorDetailer, MessageIdMetadataMap, - UserTransformationLibrary, + MetaTransferObject, + Metadata, + ProcessorTransformationOutput, ProcessorTransformationRequest, ProcessorTransformationResponse, + ProxyDeliveriesRequest, + ProxyDeliveryRequest, + ProxyRequest, RouterTransformationRequest, RouterTransformationRequestData, RouterTransformationResponse, RudderMessage, - ProcessorTransformationOutput, SourceTransformationResponse, - DeliveryResponse, - ErrorDetailer, - UserTransformationResponse, - UserTransformationServiceResponse, - MetaTransferObject, UserDeletionRequest, UserDeletionResponse, - Destination, - ComparatorInput, SourceInput, Source, + UserTransformationLibrary, + UserTransformationResponse, + UserTransformationServiceResponse, }; diff --git a/src/v0/destinations/campaign_manager/networkHandler.js b/src/v0/destinations/campaign_manager/networkHandler.js index 63efff5b506..a1fa24835c1 100644 --- a/src/v0/destinations/campaign_manager/networkHandler.js +++ b/src/v0/destinations/campaign_manager/networkHandler.js @@ -76,7 +76,7 @@ const responseHandler = (destinationResponse) => { throw new NetworkError( `Campaign Manager: ${response.error?.message} during CAMPAIGN_MANAGER response transformation 3`, - status, + 500, { [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), }, diff --git a/src/v0/destinations/salesforce/networkHandler.js b/src/v0/destinations/salesforce/networkHandler.js index dc67aff1b7d..918084cc899 100644 --- a/src/v0/destinations/salesforce/networkHandler.js +++ b/src/v0/destinations/salesforce/networkHandler.js @@ -10,7 +10,7 @@ const responseHandler = (destinationResponse, destType) => { destinationResponse, 'during Salesforce Response Handling', destinationResponse?.rudderJobMetadata?.destInfo?.authKey, - LEGACY + LEGACY, ); // else successfully return status as 200, message and original destination response diff --git a/src/v0/destinations/salesforce_oauth/networkHandler.js b/src/v0/destinations/salesforce_oauth/networkHandler.js index 2042830cb13..2bcace31c9a 100644 --- a/src/v0/destinations/salesforce_oauth/networkHandler.js +++ b/src/v0/destinations/salesforce_oauth/networkHandler.js @@ -10,7 +10,7 @@ const responseHandler = (destinationResponse, destType) => { destinationResponse, 'during Salesforce Response Handling', destinationResponse?.rudderJobMetadata?.destInfo?.authKey, - OAUTH + OAUTH, ); // else successfully return status as 200, message and original destination response diff --git a/src/v0/util/errorTypes/index.js b/src/v0/util/errorTypes/index.js index 80268c3e777..f3cef657cad 100644 --- a/src/v0/util/errorTypes/index.js +++ b/src/v0/util/errorTypes/index.js @@ -1,5 +1,7 @@ const FilteredEventsError = require('./filteredEventsError'); +const TransformerProxyError = require('./transformerProxyError'); module.exports = { FilteredEventsError, + TransformerProxyError, }; diff --git a/src/v0/util/errorTypes/transformerProxyError.js b/src/v0/util/errorTypes/transformerProxyError.js new file mode 100644 index 00000000000..9f4bb593c7e --- /dev/null +++ b/src/v0/util/errorTypes/transformerProxyError.js @@ -0,0 +1,28 @@ +const tags = require('../tags'); +const { BaseError } = require('./base'); + +const errorTypes = Object.values(tags.ERROR_TYPES); +const metaTypes = Object.values(tags.METADATA); +class TransformerProxyError extends BaseError { + constructor(message, statusCode, statTags, destResponse, authErrorCategory, response) { + const finalStatTags = { + [tags.TAG_NAMES.ERROR_CATEGORY]: tags.ERROR_CATEGORIES.NETWORK, + [tags.TAG_NAMES.ERROR_TYPE]: tags.ERROR_TYPES.ABORTED, + }; + + // Allow specifying only error type and meta tags + if (statTags && typeof statTags === 'object' && !Array.isArray(statTags)) { + if (errorTypes.includes(statTags[tags.TAG_NAMES.ERROR_TYPE])) { + finalStatTags[tags.TAG_NAMES.ERROR_TYPE] = statTags[tags.TAG_NAMES.ERROR_TYPE]; + } + + if (metaTypes.includes(statTags[tags.TAG_NAMES.META])) { + finalStatTags[tags.TAG_NAMES.META] = statTags[tags.TAG_NAMES.META]; + } + } + super(message, statusCode, finalStatTags, destResponse, authErrorCategory); + this.response = response; + } +} + +module.exports = TransformerProxyError; \ No newline at end of file diff --git a/src/v1/destinations/campaign_manager/networkHandler.js b/src/v1/destinations/campaign_manager/networkHandler.js index b3b82c8cf36..6e13a68f0a4 100644 --- a/src/v1/destinations/campaign_manager/networkHandler.js +++ b/src/v1/destinations/campaign_manager/networkHandler.js @@ -1,6 +1,6 @@ /* eslint-disable no-param-reassign */ /* eslint-disable no-restricted-syntax */ -const { NetworkError } = require('@rudderstack/integrations-lib'); +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); const { isHttpStatusSuccess, getAuthErrCategoryFromStCode } = require('../../../v0/util/index'); @@ -10,25 +10,6 @@ const { } = require('../../../adapters/utils/networkUtils'); const tags = require('../../../v0/util/tags'); -function isEventRetryableAndExtractErrMsg(element, proxyOutputObj) { - let isRetryable = false; - let errorMsg = ''; - // success event - if (!element.errors) { - return isRetryable; - } - for (const err of element.errors) { - errorMsg += `${err.message}, `; - if (err.code === 'INTERNAL') { - isRetryable = true; - } - } - if (errorMsg) { - proxyOutputObj.error = errorMsg; - } - return isRetryable; -} - function isEventAbortableAndExtractErrMsg(element, proxyOutputObj) { let isAbortable = false; let errorMsg = ''; @@ -68,10 +49,8 @@ const responseHandler = (destinationResponse) => { metadata: rudderJobMetadata[idx], error: 'success', }; - // update status of partial event as per retriable or abortable - if (isEventRetryableAndExtractErrMsg(element, proxyOutputObj)) { - proxyOutputObj.statusCode = 500; - } else if (isEventAbortableAndExtractErrMsg(element, proxyOutputObj)) { + // update status of partial event if abortable + if (isEventAbortableAndExtractErrMsg(element, proxyOutputObj)) { proxyOutputObj.statusCode = 400; } responseWithIndividualEvents.push(proxyOutputObj); @@ -95,16 +74,16 @@ const responseHandler = (destinationResponse) => { }); } - throw new NetworkError( - `Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation`, - 500, - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), - }, - destinationResponse, - getAuthErrCategoryFromStCode(status), - responseWithIndividualEvents, - ); + throw new TransformerProxyError( + `Campaign Manager: Error transformer proxy v1 during CAMPAIGN_MANAGER response transformation`, + 500, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithIndividualEvents, + ); }; function networkHandler() { diff --git a/test/__tests__/data/marketo_input.json b/test/__tests__/data/marketo_input.json index 10fff935fc7..122525560e0 100644 --- a/test/__tests__/data/marketo_input.json +++ b/test/__tests__/data/marketo_input.json @@ -55,12 +55,7 @@ "includeKeys": [], "saveDestinationResponse": true, "secretKeys": [], - "supportedMessageTypes": [ - "identify", - "page", - "screen", - "track" - ], + "supportedMessageTypes": ["identify", "page", "screen", "track"], "supportedSourceTypes": [ "android", "ios", @@ -1125,12 +1120,7 @@ "includeKeys": [], "saveDestinationResponse": true, "secretKeys": [], - "supportedMessageTypes": [ - "identify", - "page", - "screen", - "track" - ], + "supportedMessageTypes": ["identify", "page", "screen", "track"], "supportedSourceTypes": [ "android", "ios", @@ -1164,4 +1154,4 @@ "query": {} } } -] \ No newline at end of file +] diff --git a/test/integrations/destinations/braze/dataDelivery/data.ts b/test/integrations/destinations/braze/dataDelivery/data.ts index 3e1805c01d3..cd688a621b3 100644 --- a/test/integrations/destinations/braze/dataDelivery/data.ts +++ b/test/integrations/destinations/braze/dataDelivery/data.ts @@ -1,395 +1,391 @@ -import MockAdapter from "axios-mock-adapter"; +import MockAdapter from 'axios-mock-adapter'; export const data = [ { - "name": "braze", - "description": "Test 0", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test1", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 0', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test1', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request for braze Processed Successfully", - "destinationResponse": { - "response": { - "aliases_processed": 1, - "message": "success" + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: { + aliases_processed: 1, + message: 'success', }, - "status": 201 - } - } - } - } - } + status: 201, + }, + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 1", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test2", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 1', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test2', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request for braze Processed Successfully", - "destinationResponse": { - "response": { - "message": "success", - "errors": [ - "minor error message" - ] + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: { + message: 'success', + errors: ['minor error message'], }, - "status": 201 - } - } - } - } - } + status: 201, + }, + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 2", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test3", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 2', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test3', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request failed for braze with status: 201", - "destinationResponse": { - "response": { - "message": "fatal error message", - "errors": [ - "minor error message" - ] + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request failed for braze with status: 201', + destinationResponse: { + response: { + message: 'fatal error message', + errors: ['minor error message'], }, - "status": 201 + status: 201, + }, + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "aborted", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } - } + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 3", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test4", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 3', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test4', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 201, + message: 'Request for braze Processed Successfully', + destinationResponse: { + response: '', + status: 201, + }, + }, + }, + }, }, - "output": { - "response": { - "status": 200, - "body": { - "output": { - "status": 201, - "message": "Request for braze Processed Successfully", - "destinationResponse": { - "response": "", - "status": 201 - } - } - } - } - } }, { - "name": "braze", - "description": "Test 4", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test5", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 4', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test5', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 500, - "body": { - "output": { - "status": 500, - "message": "Request failed for braze with status: 500", - "destinationResponse": { - "response": "", - "status": 500 + output: { + response: { + status: 500, + body: { + output: { + status: 500, + message: 'Request failed for braze with status: 500', + destinationResponse: { + response: '', + status: 500, + }, + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "retryable", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } - } + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 5", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test6", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test 5', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test6', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 500, - "body": { - "output": { - "status": 500, - "message": "Request failed for braze with status: 500", - "destinationResponse": { - "response": "", - "status": 500 + output: { + response: { + status: 500, + body: { + output: { + status: 500, + message: 'Request failed for braze with status: 500', + destinationResponse: { + response: '', + status: 500, }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "retryable", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + }, + }, + }, + }, }, mockFns: (mockAdapter: MockAdapter) => { // params has `{ destination: salesforce }` @@ -410,79 +406,262 @@ export const data = [ 'Content-Type': 'application/json', 'User-Agent': 'RudderLabs', }, - ).replyOnce((config) => { - // @ts-ignore - const err = AxiosError.from('DNS not found', 'ENOTFOUND', config) - return Promise.reject(err); + ) + .replyOnce((config) => { + // @ts-ignore + const err = AxiosError.from('DNS not found', 'ENOTFOUND', config); + return Promise.reject(err); }); - } + }, + }, + { + name: 'braze', + description: 'Test 6', + feature: 'dataDelivery', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test7', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ + { + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + files: {}, + params: { + destination: 'braze', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 500, + body: { + output: { + status: 500, + message: 'Request failed for braze with status: 500', + destinationResponse: { + response: '', + status: 500, + }, + statTags: { + destType: 'BRAZE', + errorCategory: 'network', + destinationId: 'Non-determininable', + workspaceId: 'Non-determininable', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + }, + }, + }, + }, + }, + }, + { + name: 'braze', + description: 'Test Transformer Proxy V1 input with v0 proxy handler', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/test1', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ + { + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + metadata: [ + { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + ], + files: {}, + params: { + destination: 'braze', + }, + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '{"aliases_processed":1,"message":"success"}', + statusCode: 201, + metadata: { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + }, + ], + }, + }, + }, + }, }, { - "name": "braze", - "description": "Test 6", - "feature": "dataDelivery", - "module": "destination", - "version": "v0", - "input": { - "request": { - "body": { - "type": "REST", - "endpoint": "https://rest.iad-03.braze.com/users/identify/test7", - "method": "POST", - "userId": "gabi_userId_45", - "headers": { - "Accept": "application/json", - "Authorization": "Bearer api_key", - "Content-Type": "application/json" - }, - "body": { - "FORM": {}, - "JSON": { - "aliases_to_identify": [ + name: 'braze', + description: 'Test Transformer Proxy V1 input with v0 proxy handler Error returned', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: { + type: 'REST', + endpoint: 'https://rest.iad-03.braze.com/users/identify/testV1', + method: 'POST', + userId: 'gabi_userId_45', + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + }, + body: { + FORM: {}, + JSON: { + aliases_to_identify: [ { - "external_id": "gabi_userId_45", - "user_alias": { - "alias_label": "rudder_id", - "alias_name": "gabi_anonId_45" - } - } - ] + external_id: 'gabi_userId_45', + user_alias: { + alias_label: 'rudder_id', + alias_name: 'gabi_anonId_45', + }, + }, + ], }, - "JSON_ARRAY": {}, - "XML": {} + JSON_ARRAY: {}, + XML: {}, + }, + metadata: [ + { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + ], + files: {}, + params: { + destination: 'braze', }, - "files": {}, - "params": { - "destination": "braze" - } }, - "method": "POST" - } + method: 'POST', + }, }, - "output": { - "response": { - "status": 500, - "body": { - "output": { - "status": 500, - "message": "Request failed for braze with status: 500", - "destinationResponse": { - "response": "", - "status": 500 + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: 'Request failed for braze with status: 401', + statusCode: 401, + metadata: { + jobId: 2, + attemptNum: 0, + userId: '', + sourceId: '2Vsge2uWYdrLfG7pZb5Y82eo4lr', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', + secret: { + access_token: 'secret', + refresh_token: 'refresh', + developer_token: 'developer_Token', + }, + }, + }, + ], + statTags: { + destType: 'BRAZE', + destinationId: '2RHh08uOsXqE9KvCDg3hoaeuK2L', + errorCategory: 'network', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', + workspaceId: '2Csl0lSTbuM3qyHdaOQB2GcDH8o', }, - "statTags": { - "destType": "BRAZE", - "errorCategory": "network", - "destinationId": "Non-determininable", - "workspaceId": "Non-determininable", - "errorType": "retryable", - "feature": "dataDelivery", - "implementation": "native", - "module": "destination" - } - } - } - } - } - } -] \ No newline at end of file + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/braze/network.ts b/test/integrations/destinations/braze/network.ts index 1c39a247e43..0ef99b806d6 100644 --- a/test/integrations/destinations/braze/network.ts +++ b/test/integrations/destinations/braze/network.ts @@ -391,5 +391,34 @@ const deleteNwData = [ }, }, }, + { + httpReq: { + url: 'https://rest.iad-03.braze.com/users/identify/testV1', + data: { + aliases_to_identify: [ + { + external_id: 'gabi_userId_45', + user_alias: { alias_label: 'rudder_id', alias_name: 'gabi_anonId_45' }, + }, + ], + }, + params: { destination: 'braze' }, + headers: { + Accept: 'application/json', + Authorization: 'Bearer api_key', + 'Content-Type': 'application/json', + 'User-Agent': 'RudderLabs', + }, + method: 'POST', + }, + httpRes: { + data: { + code: 400, + message: 'Bad Req', + status: 'Fail Case', + }, + status: 401, + }, + }, ]; export const networkCallsData = [...deleteNwData, ...dataDeliveryMocksData]; From 36d3f258ed5ea4ba1899c8b2cae8203bf73d90ed Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:37:54 +0530 Subject: [PATCH 2/2] feat: onboard msl changes for new record event (#2644) * feat: onboard msl changes for new record event * chore: remove older tc for MSL * fix: metadata * fix: metadatax2 * chore: address comments * chore: address commentsx2, handle mix of audiencelist and record events * chore: address commentsx2, handle mix of audiencelist and record events * chore: add router test * fix: record to audience handling (#2813) * fix: unit case with empty request * fix: missed error lib * chore: refactor tests * refactor: address comments * chore: address commentsx2 * fix: error message, handle metadata, missing id --------- Co-authored-by: Utsab Chowdhury --- .../marketo_static_list/testData/constants.js | 45 + .../marketo_static_list/testData/testData.js | 3085 +++++++++++++++++ .../marketo_static_list/transform.js | 126 +- .../marketo_static_list/transform.test.js | 138 + .../marketo_static_list/transformV2.js | 143 + .../destinations/marketo_static_list/util.js | 1 + test/__tests__/data/marketo_static_list.json | 2542 -------------- .../marketo_static_list_router_input.json | 1209 ------- ...eto_static_list_router_metadata_input.json | 69 - ...to_static_list_router_metadata_output.json | 20 - .../marketo_static_list_router_output.json | 170 - test/__tests__/marketo_static_list.test.js | 64 - .../marketo_static_list/processor/data.ts | 8 +- .../marketo_static_list/router/data.ts | 2356 ++++++++++++- 14 files changed, 5783 insertions(+), 4193 deletions(-) create mode 100644 src/v0/destinations/marketo_static_list/testData/constants.js create mode 100644 src/v0/destinations/marketo_static_list/testData/testData.js create mode 100644 src/v0/destinations/marketo_static_list/transform.test.js create mode 100644 src/v0/destinations/marketo_static_list/transformV2.js delete mode 100644 test/__tests__/data/marketo_static_list.json delete mode 100644 test/__tests__/data/marketo_static_list_router_input.json delete mode 100644 test/__tests__/data/marketo_static_list_router_metadata_input.json delete mode 100644 test/__tests__/data/marketo_static_list_router_metadata_output.json delete mode 100644 test/__tests__/data/marketo_static_list_router_output.json delete mode 100644 test/__tests__/marketo_static_list.test.js diff --git a/src/v0/destinations/marketo_static_list/testData/constants.js b/src/v0/destinations/marketo_static_list/testData/constants.js new file mode 100644 index 00000000000..2c420c6bc2a --- /dev/null +++ b/src/v0/destinations/marketo_static_list/testData/constants.js @@ -0,0 +1,45 @@ +const EXTERNAL_ID = 'marketoStaticListId'; +const TOKEN = 'Bearer access_token_success'; +const CONTENT_TYPE = 'application/json'; +const DEST_CONFIG = { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, +}; +const DEST_DEFINITION = { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', +}; +const DEST_OBJECT = { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: DEST_DEFINITION, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, +}; +const MESSAGE_SOURCES_CONTEXT = { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', +}; + +module.exports = { + EXTERNAL_ID, + TOKEN, + CONTENT_TYPE, + DEST_OBJECT, + DEST_DEFINITION, + MESSAGE_SOURCES_CONTEXT, + DEST_CONFIG, +}; diff --git a/src/v0/destinations/marketo_static_list/testData/testData.js b/src/v0/destinations/marketo_static_list/testData/testData.js new file mode 100644 index 00000000000..45328436ce7 --- /dev/null +++ b/src/v0/destinations/marketo_static_list/testData/testData.js @@ -0,0 +1,3085 @@ +const { + EXTERNAL_ID, + TOKEN, + CONTENT_TYPE, + DEST_OBJECT, + DEST_DEFINITION, + MESSAGE_SOURCES_CONTEXT, + DEST_CONFIG, +} = require('./constants'); + +const recordInputs = [ + { + destination: DEST_OBJECT, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1001, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: DEST_OBJECT, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1002, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: 'id002', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 2, + }, + }, + { + destination: DEST_OBJECT, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1003, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 3, + }, + }, + { + destination: DEST_OBJECT, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id002', + type: EXTERNAL_ID, + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2001', + }, + type: 'record', + }, + metadata: { + jobId: 4, + }, + }, +]; + +const recordOutput = [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=1001&id=1003', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + }, + { + jobId: 3, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=2001', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 4, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=1002', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, +]; + +const audiencelistInputs = [ + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 501, + }, + { + id: 502, + }, + { + id: 503, + }, + ], + remove: [ + { + id: 704, + }, + { + id: 705, + }, + { + id: 706, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 0, + }, + { + id: 1, + }, + { + id: 2, + }, + { + id: 3, + }, + { + id: 4, + }, + { + id: 5, + }, + { + id: 6, + }, + { + id: 7, + }, + { + id: 8, + }, + { + id: 9, + }, + { + id: 10, + }, + { + id: 11, + }, + { + id: 12, + }, + { + id: 13, + }, + { + id: 14, + }, + { + id: 15, + }, + { + id: 16, + }, + { + id: 17, + }, + { + id: 18, + }, + { + id: 19, + }, + { + id: 20, + }, + { + id: 21, + }, + { + id: 22, + }, + { + id: 23, + }, + { + id: 24, + }, + { + id: 25, + }, + { + id: 26, + }, + { + id: 27, + }, + { + id: 28, + }, + { + id: 29, + }, + { + id: 30, + }, + { + id: 31, + }, + { + id: 32, + }, + { + id: 33, + }, + { + id: 34, + }, + { + id: 35, + }, + { + id: 36, + }, + { + id: 37, + }, + { + id: 38, + }, + { + id: 39, + }, + { + id: 40, + }, + { + id: 41, + }, + { + id: 42, + }, + { + id: 43, + }, + { + id: 44, + }, + { + id: 45, + }, + { + id: 46, + }, + { + id: 47, + }, + { + id: 48, + }, + { + id: 49, + }, + { + id: 50, + }, + { + id: 51, + }, + { + id: 52, + }, + { + id: 53, + }, + { + id: 54, + }, + { + id: 55, + }, + { + id: 56, + }, + { + id: 57, + }, + { + id: 58, + }, + { + id: 59, + }, + { + id: 60, + }, + { + id: 61, + }, + { + id: 62, + }, + { + id: 63, + }, + { + id: 64, + }, + { + id: 65, + }, + { + id: 66, + }, + { + id: 67, + }, + { + id: 68, + }, + { + id: 69, + }, + { + id: 70, + }, + { + id: 71, + }, + { + id: 72, + }, + { + id: 73, + }, + { + id: 74, + }, + { + id: 75, + }, + { + id: 76, + }, + { + id: 77, + }, + { + id: 78, + }, + { + id: 79, + }, + { + id: 80, + }, + { + id: 81, + }, + { + id: 82, + }, + { + id: 83, + }, + { + id: 84, + }, + { + id: 85, + }, + { + id: 86, + }, + { + id: 87, + }, + { + id: 88, + }, + { + id: 89, + }, + { + id: 90, + }, + { + id: 91, + }, + { + id: 92, + }, + { + id: 93, + }, + { + id: 94, + }, + { + id: 95, + }, + { + id: 96, + }, + { + id: 97, + }, + { + id: 98, + }, + { + id: 99, + }, + { + id: 100, + }, + { + id: 101, + }, + { + id: 102, + }, + { + id: 103, + }, + { + id: 104, + }, + { + id: 105, + }, + { + id: 106, + }, + { + id: 107, + }, + { + id: 108, + }, + { + id: 109, + }, + { + id: 110, + }, + { + id: 111, + }, + { + id: 112, + }, + { + id: 113, + }, + { + id: 114, + }, + { + id: 115, + }, + { + id: 116, + }, + { + id: 117, + }, + { + id: 118, + }, + { + id: 119, + }, + { + id: 120, + }, + { + id: 121, + }, + { + id: 122, + }, + { + id: 123, + }, + { + id: 124, + }, + { + id: 125, + }, + { + id: 126, + }, + { + id: 127, + }, + { + id: 128, + }, + { + id: 129, + }, + { + id: 130, + }, + { + id: 131, + }, + { + id: 132, + }, + { + id: 133, + }, + { + id: 134, + }, + { + id: 135, + }, + { + id: 136, + }, + { + id: 137, + }, + { + id: 138, + }, + { + id: 139, + }, + { + id: 140, + }, + { + id: 141, + }, + { + id: 142, + }, + { + id: 143, + }, + { + id: 144, + }, + { + id: 145, + }, + { + id: 146, + }, + { + id: 147, + }, + { + id: 148, + }, + { + id: 149, + }, + { + id: 150, + }, + { + id: 151, + }, + { + id: 152, + }, + { + id: 153, + }, + { + id: 154, + }, + { + id: 155, + }, + { + id: 156, + }, + { + id: 157, + }, + { + id: 158, + }, + { + id: 159, + }, + { + id: 160, + }, + { + id: 161, + }, + { + id: 162, + }, + { + id: 163, + }, + { + id: 164, + }, + { + id: 165, + }, + { + id: 166, + }, + { + id: 167, + }, + { + id: 168, + }, + { + id: 169, + }, + { + id: 170, + }, + { + id: 171, + }, + { + id: 172, + }, + { + id: 173, + }, + { + id: 174, + }, + { + id: 175, + }, + { + id: 176, + }, + { + id: 177, + }, + { + id: 178, + }, + { + id: 179, + }, + { + id: 180, + }, + { + id: 181, + }, + { + id: 182, + }, + { + id: 183, + }, + { + id: 184, + }, + { + id: 185, + }, + { + id: 186, + }, + { + id: 187, + }, + { + id: 188, + }, + { + id: 189, + }, + { + id: 190, + }, + { + id: 191, + }, + { + id: 192, + }, + { + id: 193, + }, + { + id: 194, + }, + { + id: 195, + }, + { + id: 196, + }, + { + id: 197, + }, + { + id: 198, + }, + { + id: 199, + }, + { + id: 200, + }, + { + id: 201, + }, + { + id: 202, + }, + { + id: 203, + }, + { + id: 204, + }, + { + id: 205, + }, + { + id: 206, + }, + { + id: 207, + }, + { + id: 208, + }, + { + id: 209, + }, + { + id: 210, + }, + { + id: 211, + }, + { + id: 212, + }, + { + id: 213, + }, + { + id: 214, + }, + { + id: 215, + }, + { + id: 216, + }, + { + id: 217, + }, + { + id: 218, + }, + { + id: 219, + }, + { + id: 220, + }, + { + id: 221, + }, + { + id: 222, + }, + { + id: 223, + }, + { + id: 224, + }, + { + id: 225, + }, + { + id: 226, + }, + { + id: 227, + }, + { + id: 228, + }, + { + id: 229, + }, + { + id: 230, + }, + { + id: 231, + }, + { + id: 232, + }, + { + id: 233, + }, + { + id: 234, + }, + { + id: 235, + }, + { + id: 236, + }, + { + id: 237, + }, + { + id: 238, + }, + { + id: 239, + }, + { + id: 240, + }, + { + id: 241, + }, + { + id: 242, + }, + { + id: 243, + }, + { + id: 244, + }, + { + id: 245, + }, + { + id: 246, + }, + { + id: 247, + }, + { + id: 248, + }, + { + id: 249, + }, + { + id: 250, + }, + { + id: 251, + }, + { + id: 252, + }, + { + id: 253, + }, + { + id: 254, + }, + { + id: 255, + }, + { + id: 256, + }, + { + id: 257, + }, + { + id: 258, + }, + { + id: 259, + }, + { + id: 260, + }, + { + id: 261, + }, + { + id: 262, + }, + { + id: 263, + }, + { + id: 264, + }, + { + id: 265, + }, + { + id: 266, + }, + { + id: 267, + }, + { + id: 268, + }, + { + id: 269, + }, + { + id: 270, + }, + { + id: 271, + }, + { + id: 272, + }, + { + id: 273, + }, + { + id: 274, + }, + { + id: 275, + }, + { + id: 276, + }, + { + id: 277, + }, + { + id: 278, + }, + { + id: 279, + }, + { + id: 280, + }, + { + id: 281, + }, + { + id: 282, + }, + { + id: 283, + }, + { + id: 284, + }, + { + id: 285, + }, + { + id: 286, + }, + { + id: 287, + }, + { + id: 288, + }, + { + id: 289, + }, + { + id: 290, + }, + { + id: 291, + }, + { + id: 292, + }, + { + id: 293, + }, + { + id: 294, + }, + { + id: 295, + }, + { + id: 296, + }, + { + id: 297, + }, + { + id: 298, + }, + { + id: 299, + }, + { + id: 300, + }, + { + id: 301, + }, + { + id: 302, + }, + { + id: 303, + }, + { + id: 304, + }, + { + id: 305, + }, + { + id: 306, + }, + { + id: 307, + }, + { + id: 308, + }, + { + id: 309, + }, + { + id: 310, + }, + { + id: 311, + }, + { + id: 312, + }, + { + id: 313, + }, + { + id: 314, + }, + { + id: 315, + }, + { + id: 316, + }, + { + id: 317, + }, + { + id: 318, + }, + { + id: 319, + }, + { + id: 320, + }, + { + id: 321, + }, + { + id: 322, + }, + { + id: 323, + }, + { + id: 324, + }, + { + id: 325, + }, + { + id: 326, + }, + { + id: 327, + }, + { + id: 328, + }, + { + id: 329, + }, + { + id: 330, + }, + { + id: 331, + }, + { + id: 332, + }, + { + id: 333, + }, + { + id: 334, + }, + { + id: 335, + }, + { + id: 336, + }, + { + id: 337, + }, + { + id: 338, + }, + { + id: 339, + }, + { + id: 340, + }, + { + id: 341, + }, + { + id: 342, + }, + { + id: 343, + }, + { + id: 344, + }, + { + id: 345, + }, + { + id: 346, + }, + { + id: 347, + }, + { + id: 348, + }, + { + id: 349, + }, + { + id: 350, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 2, + }, + }, +]; + +const reqMetadata = { + namespace: 'Unknown', + cluster: 'Unknown', + features: {}, +}; + +const largeRecordOutput = [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=351&id=352&id=353&id=354&id=355&id=356&id=357&id=358&id=359&id=360&id=361&id=362&id=363&id=364&id=365&id=366&id=367&id=368&id=369&id=370&id=371&id=372&id=373&id=374&id=375&id=376&id=377&id=378&id=379&id=380&id=381&id=382&id=383&id=384&id=385&id=386&id=387&id=388&id=389&id=390&id=391&id=392&id=393&id=394&id=395&id=396&id=397&id=398&id=399&id=400&id=401&id=402&id=403&id=404&id=405&id=406&id=407&id=408&id=409&id=410&id=411&id=412&id=413&id=414&id=415&id=416&id=417&id=418&id=419&id=420&id=421&id=422&id=423&id=424&id=425&id=426&id=427&id=428&id=429&id=430&id=431&id=432&id=433&id=434&id=435&id=436&id=437&id=438&id=439&id=440&id=441&id=442&id=443&id=444&id=445&id=446&id=447&id=448&id=449&id=450', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 351, + }, + { + jobId: 352, + }, + { + jobId: 353, + }, + { + jobId: 354, + }, + { + jobId: 355, + }, + { + jobId: 356, + }, + { + jobId: 357, + }, + { + jobId: 358, + }, + { + jobId: 359, + }, + { + jobId: 360, + }, + { + jobId: 361, + }, + { + jobId: 362, + }, + { + jobId: 363, + }, + { + jobId: 364, + }, + { + jobId: 365, + }, + { + jobId: 366, + }, + { + jobId: 367, + }, + { + jobId: 368, + }, + { + jobId: 369, + }, + { + jobId: 370, + }, + { + jobId: 371, + }, + { + jobId: 372, + }, + { + jobId: 373, + }, + { + jobId: 374, + }, + { + jobId: 375, + }, + { + jobId: 376, + }, + { + jobId: 377, + }, + { + jobId: 378, + }, + { + jobId: 379, + }, + { + jobId: 380, + }, + { + jobId: 381, + }, + { + jobId: 382, + }, + { + jobId: 383, + }, + { + jobId: 384, + }, + { + jobId: 385, + }, + { + jobId: 386, + }, + { + jobId: 387, + }, + { + jobId: 388, + }, + { + jobId: 389, + }, + { + jobId: 390, + }, + { + jobId: 391, + }, + { + jobId: 392, + }, + { + jobId: 393, + }, + { + jobId: 394, + }, + { + jobId: 395, + }, + { + jobId: 396, + }, + { + jobId: 397, + }, + { + jobId: 398, + }, + { + jobId: 399, + }, + { + jobId: 400, + }, + { + jobId: 401, + }, + { + jobId: 402, + }, + { + jobId: 403, + }, + { + jobId: 404, + }, + { + jobId: 405, + }, + { + jobId: 406, + }, + { + jobId: 407, + }, + { + jobId: 408, + }, + { + jobId: 409, + }, + { + jobId: 410, + }, + { + jobId: 411, + }, + { + jobId: 412, + }, + { + jobId: 413, + }, + { + jobId: 414, + }, + { + jobId: 415, + }, + { + jobId: 416, + }, + { + jobId: 417, + }, + { + jobId: 418, + }, + { + jobId: 419, + }, + { + jobId: 420, + }, + { + jobId: 421, + }, + { + jobId: 422, + }, + { + jobId: 423, + }, + { + jobId: 424, + }, + { + jobId: 425, + }, + { + jobId: 426, + }, + { + jobId: 427, + }, + { + jobId: 428, + }, + { + jobId: 429, + }, + { + jobId: 430, + }, + { + jobId: 431, + }, + { + jobId: 432, + }, + { + jobId: 433, + }, + { + jobId: 434, + }, + { + jobId: 435, + }, + { + jobId: 436, + }, + { + jobId: 437, + }, + { + jobId: 438, + }, + { + jobId: 439, + }, + { + jobId: 440, + }, + { + jobId: 441, + }, + { + jobId: 442, + }, + { + jobId: 443, + }, + { + jobId: 444, + }, + { + jobId: 445, + }, + { + jobId: 446, + }, + { + jobId: 447, + }, + { + jobId: 448, + }, + { + jobId: 449, + }, + { + jobId: 450, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299&id=300', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + }, + { + jobId: 2, + }, + { + jobId: 3, + }, + { + jobId: 4, + }, + { + jobId: 5, + }, + { + jobId: 6, + }, + { + jobId: 7, + }, + { + jobId: 8, + }, + { + jobId: 9, + }, + { + jobId: 10, + }, + { + jobId: 11, + }, + { + jobId: 12, + }, + { + jobId: 13, + }, + { + jobId: 14, + }, + { + jobId: 15, + }, + { + jobId: 16, + }, + { + jobId: 17, + }, + { + jobId: 18, + }, + { + jobId: 19, + }, + { + jobId: 20, + }, + { + jobId: 21, + }, + { + jobId: 22, + }, + { + jobId: 23, + }, + { + jobId: 24, + }, + { + jobId: 25, + }, + { + jobId: 26, + }, + { + jobId: 27, + }, + { + jobId: 28, + }, + { + jobId: 29, + }, + { + jobId: 30, + }, + { + jobId: 31, + }, + { + jobId: 32, + }, + { + jobId: 33, + }, + { + jobId: 34, + }, + { + jobId: 35, + }, + { + jobId: 36, + }, + { + jobId: 37, + }, + { + jobId: 38, + }, + { + jobId: 39, + }, + { + jobId: 40, + }, + { + jobId: 41, + }, + { + jobId: 42, + }, + { + jobId: 43, + }, + { + jobId: 44, + }, + { + jobId: 45, + }, + { + jobId: 46, + }, + { + jobId: 47, + }, + { + jobId: 48, + }, + { + jobId: 49, + }, + { + jobId: 50, + }, + { + jobId: 51, + }, + { + jobId: 52, + }, + { + jobId: 53, + }, + { + jobId: 54, + }, + { + jobId: 55, + }, + { + jobId: 56, + }, + { + jobId: 57, + }, + { + jobId: 58, + }, + { + jobId: 59, + }, + { + jobId: 60, + }, + { + jobId: 61, + }, + { + jobId: 62, + }, + { + jobId: 63, + }, + { + jobId: 64, + }, + { + jobId: 65, + }, + { + jobId: 66, + }, + { + jobId: 67, + }, + { + jobId: 68, + }, + { + jobId: 69, + }, + { + jobId: 70, + }, + { + jobId: 71, + }, + { + jobId: 72, + }, + { + jobId: 73, + }, + { + jobId: 74, + }, + { + jobId: 75, + }, + { + jobId: 76, + }, + { + jobId: 77, + }, + { + jobId: 78, + }, + { + jobId: 79, + }, + { + jobId: 80, + }, + { + jobId: 81, + }, + { + jobId: 82, + }, + { + jobId: 83, + }, + { + jobId: 84, + }, + { + jobId: 85, + }, + { + jobId: 86, + }, + { + jobId: 87, + }, + { + jobId: 88, + }, + { + jobId: 89, + }, + { + jobId: 90, + }, + { + jobId: 91, + }, + { + jobId: 92, + }, + { + jobId: 93, + }, + { + jobId: 94, + }, + { + jobId: 95, + }, + { + jobId: 96, + }, + { + jobId: 97, + }, + { + jobId: 98, + }, + { + jobId: 99, + }, + { + jobId: 100, + }, + { + jobId: 101, + }, + { + jobId: 102, + }, + { + jobId: 103, + }, + { + jobId: 104, + }, + { + jobId: 105, + }, + { + jobId: 106, + }, + { + jobId: 107, + }, + { + jobId: 108, + }, + { + jobId: 109, + }, + { + jobId: 110, + }, + { + jobId: 111, + }, + { + jobId: 112, + }, + { + jobId: 113, + }, + { + jobId: 114, + }, + { + jobId: 115, + }, + { + jobId: 116, + }, + { + jobId: 117, + }, + { + jobId: 118, + }, + { + jobId: 119, + }, + { + jobId: 120, + }, + { + jobId: 121, + }, + { + jobId: 122, + }, + { + jobId: 123, + }, + { + jobId: 124, + }, + { + jobId: 125, + }, + { + jobId: 126, + }, + { + jobId: 127, + }, + { + jobId: 128, + }, + { + jobId: 129, + }, + { + jobId: 130, + }, + { + jobId: 131, + }, + { + jobId: 132, + }, + { + jobId: 133, + }, + { + jobId: 134, + }, + { + jobId: 135, + }, + { + jobId: 136, + }, + { + jobId: 137, + }, + { + jobId: 138, + }, + { + jobId: 139, + }, + { + jobId: 140, + }, + { + jobId: 141, + }, + { + jobId: 142, + }, + { + jobId: 143, + }, + { + jobId: 144, + }, + { + jobId: 145, + }, + { + jobId: 146, + }, + { + jobId: 147, + }, + { + jobId: 148, + }, + { + jobId: 149, + }, + { + jobId: 150, + }, + { + jobId: 151, + }, + { + jobId: 152, + }, + { + jobId: 153, + }, + { + jobId: 154, + }, + { + jobId: 155, + }, + { + jobId: 156, + }, + { + jobId: 157, + }, + { + jobId: 158, + }, + { + jobId: 159, + }, + { + jobId: 160, + }, + { + jobId: 161, + }, + { + jobId: 162, + }, + { + jobId: 163, + }, + { + jobId: 164, + }, + { + jobId: 165, + }, + { + jobId: 166, + }, + { + jobId: 167, + }, + { + jobId: 168, + }, + { + jobId: 169, + }, + { + jobId: 170, + }, + { + jobId: 171, + }, + { + jobId: 172, + }, + { + jobId: 173, + }, + { + jobId: 174, + }, + { + jobId: 175, + }, + { + jobId: 176, + }, + { + jobId: 177, + }, + { + jobId: 178, + }, + { + jobId: 179, + }, + { + jobId: 180, + }, + { + jobId: 181, + }, + { + jobId: 182, + }, + { + jobId: 183, + }, + { + jobId: 184, + }, + { + jobId: 185, + }, + { + jobId: 186, + }, + { + jobId: 187, + }, + { + jobId: 188, + }, + { + jobId: 189, + }, + { + jobId: 190, + }, + { + jobId: 191, + }, + { + jobId: 192, + }, + { + jobId: 193, + }, + { + jobId: 194, + }, + { + jobId: 195, + }, + { + jobId: 196, + }, + { + jobId: 197, + }, + { + jobId: 198, + }, + { + jobId: 199, + }, + { + jobId: 200, + }, + { + jobId: 201, + }, + { + jobId: 202, + }, + { + jobId: 203, + }, + { + jobId: 204, + }, + { + jobId: 205, + }, + { + jobId: 206, + }, + { + jobId: 207, + }, + { + jobId: 208, + }, + { + jobId: 209, + }, + { + jobId: 210, + }, + { + jobId: 211, + }, + { + jobId: 212, + }, + { + jobId: 213, + }, + { + jobId: 214, + }, + { + jobId: 215, + }, + { + jobId: 216, + }, + { + jobId: 217, + }, + { + jobId: 218, + }, + { + jobId: 219, + }, + { + jobId: 220, + }, + { + jobId: 221, + }, + { + jobId: 222, + }, + { + jobId: 223, + }, + { + jobId: 224, + }, + { + jobId: 225, + }, + { + jobId: 226, + }, + { + jobId: 227, + }, + { + jobId: 228, + }, + { + jobId: 229, + }, + { + jobId: 230, + }, + { + jobId: 231, + }, + { + jobId: 232, + }, + { + jobId: 233, + }, + { + jobId: 234, + }, + { + jobId: 235, + }, + { + jobId: 236, + }, + { + jobId: 237, + }, + { + jobId: 238, + }, + { + jobId: 239, + }, + { + jobId: 240, + }, + { + jobId: 241, + }, + { + jobId: 242, + }, + { + jobId: 243, + }, + { + jobId: 244, + }, + { + jobId: 245, + }, + { + jobId: 246, + }, + { + jobId: 247, + }, + { + jobId: 248, + }, + { + jobId: 249, + }, + { + jobId: 250, + }, + { + jobId: 251, + }, + { + jobId: 252, + }, + { + jobId: 253, + }, + { + jobId: 254, + }, + { + jobId: 255, + }, + { + jobId: 256, + }, + { + jobId: 257, + }, + { + jobId: 258, + }, + { + jobId: 259, + }, + { + jobId: 260, + }, + { + jobId: 261, + }, + { + jobId: 262, + }, + { + jobId: 263, + }, + { + jobId: 264, + }, + { + jobId: 265, + }, + { + jobId: 266, + }, + { + jobId: 267, + }, + { + jobId: 268, + }, + { + jobId: 269, + }, + { + jobId: 270, + }, + { + jobId: 271, + }, + { + jobId: 272, + }, + { + jobId: 273, + }, + { + jobId: 274, + }, + { + jobId: 275, + }, + { + jobId: 276, + }, + { + jobId: 277, + }, + { + jobId: 278, + }, + { + jobId: 279, + }, + { + jobId: 280, + }, + { + jobId: 281, + }, + { + jobId: 282, + }, + { + jobId: 283, + }, + { + jobId: 284, + }, + { + jobId: 285, + }, + { + jobId: 286, + }, + { + jobId: 287, + }, + { + jobId: 288, + }, + { + jobId: 289, + }, + { + jobId: 290, + }, + { + jobId: 291, + }, + { + jobId: 292, + }, + { + jobId: 293, + }, + { + jobId: 294, + }, + { + jobId: 295, + }, + { + jobId: 296, + }, + { + jobId: 297, + }, + { + jobId: 298, + }, + { + jobId: 299, + }, + { + jobId: 300, + }, + { + jobId: 301, + }, + { + jobId: 302, + }, + { + jobId: 303, + }, + { + jobId: 304, + }, + { + jobId: 305, + }, + { + jobId: 306, + }, + { + jobId: 307, + }, + { + jobId: 308, + }, + { + jobId: 309, + }, + { + jobId: 310, + }, + { + jobId: 311, + }, + { + jobId: 312, + }, + { + jobId: 313, + }, + { + jobId: 314, + }, + { + jobId: 315, + }, + { + jobId: 316, + }, + { + jobId: 317, + }, + { + jobId: 318, + }, + { + jobId: 319, + }, + { + jobId: 320, + }, + { + jobId: 321, + }, + { + jobId: 322, + }, + { + jobId: 323, + }, + { + jobId: 324, + }, + { + jobId: 325, + }, + { + jobId: 326, + }, + { + jobId: 327, + }, + { + jobId: 328, + }, + { + jobId: 329, + }, + { + jobId: 330, + }, + { + jobId: 331, + }, + { + jobId: 332, + }, + { + jobId: 333, + }, + { + jobId: 334, + }, + { + jobId: 335, + }, + { + jobId: 336, + }, + { + jobId: 337, + }, + { + jobId: 338, + }, + { + jobId: 339, + }, + { + jobId: 340, + }, + { + jobId: 341, + }, + { + jobId: 342, + }, + { + jobId: 343, + }, + { + jobId: 344, + }, + { + jobId: 345, + }, + { + jobId: 346, + }, + { + jobId: 347, + }, + { + jobId: 348, + }, + { + jobId: 349, + }, + { + jobId: 350, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, +]; + +const mixedBatchOutput = [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=911&id=912&id=913&id=914&id=915&id=916&id=917&id=918&id=919&id=920', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 911, + }, + { + jobId: 912, + }, + { + jobId: 913, + }, + { + jobId: 914, + }, + { + jobId: 915, + }, + { + jobId: 916, + }, + { + jobId: 917, + }, + { + jobId: 918, + }, + { + jobId: 919, + }, + { + jobId: 920, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=901&id=902&id=903&id=904&id=905&id=906&id=907&id=908&id=909&id=910', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 901, + }, + { + jobId: 902, + }, + { + jobId: 903, + }, + { + jobId: 904, + }, + { + jobId: 905, + }, + { + jobId: 906, + }, + { + jobId: 907, + }, + { + jobId: 908, + }, + { + jobId: 909, + }, + { + jobId: 910, + }, + ], + batched: true, + statusCode: 200, + destination: DEST_OBJECT, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=704&id=705&id=706', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=501&id=502&id=503', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + headers: { + Authorization: TOKEN, + 'Content-Type': CONTENT_TYPE, + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 2, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo', + DestinationDefinition: DEST_DEFINITION, + Config: DEST_CONFIG, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, +]; + +const recordEventGenerator = (id, action, externalId) => { + // this function is used to generate record events for testing + const testRecordEvent = { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: DEST_DEFINITION, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action, + fields: { + id, + }, + channel: 'sources', + context: { + sources: MESSAGE_SOURCES_CONTEXT, + externalId: [ + { + type: EXTERNAL_ID, + id: externalId, + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: id, + }, + }; + return testRecordEvent; +}; + +module.exports = { + recordInputs, + audiencelistInputs, + reqMetadata, + recordOutput, + largeRecordOutput, + mixedBatchOutput, + recordEventGenerator, +}; diff --git a/src/v0/destinations/marketo_static_list/transform.js b/src/v0/destinations/marketo_static_list/transform.js index 856947c33c7..cbdb89f532e 100644 --- a/src/v0/destinations/marketo_static_list/transform.js +++ b/src/v0/destinations/marketo_static_list/transform.js @@ -5,6 +5,7 @@ const { defaultPostRequestConfig, defaultDeleteRequestConfig, generateErrorObject, + simpleProcessRouterDest, } = require('../../util'); const { AUTH_CACHE_TTL, JSON_MIME_TYPE } = require('../../util/constant'); const { getIds, validateMessageType } = require('./util'); @@ -12,11 +13,11 @@ const { getDestinationExternalID, defaultRequestConfig, getErrorRespEvents, - simpleProcessRouterDest, } = require('../../util'); const { formatConfig, MAX_LEAD_IDS_SIZE } = require('./config'); const Cache = require('../../util/cache'); const { getAuthToken } = require('../marketo/transform'); +const { processRecordInputs } = require('./transformV2'); const authCache = new Cache(AUTH_CACHE_TTL); // 1 hr @@ -57,8 +58,8 @@ const batchResponseBuilder = (message, Config, token, leadIds, operation) => { return response; }; -const processEvent = (input) => { - const { token, message, destination } = input; +const processEvent = (event) => { + const { token, message, destination } = event; const { Config } = destination; validateMessageType(message, ['audiencelist']); const response = []; @@ -70,77 +71,92 @@ const processEvent = (input) => { if (message.properties?.listData?.remove) { toRemove = getIds(message.properties.listData.remove); } - if ( - (Array.isArray(toAdd) && toAdd.length > 0) || - (Array.isArray(toRemove) && toRemove.length > 0) - ) { - if (Array.isArray(toAdd) && toAdd.length > 0) { - const payload = batchResponseBuilder(message, Config, token, toAdd, 'add'); - if (payload) { - response.push(...payload); - } + if (Array.isArray(toRemove) && toRemove.length > 0) { + const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); + if (payload) { + response.push(...payload); } - if (Array.isArray(toRemove) && toRemove.length > 0) { - const payload = batchResponseBuilder(message, Config, token, toRemove, 'remove'); - if (payload) { - response.push(...payload); - } + } + if (Array.isArray(toAdd) && toAdd.length > 0) { + const payload = batchResponseBuilder(message, Config, token, toAdd, 'add'); + if (payload) { + response.push(...payload); } - } else { + } + if (response.length === 0) { throw new InstrumentationError( 'Invalid leadIds format or no leadIds found neither to add nor to remove', ); } return response; }; -const process = async (event) => { - const token = await getAuthToken(formatConfig(event.destination)); +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const process = async (event, _processParams) => { + const token = await getAuthToken(formatConfig(event.destination)); if (!token) { throw new UnauthorizedError('Authorization failed'); } - const response = processEvent({ ...event, token }); + const updatedEvent = { ...event, token }; + const response = processEvent(updatedEvent); return response; }; const processRouterDest = async (inputs, reqMetadata) => { // Token needs to be generated for marketo which will be done on input level. // If destination information is not present Error should be thrown - let token; + + const { destination } = inputs[0]; try { - token = await getAuthToken(formatConfig(inputs[0].destination)); + const token = await getAuthToken(formatConfig(destination)); if (!token) { - const errResp = { - status: 400, - message: 'Authorisation failed', - responseTransformFailure: true, - statTags: {}, - }; - const respEvents = getErrorRespEvents( - inputs.map((input) => input.metadata), - errResp.status, - errResp.message, - errResp.statTags, - ); - return [{ ...respEvents, destination: inputs?.[0]?.destination }]; + throw new UnauthorizedError('Could not retrieve authorisation token'); } } catch (error) { - // Not using handleRtTfSingleEventError here as this is for multiple events - const errObj = generateErrorObject(error); - const respEvents = getErrorRespEvents( - inputs.map((input) => input.metadata), - errObj.status, - errObj.message, - errObj.statTags, + const errorObj = generateErrorObject(error); + const errResponses = inputs.map((input) => + getErrorRespEvents(input.metadata, errorObj.status, errorObj.message, errorObj.statTags), ); - return [{ ...respEvents, destination: inputs?.[0]?.destination }]; + + return errResponses; } - // Checking previous status Code. Initially setting to false. - // If true then previous status is 500 and every subsequent event output should be - // sent with status code 500 to the router to be retried. - const tokenisedInputs = inputs.map((input) => ({ ...input, token })); - const respList = await simpleProcessRouterDest(tokenisedInputs, processEvent, reqMetadata); + // use lodash.groupby to group the inputs based on message type + const transformedRecordEvent = []; + let transformedAudienceEvent = []; + const groupedInputs = lodash.groupBy(inputs, (input) => input.message.type); + + const respList = []; + // process record events + if (groupedInputs.record) { + const groupedRecordInputs = groupedInputs.record; + const { staticListId } = destination.Config; + const externalIdGroupedRecordInputs = lodash.groupBy( + groupedRecordInputs, + (input) => getDestinationExternalID(input.message, 'marketoStaticListId') || staticListId, + ); + const alltransformedGroupedRecordEvent = await Promise.all( + Object.keys(externalIdGroupedRecordInputs).map(async (key) => { + const transformedGroupedRecordEvent = await processRecordInputs( + externalIdGroupedRecordInputs[key], + destination, + key, + ); + return transformedGroupedRecordEvent; + }), + ); + + transformedRecordEvent.push(...alltransformedGroupedRecordEvent.flat()); + } + // process audiencelist events + if (groupedInputs.audiencelist) { + transformedAudienceEvent = await simpleProcessRouterDest( + groupedInputs.audiencelist, + process, + reqMetadata, + ); + } + respList.push(...transformedRecordEvent, ...transformedAudienceEvent); return respList; }; @@ -152,16 +168,20 @@ const processRouterDest = async (inputs, reqMetadata) => { function processMetadataForRouter(output) { const { metadata, destination } = output; const clonedMetadata = cloneDeep(metadata); - clonedMetadata.forEach((metadataElement) => { - // eslint-disable-next-line no-param-reassign - metadataElement.destInfo = { authKey: destination.ID }; - }); + if (Array.isArray(clonedMetadata)) { + clonedMetadata.forEach((metadataElement) => { + // eslint-disable-next-line no-param-reassign + metadataElement.destInfo = { authKey: destination?.ID }; + }); + } return clonedMetadata; } module.exports = { process, + processEvent, processRouterDest, processMetadataForRouter, authCache, + batchResponseBuilder, }; diff --git a/src/v0/destinations/marketo_static_list/transform.test.js b/src/v0/destinations/marketo_static_list/transform.test.js new file mode 100644 index 00000000000..3c387c42957 --- /dev/null +++ b/src/v0/destinations/marketo_static_list/transform.test.js @@ -0,0 +1,138 @@ +const { processRouterDest } = require('./transform'); +const axios = require('axios'); +const MockAxiosAdapter = require('axios-mock-adapter'); +const { + recordInputs, + audiencelistInputs, + reqMetadata, + recordOutput, + largeRecordOutput, + mixedBatchOutput, + recordEventGenerator, +} = require('./testData/testData'); + +const mockAdapter = new MockAxiosAdapter(axios, { onNoMatch: 'throwException' }); +beforeAll(() => { + mockAdapter + .onGet('https://marketo_acct_id_success.mktorest.com/identity/oauth/token') + .reply(200, { access_token: 'access_token_success' }); +}); +describe('Unit cases covering the router flow for both record and audiencelist event types', () => { + it('Sending a small batch of only record events (4 events)', async () => { + const inputs = recordInputs; + const result = await processRouterDest(inputs, reqMetadata); + + // assert that the result is as expected + expect(result.length).toEqual(3); + expect(result[0].batchedRequest.length).toEqual(1); // 1 batched request for 2 record events + expect(result[1].batchedRequest.length).toEqual(1); // 1 batched request for 1 record events + expect(result[2].batchedRequest.length).toEqual(1); // 1 batched request for 1 record events + expect(result[0].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=1001&id=1003', + ); // 1 api call for 2 leadIds + expect(result[1].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=2001', + ); // 1 api call for 1 leadId + expect(result[2].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=1002', + ); // 1 api call for 1 leadId + expect(result[1].batchedRequest[0].method).toEqual('DELETE'); // DELETE requests are sent first + expect(result[0].metadata.length).toEqual(2); // 2 metadata objects for 2 record events + expect(result[1].metadata.length).toEqual(1); // 1 metadata object for 1 record event + expect(result[2].metadata.length).toEqual(1); // 1 metadata object for 1 record event + expect(result).toEqual(recordOutput); // overall result should be equal to the expected output + }); + + it('Sending a large batch of only record events (450 events: 350 inserts | 100 deletes )', async () => { + const largeRecordInputs = []; + for (let index = 0; index < 350; index++) { + largeRecordInputs.push(recordEventGenerator(index + 1, 'insert', 1122)); + } + for (let index = 350; index < 450; index++) { + largeRecordInputs.push(recordEventGenerator(index + 1, 'delete', 1122)); + } + const result = await processRouterDest(largeRecordInputs, reqMetadata); + + // assert that the result is as expected + /* + Total 3 API calls + 1. 1 API call for 100 DELETE requests + 2. 1 API call for 100 POST requests = limit reached for Marketo, leads split to next API call + 3. 1 API call for 50 POST requests + */ + expect(result.length).toEqual(2); + expect(result[0].batchedRequest.length).toEqual(1); // 1 batched request for 1 record event + expect(result[1].batchedRequest.length).toEqual(2); // 1 batched request for 2 record events + expect(result[0].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=351&id=352&id=353&id=354&id=355&id=356&id=357&id=358&id=359&id=360&id=361&id=362&id=363&id=364&id=365&id=366&id=367&id=368&id=369&id=370&id=371&id=372&id=373&id=374&id=375&id=376&id=377&id=378&id=379&id=380&id=381&id=382&id=383&id=384&id=385&id=386&id=387&id=388&id=389&id=390&id=391&id=392&id=393&id=394&id=395&id=396&id=397&id=398&id=399&id=400&id=401&id=402&id=403&id=404&id=405&id=406&id=407&id=408&id=409&id=410&id=411&id=412&id=413&id=414&id=415&id=416&id=417&id=418&id=419&id=420&id=421&id=422&id=423&id=424&id=425&id=426&id=427&id=428&id=429&id=430&id=431&id=432&id=433&id=434&id=435&id=436&id=437&id=438&id=439&id=440&id=441&id=442&id=443&id=444&id=445&id=446&id=447&id=448&id=449&id=450', + ); + expect(result[1].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299&id=300', + ); + expect(result[1].batchedRequest[1].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + ); + expect(result[0].batchedRequest[0].method).toEqual('DELETE'); // DELETE requests are sent first + expect(result[1].batchedRequest[0].method).toEqual('POST'); + expect(result[1].batchedRequest[1].method).toEqual('POST'); + expect(result[0].metadata.length).toEqual(100); // 100 metadata objects for 100 record events + expect(result[1].metadata.length).toEqual(350); // 350 metadata objects for 350 record events + expect(result).toEqual(largeRecordOutput); // overall result should be equal to the expected output + }); + + it('Sending a mixed batch of record and audiencelist events (22 events: 10 inserts | 10 deletes | 2 audiencelist (⌐■_■) )', async () => { + const mixedBatchInputs = []; + for (let index = 900; index < 910; index++) { + mixedBatchInputs.push(recordEventGenerator(index + 1, 'insert', 1122)); + } + for (let index = 910; index < 920; index++) { + mixedBatchInputs.push(recordEventGenerator(index + 1, 'delete', 1122)); + } + mixedBatchInputs.push(...audiencelistInputs); + const result = await processRouterDest(mixedBatchInputs, reqMetadata); + + // assert that the result is as expected + /* + Total 4 API calls + 1. 1 API call for 10 DELETE requests + 2. 1 API call for 10 POST requests = limit reached for Marketo, leads split to next API call + 3. 1 API call for 2 POST requests + 4. 1 API call for 2 POST requests + */ + expect(result.length).toEqual(4); + expect(result[0].batchedRequest.length).toEqual(1); // 1 batched request for 1 record event + expect(result[1].batchedRequest.length).toEqual(1); // 1 batched request for 1 record event + expect(result[2].batchedRequest.length).toEqual(2); // 1 batched request for 2 audiencelist events + expect(result[3].batchedRequest.length).toEqual(2); // 1 batched request for 2 audiencelist events + expect(result[0].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=911&id=912&id=913&id=914&id=915&id=916&id=917&id=918&id=919&id=920', + ); + expect(result[1].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=901&id=902&id=903&id=904&id=905&id=906&id=907&id=908&id=909&id=910', + ); + expect(result[2].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=704&id=705&id=706', + ); + expect(result[2].batchedRequest[1].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=501&id=502&id=503', + ); + expect(result[3].batchedRequest[0].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299', + ); + expect(result[2].batchedRequest[1].endpoint).toEqual( + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=501&id=502&id=503', + ); + expect(result[0].batchedRequest[0].method).toEqual('DELETE'); // DELETE requests are sent first + expect(result[1].batchedRequest[0].method).toEqual('POST'); + expect(result[1].batchedRequest[0].method).toEqual('POST'); + expect(result[2].batchedRequest[0].method).toEqual('DELETE'); + expect(result[0].metadata.length).toEqual(10); + expect(result[1].metadata.length).toEqual(10); + expect(result[2].metadata.length).toEqual(1); + expect(result[3].metadata.length).toEqual(1); + expect(result).toEqual(mixedBatchOutput); // overall result should be equal to the expected output + }); +}); +afterAll(() => { + mockAdapter.restore(); +}); diff --git a/src/v0/destinations/marketo_static_list/transformV2.js b/src/v0/destinations/marketo_static_list/transformV2.js new file mode 100644 index 00000000000..912d548d098 --- /dev/null +++ b/src/v0/destinations/marketo_static_list/transformV2.js @@ -0,0 +1,143 @@ +const lodash = require('lodash'); +const { InstrumentationError, UnauthorizedError } = require('@rudderstack/integrations-lib'); +const { + defaultPostRequestConfig, + defaultDeleteRequestConfig, + defaultRequestConfig, + getSuccessRespEvents, + isDefinedAndNotNull, + generateErrorObject, + getErrorRespEvents, +} = require('../../util'); +const { JSON_MIME_TYPE } = require('../../util/constant'); +const { MAX_LEAD_IDS_SIZE } = require('./config'); +const { getAuthToken } = require('../marketo/transform'); +const { formatConfig } = require('../marketo/config'); + +/** + * Generates the final response structure to be sent to the destination + * @param {*} endPoint + * @param {*} leadIds + * @param {*} operation + * @param {*} token + * @returns batched response + */ +const responseBuilder = (endPoint, leadIds, operation, token) => { + let updatedEndpoint = endPoint; + if (leadIds.length > 0) { + leadIds.forEach((id) => { + updatedEndpoint = `${updatedEndpoint}id=${id}&`; + }); + } + updatedEndpoint = updatedEndpoint.slice(0, -1); + const response = defaultRequestConfig(); + response.endpoint = updatedEndpoint; + if (operation === 'insert') { + response.method = defaultPostRequestConfig.requestMethod; + } else { + response.method = defaultDeleteRequestConfig.requestMethod; + } + response.headers = { + Authorization: `Bearer ${token}`, + 'Content-Type': JSON_MIME_TYPE, + }; + return response; +}; + +/** + * The function is responsible for building the batched response for a given set of record inputs. + * @param {*} groupedRecordInputs + * @param {*} Config + * @param {*} token + * @param {*} leadIds + * @param {*} operation + * @returns an array of response objects, where each object represents a batched response for a chunk of lead IDs. + */ +const batchResponseBuilder = (listId, Config, token, leadIds, operation) => { + const { accountId } = Config; + const endpoint = `https://${accountId}.mktorest.com/rest/v1/lists/${listId}/leads.json?`; + const response = []; + const leadIdsChunks = lodash.chunk(leadIds, MAX_LEAD_IDS_SIZE); + leadIdsChunks.forEach((ids) => { + response.push(responseBuilder(endpoint, ids, operation, token)); + }); + return response; +}; + +/** + * A function that processes a list of grouped record inputs. + * It iterates through each input and groups the field IDs based on the action. + * @param {*} groupedRecordInputs + * @param {*} destination + * @param {*} listId + * @returns An array containing the batched responses for the insert and delete actions along with the metadata. + */ +async function processRecordInputs(groupedRecordInputs, destination, listId) { + const token = await getAuthToken(formatConfig(destination)); + if (!token) { + throw new UnauthorizedError('Authorization failed'); + } + const { Config } = destination; + + // iterate through each input and group field id based on action + const insertFields = []; + const deleteFields = []; + const successMetadataForInsert = []; + const successMetadataForDelete = []; + const errorMetadata = []; + + groupedRecordInputs.forEach((input) => { + const { fields, action } = input.message; + const fieldId = fields?.id; + if (action === 'insert' && isDefinedAndNotNull(fieldId)) { + insertFields.push(fieldId); + successMetadataForInsert.push(input.metadata); + } else if (action === 'delete' && isDefinedAndNotNull(fieldId)) { + deleteFields.push(fieldId); + successMetadataForDelete.push(input.metadata); + } else { + errorMetadata.push(input.metadata); + } + }); + const deletePayloads = batchResponseBuilder(listId, Config, token, deleteFields, 'delete'); + + const deleteResponse = getSuccessRespEvents( + deletePayloads, + successMetadataForDelete, + destination, + true, + ); + + const insertPayloads = batchResponseBuilder(listId, Config, token, insertFields, 'insert'); + + const insertResponse = getSuccessRespEvents( + insertPayloads, + successMetadataForInsert, + destination, + true, + ); + + const error = new InstrumentationError( + 'Invalid action type or no leadIds found neither to add nor to remove', + ); + const errorObj = generateErrorObject(error); + const errorResponseList = errorMetadata.map((metadata) => + getErrorRespEvents(metadata, errorObj.status, errorObj.message, errorObj.statTags), + ); + const finalResponse = []; + if (deleteResponse.batchedRequest.length > 0) { + finalResponse.push(deleteResponse); + } + if (insertResponse.batchedRequest.length > 0) { + finalResponse.push(insertResponse); + } + if (errorResponseList.length > 0) { + finalResponse.push(...errorResponseList); + } + return finalResponse; + // return [deleteResponse, insertResponse, ...errorResponseList]; +} + +module.exports = { + processRecordInputs, +}; diff --git a/src/v0/destinations/marketo_static_list/util.js b/src/v0/destinations/marketo_static_list/util.js index 3db6180d241..c6959cb2564 100644 --- a/src/v0/destinations/marketo_static_list/util.js +++ b/src/v0/destinations/marketo_static_list/util.js @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/consistent-destructuring */ const { InstrumentationError } = require('@rudderstack/integrations-lib'); /** diff --git a/test/__tests__/data/marketo_static_list.json b/test/__tests__/data/marketo_static_list.json deleted file mode 100644 index 701868cab06..00000000000 --- a/test/__tests__/data/marketo_static_list.json +++ /dev/null @@ -1,2542 +0,0 @@ -[ - { - "description": "adding and removing users and getting staticListId from externalId", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 3421 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - } - ], - "remove": [ - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - }, - "externalId": [ - { - "type": "marketoStaticListId", - "id": 1234 - } - ] - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - }, - { - "description": "adding more than max limit users", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 0 - }, - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - }, - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - }, - { - "id": 14 - }, - { - "id": 15 - }, - { - "id": 16 - }, - { - "id": 17 - }, - { - "id": 18 - }, - { - "id": 19 - }, - { - "id": 20 - }, - { - "id": 21 - }, - { - "id": 22 - }, - { - "id": 23 - }, - { - "id": 24 - }, - { - "id": 25 - }, - { - "id": 26 - }, - { - "id": 27 - }, - { - "id": 28 - }, - { - "id": 29 - }, - { - "id": 30 - }, - { - "id": 31 - }, - { - "id": 32 - }, - { - "id": 33 - }, - { - "id": 34 - }, - { - "id": 35 - }, - { - "id": 36 - }, - { - "id": 37 - }, - { - "id": 38 - }, - { - "id": 39 - }, - { - "id": 40 - }, - { - "id": 41 - }, - { - "id": 42 - }, - { - "id": 43 - }, - { - "id": 44 - }, - { - "id": 45 - }, - { - "id": 46 - }, - { - "id": 47 - }, - { - "id": 48 - }, - { - "id": 49 - }, - { - "id": 50 - }, - { - "id": 51 - }, - { - "id": 52 - }, - { - "id": 53 - }, - { - "id": 54 - }, - { - "id": 55 - }, - { - "id": 56 - }, - { - "id": 57 - }, - { - "id": 58 - }, - { - "id": 59 - }, - { - "id": 60 - }, - { - "id": 61 - }, - { - "id": 62 - }, - { - "id": 63 - }, - { - "id": 64 - }, - { - "id": 65 - }, - { - "id": 66 - }, - { - "id": 67 - }, - { - "id": 68 - }, - { - "id": 69 - }, - { - "id": 70 - }, - { - "id": 71 - }, - { - "id": 72 - }, - { - "id": 73 - }, - { - "id": 74 - }, - { - "id": 75 - }, - { - "id": 76 - }, - { - "id": 77 - }, - { - "id": 78 - }, - { - "id": 79 - }, - { - "id": 80 - }, - { - "id": 81 - }, - { - "id": 82 - }, - { - "id": 83 - }, - { - "id": 84 - }, - { - "id": 85 - }, - { - "id": 86 - }, - { - "id": 87 - }, - { - "id": 88 - }, - { - "id": 89 - }, - { - "id": 90 - }, - { - "id": 91 - }, - { - "id": 92 - }, - { - "id": 93 - }, - { - "id": 94 - }, - { - "id": 95 - }, - { - "id": 96 - }, - { - "id": 97 - }, - { - "id": 98 - }, - { - "id": 99 - }, - { - "id": 100 - }, - { - "id": 101 - }, - { - "id": 102 - }, - { - "id": 103 - }, - { - "id": 104 - }, - { - "id": 105 - }, - { - "id": 106 - }, - { - "id": 107 - }, - { - "id": 108 - }, - { - "id": 109 - }, - { - "id": 110 - }, - { - "id": 111 - }, - { - "id": 112 - }, - { - "id": 113 - }, - { - "id": 114 - }, - { - "id": 115 - }, - { - "id": 116 - }, - { - "id": 117 - }, - { - "id": 118 - }, - { - "id": 119 - }, - { - "id": 120 - }, - { - "id": 121 - }, - { - "id": 122 - }, - { - "id": 123 - }, - { - "id": 124 - }, - { - "id": 125 - }, - { - "id": 126 - }, - { - "id": 127 - }, - { - "id": 128 - }, - { - "id": 129 - }, - { - "id": 130 - }, - { - "id": 131 - }, - { - "id": 132 - }, - { - "id": 133 - }, - { - "id": 134 - }, - { - "id": 135 - }, - { - "id": 136 - }, - { - "id": 137 - }, - { - "id": 138 - }, - { - "id": 139 - }, - { - "id": 140 - }, - { - "id": 141 - }, - { - "id": 142 - }, - { - "id": 143 - }, - { - "id": 144 - }, - { - "id": 145 - }, - { - "id": 146 - }, - { - "id": 147 - }, - { - "id": 148 - }, - { - "id": 149 - }, - { - "id": 150 - }, - { - "id": 151 - }, - { - "id": 152 - }, - { - "id": 153 - }, - { - "id": 154 - }, - { - "id": 155 - }, - { - "id": 156 - }, - { - "id": 157 - }, - { - "id": 158 - }, - { - "id": 159 - }, - { - "id": 160 - }, - { - "id": 161 - }, - { - "id": 162 - }, - { - "id": 163 - }, - { - "id": 164 - }, - { - "id": 165 - }, - { - "id": 166 - }, - { - "id": 167 - }, - { - "id": 168 - }, - { - "id": 169 - }, - { - "id": 170 - }, - { - "id": 171 - }, - { - "id": 172 - }, - { - "id": 173 - }, - { - "id": 174 - }, - { - "id": 175 - }, - { - "id": 176 - }, - { - "id": 177 - }, - { - "id": 178 - }, - { - "id": 179 - }, - { - "id": 180 - }, - { - "id": 181 - }, - { - "id": 182 - }, - { - "id": 183 - }, - { - "id": 184 - }, - { - "id": 185 - }, - { - "id": 186 - }, - { - "id": 187 - }, - { - "id": 188 - }, - { - "id": 189 - }, - { - "id": 190 - }, - { - "id": 191 - }, - { - "id": 192 - }, - { - "id": 193 - }, - { - "id": 194 - }, - { - "id": 195 - }, - { - "id": 196 - }, - { - "id": 197 - }, - { - "id": 198 - }, - { - "id": 199 - }, - { - "id": 200 - }, - { - "id": 201 - }, - { - "id": 202 - }, - { - "id": 203 - }, - { - "id": 204 - }, - { - "id": 205 - }, - { - "id": 206 - }, - { - "id": 207 - }, - { - "id": 208 - }, - { - "id": 209 - }, - { - "id": 210 - }, - { - "id": 211 - }, - { - "id": 212 - }, - { - "id": 213 - }, - { - "id": 214 - }, - { - "id": 215 - }, - { - "id": 216 - }, - { - "id": 217 - }, - { - "id": 218 - }, - { - "id": 219 - }, - { - "id": 220 - }, - { - "id": 221 - }, - { - "id": 222 - }, - { - "id": 223 - }, - { - "id": 224 - }, - { - "id": 225 - }, - { - "id": 226 - }, - { - "id": 227 - }, - { - "id": 228 - }, - { - "id": 229 - }, - { - "id": 230 - }, - { - "id": 231 - }, - { - "id": 232 - }, - { - "id": 233 - }, - { - "id": 234 - }, - { - "id": 235 - }, - { - "id": 236 - }, - { - "id": 237 - }, - { - "id": 238 - }, - { - "id": 239 - }, - { - "id": 240 - }, - { - "id": 241 - }, - { - "id": 242 - }, - { - "id": 243 - }, - { - "id": 244 - }, - { - "id": 245 - }, - { - "id": 246 - }, - { - "id": 247 - }, - { - "id": 248 - }, - { - "id": 249 - }, - { - "id": 250 - }, - { - "id": 251 - }, - { - "id": 252 - }, - { - "id": 253 - }, - { - "id": 254 - }, - { - "id": 255 - }, - { - "id": 256 - }, - { - "id": 257 - }, - { - "id": 258 - }, - { - "id": 259 - }, - { - "id": 260 - }, - { - "id": 261 - }, - { - "id": 262 - }, - { - "id": 263 - }, - { - "id": 264 - }, - { - "id": 265 - }, - { - "id": 266 - }, - { - "id": 267 - }, - { - "id": 268 - }, - { - "id": 269 - }, - { - "id": 270 - }, - { - "id": 271 - }, - { - "id": 272 - }, - { - "id": 273 - }, - { - "id": 274 - }, - { - "id": 275 - }, - { - "id": 276 - }, - { - "id": 277 - }, - { - "id": 278 - }, - { - "id": 279 - }, - { - "id": 280 - }, - { - "id": 281 - }, - { - "id": 282 - }, - { - "id": 283 - }, - { - "id": 284 - }, - { - "id": 285 - }, - { - "id": 286 - }, - { - "id": 287 - }, - { - "id": 288 - }, - { - "id": 289 - }, - { - "id": 290 - }, - { - "id": 291 - }, - { - "id": 292 - }, - { - "id": 293 - }, - { - "id": 294 - }, - { - "id": 295 - }, - { - "id": 296 - }, - { - "id": 297 - }, - { - "id": 298 - }, - { - "id": 299 - }, - { - "id": 300 - }, - { - "id": 301 - }, - { - "id": 302 - }, - { - "id": 303 - }, - { - "id": 304 - }, - { - "id": 305 - }, - { - "id": 306 - }, - { - "id": 307 - }, - { - "id": 308 - }, - { - "id": 309 - }, - { - "id": 310 - }, - { - "id": 311 - }, - { - "id": 312 - }, - { - "id": 313 - }, - { - "id": 314 - }, - { - "id": 315 - }, - { - "id": 316 - }, - { - "id": 317 - }, - { - "id": 318 - }, - { - "id": 319 - }, - { - "id": 320 - }, - { - "id": 321 - }, - { - "id": 322 - }, - { - "id": 323 - }, - { - "id": 324 - }, - { - "id": 325 - }, - { - "id": 326 - }, - { - "id": 327 - }, - { - "id": 328 - }, - { - "id": 329 - }, - { - "id": 330 - }, - { - "id": 331 - }, - { - "id": 332 - }, - { - "id": 333 - }, - { - "id": 334 - }, - { - "id": 335 - }, - { - "id": 336 - }, - { - "id": 337 - }, - { - "id": 338 - }, - { - "id": 339 - }, - { - "id": 340 - }, - { - "id": 341 - }, - { - "id": 342 - }, - { - "id": 343 - }, - { - "id": 344 - }, - { - "id": 345 - }, - { - "id": 346 - }, - { - "id": 347 - }, - { - "id": 348 - }, - { - "id": 349 - }, - { - "id": 350 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - }, - { - "description": "removing more than max limit users", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "remove": [ - { - "id": 0 - }, - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - }, - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - }, - { - "id": 14 - }, - { - "id": 15 - }, - { - "id": 16 - }, - { - "id": 17 - }, - { - "id": 18 - }, - { - "id": 19 - }, - { - "id": 20 - }, - { - "id": 21 - }, - { - "id": 22 - }, - { - "id": 23 - }, - { - "id": 24 - }, - { - "id": 25 - }, - { - "id": 26 - }, - { - "id": 27 - }, - { - "id": 28 - }, - { - "id": 29 - }, - { - "id": 30 - }, - { - "id": 31 - }, - { - "id": 32 - }, - { - "id": 33 - }, - { - "id": 34 - }, - { - "id": 35 - }, - { - "id": 36 - }, - { - "id": 37 - }, - { - "id": 38 - }, - { - "id": 39 - }, - { - "id": 40 - }, - { - "id": 41 - }, - { - "id": 42 - }, - { - "id": 43 - }, - { - "id": 44 - }, - { - "id": 45 - }, - { - "id": 46 - }, - { - "id": 47 - }, - { - "id": 48 - }, - { - "id": 49 - }, - { - "id": 50 - }, - { - "id": 51 - }, - { - "id": 52 - }, - { - "id": 53 - }, - { - "id": 54 - }, - { - "id": 55 - }, - { - "id": 56 - }, - { - "id": 57 - }, - { - "id": 58 - }, - { - "id": 59 - }, - { - "id": 60 - }, - { - "id": 61 - }, - { - "id": 62 - }, - { - "id": 63 - }, - { - "id": 64 - }, - { - "id": 65 - }, - { - "id": 66 - }, - { - "id": 67 - }, - { - "id": 68 - }, - { - "id": 69 - }, - { - "id": 70 - }, - { - "id": 71 - }, - { - "id": 72 - }, - { - "id": 73 - }, - { - "id": 74 - }, - { - "id": 75 - }, - { - "id": 76 - }, - { - "id": 77 - }, - { - "id": 78 - }, - { - "id": 79 - }, - { - "id": 80 - }, - { - "id": 81 - }, - { - "id": 82 - }, - { - "id": 83 - }, - { - "id": 84 - }, - { - "id": 85 - }, - { - "id": 86 - }, - { - "id": 87 - }, - { - "id": 88 - }, - { - "id": 89 - }, - { - "id": 90 - }, - { - "id": 91 - }, - { - "id": 92 - }, - { - "id": 93 - }, - { - "id": 94 - }, - { - "id": 95 - }, - { - "id": 96 - }, - { - "id": 97 - }, - { - "id": 98 - }, - { - "id": 99 - }, - { - "id": 100 - }, - { - "id": 101 - }, - { - "id": 102 - }, - { - "id": 103 - }, - { - "id": 104 - }, - { - "id": 105 - }, - { - "id": 106 - }, - { - "id": 107 - }, - { - "id": 108 - }, - { - "id": 109 - }, - { - "id": 110 - }, - { - "id": 111 - }, - { - "id": 112 - }, - { - "id": 113 - }, - { - "id": 114 - }, - { - "id": 115 - }, - { - "id": 116 - }, - { - "id": 117 - }, - { - "id": 118 - }, - { - "id": 119 - }, - { - "id": 120 - }, - { - "id": 121 - }, - { - "id": 122 - }, - { - "id": 123 - }, - { - "id": 124 - }, - { - "id": 125 - }, - { - "id": 126 - }, - { - "id": 127 - }, - { - "id": 128 - }, - { - "id": 129 - }, - { - "id": 130 - }, - { - "id": 131 - }, - { - "id": 132 - }, - { - "id": 133 - }, - { - "id": 134 - }, - { - "id": 135 - }, - { - "id": 136 - }, - { - "id": 137 - }, - { - "id": 138 - }, - { - "id": 139 - }, - { - "id": 140 - }, - { - "id": 141 - }, - { - "id": 142 - }, - { - "id": 143 - }, - { - "id": 144 - }, - { - "id": 145 - }, - { - "id": 146 - }, - { - "id": 147 - }, - { - "id": 148 - }, - { - "id": 149 - }, - { - "id": 150 - }, - { - "id": 151 - }, - { - "id": 152 - }, - { - "id": 153 - }, - { - "id": 154 - }, - { - "id": 155 - }, - { - "id": 156 - }, - { - "id": 157 - }, - { - "id": 158 - }, - { - "id": 159 - }, - { - "id": 160 - }, - { - "id": 161 - }, - { - "id": 162 - }, - { - "id": 163 - }, - { - "id": 164 - }, - { - "id": 165 - }, - { - "id": 166 - }, - { - "id": 167 - }, - { - "id": 168 - }, - { - "id": 169 - }, - { - "id": 170 - }, - { - "id": 171 - }, - { - "id": 172 - }, - { - "id": 173 - }, - { - "id": 174 - }, - { - "id": 175 - }, - { - "id": 176 - }, - { - "id": 177 - }, - { - "id": 178 - }, - { - "id": 179 - }, - { - "id": 180 - }, - { - "id": 181 - }, - { - "id": 182 - }, - { - "id": 183 - }, - { - "id": 184 - }, - { - "id": 185 - }, - { - "id": 186 - }, - { - "id": 187 - }, - { - "id": 188 - }, - { - "id": 189 - }, - { - "id": 190 - }, - { - "id": 191 - }, - { - "id": 192 - }, - { - "id": 193 - }, - { - "id": 194 - }, - { - "id": 195 - }, - { - "id": 196 - }, - { - "id": 197 - }, - { - "id": 198 - }, - { - "id": 199 - }, - { - "id": 200 - }, - { - "id": 201 - }, - { - "id": 202 - }, - { - "id": 203 - }, - { - "id": 204 - }, - { - "id": 205 - }, - { - "id": 206 - }, - { - "id": 207 - }, - { - "id": 208 - }, - { - "id": 209 - }, - { - "id": 210 - }, - { - "id": 211 - }, - { - "id": 212 - }, - { - "id": 213 - }, - { - "id": 214 - }, - { - "id": 215 - }, - { - "id": 216 - }, - { - "id": 217 - }, - { - "id": 218 - }, - { - "id": 219 - }, - { - "id": 220 - }, - { - "id": 221 - }, - { - "id": 222 - }, - { - "id": 223 - }, - { - "id": 224 - }, - { - "id": 225 - }, - { - "id": 226 - }, - { - "id": 227 - }, - { - "id": 228 - }, - { - "id": 229 - }, - { - "id": 230 - }, - { - "id": 231 - }, - { - "id": 232 - }, - { - "id": 233 - }, - { - "id": 234 - }, - { - "id": 235 - }, - { - "id": 236 - }, - { - "id": 237 - }, - { - "id": 238 - }, - { - "id": 239 - }, - { - "id": 240 - }, - { - "id": 241 - }, - { - "id": 242 - }, - { - "id": 243 - }, - { - "id": 244 - }, - { - "id": 245 - }, - { - "id": 246 - }, - { - "id": 247 - }, - { - "id": 248 - }, - { - "id": 249 - }, - { - "id": 250 - }, - { - "id": 251 - }, - { - "id": 252 - }, - { - "id": 253 - }, - { - "id": 254 - }, - { - "id": 255 - }, - { - "id": 256 - }, - { - "id": 257 - }, - { - "id": 258 - }, - { - "id": 259 - }, - { - "id": 260 - }, - { - "id": 261 - }, - { - "id": 262 - }, - { - "id": 263 - }, - { - "id": 264 - }, - { - "id": 265 - }, - { - "id": 266 - }, - { - "id": 267 - }, - { - "id": 268 - }, - { - "id": 269 - }, - { - "id": 270 - }, - { - "id": 271 - }, - { - "id": 272 - }, - { - "id": 273 - }, - { - "id": 274 - }, - { - "id": 275 - }, - { - "id": 276 - }, - { - "id": 277 - }, - { - "id": 278 - }, - { - "id": 279 - }, - { - "id": 280 - }, - { - "id": 281 - }, - { - "id": 282 - }, - { - "id": 283 - }, - { - "id": 284 - }, - { - "id": 285 - }, - { - "id": 286 - }, - { - "id": 287 - }, - { - "id": 288 - }, - { - "id": 289 - }, - { - "id": 290 - }, - { - "id": 291 - }, - { - "id": 292 - }, - { - "id": 293 - }, - { - "id": 294 - }, - { - "id": 295 - }, - { - "id": 296 - }, - { - "id": 297 - }, - { - "id": 298 - }, - { - "id": 299 - }, - { - "id": 300 - }, - { - "id": 301 - }, - { - "id": 302 - }, - { - "id": 303 - }, - { - "id": 304 - }, - { - "id": 305 - }, - { - "id": 306 - }, - { - "id": 307 - }, - { - "id": 308 - }, - { - "id": 309 - }, - { - "id": 310 - }, - { - "id": 311 - }, - { - "id": 312 - }, - { - "id": 313 - }, - { - "id": 314 - }, - { - "id": 315 - }, - { - "id": 316 - }, - { - "id": 317 - }, - { - "id": 318 - }, - { - "id": 319 - }, - { - "id": 320 - }, - { - "id": 321 - }, - { - "id": 322 - }, - { - "id": 323 - }, - { - "id": 324 - }, - { - "id": 325 - }, - { - "id": 326 - }, - { - "id": 327 - }, - { - "id": 328 - }, - { - "id": 329 - }, - { - "id": 330 - }, - { - "id": 331 - }, - { - "id": 332 - }, - { - "id": 333 - }, - { - "id": 334 - }, - { - "id": 335 - }, - { - "id": 336 - }, - { - "id": 337 - }, - { - "id": 338 - }, - { - "id": 339 - }, - { - "id": 340 - }, - { - "id": 341 - }, - { - "id": 342 - }, - { - "id": 343 - }, - { - "id": 344 - }, - { - "id": 345 - }, - { - "id": 346 - }, - { - "id": 347 - }, - { - "id": 348 - }, - { - "id": 349 - }, - { - "id": 350 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - }, - { - "description": "unsupported message Type", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "track", - "properties": { - "listData": { - "remove": [1] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": { - "error": "Event type track is not supported" - } - }, - { - "description": "Invalid leadIds format ", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "remove": 1 - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": { - "error": "Invalid leadIds format or no leadIds found neither to add nor to remove" - } - }, - { - "description": "Only adding with remove not an array", - "input": { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [{ "id": 1 }], - "remove": 2 - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - } - }, - "output": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ] - } -] diff --git a/test/__tests__/data/marketo_static_list_router_input.json b/test/__tests__/data/marketo_static_list_router_input.json deleted file mode 100644 index ed3a7591e6d..00000000000 --- a/test/__tests__/data/marketo_static_list_router_input.json +++ /dev/null @@ -1,1209 +0,0 @@ -[ - { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - } - ], - "remove": [ - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - }, - "metadata": { - "jobId": 1 - } - }, - { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "audiencelist", - "properties": { - "listData": { - "add": [ - { - "id": 0 - }, - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - }, - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - }, - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - }, - { - "id": 14 - }, - { - "id": 15 - }, - { - "id": 16 - }, - { - "id": 17 - }, - { - "id": 18 - }, - { - "id": 19 - }, - { - "id": 20 - }, - { - "id": 21 - }, - { - "id": 22 - }, - { - "id": 23 - }, - { - "id": 24 - }, - { - "id": 25 - }, - { - "id": 26 - }, - { - "id": 27 - }, - { - "id": 28 - }, - { - "id": 29 - }, - { - "id": 30 - }, - { - "id": 31 - }, - { - "id": 32 - }, - { - "id": 33 - }, - { - "id": 34 - }, - { - "id": 35 - }, - { - "id": 36 - }, - { - "id": 37 - }, - { - "id": 38 - }, - { - "id": 39 - }, - { - "id": 40 - }, - { - "id": 41 - }, - { - "id": 42 - }, - { - "id": 43 - }, - { - "id": 44 - }, - { - "id": 45 - }, - { - "id": 46 - }, - { - "id": 47 - }, - { - "id": 48 - }, - { - "id": 49 - }, - { - "id": 50 - }, - { - "id": 51 - }, - { - "id": 52 - }, - { - "id": 53 - }, - { - "id": 54 - }, - { - "id": 55 - }, - { - "id": 56 - }, - { - "id": 57 - }, - { - "id": 58 - }, - { - "id": 59 - }, - { - "id": 60 - }, - { - "id": 61 - }, - { - "id": 62 - }, - { - "id": 63 - }, - { - "id": 64 - }, - { - "id": 65 - }, - { - "id": 66 - }, - { - "id": 67 - }, - { - "id": 68 - }, - { - "id": 69 - }, - { - "id": 70 - }, - { - "id": 71 - }, - { - "id": 72 - }, - { - "id": 73 - }, - { - "id": 74 - }, - { - "id": 75 - }, - { - "id": 76 - }, - { - "id": 77 - }, - { - "id": 78 - }, - { - "id": 79 - }, - { - "id": 80 - }, - { - "id": 81 - }, - { - "id": 82 - }, - { - "id": 83 - }, - { - "id": 84 - }, - { - "id": 85 - }, - { - "id": 86 - }, - { - "id": 87 - }, - { - "id": 88 - }, - { - "id": 89 - }, - { - "id": 90 - }, - { - "id": 91 - }, - { - "id": 92 - }, - { - "id": 93 - }, - { - "id": 94 - }, - { - "id": 95 - }, - { - "id": 96 - }, - { - "id": 97 - }, - { - "id": 98 - }, - { - "id": 99 - }, - { - "id": 100 - }, - { - "id": 101 - }, - { - "id": 102 - }, - { - "id": 103 - }, - { - "id": 104 - }, - { - "id": 105 - }, - { - "id": 106 - }, - { - "id": 107 - }, - { - "id": 108 - }, - { - "id": 109 - }, - { - "id": 110 - }, - { - "id": 111 - }, - { - "id": 112 - }, - { - "id": 113 - }, - { - "id": 114 - }, - { - "id": 115 - }, - { - "id": 116 - }, - { - "id": 117 - }, - { - "id": 118 - }, - { - "id": 119 - }, - { - "id": 120 - }, - { - "id": 121 - }, - { - "id": 122 - }, - { - "id": 123 - }, - { - "id": 124 - }, - { - "id": 125 - }, - { - "id": 126 - }, - { - "id": 127 - }, - { - "id": 128 - }, - { - "id": 129 - }, - { - "id": 130 - }, - { - "id": 131 - }, - { - "id": 132 - }, - { - "id": 133 - }, - { - "id": 134 - }, - { - "id": 135 - }, - { - "id": 136 - }, - { - "id": 137 - }, - { - "id": 138 - }, - { - "id": 139 - }, - { - "id": 140 - }, - { - "id": 141 - }, - { - "id": 142 - }, - { - "id": 143 - }, - { - "id": 144 - }, - { - "id": 145 - }, - { - "id": 146 - }, - { - "id": 147 - }, - { - "id": 148 - }, - { - "id": 149 - }, - { - "id": 150 - }, - { - "id": 151 - }, - { - "id": 152 - }, - { - "id": 153 - }, - { - "id": 154 - }, - { - "id": 155 - }, - { - "id": 156 - }, - { - "id": 157 - }, - { - "id": 158 - }, - { - "id": 159 - }, - { - "id": 160 - }, - { - "id": 161 - }, - { - "id": 162 - }, - { - "id": 163 - }, - { - "id": 164 - }, - { - "id": 165 - }, - { - "id": 166 - }, - { - "id": 167 - }, - { - "id": 168 - }, - { - "id": 169 - }, - { - "id": 170 - }, - { - "id": 171 - }, - { - "id": 172 - }, - { - "id": 173 - }, - { - "id": 174 - }, - { - "id": 175 - }, - { - "id": 176 - }, - { - "id": 177 - }, - { - "id": 178 - }, - { - "id": 179 - }, - { - "id": 180 - }, - { - "id": 181 - }, - { - "id": 182 - }, - { - "id": 183 - }, - { - "id": 184 - }, - { - "id": 185 - }, - { - "id": 186 - }, - { - "id": 187 - }, - { - "id": 188 - }, - { - "id": 189 - }, - { - "id": 190 - }, - { - "id": 191 - }, - { - "id": 192 - }, - { - "id": 193 - }, - { - "id": 194 - }, - { - "id": 195 - }, - { - "id": 196 - }, - { - "id": 197 - }, - { - "id": 198 - }, - { - "id": 199 - }, - { - "id": 200 - }, - { - "id": 201 - }, - { - "id": 202 - }, - { - "id": 203 - }, - { - "id": 204 - }, - { - "id": 205 - }, - { - "id": 206 - }, - { - "id": 207 - }, - { - "id": 208 - }, - { - "id": 209 - }, - { - "id": 210 - }, - { - "id": 211 - }, - { - "id": 212 - }, - { - "id": 213 - }, - { - "id": 214 - }, - { - "id": 215 - }, - { - "id": 216 - }, - { - "id": 217 - }, - { - "id": 218 - }, - { - "id": 219 - }, - { - "id": 220 - }, - { - "id": 221 - }, - { - "id": 222 - }, - { - "id": 223 - }, - { - "id": 224 - }, - { - "id": 225 - }, - { - "id": 226 - }, - { - "id": 227 - }, - { - "id": 228 - }, - { - "id": 229 - }, - { - "id": 230 - }, - { - "id": 231 - }, - { - "id": 232 - }, - { - "id": 233 - }, - { - "id": 234 - }, - { - "id": 235 - }, - { - "id": 236 - }, - { - "id": 237 - }, - { - "id": 238 - }, - { - "id": 239 - }, - { - "id": 240 - }, - { - "id": 241 - }, - { - "id": 242 - }, - { - "id": 243 - }, - { - "id": 244 - }, - { - "id": 245 - }, - { - "id": 246 - }, - { - "id": 247 - }, - { - "id": 248 - }, - { - "id": 249 - }, - { - "id": 250 - }, - { - "id": 251 - }, - { - "id": 252 - }, - { - "id": 253 - }, - { - "id": 254 - }, - { - "id": 255 - }, - { - "id": 256 - }, - { - "id": 257 - }, - { - "id": 258 - }, - { - "id": 259 - }, - { - "id": 260 - }, - { - "id": 261 - }, - { - "id": 262 - }, - { - "id": 263 - }, - { - "id": 264 - }, - { - "id": 265 - }, - { - "id": 266 - }, - { - "id": 267 - }, - { - "id": 268 - }, - { - "id": 269 - }, - { - "id": 270 - }, - { - "id": 271 - }, - { - "id": 272 - }, - { - "id": 273 - }, - { - "id": 274 - }, - { - "id": 275 - }, - { - "id": 276 - }, - { - "id": 277 - }, - { - "id": 278 - }, - { - "id": 279 - }, - { - "id": 280 - }, - { - "id": 281 - }, - { - "id": 282 - }, - { - "id": 283 - }, - { - "id": 284 - }, - { - "id": 285 - }, - { - "id": 286 - }, - { - "id": 287 - }, - { - "id": 288 - }, - { - "id": 289 - }, - { - "id": 290 - }, - { - "id": 291 - }, - { - "id": 292 - }, - { - "id": 293 - }, - { - "id": 294 - }, - { - "id": 295 - }, - { - "id": 296 - }, - { - "id": 297 - }, - { - "id": 298 - }, - { - "id": 299 - }, - { - "id": 300 - }, - { - "id": 301 - }, - { - "id": 302 - }, - { - "id": 303 - }, - { - "id": 304 - }, - { - "id": 305 - }, - { - "id": 306 - }, - { - "id": 307 - }, - { - "id": 308 - }, - { - "id": 309 - }, - { - "id": 310 - }, - { - "id": 311 - }, - { - "id": 312 - }, - { - "id": 313 - }, - { - "id": 314 - }, - { - "id": 315 - }, - { - "id": 316 - }, - { - "id": 317 - }, - { - "id": 318 - }, - { - "id": 319 - }, - { - "id": 320 - }, - { - "id": 321 - }, - { - "id": 322 - }, - { - "id": 323 - }, - { - "id": 324 - }, - { - "id": 325 - }, - { - "id": 326 - }, - { - "id": 327 - }, - { - "id": 328 - }, - { - "id": 329 - }, - { - "id": 330 - }, - { - "id": 331 - }, - { - "id": 332 - }, - { - "id": 333 - }, - { - "id": 334 - }, - { - "id": 335 - }, - { - "id": 336 - }, - { - "id": 337 - }, - { - "id": 338 - }, - { - "id": 339 - }, - { - "id": 340 - }, - { - "id": 341 - }, - { - "id": 342 - }, - { - "id": 343 - }, - { - "id": 344 - }, - { - "id": 345 - }, - { - "id": 346 - }, - { - "id": 347 - }, - { - "id": 348 - }, - { - "id": 349 - }, - { - "id": 350 - } - ] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - }, - "metadata": { - "jobId": 2 - } - }, - { - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "message": { - "userId": "user 1", - "anonymousId": "anon-id-new", - "event": "event1", - "type": "track", - "properties": { - "listData": { - "remove": [1] - }, - "enablePartialFailure": true - }, - "context": { - "ip": "14.5.67.21", - "library": { - "name": "http" - } - }, - "timestamp": "2020-02-02T00:23:09.544Z" - }, - "metadata": { - "jobId": 3 - } - } -] diff --git a/test/__tests__/data/marketo_static_list_router_metadata_input.json b/test/__tests__/data/marketo_static_list_router_metadata_input.json deleted file mode 100644 index d006aa184be..00000000000 --- a/test/__tests__/data/marketo_static_list_router_metadata_input.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "batchedRequest": { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer access_token_success" - }, - "params": {}, - "body": { - "JSON": {}, - "XML": {}, - "JSON_ARRAY": {}, - "FORM": {} - }, - "files": {} - }, - "metadata": [ - { - "jobId": 1 - }, - { - "jobId": 2 - }, - { - "jobId": 3 - } - ], - "batched": false, - "statusCode": 200, - "destination": { - "Config": { - "accountId": "marketo_acct_id_success", - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "staticListId": 3027 - }, - "secretConfig": {}, - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "name": "Marketo Static List", - "enabled": true, - "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmLd", - "deleted": false, - "createdAt": "2020-12-30T08:39:32.005Z", - "updatedAt": "2021-02-03T16:22:31.374Z", - "destinationDefinition": { - "config": { - "destConfig": { - "defaultConfig": ["accountId", "clientId", "clientSecret", "staticListId"] - }, - "secretKeys": ["clientSecret"], - "excludeKeys": [], - "includeKeys": [], - "routerTransform": true, - "supportedSourceTypes": ["android", "ios", "web", "unity", "amp", "cloud", "reactnative"] - }, - "id": "1aIXqM806xAVm92nx07YwKbRrO9", - "name": "MARKETO_STATIC_LIST", - "displayName": "Marketo Static List", - "createdAt": "2020-04-09T09:24:31.794Z", - "updatedAt": "2021-01-11T11:03:28.103Z" - }, - "transformations": [], - "isConnectionEnabled": true, - "isProcessorEnabled": true - } -} diff --git a/test/__tests__/data/marketo_static_list_router_metadata_output.json b/test/__tests__/data/marketo_static_list_router_metadata_output.json deleted file mode 100644 index bc560d73a7f..00000000000 --- a/test/__tests__/data/marketo_static_list_router_metadata_output.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "destInfo": { - "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" - }, - "jobId": 1 - }, - { - "destInfo": { - "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" - }, - "jobId": 2 - }, - { - "destInfo": { - "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" - }, - "jobId": 3 - } -] diff --git a/test/__tests__/data/marketo_static_list_router_output.json b/test/__tests__/data/marketo_static_list_router_output.json deleted file mode 100644 index 9c096b84c64..00000000000 --- a/test/__tests__/data/marketo_static_list_router_output.json +++ /dev/null @@ -1,170 +0,0 @@ -[ - { - "batchedRequest": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "DELETE", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ], - "metadata": [ - { - "jobId": 1 - } - ], - "batched": false, - "statusCode": 200, - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - } - }, - { - "batchedRequest": [ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", - "headers": { - "Authorization": "Bearer access_token_success", - "Content-Type": "application/json" - }, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } - ], - "metadata": [ - { - "jobId": 2 - } - ], - "batched": false, - "statusCode": 200, - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - } - }, - { - "metadata": [ - { - "jobId": 3 - } - ], - "destination": { - "ID": "1zia9wKshXt80YksLmUdJnr7IHI", - "Name": "test_marketo", - "DestinationDefinition": { - "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", - "Name": "MARKETO", - "DisplayName": "Marketo", - "transformAt": "processor", - "transformAtV1": "processor" - }, - "Config": { - "clientId": "marketo_client_id_success", - "clientSecret": "marketo_client_secret_success", - "accountId": "marketo_acct_id_success", - "staticListId": 1234 - }, - "Enabled": true, - "Transformations": [], - "IsProcessorEnabled": true - }, - "batched": false, - "statusCode": 400, - "error": "Event type track is not supported", - "statTags": { - "errorCategory": "dataValidation", - "errorType": "instrumentation" - } - } -] diff --git a/test/__tests__/marketo_static_list.test.js b/test/__tests__/marketo_static_list.test.js deleted file mode 100644 index 43300f243ae..00000000000 --- a/test/__tests__/marketo_static_list.test.js +++ /dev/null @@ -1,64 +0,0 @@ -const integration = "marketo_static_list"; -const name = "marketo_static_list"; -const version = "v0"; - -const fs = require("fs"); -const path = require("path"); - -const transformer = require(`../../src/${version}/destinations/${integration}/transform`); - -// Processor Test Data -const testDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}.json`) -); -const testData = JSON.parse(testDataFile); - -// Router Test Data -const inputRouterDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_input.json`) -); -const outputRouterDataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_output.json`) -); -const inputRouterData = JSON.parse(inputRouterDataFile); -const expectedRouterData = JSON.parse(outputRouterDataFile); - -// Router Metadata Test files -const inputRouterMetadataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_metadata_input.json`) -); -const outputRouterMetadataFile = fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_router_metadata_output.json`) -); -const inputRouterMetadata = JSON.parse(inputRouterMetadataFile); -const expectedRouterMetadata = JSON.parse(outputRouterMetadataFile); - -describe(`${name} Tests`, () => { - describe("Processor", () => { - testData.forEach((dataPoint, index) => { - it(`${index}. ${integration} - ${dataPoint.description}`, async () => { - try { - const output = await transformer.process(dataPoint.input); - expect(output).toEqual(dataPoint.output); - } catch (error) { - expect(error.message).toEqual(dataPoint.output.error); - } - }); - }); - }); - - describe("Router Tests", () => { - it("Payload", async () => { - const routerOutput = await transformer.processRouterDest(inputRouterData); - expect(routerOutput).toEqual(expectedRouterData); - }); - }); - describe("Router Metadata Tests", () => { - it("Payload", async () => { - const routerMetadataOutput = await transformer.processMetadataForRouter( - inputRouterMetadata - ); - expect(routerMetadataOutput).toEqual(expectedRouterMetadata); - }); - }); -}); diff --git a/test/integrations/destinations/marketo_static_list/processor/data.ts b/test/integrations/destinations/marketo_static_list/processor/data.ts index 51e4e878242..05fba54f6a5 100644 --- a/test/integrations/destinations/marketo_static_list/processor/data.ts +++ b/test/integrations/destinations/marketo_static_list/processor/data.ts @@ -62,9 +62,9 @@ export const data = [ output: { version: '1', type: 'REST', - method: 'POST', + method: 'DELETE', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', @@ -80,9 +80,9 @@ export const data = [ output: { version: '1', type: 'REST', - method: 'DELETE', + method: 'POST', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', diff --git a/test/integrations/destinations/marketo_static_list/router/data.ts b/test/integrations/destinations/marketo_static_list/router/data.ts index 23b2ea8ea34..fa97ba49f2f 100644 --- a/test/integrations/destinations/marketo_static_list/router/data.ts +++ b/test/integrations/destinations/marketo_static_list/router/data.ts @@ -1,7 +1,7 @@ export const data = [ { name: 'marketo_static_list', - description: 'Test 0', + description: 'Test 0: Test audiencelist event with add and remove', feature: 'router', module: 'destination', version: 'v0', @@ -1172,50 +1172,6 @@ export const data = [ jobId: 2, }, }, - { - destination: { - ID: '1zia9wKshXt80YksLmUdJnr7IHI', - Name: 'test_marketo', - DestinationDefinition: { - ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', - Name: 'MARKETO', - DisplayName: 'Marketo', - transformAt: 'processor', - transformAtV1: 'processor', - }, - Config: { - clientId: 'marketo_client_id_success', - clientSecret: 'marketo_client_secret_success', - accountId: 'marketo_acct_id_success', - staticListId: 1234, - }, - Enabled: true, - Transformations: [], - IsProcessorEnabled: true, - }, - message: { - userId: 'user 1', - anonymousId: 'anon-id-new', - event: 'event1', - type: 'track', - properties: { - listData: { - remove: [1], - }, - enablePartialFailure: true, - }, - context: { - ip: '14.5.67.21', - library: { - name: 'http', - }, - }, - timestamp: '2020-02-02T00:23:09.544Z', - }, - metadata: { - jobId: 3, - }, - }, ], destType: 'marketo_static_list', }, @@ -1232,9 +1188,9 @@ export const data = [ { version: '1', type: 'REST', - method: 'POST', + method: 'DELETE', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', @@ -1251,9 +1207,9 @@ export const data = [ { version: '1', type: 'REST', - method: 'DELETE', + method: 'POST', endpoint: - 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3', headers: { Authorization: 'Bearer access_token_success', 'Content-Type': 'application/json', @@ -1371,18 +1327,325 @@ export const data = [ IsProcessorEnabled: true, }, }, + ], + }, + }, + }, + }, + { + name: 'marketo_static_list', + description: 'Test 1: Test record and audiencelist events in a single batch', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1001, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + identifierType: 'id', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 1, + }, + }, { - metadata: [ - { - destInfo: { - authKey: '1zia9wKshXt80YksLmUdJnr7IHI', + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1002, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', }, - jobId: 3, + externalId: [ + { + type: 'marketoStaticListId', + identifierType: 'id', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', }, - ], + recordId: '3', + }, + metadata: { + jobId: 2, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1003, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + identifierType: 'id', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 3, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + identifierType: 'id', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2001', + }, + type: 'record', + }, + metadata: { + jobId: 4, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + identifierType: 'id', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2002', + }, + type: 'record', + }, + metadata: { + jobId: 5, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + identifierType: 'id', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2003', + }, + type: 'record', + }, + metadata: { + jobId: 6, + }, + }, + { destination: { ID: '1zia9wKshXt80YksLmUdJnr7IHI', - Name: 'test_marketo', + Name: 'test_marketo_al', DestinationDefinition: { ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', Name: 'MARKETO', @@ -1400,12 +1663,1981 @@ export const data = [ Transformations: [], IsProcessorEnabled: true, }, - batched: false, - statusCode: 400, - error: 'Event type track is not supported', - statTags: { - errorCategory: 'dataValidation', - errorType: 'instrumentation', + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 101, + }, + { + id: 102, + }, + { + id: 103, + }, + ], + remove: [ + { + id: 4, + }, + { + id: 5, + }, + { + id: 6, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 7, + }, + }, + { + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + userId: 'user 1', + anonymousId: 'anon-id-new', + event: 'event1', + type: 'audiencelist', + properties: { + listData: { + add: [ + { + id: 0, + }, + { + id: 1, + }, + { + id: 2, + }, + { + id: 3, + }, + { + id: 4, + }, + { + id: 5, + }, + { + id: 6, + }, + { + id: 7, + }, + { + id: 8, + }, + { + id: 9, + }, + { + id: 10, + }, + { + id: 11, + }, + { + id: 12, + }, + { + id: 13, + }, + { + id: 14, + }, + { + id: 15, + }, + { + id: 16, + }, + { + id: 17, + }, + { + id: 18, + }, + { + id: 19, + }, + { + id: 20, + }, + { + id: 21, + }, + { + id: 22, + }, + { + id: 23, + }, + { + id: 24, + }, + { + id: 25, + }, + { + id: 26, + }, + { + id: 27, + }, + { + id: 28, + }, + { + id: 29, + }, + { + id: 30, + }, + { + id: 31, + }, + { + id: 32, + }, + { + id: 33, + }, + { + id: 34, + }, + { + id: 35, + }, + { + id: 36, + }, + { + id: 37, + }, + { + id: 38, + }, + { + id: 39, + }, + { + id: 40, + }, + { + id: 41, + }, + { + id: 42, + }, + { + id: 43, + }, + { + id: 44, + }, + { + id: 45, + }, + { + id: 46, + }, + { + id: 47, + }, + { + id: 48, + }, + { + id: 49, + }, + { + id: 50, + }, + { + id: 51, + }, + { + id: 52, + }, + { + id: 53, + }, + { + id: 54, + }, + { + id: 55, + }, + { + id: 56, + }, + { + id: 57, + }, + { + id: 58, + }, + { + id: 59, + }, + { + id: 60, + }, + { + id: 61, + }, + { + id: 62, + }, + { + id: 63, + }, + { + id: 64, + }, + { + id: 65, + }, + { + id: 66, + }, + { + id: 67, + }, + { + id: 68, + }, + { + id: 69, + }, + { + id: 70, + }, + { + id: 71, + }, + { + id: 72, + }, + { + id: 73, + }, + { + id: 74, + }, + { + id: 75, + }, + { + id: 76, + }, + { + id: 77, + }, + { + id: 78, + }, + { + id: 79, + }, + { + id: 80, + }, + { + id: 81, + }, + { + id: 82, + }, + { + id: 83, + }, + { + id: 84, + }, + { + id: 85, + }, + { + id: 86, + }, + { + id: 87, + }, + { + id: 88, + }, + { + id: 89, + }, + { + id: 90, + }, + { + id: 91, + }, + { + id: 92, + }, + { + id: 93, + }, + { + id: 94, + }, + { + id: 95, + }, + { + id: 96, + }, + { + id: 97, + }, + { + id: 98, + }, + { + id: 99, + }, + { + id: 100, + }, + { + id: 101, + }, + { + id: 102, + }, + { + id: 103, + }, + { + id: 104, + }, + { + id: 105, + }, + { + id: 106, + }, + { + id: 107, + }, + { + id: 108, + }, + { + id: 109, + }, + { + id: 110, + }, + { + id: 111, + }, + { + id: 112, + }, + { + id: 113, + }, + { + id: 114, + }, + { + id: 115, + }, + { + id: 116, + }, + { + id: 117, + }, + { + id: 118, + }, + { + id: 119, + }, + { + id: 120, + }, + { + id: 121, + }, + { + id: 122, + }, + { + id: 123, + }, + { + id: 124, + }, + { + id: 125, + }, + { + id: 126, + }, + { + id: 127, + }, + { + id: 128, + }, + { + id: 129, + }, + { + id: 130, + }, + { + id: 131, + }, + { + id: 132, + }, + { + id: 133, + }, + { + id: 134, + }, + { + id: 135, + }, + { + id: 136, + }, + { + id: 137, + }, + { + id: 138, + }, + { + id: 139, + }, + { + id: 140, + }, + { + id: 141, + }, + { + id: 142, + }, + { + id: 143, + }, + { + id: 144, + }, + { + id: 145, + }, + { + id: 146, + }, + { + id: 147, + }, + { + id: 148, + }, + { + id: 149, + }, + { + id: 150, + }, + { + id: 151, + }, + { + id: 152, + }, + { + id: 153, + }, + { + id: 154, + }, + { + id: 155, + }, + { + id: 156, + }, + { + id: 157, + }, + { + id: 158, + }, + { + id: 159, + }, + { + id: 160, + }, + { + id: 161, + }, + { + id: 162, + }, + { + id: 163, + }, + { + id: 164, + }, + { + id: 165, + }, + { + id: 166, + }, + { + id: 167, + }, + { + id: 168, + }, + { + id: 169, + }, + { + id: 170, + }, + { + id: 171, + }, + { + id: 172, + }, + { + id: 173, + }, + { + id: 174, + }, + { + id: 175, + }, + { + id: 176, + }, + { + id: 177, + }, + { + id: 178, + }, + { + id: 179, + }, + { + id: 180, + }, + { + id: 181, + }, + { + id: 182, + }, + { + id: 183, + }, + { + id: 184, + }, + { + id: 185, + }, + { + id: 186, + }, + { + id: 187, + }, + { + id: 188, + }, + { + id: 189, + }, + { + id: 190, + }, + { + id: 191, + }, + { + id: 192, + }, + { + id: 193, + }, + { + id: 194, + }, + { + id: 195, + }, + { + id: 196, + }, + { + id: 197, + }, + { + id: 198, + }, + { + id: 199, + }, + { + id: 200, + }, + { + id: 201, + }, + { + id: 202, + }, + { + id: 203, + }, + { + id: 204, + }, + { + id: 205, + }, + { + id: 206, + }, + { + id: 207, + }, + { + id: 208, + }, + { + id: 209, + }, + { + id: 210, + }, + { + id: 211, + }, + { + id: 212, + }, + { + id: 213, + }, + { + id: 214, + }, + { + id: 215, + }, + { + id: 216, + }, + { + id: 217, + }, + { + id: 218, + }, + { + id: 219, + }, + { + id: 220, + }, + { + id: 221, + }, + { + id: 222, + }, + { + id: 223, + }, + { + id: 224, + }, + { + id: 225, + }, + { + id: 226, + }, + { + id: 227, + }, + { + id: 228, + }, + { + id: 229, + }, + { + id: 230, + }, + { + id: 231, + }, + { + id: 232, + }, + { + id: 233, + }, + { + id: 234, + }, + { + id: 235, + }, + { + id: 236, + }, + { + id: 237, + }, + { + id: 238, + }, + { + id: 239, + }, + { + id: 240, + }, + { + id: 241, + }, + { + id: 242, + }, + { + id: 243, + }, + { + id: 244, + }, + { + id: 245, + }, + { + id: 246, + }, + { + id: 247, + }, + { + id: 248, + }, + { + id: 249, + }, + { + id: 250, + }, + { + id: 251, + }, + { + id: 252, + }, + { + id: 253, + }, + { + id: 254, + }, + { + id: 255, + }, + { + id: 256, + }, + { + id: 257, + }, + { + id: 258, + }, + { + id: 259, + }, + { + id: 260, + }, + { + id: 261, + }, + { + id: 262, + }, + { + id: 263, + }, + { + id: 264, + }, + { + id: 265, + }, + { + id: 266, + }, + { + id: 267, + }, + { + id: 268, + }, + { + id: 269, + }, + { + id: 270, + }, + { + id: 271, + }, + { + id: 272, + }, + { + id: 273, + }, + { + id: 274, + }, + { + id: 275, + }, + { + id: 276, + }, + { + id: 277, + }, + { + id: 278, + }, + { + id: 279, + }, + { + id: 280, + }, + { + id: 281, + }, + { + id: 282, + }, + { + id: 283, + }, + { + id: 284, + }, + { + id: 285, + }, + { + id: 286, + }, + { + id: 287, + }, + { + id: 288, + }, + { + id: 289, + }, + { + id: 290, + }, + { + id: 291, + }, + { + id: 292, + }, + { + id: 293, + }, + { + id: 294, + }, + { + id: 295, + }, + { + id: 296, + }, + { + id: 297, + }, + { + id: 298, + }, + { + id: 299, + }, + { + id: 300, + }, + { + id: 301, + }, + { + id: 302, + }, + { + id: 303, + }, + { + id: 304, + }, + { + id: 305, + }, + { + id: 306, + }, + { + id: 307, + }, + { + id: 308, + }, + { + id: 309, + }, + { + id: 310, + }, + { + id: 311, + }, + { + id: 312, + }, + { + id: 313, + }, + { + id: 314, + }, + { + id: 315, + }, + { + id: 316, + }, + { + id: 317, + }, + { + id: 318, + }, + { + id: 319, + }, + { + id: 320, + }, + { + id: 321, + }, + { + id: 322, + }, + { + id: 323, + }, + { + id: 324, + }, + { + id: 325, + }, + { + id: 326, + }, + { + id: 327, + }, + { + id: 328, + }, + { + id: 329, + }, + { + id: 330, + }, + { + id: 331, + }, + { + id: 332, + }, + { + id: 333, + }, + { + id: 334, + }, + { + id: 335, + }, + { + id: 336, + }, + { + id: 337, + }, + { + id: 338, + }, + { + id: 339, + }, + { + id: 340, + }, + { + id: 341, + }, + { + id: 342, + }, + { + id: 343, + }, + { + id: 344, + }, + { + id: 345, + }, + { + id: 346, + }, + { + id: 347, + }, + { + id: 348, + }, + { + id: 349, + }, + { + id: 350, + }, + ], + }, + enablePartialFailure: true, + }, + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + timestamp: '2020-02-02T00:23:09.544Z', + }, + metadata: { + jobId: 8, + }, + }, + ], + destType: 'marketo_static_list', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=2001&id=2002&id=2003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 4, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 5, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 6, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1122/leads.json?id=1001&id=1002&id=1003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 2, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 3, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=101&id=102&id=103', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 7, + destInfo: { + authKey: '1zia9wKshXt80YksLmUdJnr7IHI', + }, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 8, + destInfo: { + authKey: '1zia9wKshXt80YksLmUdJnr7IHI', + }, + }, + ], + batched: false, + statusCode: 200, + destination: { + ID: '1zia9wKshXt80YksLmUdJnr7IHI', + Name: 'test_marketo_al', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1234, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + ], + }, + }, + }, + }, + { + name: 'marketo_static_list', + description: 'Test 2: Test record evemts', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + destType: 'marketo_static_list', + input: [ + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1001, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 1, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1002, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + id: 'id002', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 2, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + type: 'record', + action: 'insert', + fields: { + id: 1003, + }, + channel: 'sources', + context: { + sources: { + job_id: '2VsZs4hyPpq7f1p8igrpmHsibHl', + version: 'feat.mirrorRetlEvent', + job_run_id: 'ck99nbd2kqiljdihhkh0', + task_run_id: 'ck99nbd2kqiljdihhkhg', + }, + externalId: [ + { + type: 'marketoStaticListId', + id: 'id001', + }, + ], + destinationFields: 'id', + mappedToDestination: 'true', + }, + recordId: '3', + }, + metadata: { + jobId: 3, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id002', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2001', + }, + type: 'record', + }, + metadata: { + jobId: 4, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id001', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2002', + }, + type: 'record', + }, + metadata: { + jobId: 5, + }, + }, + { + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + message: { + action: 'delete', + context: { + destinationFields: 'id', + externalId: [ + { + id: 'id002', + type: 'marketoStaticListId', + }, + ], + mappedToDestination: 'true', + sources: { + job_id: 'sf', + job_run_id: 'ck985bve58cvnti48120', + task_run_id: 'ck985bve58cvnti4812g', + version: '', + }, + }, + recordId: '2', + rudderId: '2', + fields: { + id: '2003', + }, + type: 'record', + }, + metadata: { + jobId: 6, + }, + }, + ], + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=2002', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 5, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id001/leads.json?id=1001&id=1003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 1, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 3, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=2001&id=2003', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 4, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + { + jobId: 6, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, + }, + }, + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://marketo_acct_id_success.mktorest.com/rest/v1/lists/id002/leads.json?id=1002', + headers: { + Authorization: 'Bearer access_token_success', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + jobId: 2, + destInfo: { + authKey: '1zwa1wKshSt81YksKmUdJnr4IOK', + }, + }, + ], + batched: true, + statusCode: 200, + destination: { + ID: '1zwa1wKshSt81YksKmUdJnr4IOK', + Name: 'test_marketo_rc', + DestinationDefinition: { + ID: '1iVQvTRMsPPyJzwol0ifH93QTQ6', + Name: 'MARKETO', + DisplayName: 'Marketo', + transformAt: 'processor', + transformAtV1: 'processor', + }, + Config: { + clientId: 'marketo_client_id_success', + clientSecret: 'marketo_client_secret_success', + accountId: 'marketo_acct_id_success', + staticListId: 1122, + }, + Enabled: true, + Transformations: [], + IsProcessorEnabled: true, }, }, ],