diff --git a/src/cdk/v2/destinations/the_trade_desk/config.js b/src/cdk/v2/destinations/the_trade_desk/config.js index 6f7ef37350b..9455c818fdd 100644 --- a/src/cdk/v2/destinations/the_trade_desk/config.js +++ b/src/cdk/v2/destinations/the_trade_desk/config.js @@ -1,7 +1,6 @@ const SUPPORTED_EVENT_TYPE = 'record'; const ACTION_TYPES = ['insert', 'delete']; const DATA_PROVIDER_ID = 'rudderstack'; -const MAX_REQUEST_SIZE_IN_BYTES = 2500000; // ref:- https://partner.thetradedesk.com/v3/portal/data/doc/DataEnvironments const DATA_SERVERS_BASE_ENDPOINTS_MAP = { @@ -17,6 +16,6 @@ module.exports = { SUPPORTED_EVENT_TYPE, ACTION_TYPES, DATA_PROVIDER_ID, - MAX_REQUEST_SIZE_IN_BYTES, + MAX_REQUEST_SIZE_IN_BYTES: 2500000, DATA_SERVERS_BASE_ENDPOINTS_MAP, }; diff --git a/src/cdk/v2/destinations/the_trade_desk/utils.js b/src/cdk/v2/destinations/the_trade_desk/utils.js index d0400d7a9b1..0f1c3fb0c1c 100644 --- a/src/cdk/v2/destinations/the_trade_desk/utils.js +++ b/src/cdk/v2/destinations/the_trade_desk/utils.js @@ -1,6 +1,5 @@ const lodash = require('lodash'); const CryptoJS = require('crypto-js'); -const jsonSize = require('json-size'); const { InstrumentationError, AbortedError } = require('@rudderstack/integrations-lib'); const { BatchUtils } = require('@rudderstack/workflow-engine'); const { @@ -10,11 +9,9 @@ const { removeUndefinedAndNullValues, handleRtTfSingleEventError, } = require('../../../../v0/util'); -const { - DATA_PROVIDER_ID, - MAX_REQUEST_SIZE_IN_BYTES, - DATA_SERVERS_BASE_ENDPOINTS_MAP, -} = require('./config'); +const tradeDeskConfig = require('./config'); + +const { DATA_PROVIDER_ID, DATA_SERVERS_BASE_ENDPOINTS_MAP } = tradeDeskConfig; const ttlInMin = (ttl) => parseInt(ttl, 10) * 1440; const getBaseEndpoint = (dataServer) => DATA_SERVERS_BASE_ENDPOINTS_MAP[dataServer]; @@ -41,40 +38,12 @@ const responseBuilder = (items, config) => { return response; }; -const splitItemsBasedOnMaxSizeInBytes = (items, maxSize) => { - if (!items || items.length === 0) return []; - - const itemsSize = jsonSize(items); - - if (itemsSize <= maxSize) { - return [items]; - } - - const batches = []; - let currentBatch = []; - - items.forEach((item) => { - const itemSize = jsonSize(item); - - if (jsonSize(currentBatch) + itemSize <= maxSize) { - currentBatch.push(item); - } else { - batches.push([...currentBatch]); - currentBatch = [item]; - } - }); - - if (currentBatch.length > 0) { - batches.push(currentBatch); - } - - return batches; -}; - const batchResponseBuilder = (items, config) => { const response = []; const itemsChunks = BatchUtils.chunkArrayBySizeAndLength(items, { - maxSizeInBytes: MAX_REQUEST_SIZE_IN_BYTES, + // TODO: use destructuring at the top of file once proper 'mocking' is implemented. + // eslint-disable-next-line unicorn/consistent-destructuring + maxSizeInBytes: tradeDeskConfig.MAX_REQUEST_SIZE_IN_BYTES, }); itemsChunks.items.forEach((chunk) => { @@ -104,8 +73,13 @@ const processRecordInputs = (inputs, destination) => { TTLInMinutes: action === 'insert' ? ttlInMin(Config.ttlInDays) : 0, }, ]; + Object.keys(fields).forEach((id) => { - items.push({ [id]: fields[id], Data: data }); + const value = fields[id]; + if (value) { + // adding only non empty ID's + items.push({ [id]: value, Data: data }); + } }); } else { errorResponseList.push(handleRtTfSingleEventError(input, error, {})); @@ -130,4 +104,4 @@ const processRouterDest = (inputs) => { return respList; }; -module.exports = { getSignatureHeader, splitItemsBasedOnMaxSizeInBytes, processRouterDest }; +module.exports = { getSignatureHeader, processRouterDest }; diff --git a/src/cdk/v2/destinations/the_trade_desk/utils.test.js b/src/cdk/v2/destinations/the_trade_desk/utils.test.js index b5e006bfca6..81fd7cf17df 100644 --- a/src/cdk/v2/destinations/the_trade_desk/utils.test.js +++ b/src/cdk/v2/destinations/the_trade_desk/utils.test.js @@ -1,5 +1,5 @@ const { AbortedError } = require('@rudderstack/integrations-lib'); -const { getSignatureHeader, splitItemsBasedOnMaxSizeInBytes } = require('./utils'); +const { getSignatureHeader } = require('./utils'); describe('getSignatureHeader', () => { it('should calculate the signature header for a valid request and secret key', () => { @@ -47,45 +47,3 @@ describe('getSignatureHeader', () => { }).toThrow(AbortedError); }); }); - -describe('splitItemsBasedOnMaxSizeInBytes', () => { - it('should return an array with a single element when the total size of the items is less than or equal to the maximum size', () => { - const items = [ - { id: 1, name: 'item1' }, - { id: 2, name: 'item2' }, - { id: 3, name: 'item3' }, - ]; - const maxSize = 100; - const result = splitItemsBasedOnMaxSizeInBytes(items, maxSize); - expect(result).toEqual([items]); - }); - - it('should split the items into batches of maximum size and return an array of batches', () => { - // size of this items array is 121 bytes - const items = [ - { id: 1, name: 'item1' }, - { id: 2, name: 'item2' }, - { id: 3, name: 'item3' }, - { id: 4, name: 'item4' }, - { id: 5, name: 'item5' }, - ]; - const maxSize = 100; - const result = splitItemsBasedOnMaxSizeInBytes(items, maxSize); - expect(result).toEqual([ - [ - { id: 1, name: 'item1' }, - { id: 2, name: 'item2' }, - { id: 3, name: 'item3' }, - { id: 4, name: 'item4' }, - ], - [{ id: 5, name: 'item5' }], - ]); - }); - - it('should return an empty array when the items array is empty', () => { - const items = []; - const maxSize = 100; - const result = splitItemsBasedOnMaxSizeInBytes(items, maxSize); - expect(result).toEqual([]); - }); -}); diff --git a/src/features.json b/src/features.json index 3c10948b664..cc75b16a74a 100644 --- a/src/features.json +++ b/src/features.json @@ -68,5 +68,5 @@ "INTERCOM": true }, "supportSourceTransformV1": true, - "supportTransformerProxyV1": true + "supportTransformerProxyV1": false } diff --git a/test/integrations/destinations/the_trade_desk/mocks.ts b/test/integrations/destinations/the_trade_desk/mocks.ts new file mode 100644 index 00000000000..ddcbebae880 --- /dev/null +++ b/test/integrations/destinations/the_trade_desk/mocks.ts @@ -0,0 +1,5 @@ +import config from '../../../../src/cdk/v2/destinations/the_trade_desk/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_REQUEST_SIZE_IN_BYTES', 250); +}; diff --git a/test/integrations/destinations/the_trade_desk/router/data.ts b/test/integrations/destinations/the_trade_desk/router/data.ts index d81454d5b7a..3c9cb1cc704 100644 --- a/test/integrations/destinations/the_trade_desk/router/data.ts +++ b/test/integrations/destinations/the_trade_desk/router/data.ts @@ -1,4 +1,5 @@ import { overrideDestination } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; import { destType, destTypeInUpperCase, @@ -43,7 +44,7 @@ export const data = [ action: 'insert', fields: { DAID: 'test-daid-2', - UID2: 'test-uid2-2', + UID2: null, }, channel: 'sources', context: sampleContext, @@ -106,15 +107,6 @@ export const data = [ }, ], }, - { - UID2: 'test-uid2-2', - Data: [ - { - Name: segmentName, - TTLInMinutes: 43200, - }, - ], - }, ], }, JSON_ARRAY: {}, @@ -140,10 +132,12 @@ export const data = [ }, }, }, + mockFns: defaultMockFns, }, { name: destType, - description: 'Add/Remove IDs to/from the segment', + description: + 'Add/Remove IDs to/from the segment and split into multiple requests based on size', feature: 'router', module: 'destination', version: 'v0', @@ -237,6 +231,26 @@ export const data = [ }, ], }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://sin-data.adsrvr.org/data/advertiser', + headers: {}, + params: {}, + body: { + JSON: { + DataProviderId: dataProviderId, + AdvertiserId: advertiserId, + Items: [ { UID2: 'test-uid2-2', Data: [ @@ -271,6 +285,7 @@ export const data = [ }, }, }, + mockFns: defaultMockFns, }, { name: destType,