From 04c069486bdd3c101906fa6c621e983090fcab25 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Thu, 17 Oct 2024 18:25:20 +0530 Subject: [PATCH 01/12] feat: sources v2 spec support along with adapters --- src/controllers/__tests__/source.test.ts | 111 +++++++++++- src/controllers/util/index.test.ts | 163 ++++++++++++++++-- src/controllers/util/index.ts | 62 ++++++- src/interfaces/SourceService.ts | 8 +- src/middlewares/routeActivation.ts | 15 ++ .../__tests__/nativeIntegration.test.ts | 20 ++- src/services/source/nativeIntegration.ts | 18 +- src/types/index.ts | 22 +++ test/apitests/service.api.test.ts | 7 + 9 files changed, 390 insertions(+), 36 deletions(-) diff --git a/src/controllers/__tests__/source.test.ts b/src/controllers/__tests__/source.test.ts index 565f39d559..72bee83282 100644 --- a/src/controllers/__tests__/source.test.ts +++ b/src/controllers/__tests__/source.test.ts @@ -6,6 +6,7 @@ import { applicationRoutes } from '../../routes'; import { NativeIntegrationSourceService } from '../../services/source/nativeIntegration'; import { ServiceSelector } from '../../helpers/serviceSelector'; import { ControllerUtility } from '../util/index'; +import { SourceInputConversionResult } from '../../types'; let server: any; const OLD_ENV = process.env; @@ -38,6 +39,19 @@ const getData = () => { return [{ event: { a: 'b1' } }, { event: { a: 'b2' } }]; }; +const getV2Data = () => { + return [ + { request: { body: '{"a": "b"}' }, source: { id: 1 } }, + { request: { body: '{"a": "b"}' }, source: { id: 1 } }, + ]; +}; + +const getConvertedData = () => { + return getData().map((eventInstance) => { + return { output: eventInstance } as SourceInputConversionResult; + }); +}; + describe('Source controller tests', () => { describe('V0 Source transform tests', () => { test('successful source transform', async () => { @@ -49,7 +63,7 @@ describe('Source controller tests', () => { mockSourceService.sourceTransformRoutine = jest .fn() .mockImplementation((i, s, v, requestMetadata) => { - expect(i).toEqual(getData()); + expect(i).toEqual(getConvertedData()); expect(s).toEqual(sourceType); expect(v).toEqual(version); return testOutput; @@ -66,7 +80,7 @@ describe('Source controller tests', () => { expect(s).toEqual(sourceType); expect(v).toEqual(version); expect(e).toEqual(getData()); - return { implementationVersion: version, input: e }; + return { implementationVersion: version, input: getConvertedData() }; }); const response = await request(server) @@ -139,7 +153,7 @@ describe('Source controller tests', () => { mockSourceService.sourceTransformRoutine = jest .fn() .mockImplementation((i, s, v, requestMetadata) => { - expect(i).toEqual(getData()); + expect(i).toEqual(getConvertedData()); expect(s).toEqual(sourceType); expect(v).toEqual(version); return testOutput; @@ -156,7 +170,7 @@ describe('Source controller tests', () => { expect(s).toEqual(sourceType); expect(v).toEqual(version); expect(e).toEqual(getData()); - return { implementationVersion: version, input: e }; + return { implementationVersion: version, input: getConvertedData() }; }); const response = await request(server) @@ -217,4 +231,93 @@ describe('Source controller tests', () => { expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); }); }); + + describe('V2 Source transform tests', () => { + test('successful source transform', async () => { + const sourceType = '__rudder_test__'; + const version = 'v2'; + const testOutput = [{ event: { a: 'b' }, source: { id: 'id' } }]; + + const mockSourceService = new NativeIntegrationSourceService(); + mockSourceService.sourceTransformRoutine = jest + .fn() + .mockImplementation((i, s, v, requestMetadata) => { + expect(i).toEqual(getConvertedData()); + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + return testOutput; + }); + const getNativeSourceServiceSpy = jest + .spyOn(ServiceSelector, 'getNativeSourceService') + .mockImplementation(() => { + return mockSourceService; + }); + + const adaptInputToVersionSpy = jest + .spyOn(ControllerUtility, 'adaptInputToVersion') + .mockImplementation((s, v, e) => { + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + expect(e).toEqual(getV2Data()); + return { implementationVersion: version, input: getConvertedData() }; + }); + + const response = await request(server) + .post('/v2/sources/__rudder_test__') + .set('Accept', 'application/json') + .send(getV2Data()); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(testOutput); + + expect(response.header['apiversion']).toEqual('2'); + + expect(getNativeSourceServiceSpy).toHaveBeenCalledTimes(1); + expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); + expect(mockSourceService.sourceTransformRoutine).toHaveBeenCalledTimes(1); + }); + + test('failing source transform', async () => { + const sourceType = '__rudder_test__'; + const version = 'v2'; + const mockSourceService = new NativeIntegrationSourceService(); + const getNativeSourceServiceSpy = jest + .spyOn(ServiceSelector, 'getNativeSourceService') + .mockImplementation(() => { + return mockSourceService; + }); + + const adaptInputToVersionSpy = jest + .spyOn(ControllerUtility, 'adaptInputToVersion') + .mockImplementation((s, v, e) => { + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + expect(e).toEqual(getV2Data()); + throw new Error('test error'); + }); + + const response = await request(server) + .post('/v2/sources/__rudder_test__') + .set('Accept', 'application/json') + .send(getV2Data()); + + const expectedResp = [ + { + error: 'test error', + statTags: { + errorCategory: 'transformation', + }, + statusCode: 500, + }, + ]; + + expect(response.status).toEqual(200); + expect(response.body).toEqual(expectedResp); + + expect(response.header['apiversion']).toEqual('2'); + + expect(getNativeSourceServiceSpy).toHaveBeenCalledTimes(1); + expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 6065920846..6ab2336b71 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -19,9 +19,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: undefined, input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, ], }; @@ -40,9 +40,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, ], }; @@ -71,16 +71,22 @@ describe('adaptInputToVersion', () => { implementationVersion: 'v1', input: [ { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, ], }; @@ -100,9 +106,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v1', input: [ - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, ], }; @@ -131,9 +137,130 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should convert input from v2 to v0 format when the request version is v2 and the implementation version is v0', () => { + const sourceType = 'pipedream'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v0', + input: [ + { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { + const sourceType = 'webhook'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v1', + input: [ + { + output: { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, ], }; diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index c5bf7ab358..b562381ed6 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -10,6 +10,8 @@ import { RouterTransformationRequestData, RudderMessage, SourceInput, + SourceInputConversionResult, + SourceInputV2, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; @@ -45,28 +47,72 @@ export class ControllerUtility { return this.sourceVersionMap; } - private static convertSourceInputv1Tov0(sourceEvents: SourceInput[]): NonNullable[] { - return sourceEvents.map((sourceEvent) => sourceEvent.event); + private static convertSourceInputv1Tov0( + sourceEvents: SourceInput[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent.event as NonNullable, + })); } - private static convertSourceInputv0Tov1(sourceEvents: unknown[]): SourceInput[] { - return sourceEvents.map( - (sourceEvent) => ({ event: sourceEvent, source: undefined }) as SourceInput, - ); + private static convertSourceInputv0Tov1( + sourceEvents: unknown[], + ): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => ({ + output: { event: sourceEvent, source: undefined } as SourceInput, + })); + } + + private static convertSourceInputv2Tov0( + sourceEvents: SourceInputV2[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => { + try { + const v0Event = JSON.parse(sourceEvent.request.body); + v0Event.query_parameters = sourceEvent.request.query_parameters; + return { output: v0Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v0 spec'); + return { output: {} as NonNullable, conversionError }; + } + }); + } + + private static convertSourceInputv2Tov1( + sourceEvents: SourceInputV2[], + ): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => { + try { + const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; + v1Event.event.query_parameters = sourceEvent.request.query_parameters; + return { output: v1Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v1 spec'); + return { output: {} as SourceInput, conversionError }; + } + }); } public static adaptInputToVersion( sourceType: string, requestVersion: string, input: NonNullable[], - ): { implementationVersion: string; input: NonNullable[] } { + ): { implementationVersion: string; input: SourceInputConversionResult>[] } { const sourceToVersionMap = this.getSourceVersionsMap(); const implementationVersion = sourceToVersionMap.get(sourceType); - let updatedInput: NonNullable[] = input; + let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ + output: event, + })); if (requestVersion === 'v0' && implementationVersion === 'v1') { updatedInput = this.convertSourceInputv0Tov1(input); } else if (requestVersion === 'v1' && implementationVersion === 'v0') { updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); + } else if (requestVersion === 'v2' && implementationVersion === 'v0') { + updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); + } else if (requestVersion === 'v2' && implementationVersion === 'v1') { + updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); } return { implementationVersion, input: updatedInput }; } diff --git a/src/interfaces/SourceService.ts b/src/interfaces/SourceService.ts index c7de8cfe8b..32a7125e7a 100644 --- a/src/interfaces/SourceService.ts +++ b/src/interfaces/SourceService.ts @@ -1,10 +1,14 @@ -import { MetaTransferObject, SourceTransformationResponse } from '../types/index'; +import { + MetaTransferObject, + SourceInputConversionResult, + SourceTransformationResponse, +} from '../types/index'; export interface SourceService { getTags(): MetaTransferObject; sourceTransformRoutine( - sourceEvents: NonNullable[], + sourceEvents: SourceInputConversionResult>[], sourceType: string, version: string, requestMetadata: NonNullable, diff --git a/src/middlewares/routeActivation.ts b/src/middlewares/routeActivation.ts index ffb1e15e80..126749b083 100644 --- a/src/middlewares/routeActivation.ts +++ b/src/middlewares/routeActivation.ts @@ -106,4 +106,19 @@ export class RouteActivationMiddleware { RouteActivationMiddleware.shouldActivateRoute(destination, deliveryFilterList), ); } + + // This middleware will be used by source endpoint when we completely deprecate v0, v1 versions. + public static isVersionAllowed(ctx: Context, next: Next) { + const { version } = ctx.params; + if (version === 'v0' || version === 'v1') { + ctx.status = 500; + ctx.body = + '/v0, /v1 versioned endpoints are deprecated. Use /v2 version endpoint. This is probably caused because of source transformation call from an outdated rudder-server version. Please upgrade rudder-server to a minimum of 1.xx.xx version.'; + } else if (version === 'v2') { + next(); + } else { + ctx.status = 404; + ctx.body = 'Path not found. Verify the version of your api call.'; + } + } } diff --git a/src/services/source/__tests__/nativeIntegration.test.ts b/src/services/source/__tests__/nativeIntegration.test.ts index 2ef8129cdc..51bb37f5f1 100644 --- a/src/services/source/__tests__/nativeIntegration.test.ts +++ b/src/services/source/__tests__/nativeIntegration.test.ts @@ -44,7 +44,15 @@ describe('NativeIntegration Source Service', () => { }); const service = new NativeIntegrationSourceService(); - const resp = await service.sourceTransformRoutine(events, sourceType, version, requestMetadata); + const adapterConvertedEvents = events.map((eventInstance) => { + return { output: eventInstance }; + }); + const resp = await service.sourceTransformRoutine( + adapterConvertedEvents, + sourceType, + version, + requestMetadata, + ); expect(resp).toEqual(tresponse); @@ -81,7 +89,15 @@ describe('NativeIntegration Source Service', () => { jest.spyOn(stats, 'increment').mockImplementation(() => {}); const service = new NativeIntegrationSourceService(); - const resp = await service.sourceTransformRoutine(events, sourceType, version, requestMetadata); + const adapterConvertedEvents = events.map((eventInstance) => { + return { output: eventInstance }; + }); + const resp = await service.sourceTransformRoutine( + adapterConvertedEvents, + sourceType, + version, + requestMetadata, + ); expect(resp).toEqual(tresponse); diff --git a/src/services/source/nativeIntegration.ts b/src/services/source/nativeIntegration.ts index 5c89de7b92..58a6a19649 100644 --- a/src/services/source/nativeIntegration.ts +++ b/src/services/source/nativeIntegration.ts @@ -4,6 +4,7 @@ import { ErrorDetailer, MetaTransferObject, RudderMessage, + SourceInputConversionResult, SourceTransformationEvent, SourceTransformationResponse, } from '../../types/index'; @@ -28,7 +29,7 @@ export class NativeIntegrationSourceService implements SourceService { } public async sourceTransformRoutine( - sourceEvents: NonNullable[], + sourceEvents: SourceInputConversionResult>[], sourceType: string, version: string, // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -39,7 +40,20 @@ export class NativeIntegrationSourceService implements SourceService { const respList: SourceTransformationResponse[] = await Promise.all( sourceEvents.map(async (sourceEvent) => { try { - const newSourceEvent = sourceEvent; + if (sourceEvent.conversionError) { + stats.increment('source_transform_errors', { + source: sourceType, + version, + }); + logger.debug(`Error during source Transform: ${sourceEvent.conversionError}`, { + ...logger.getLogMetadata(metaTO.errorDetails), + }); + return SourcePostTransformationService.handleFailureEventsSource( + sourceEvent.conversionError, + metaTO, + ); + } + const newSourceEvent = sourceEvent.output; const { headers } = newSourceEvent; delete newSourceEvent.headers; const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = diff --git a/src/types/index.ts b/src/types/index.ts index 45ec7445c3..0bc2cbc33b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -355,6 +355,26 @@ type SourceInput = { event: NonNullable[]; source?: Source; }; + +type SourceRequestV2 = { + method: string; + url: string; + proto: string; + body: string; + headers: NonNullable; + query_parameters: NonNullable; +}; + +type SourceInputV2 = { + request: SourceRequestV2; + source?: Source; +}; + +type SourceInputConversionResult = { + output: T; + conversionError?: Error; +}; + export { ComparatorInput, DeliveryJobState, @@ -382,7 +402,9 @@ export { UserDeletionRequest, UserDeletionResponse, SourceInput, + SourceInputV2, Source, + SourceInputConversionResult, UserTransformationLibrary, UserTransformationResponse, UserTransformationServiceResponse, diff --git a/test/apitests/service.api.test.ts b/test/apitests/service.api.test.ts index 9c1d96e7fe..2ad1f323ac 100644 --- a/test/apitests/service.api.test.ts +++ b/test/apitests/service.api.test.ts @@ -78,6 +78,13 @@ describe('features tests', () => { const supportTransformerProxyV1 = JSON.parse(response.text).supportTransformerProxyV1; expect(typeof supportTransformerProxyV1).toBe('boolean'); }); + + test('features upgradedToSourceTransformV2 to be boolean', async () => { + const response = await request(server).get('/features'); + expect(response.status).toEqual(200); + const upgradedToSourceTransformV2 = JSON.parse(response.text).upgradedToSourceTransformV2; + expect(typeof upgradedToSourceTransformV2).toBe('boolean'); + }); }); describe('Api tests with a mock source/destination', () => { From 778b028cb0ba0f9a3b5feefbc11bfb72901fe01b Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Thu, 17 Oct 2024 18:48:48 +0530 Subject: [PATCH 02/12] chore: unwanted file .python-version removed, updated .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 09c536ebb8..624a40d751 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ dist # Stores VSCode versions used for testing VSCode extensions .vscode-test +.vscode # yarn v2 .yarn/cache @@ -133,7 +134,7 @@ dist # Others **/.DS_Store .dccache - +.python-version .idea # component test report From de8faba7ed4f908f69485aa2776c71e806fbcc44 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Mon, 21 Oct 2024 13:38:24 +0530 Subject: [PATCH 03/12] chore: lint check github workflow issue for non js files fixed --- .github/workflows/verify.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 115cad4248..e8b1920b87 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -32,11 +32,18 @@ jobs: uses: Ana06/get-changed-files@v1.2 with: token: ${{ secrets.GITHUB_TOKEN }} + + - name: Filter JS/TS Files + run: | + echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt + if [ ! -s changed_files.txt ]; then + echo "No JS/TS files to format or lint." + exit 0 + fi - name: Run format Checks run: | - npx prettier ${{steps.files.outputs.added_modified}} --write - + npx prettier --write $(cat changed_files.txt) - run: git diff --exit-code - name: Formatting Error message From 3f8e75d984258bb88b9458ccfbc101ddcbb98c05 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Fri, 25 Oct 2024 11:46:47 +0530 Subject: [PATCH 04/12] chore: refactoring version conversion adapter to readable format --- .github/workflows/verify.yml | 3 +- src/controllers/source.ts | 1 + .../util/conversionStrategies/abstractions.ts | 5 ++ .../conversionStrategies/strategyDefault.ts | 15 +++++ .../conversionStrategies/strategyV0ToV1.ts | 11 +++ .../conversionStrategies/strategyV1ToV0.ts | 10 +++ .../conversionStrategies/strategyV1ToV2.ts | 37 ++++++++++ .../conversionStrategies/strategyV2ToV0.ts | 18 +++++ .../conversionStrategies/strategyV2ToV1.ts | 18 +++++ src/controllers/util/index.ts | 67 +++++++++++++++---- src/controllers/util/versionConversion.ts | 65 ++++++++++++++++++ src/types/index.ts | 1 + 12 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 src/controllers/util/conversionStrategies/abstractions.ts create mode 100644 src/controllers/util/conversionStrategies/strategyDefault.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV0ToV1.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV1ToV0.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV1ToV2.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV2ToV0.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV2ToV1.ts create mode 100644 src/controllers/util/versionConversion.ts diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index e8b1920b87..4fca34673a 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -32,7 +32,7 @@ jobs: uses: Ana06/get-changed-files@v1.2 with: token: ${{ secrets.GITHUB_TOKEN }} - + - name: Filter JS/TS Files run: | echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt @@ -45,7 +45,6 @@ jobs: run: | npx prettier --write $(cat changed_files.txt) - run: git diff --exit-code - - name: Formatting Error message if: ${{ failure() }} run: | diff --git a/src/controllers/source.ts b/src/controllers/source.ts index 230636f193..8b6d2d70f8 100644 --- a/src/controllers/source.ts +++ b/src/controllers/source.ts @@ -12,6 +12,7 @@ export class SourceController { const events = ctx.request.body as object[]; const { version, source }: { version: string; source: string } = ctx.params; const integrationService = ServiceSelector.getNativeSourceService(); + try { const { implementationVersion, input } = ControllerUtility.adaptInputToVersion( source, diff --git a/src/controllers/util/conversionStrategies/abstractions.ts b/src/controllers/util/conversionStrategies/abstractions.ts new file mode 100644 index 0000000000..f25bc374a2 --- /dev/null +++ b/src/controllers/util/conversionStrategies/abstractions.ts @@ -0,0 +1,5 @@ +import { SourceInputConversionResult } from '../../../types'; + +export abstract class VersionConversionStrategy { + abstract convert(sourceEvents: I[]): SourceInputConversionResult[]; +} diff --git a/src/controllers/util/conversionStrategies/strategyDefault.ts b/src/controllers/util/conversionStrategies/strategyDefault.ts new file mode 100644 index 0000000000..44b9fbf312 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyDefault.ts @@ -0,0 +1,15 @@ +import { SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyDefault extends VersionConversionStrategy< + NonNullable, + NonNullable +> { + convert( + sourceEvents: NonNullable[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV0ToV1.ts b/src/controllers/util/conversionStrategies/strategyV0ToV1.ts new file mode 100644 index 0000000000..28f170c4dd --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV0ToV1.ts @@ -0,0 +1,11 @@ +import { SourceInput, SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV0ToV1 extends VersionConversionStrategy, SourceInput> { + convert(sourceEvents: NonNullable[]): SourceInputConversionResult[] { + // This should be deprecated along with v0-webhook-rudder-server deprecation + return sourceEvents.map((sourceEvent) => ({ + output: { event: sourceEvent, source: undefined } as SourceInput, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV0.ts b/src/controllers/util/conversionStrategies/strategyV1ToV0.ts new file mode 100644 index 0000000000..d0894099a5 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV1ToV0.ts @@ -0,0 +1,10 @@ +import { SourceInput, SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV1ToV0 extends VersionConversionStrategy> { + convert(sourceEvents: SourceInput[]): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent.event as NonNullable, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts new file mode 100644 index 0000000000..0db03cc811 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -0,0 +1,37 @@ +import { + SourceInput, + SourceInputConversionResult, + SourceInputV2, + SourceRequestV2, +} from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV1ToV2 extends VersionConversionStrategy { + convert(sourceEvents: SourceInput[]): SourceInputConversionResult[] { + // Currently this is not being used + // Hold off on testing this until atleast one v2 source has been implemented + return sourceEvents.map((sourceEvent) => { + try { + const sourceRequest: SourceRequestV2 = { + method: '', + url: '', + proto: '', + headers: {}, + query_parameters: {}, + body: JSON.stringify(sourceEvent.event), + }; + const sourceInputV2: SourceInputV2 = { + request: sourceRequest, + source: sourceEvent.source, + }; + return { + output: sourceInputV2, + }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v1 to v2 spec'); + return { output: {} as SourceInputV2, conversionError }; + } + }); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts new file mode 100644 index 0000000000..031039e538 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts @@ -0,0 +1,18 @@ +import { SourceInputConversionResult, SourceInputV2 } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV2ToV0 extends VersionConversionStrategy> { + convert(sourceEvents: SourceInputV2[]): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => { + try { + const v0Event = JSON.parse(sourceEvent.request.body); + v0Event.query_parameters = sourceEvent.request.query_parameters; + return { output: v0Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v0 spec'); + return { output: {} as NonNullable, conversionError }; + } + }); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts new file mode 100644 index 0000000000..7ddafd782e --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -0,0 +1,18 @@ +import { SourceInput, SourceInputConversionResult, SourceInputV2 } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV2ToV1 extends VersionConversionStrategy { + convert(sourceEvents: SourceInputV2[]): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => { + try { + const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; + v1Event.event.query_parameters = sourceEvent.request.query_parameters; + return { output: v1Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v1 spec'); + return { output: {} as SourceInput, conversionError }; + } + }); + } +} diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index b562381ed6..b6fa909d27 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -12,10 +12,12 @@ import { SourceInput, SourceInputConversionResult, SourceInputV2, + SourceRequestV2, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; import { EventType, MappedToDestinationKey } from '../../constants'; +import { versionConversionFactory } from './versionConversion'; export class ControllerUtility { private static sourceVersionMap: Map = new Map(); @@ -55,6 +57,36 @@ export class ControllerUtility { })); } + private static convertSourceInputv1Tov2( + sourceEvents: SourceInput[], + ): SourceInputConversionResult[] { + // Currently this is not being used + // Hold off on testing this until atleast one v2 source has been implemented + return sourceEvents.map((sourceEvent) => { + try { + const sourceRequest: SourceRequestV2 = { + method: '', + url: '', + proto: '', + headers: {}, + query_parameters: {}, + body: JSON.stringify(sourceEvent.event), + }; + const sourceInputV2: SourceInputV2 = { + request: sourceRequest, + source: sourceEvent.source, + }; + return { + output: sourceInputV2, + }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v1 to v2 spec'); + return { output: {} as SourceInputV2, conversionError }; + } + }); + } + private static convertSourceInputv0Tov1( sourceEvents: unknown[], ): SourceInputConversionResult[] { @@ -102,19 +134,28 @@ export class ControllerUtility { ): { implementationVersion: string; input: SourceInputConversionResult>[] } { const sourceToVersionMap = this.getSourceVersionsMap(); const implementationVersion = sourceToVersionMap.get(sourceType); - let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ - output: event, - })); - if (requestVersion === 'v0' && implementationVersion === 'v1') { - updatedInput = this.convertSourceInputv0Tov1(input); - } else if (requestVersion === 'v1' && implementationVersion === 'v0') { - updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); - } else if (requestVersion === 'v2' && implementationVersion === 'v0') { - updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); - } else if (requestVersion === 'v2' && implementationVersion === 'v1') { - updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); - } - return { implementationVersion, input: updatedInput }; + + const conversionStrategy = versionConversionFactory.getStrategy( + requestVersion, + implementationVersion, + ); + return { implementationVersion, input: conversionStrategy.convert(input) }; + + // let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ + // output: event, + // })); + // if (requestVersion === 'v0' && implementationVersion === 'v1') { + // updatedInput = this.convertSourceInputv0Tov1(input); + // } else if (requestVersion === 'v1' && implementationVersion === 'v0') { + // updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); + // } else if (requestVersion === 'v1' && implementationVersion === 'v2') { + // updatedInput = this.convertSourceInputv1Tov2(input as SourceInput[]); + // } else if (requestVersion === 'v2' && implementationVersion === 'v0') { + // updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); + // } else if (requestVersion === 'v2' && implementationVersion === 'v1') { + // updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); + // } + // return { implementationVersion, input: updatedInput }; } private static getCompatibleStatusCode(status: number): number { diff --git a/src/controllers/util/versionConversion.ts b/src/controllers/util/versionConversion.ts new file mode 100644 index 0000000000..3058531f57 --- /dev/null +++ b/src/controllers/util/versionConversion.ts @@ -0,0 +1,65 @@ +import { VersionConversionStrategy } from './conversionStrategies/abstractions'; +import { StrategyDefault } from './conversionStrategies/strategyDefault'; +import { StrategyV0ToV1 } from './conversionStrategies/strategyV0ToV1'; +import { StrategyV1ToV0 } from './conversionStrategies/strategyV1ToV0'; +import { StrategyV1ToV2 } from './conversionStrategies/strategyV1ToV2'; +import { StrategyV2ToV0 } from './conversionStrategies/strategyV2ToV0'; +import { StrategyV2ToV1 } from './conversionStrategies/strategyV2ToV1'; + +export class VersionConversionFactory { + private strategyCache: Map> = new Map(); + + private getCase(requestVersion: string, implementationVersion: string) { + return `${String(requestVersion)}-to-${String(implementationVersion)}`; + } + + public getStrategy( + requestVersion: string, + implementationVersion: string, + ): VersionConversionStrategy { + const versionCase = this.getCase(requestVersion, implementationVersion); + + if (this.strategyCache.has(versionCase)) { + const cachedStrategy = this.strategyCache.get(versionCase); + if (cachedStrategy) { + return cachedStrategy; + } + } + + let strategy: VersionConversionStrategy; + + switch (versionCase) { + case 'v0-to-v1': + strategy = new StrategyV0ToV1(); + break; + + case 'v1-to-v0': + strategy = new StrategyV1ToV0(); + break; + + case 'v1-to-v2': + strategy = new StrategyV1ToV2(); + break; + + case 'v2-to-v0': + strategy = new StrategyV2ToV0(); + break; + + case 'v2-to-v1': + strategy = new StrategyV2ToV1(); + break; + + default: + strategy = new StrategyDefault(); + break; + } + + if (strategy) { + this.strategyCache[versionCase] = strategy; + } + + return strategy; + } +} + +export const versionConversionFactory = new VersionConversionFactory(); diff --git a/src/types/index.ts b/src/types/index.ts index 0bc2cbc33b..54ff3a994e 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -403,6 +403,7 @@ export { UserDeletionResponse, SourceInput, SourceInputV2, + SourceRequestV2, Source, SourceInputConversionResult, UserTransformationLibrary, From 5705f2e28a5c8e123a4018b2c6815f4143b4df67 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Mon, 28 Oct 2024 21:16:06 +0530 Subject: [PATCH 05/12] chore: stricter types, extra test cases for v2 --- .gitignore | 2 +- .../conversionStrategies/strategyV1ToV2.ts | 23 ++-- .../conversionStrategies/strategyV2ToV1.ts | 4 +- src/controllers/util/index.test.ts | 124 ++++++++++++++++++ src/controllers/util/index.ts | 97 -------------- src/middlewares/routeActivation.ts | 15 --- src/types/index.ts | 15 ++- 7 files changed, 151 insertions(+), 129 deletions(-) diff --git a/.gitignore b/.gitignore index 624a40d751..84421f49d9 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,4 @@ dist # component test report test_reports/ -temp/ +temp/ \ No newline at end of file diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts index 0db03cc811..b4f04ef858 100644 --- a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -8,21 +8,26 @@ import { VersionConversionStrategy } from './abstractions'; export class StrategyV1ToV2 extends VersionConversionStrategy { convert(sourceEvents: SourceInput[]): SourceInputConversionResult[] { - // Currently this is not being used - // Hold off on testing this until atleast one v2 source has been implemented return sourceEvents.map((sourceEvent) => { try { + const sourceEventParam = { ...sourceEvent }; + + let queryParameters: Record | undefined; + if (sourceEventParam.event && sourceEventParam.event.query_parameters) { + queryParameters = sourceEventParam.event.query_parameters; + delete sourceEventParam.event.query_parameters; + } + const sourceRequest: SourceRequestV2 = { - method: '', - url: '', - proto: '', - headers: {}, - query_parameters: {}, - body: JSON.stringify(sourceEvent.event), + body: JSON.stringify(sourceEventParam.event), }; + if (queryParameters) { + sourceRequest.query_parameters = queryParameters; + } + const sourceInputV2: SourceInputV2 = { request: sourceRequest, - source: sourceEvent.source, + source: sourceEventParam.source, }; return { output: sourceInputV2, diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts index 7ddafd782e..0872d549f0 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -6,7 +6,9 @@ export class StrategyV2ToV1 extends VersionConversionStrategy { try { const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; - v1Event.event.query_parameters = sourceEvent.request.query_parameters; + if (sourceEvent.request) { + v1Event.event.query_parameters = sourceEvent.request.query_parameters; + } return { output: v1Event }; } catch (err) { const conversionError = diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 6ab2336b71..138572a8ea 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -201,6 +201,38 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + it('should fail trying to convert input from v2 to v0 format when the request version is v2 and the implementation version is v0', () => { + const sourceType = 'pipedream'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v0', + input: [ + { + output: {}, + conversionError: new SyntaxError('Unexpected end of JSON input'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + it('should convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { const sourceType = 'webhook'; const requestVersion = 'v2'; @@ -269,6 +301,38 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + it('should fail trying to convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { + const sourceType = 'webhook'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v1', + input: [ + { + output: {}, + conversionError: new SyntaxError('Unexpected end of JSON input'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + // Should return an empty array when the input is an empty array it('should return an empty array when the input is an empty array', () => { const sourceType = 'pipedream'; @@ -280,6 +344,66 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + + it('should convert input from v1 to v2 format when the request version is v1 and the implementation version is v2', () => { + const sourceType = 'someSourceType'; + const requestVersion = 'v1'; + + // Mock return value for getSourceVersionsMap + jest + .spyOn(ControllerUtility as any, 'getSourceVersionsMap') + .mockReturnValue(new Map([['someSourceType', 'v2']])); + + const input = [ + { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: { key: 'value' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: {}, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + + const expected = { + implementationVersion: 'v2', + input: [ + { + output: { + request: { + body: '{"key":"value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + request: { + body: '{"key":"value"}', + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + request: { + body: '{}', + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); }); type timestampTestCases = { diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index b6fa909d27..ab2a0f5dc3 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -9,10 +9,7 @@ import { ProcessorTransformationRequest, RouterTransformationRequestData, RudderMessage, - SourceInput, SourceInputConversionResult, - SourceInputV2, - SourceRequestV2, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; @@ -49,84 +46,6 @@ export class ControllerUtility { return this.sourceVersionMap; } - private static convertSourceInputv1Tov0( - sourceEvents: SourceInput[], - ): SourceInputConversionResult>[] { - return sourceEvents.map((sourceEvent) => ({ - output: sourceEvent.event as NonNullable, - })); - } - - private static convertSourceInputv1Tov2( - sourceEvents: SourceInput[], - ): SourceInputConversionResult[] { - // Currently this is not being used - // Hold off on testing this until atleast one v2 source has been implemented - return sourceEvents.map((sourceEvent) => { - try { - const sourceRequest: SourceRequestV2 = { - method: '', - url: '', - proto: '', - headers: {}, - query_parameters: {}, - body: JSON.stringify(sourceEvent.event), - }; - const sourceInputV2: SourceInputV2 = { - request: sourceRequest, - source: sourceEvent.source, - }; - return { - output: sourceInputV2, - }; - } catch (err) { - const conversionError = - err instanceof Error ? err : new Error('error converting v1 to v2 spec'); - return { output: {} as SourceInputV2, conversionError }; - } - }); - } - - private static convertSourceInputv0Tov1( - sourceEvents: unknown[], - ): SourceInputConversionResult[] { - return sourceEvents.map((sourceEvent) => ({ - output: { event: sourceEvent, source: undefined } as SourceInput, - })); - } - - private static convertSourceInputv2Tov0( - sourceEvents: SourceInputV2[], - ): SourceInputConversionResult>[] { - return sourceEvents.map((sourceEvent) => { - try { - const v0Event = JSON.parse(sourceEvent.request.body); - v0Event.query_parameters = sourceEvent.request.query_parameters; - return { output: v0Event }; - } catch (err) { - const conversionError = - err instanceof Error ? err : new Error('error converting v2 to v0 spec'); - return { output: {} as NonNullable, conversionError }; - } - }); - } - - private static convertSourceInputv2Tov1( - sourceEvents: SourceInputV2[], - ): SourceInputConversionResult[] { - return sourceEvents.map((sourceEvent) => { - try { - const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; - v1Event.event.query_parameters = sourceEvent.request.query_parameters; - return { output: v1Event }; - } catch (err) { - const conversionError = - err instanceof Error ? err : new Error('error converting v2 to v1 spec'); - return { output: {} as SourceInput, conversionError }; - } - }); - } - public static adaptInputToVersion( sourceType: string, requestVersion: string, @@ -140,22 +59,6 @@ export class ControllerUtility { implementationVersion, ); return { implementationVersion, input: conversionStrategy.convert(input) }; - - // let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ - // output: event, - // })); - // if (requestVersion === 'v0' && implementationVersion === 'v1') { - // updatedInput = this.convertSourceInputv0Tov1(input); - // } else if (requestVersion === 'v1' && implementationVersion === 'v0') { - // updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); - // } else if (requestVersion === 'v1' && implementationVersion === 'v2') { - // updatedInput = this.convertSourceInputv1Tov2(input as SourceInput[]); - // } else if (requestVersion === 'v2' && implementationVersion === 'v0') { - // updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); - // } else if (requestVersion === 'v2' && implementationVersion === 'v1') { - // updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); - // } - // return { implementationVersion, input: updatedInput }; } private static getCompatibleStatusCode(status: number): number { diff --git a/src/middlewares/routeActivation.ts b/src/middlewares/routeActivation.ts index 126749b083..ffb1e15e80 100644 --- a/src/middlewares/routeActivation.ts +++ b/src/middlewares/routeActivation.ts @@ -106,19 +106,4 @@ export class RouteActivationMiddleware { RouteActivationMiddleware.shouldActivateRoute(destination, deliveryFilterList), ); } - - // This middleware will be used by source endpoint when we completely deprecate v0, v1 versions. - public static isVersionAllowed(ctx: Context, next: Next) { - const { version } = ctx.params; - if (version === 'v0' || version === 'v1') { - ctx.status = 500; - ctx.body = - '/v0, /v1 versioned endpoints are deprecated. Use /v2 version endpoint. This is probably caused because of source transformation call from an outdated rudder-server version. Please upgrade rudder-server to a minimum of 1.xx.xx version.'; - } else if (version === 'v2') { - next(); - } else { - ctx.status = 404; - ctx.body = 'Path not found. Verify the version of your api call.'; - } - } } diff --git a/src/types/index.ts b/src/types/index.ts index 54ff3a994e..ee225bb0c0 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -352,17 +352,20 @@ type Source = { }; type SourceInput = { - event: NonNullable[]; + event: { + query_parameters?: any; + [key: string]: any; + }; source?: Source; }; type SourceRequestV2 = { - method: string; - url: string; - proto: string; + method?: string; + url?: string; + proto?: string; body: string; - headers: NonNullable; - query_parameters: NonNullable; + headers?: Record; + query_parameters?: Record; }; type SourceInputV2 = { From b10a39f1e329f4b2efe2d433e064febcbb13354d Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 11 Nov 2024 11:12:07 +0000 Subject: [PATCH 06/12] chore(release): 1.84.0 --- CHANGELOG.md | 17 +++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7591edec..c3c5528ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.84.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.2...v1.84.0) (2024-11-11) + + +### Features + +* gaec migration ([#3855](https://github.com/rudderlabs/rudder-transformer/issues/3855)) ([7a26459](https://github.com/rudderlabs/rudder-transformer/commit/7a264590b61d3d31d5559c8ac53fd572b40cddec)) +* **GARL:** support vdm next for GARL ([#3835](https://github.com/rudderlabs/rudder-transformer/issues/3835)) ([f4b38eb](https://github.com/rudderlabs/rudder-transformer/commit/f4b38eba3ca8dff602915853fda5cd7ca284bba3)) +* update on twitter_ads ([#3856](https://github.com/rudderlabs/rudder-transformer/issues/3856)) ([adc8976](https://github.com/rudderlabs/rudder-transformer/commit/adc8976990fa98c5b874472aee180cadfabb0088)) + + +### Bug Fixes + +* adding throttled status code for server unavailable error in salesforce ([#3862](https://github.com/rudderlabs/rudder-transformer/issues/3862)) ([fa93f09](https://github.com/rudderlabs/rudder-transformer/commit/fa93f0917d4f75fc197a6ea4c574d37faa0a3f77)) +* linkedin ads conversionValue object as well as price is not mandatory ([#3860](https://github.com/rudderlabs/rudder-transformer/issues/3860)) ([bfd7edc](https://github.com/rudderlabs/rudder-transformer/commit/bfd7edc5608c60a39644a1d4ad6e15e5dbcbea0e)) +* marketo bulk upload handle special chars ([#3859](https://github.com/rudderlabs/rudder-transformer/issues/3859)) ([f959a7d](https://github.com/rudderlabs/rudder-transformer/commit/f959a7dc2487dc7e36377f5f2e265014f692f476)) +* unsafe property getting set via set value library ([#3853](https://github.com/rudderlabs/rudder-transformer/issues/3853)) ([80d7b41](https://github.com/rudderlabs/rudder-transformer/commit/80d7b417be7a0e459de49caca25aba43ffdba337)) + ### [1.83.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.1...v1.83.2) (2024-11-05) diff --git a/package-lock.json b/package-lock.json index 0965688626..0422078f21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.84.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.84.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 65b7313e88..907c340cbf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.84.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 7ad4423499959684ecabff6b50a77695c1749f35 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Tue, 12 Nov 2024 12:42:44 +0530 Subject: [PATCH 07/12] chore: remove query_parameter injection --- .../util/conversionStrategies/strategyV2ToV0.ts | 1 - .../util/conversionStrategies/strategyV2ToV1.ts | 3 --- src/controllers/util/index.test.ts | 12 ++++++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts index 031039e538..5d90b8cdda 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts @@ -6,7 +6,6 @@ export class StrategyV2ToV0 extends VersionConversionStrategy { try { const v0Event = JSON.parse(sourceEvent.request.body); - v0Event.query_parameters = sourceEvent.request.query_parameters; return { output: v0Event }; } catch (err) { const conversionError = diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts index 0872d549f0..d651917096 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -6,9 +6,6 @@ export class StrategyV2ToV1 extends VersionConversionStrategy { try { const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; - if (sourceEvent.request) { - v1Event.event.query_parameters = sourceEvent.request.query_parameters; - } return { output: v1Event }; } catch (err) { const conversionError = diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 138572a8ea..f1503f7f81 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -190,9 +190,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, - { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, - { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + { output: { key: 'value' } }, + { output: { key: 'value' } }, + { output: { key: 'value' } }, ], }; @@ -277,19 +277,19 @@ describe('adaptInputToVersion', () => { input: [ { output: { - event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + event: { key: 'value' }, source: { id: 'source_id', config: { configField1: 'configVal1' } }, }, }, { output: { - event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + event: { key: 'value' }, source: { id: 'source_id', config: { configField1: 'configVal1' } }, }, }, { output: { - event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + event: { key: 'value' }, source: { id: 'source_id', config: { configField1: 'configVal1' } }, }, }, From 70ab9c814c295fc0e7d0b606e5b981510b0bbe3f Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Tue, 12 Nov 2024 14:50:45 +0530 Subject: [PATCH 08/12] chore: fix workflow to consider empty commits --- .github/workflows/verify.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4fca34673a..4bd66285bd 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -43,8 +43,12 @@ jobs: - name: Run format Checks run: | - npx prettier --write $(cat changed_files.txt) + if [ -s changed_files.txt ]; then + npx prettier --write $(cat changed_files.txt) + fi + - run: git diff --exit-code + - name: Formatting Error message if: ${{ failure() }} run: | From 1c134f84601aaea78581078137cb9955de576f9e Mon Sep 17 00:00:00 2001 From: Aanshi Lahoti <110057617+aanshi07@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:36:09 +0530 Subject: [PATCH 09/12] feat: iterable EUDC (#3828) * feat: iterable EUDC * chore: simplyfied functions * chore: modifications added * chore: fix * chore: modification for endpoint * chore: removed dataCenter to check default --- src/v0/destinations/iterable/config.js | 36 ++-- src/v0/destinations/iterable/transform.js | 21 ++- src/v0/destinations/iterable/util.js | 32 ++-- .../iterable/processor/aliasTestData.ts | 59 ++++++- .../iterable/processor/identifyTestData.ts | 57 +++++++ .../iterable/processor/pageScreenTestData.ts | 65 +++++++- .../iterable/processor/trackTestData.ts | 55 +++++++ .../destinations/iterable/router/data.ts | 155 ++++++++++++++++++ 8 files changed, 439 insertions(+), 41 deletions(-) diff --git a/src/v0/destinations/iterable/config.js b/src/v0/destinations/iterable/config.js index f74fdb4975..125367875f 100644 --- a/src/v0/destinations/iterable/config.js +++ b/src/v0/destinations/iterable/config.js @@ -1,42 +1,45 @@ const { getMappingConfig } = require('../../util'); -const BASE_URL = 'https://api.iterable.com/api/'; +const BASE_URL = { + USDC: 'https://api.iterable.com/api/', + EUDC: 'https://api.eu.iterable.com/api/', +}; const ConfigCategory = { IDENTIFY_BROWSER: { name: 'IterableRegisterBrowserTokenConfig', action: 'identifyBrowser', - endpoint: `${BASE_URL}users/registerBrowserToken`, + endpoint: `users/registerBrowserToken`, }, IDENTIFY_DEVICE: { name: 'IterableRegisterDeviceTokenConfig', action: 'identifyDevice', - endpoint: `${BASE_URL}users/registerDeviceToken`, + endpoint: `users/registerDeviceToken`, }, IDENTIFY: { name: 'IterableIdentifyConfig', action: 'identify', - endpoint: `${BASE_URL}users/update`, + endpoint: `users/update`, }, PAGE: { name: 'IterablePageConfig', action: 'page', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, SCREEN: { name: 'IterablePageConfig', action: 'screen', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, TRACK: { name: 'IterableTrackConfig', action: 'track', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, TRACK_PURCHASE: { name: 'IterableTrackPurchaseConfig', action: 'trackPurchase', - endpoint: `${BASE_URL}commerce/trackPurchase`, + endpoint: `commerce/trackPurchase`, }, PRODUCT: { name: 'IterableProductConfig', @@ -46,7 +49,7 @@ const ConfigCategory = { UPDATE_CART: { name: 'IterableProductConfig', action: 'updateCart', - endpoint: `${BASE_URL}commerce/updateCart`, + endpoint: `commerce/updateCart`, }, DEVICE: { name: 'IterableDeviceConfig', @@ -56,30 +59,33 @@ const ConfigCategory = { ALIAS: { name: 'IterableAliasConfig', action: 'alias', - endpoint: `${BASE_URL}users/updateEmail`, + endpoint: `users/updateEmail`, }, CATALOG: { name: 'IterableCatalogConfig', action: 'catalogs', - endpoint: `${BASE_URL}catalogs`, + endpoint: `catalogs`, }, }; const mappingConfig = getMappingConfig(ConfigCategory, __dirname); +// Function to construct endpoint based on the selected data center +const constructEndpoint = (dataCenter, category) => { + const baseUrl = BASE_URL[dataCenter] || BASE_URL.USDC; // Default to USDC if not found + return `${baseUrl}${category.endpoint}`; +}; + const IDENTIFY_MAX_BATCH_SIZE = 1000; const IDENTIFY_MAX_BODY_SIZE_IN_BYTES = 4000000; -const IDENTIFY_BATCH_ENDPOINT = 'https://api.iterable.com/api/users/bulkUpdate'; const TRACK_MAX_BATCH_SIZE = 8000; -const TRACK_BATCH_ENDPOINT = 'https://api.iterable.com/api/events/trackBulk'; module.exports = { mappingConfig, ConfigCategory, - TRACK_BATCH_ENDPOINT, + constructEndpoint, TRACK_MAX_BATCH_SIZE, IDENTIFY_MAX_BATCH_SIZE, - IDENTIFY_BATCH_ENDPOINT, IDENTIFY_MAX_BODY_SIZE_IN_BYTES, }; diff --git a/src/v0/destinations/iterable/transform.js b/src/v0/destinations/iterable/transform.js index 207a8d1186..dd67deef69 100644 --- a/src/v0/destinations/iterable/transform.js +++ b/src/v0/destinations/iterable/transform.js @@ -14,6 +14,7 @@ const { filterEventsAndPrepareBatchRequests, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + getCategoryWithEndpoint, } = require('./util'); const { constructPayload, @@ -116,12 +117,11 @@ const responseBuilderForRegisterDeviceOrBrowserTokenEvents = (message, destinati /** * Function to find category value - * @param {*} messageType * @param {*} message * @returns */ -const getCategory = (messageType, message) => { - const eventType = messageType.toLowerCase(); +const getCategory = (message, dataCenter) => { + const eventType = message.type.toLowerCase(); switch (eventType) { case EventType.IDENTIFY: @@ -129,17 +129,17 @@ const getCategory = (messageType, message) => { get(message, MappedToDestinationKey) && getDestinationExternalIDInfoForRetl(message, 'ITERABLE').objectType !== 'users' ) { - return ConfigCategory.CATALOG; + return getCategoryWithEndpoint(ConfigCategory.CATALOG, dataCenter); } - return ConfigCategory.IDENTIFY; + return getCategoryWithEndpoint(ConfigCategory.IDENTIFY, dataCenter); case EventType.PAGE: - return ConfigCategory.PAGE; + return getCategoryWithEndpoint(ConfigCategory.PAGE, dataCenter); case EventType.SCREEN: - return ConfigCategory.SCREEN; + return getCategoryWithEndpoint(ConfigCategory.SCREEN, dataCenter); case EventType.TRACK: - return getCategoryUsingEventName(message); + return getCategoryUsingEventName(message, dataCenter); case EventType.ALIAS: - return ConfigCategory.ALIAS; + return getCategoryWithEndpoint(ConfigCategory.ALIAS, dataCenter); default: throw new InstrumentationError(`Message type ${eventType} not supported`); } @@ -150,8 +150,7 @@ const process = (event) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } - const messageType = message.type.toLowerCase(); - const category = getCategory(messageType, message); + const category = getCategory(message, destination.Config.dataCenter); const response = responseBuilder(message, category, destination); if (hasMultipleResponses(message, category, destination.Config)) { diff --git a/src/v0/destinations/iterable/util.js b/src/v0/destinations/iterable/util.js index 7c1509c2b7..b918600253 100644 --- a/src/v0/destinations/iterable/util.js +++ b/src/v0/destinations/iterable/util.js @@ -14,10 +14,9 @@ const { ConfigCategory, mappingConfig, TRACK_MAX_BATCH_SIZE, - TRACK_BATCH_ENDPOINT, IDENTIFY_MAX_BATCH_SIZE, - IDENTIFY_BATCH_ENDPOINT, IDENTIFY_MAX_BODY_SIZE_IN_BYTES, + constructEndpoint, } = require('./config'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { EventType, MappedToDestinationKey } = require('../../../constants'); @@ -88,12 +87,17 @@ const hasMultipleResponses = (message, category, config) => { return isIdentifyEvent && isIdentifyCategory && hasToken && hasRegisterDeviceOrBrowserKey; }; +const getCategoryWithEndpoint = (categoryConfig, dataCenter) => ({ + ...categoryConfig, + endpoint: constructEndpoint(dataCenter, categoryConfig), +}); + /** * Returns category value * @param {*} message * @returns */ -const getCategoryUsingEventName = (message) => { +const getCategoryUsingEventName = (message, dataCenter) => { let { event } = message; if (typeof event === 'string') { event = event.toLowerCase(); @@ -101,12 +105,12 @@ const getCategoryUsingEventName = (message) => { switch (event) { case 'order completed': - return ConfigCategory.TRACK_PURCHASE; + return getCategoryWithEndpoint(ConfigCategory.TRACK_PURCHASE, dataCenter); case 'product added': case 'product removed': - return ConfigCategory.UPDATE_CART; + return getCategoryWithEndpoint(ConfigCategory.UPDATE_CART, dataCenter); default: - return ConfigCategory.TRACK; + return getCategoryWithEndpoint(ConfigCategory.TRACK, dataCenter); } }; @@ -444,8 +448,8 @@ const processUpdateUserBatch = (chunk, registerDeviceOrBrowserTokenEvents) => { batchEventResponse.batchedRequest.body.JSON = { users: batch.users }; const { destination, metadata, nonBatchedRequests } = batch; - const { apiKey } = destination.Config; - + const { apiKey, dataCenter } = destination.Config; + const IDENTIFY_BATCH_ENDPOINT = constructEndpoint(dataCenter, { endpoint: 'users/bulkUpdate' }); const batchedResponse = combineBatchedAndNonBatchedEvents( apiKey, metadata, @@ -552,8 +556,8 @@ const processTrackBatch = (chunk) => { const metadata = []; const { destination } = chunk[0]; - const { apiKey } = destination.Config; - + const { apiKey, dataCenter } = destination.Config; + const TRACK_BATCH_ENDPOINT = constructEndpoint(dataCenter, { endpoint: 'events/trackBulk' }); chunk.forEach((event) => { metadata.push(event.metadata); events.push(get(event, `${MESSAGE_JSON_PATH}`)); @@ -653,12 +657,13 @@ const mapRegisterDeviceOrBrowserTokenEventsWithJobId = (events) => { */ const categorizeEvent = (event) => { const { message, metadata, destination, error } = event; + const { dataCenter } = destination.Config; if (error) { return { type: 'error', data: event }; } - if (message.endpoint === ConfigCategory.IDENTIFY.endpoint) { + if (message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY)) { return { type: 'updateUser', data: { message, metadata, destination } }; } @@ -667,8 +672,8 @@ const categorizeEvent = (event) => { } if ( - message.endpoint === ConfigCategory.IDENTIFY_BROWSER.endpoint || - message.endpoint === ConfigCategory.IDENTIFY_DEVICE.endpoint + message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY_BROWSER) || + message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY_DEVICE) ) { return { type: 'registerDeviceOrBrowser', data: { message, metadata, destination } }; } @@ -753,4 +758,5 @@ module.exports = { filterEventsAndPrepareBatchRequests, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + getCategoryWithEndpoint, }; diff --git a/test/integrations/destinations/iterable/processor/aliasTestData.ts b/test/integrations/destinations/iterable/processor/aliasTestData.ts index cac43767bb..1ee4134859 100644 --- a/test/integrations/destinations/iterable/processor/aliasTestData.ts +++ b/test/integrations/destinations/iterable/processor/aliasTestData.ts @@ -1,4 +1,8 @@ -import { generateMetadata, transformResultBuilder } from './../../../testUtils'; +import { + generateMetadata, + overrideDestination, + transformResultBuilder, +} from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -15,6 +19,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -94,4 +99,56 @@ export const aliasTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-alias-test-1', + name: 'iterable', + description: 'Alias call with dataCenter as EUDC', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update email payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId: 'anonId', + userId: 'new@email.com', + previousId: 'old@email.com', + name: 'ApplicationLoaded', + context: {}, + properties, + type: 'alias', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: 'https://api.eu.iterable.com/api/users/updateEmail', + JSON: { + currentEmail: 'old@email.com', + newEmail: 'new@email.com', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/identifyTestData.ts b/test/integrations/destinations/iterable/processor/identifyTestData.ts index d05f87a11f..21d294e232 100644 --- a/test/integrations/destinations/iterable/processor/identifyTestData.ts +++ b/test/integrations/destinations/iterable/processor/identifyTestData.ts @@ -2,6 +2,7 @@ import { generateMetadata, transformResultBuilder, generateIndentifyPayload, + overrideDestination, } from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -19,6 +20,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -55,6 +57,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const updateUserEndpoint = 'https://api.iterable.com/api/users/update'; +const updateUserEndpointEUDC = 'https://api.eu.iterable.com/api/users/update'; export const identifyTestData: ProcessorTestData[] = [ { @@ -404,4 +407,58 @@ export const identifyTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-identify-test-7', + name: 'iterable', + description: 'Indentify call to update user in iterable with EUDC dataCenter', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update user payload with all user traits and updateUserEndpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + context: { + traits: user1Traits, + }, + traits: user1Traits, + type: 'identify', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: updateUserEndpointEUDC, + JSON: { + email: user1Traits.email, + userId: anonymousId, + dataFields: user1Traits, + preferUserId: false, + mergeNestedObjects: true, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/pageScreenTestData.ts b/test/integrations/destinations/iterable/processor/pageScreenTestData.ts index 074d6b56df..a27cf9fe3b 100644 --- a/test/integrations/destinations/iterable/processor/pageScreenTestData.ts +++ b/test/integrations/destinations/iterable/processor/pageScreenTestData.ts @@ -1,4 +1,8 @@ -import { generateMetadata, transformResultBuilder } from './../../../testUtils'; +import { + generateMetadata, + overrideDestination, + transformResultBuilder, +} from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -15,6 +19,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -43,6 +48,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const pageEndpoint = 'https://api.iterable.com/api/events/track'; +const pageEndpointEUDC = 'https://api.eu.iterable.com/api/events/track'; export const pageScreenTestData: ProcessorTestData[] = [ { @@ -406,4 +412,61 @@ export const pageScreenTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-page-test-4', + name: 'iterable', + description: 'Page call with dataCenter as EUDC', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain endpoint as pageEndpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + name: 'ApplicationLoaded', + context: { + traits: { + email: 'sayan@gmail.com', + }, + }, + properties, + type: 'page', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: pageEndpointEUDC, + JSON: { + userId: anonymousId, + dataFields: properties, + email: 'sayan@gmail.com', + createdAt: 1598631966468, + eventName: 'ApplicationLoaded page', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/trackTestData.ts b/test/integrations/destinations/iterable/processor/trackTestData.ts index 296275ad77..2b7d2a9c47 100644 --- a/test/integrations/destinations/iterable/processor/trackTestData.ts +++ b/test/integrations/destinations/iterable/processor/trackTestData.ts @@ -1,6 +1,7 @@ import { generateMetadata, generateTrackPayload, + overrideDestination, transformResultBuilder, } from './../../../testUtils'; import { Destination } from '../../../../../src/types'; @@ -19,6 +20,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -126,6 +128,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const endpoint = 'https://api.iterable.com/api/events/track'; +const endpointEUDC = 'https://api.eu.iterable.com/api/events/track'; const updateCartEndpoint = 'https://api.iterable.com/api/commerce/updateCart'; const trackPurchaseEndpoint = 'https://api.iterable.com/api/commerce/trackPurchase'; @@ -714,4 +717,56 @@ export const trackTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-track-test-9', + name: 'iterable', + description: 'Track call to add event with user with EUDC dataCenter', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain event properties, event name and endpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + event: 'Email Opened', + type: 'track', + context: {}, + properties, + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: endpointEUDC, + JSON: { + userId: 'anonId', + createdAt: 1598631966468, + eventName: 'Email Opened', + dataFields: properties, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/router/data.ts b/test/integrations/destinations/iterable/router/data.ts index 09eedc8eb8..1917c078eb 100644 --- a/test/integrations/destinations/iterable/router/data.ts +++ b/test/integrations/destinations/iterable/router/data.ts @@ -247,6 +247,7 @@ export const data = [ destination: { Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -308,6 +309,7 @@ export const data = [ destConfig: { defaultConfig: [ 'apiKey', + 'dataCenter', 'mapToSingleEvent', 'trackAllPages', 'trackCategorisedPages', @@ -339,6 +341,7 @@ export const data = [ }, Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: true, trackAllPages: false, trackCategorisedPages: true, @@ -414,6 +417,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -442,6 +446,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -472,6 +477,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -623,6 +629,7 @@ export const data = [ destination: { Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -686,6 +693,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -732,6 +740,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -765,6 +774,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -821,6 +831,7 @@ export const data = [ destConfig: { defaultConfig: [ 'apiKey', + 'dataCenter', 'mapToSingleEvent', 'trackAllPages', 'trackCategorisedPages', @@ -852,6 +863,7 @@ export const data = [ }, Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: true, trackAllPages: false, trackCategorisedPages: true, @@ -867,4 +879,147 @@ export const data = [ }, }, }, + { + name: 'iterable', + description: 'Simple identify call with EUDC dataCenter', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + receivedAt: '2022-09-27T11:12:59.080Z', + sentAt: '2022-09-27T11:13:03.777Z', + messageId: '9ad41366-8060-4c9f-b181-f6bea67d5469', + originalTimestamp: '2022-09-27T11:13:03.777Z', + traits: { ruchira: 'donaldbaker@ellis.com', new_field2: 'GB' }, + channel: 'sources', + rudderId: '3d51640c-ab09-42c1-b7b2-db6ab433b35e', + context: { + sources: { + version: 'feat.SupportForTrack', + job_run_id: 'ccpdlajh6cfi19mr1vs0', + task_run_id: 'ccpdlajh6cfi19mr1vsg', + batch_id: '4917ad78-280b-40d2-a30d-119434152a0f', + job_id: '2FLKJDcTdjPHQpq7pUjB34dQ5w6/Syncher', + task_id: 'rows_100', + }, + mappedToDestination: 'true', + externalId: [ + { id: 'Tiffany', type: 'ITERABLE-test-ruchira', identifierType: 'itemId' }, + ], + }, + timestamp: '2022-09-27T11:12:59.079Z', + type: 'identify', + userId: 'Tiffany', + recordId: '10', + request_ip: '10.1.86.248', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + { + message: { + receivedAt: '2022-09-27T11:12:59.080Z', + sentAt: '2022-09-27T11:13:03.777Z', + messageId: '9ad41366-8060-4c9f-b181-f6bea67d5469', + originalTimestamp: '2022-09-27T11:13:03.777Z', + traits: { ruchira: 'abc@ellis.com', new_field2: 'GB1' }, + channel: 'sources', + rudderId: '3d51640c-ab09-42c1-b7b2-db6ab433b35e', + context: { + sources: { + version: 'feat.SupportForTrack', + job_run_id: 'ccpdlajh6cfi19mr1vs0', + task_run_id: 'ccpdlajh6cfi19mr1vsg', + batch_id: '4917ad78-280b-40d2-a30d-119434152a0f', + job_id: '2FLKJDcTdjPHQpq7pUjB34dQ5w6/Syncher', + task_id: 'rows_100', + }, + mappedToDestination: 'true', + externalId: [ + { id: 'ABC', type: 'ITERABLE-test-ruchira', identifierType: 'itemId' }, + ], + }, + timestamp: '2022-09-27T11:12:59.079Z', + type: 'identify', + userId: 'Tiffany', + recordId: '10', + request_ip: '10.1.86.248', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + ], + destType: 'iterable', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.eu.iterable.com/api/catalogs/test-ruchira/items', + headers: { + 'Content-Type': 'application/json', + api_key: '583af2f8-15ba-49c0-8511-76383e7de07e', + }, + params: {}, + body: { + JSON: { + documents: { + Tiffany: { ruchira: 'donaldbaker@ellis.com', new_field2: 'GB' }, + ABC: { ruchira: 'abc@ellis.com', new_field2: 'GB1' }, + }, + replaceUploadedFieldsOnly: true, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { jobId: 2, userId: 'u1' }, + { jobId: 2, userId: 'u1' }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + ], + }, + }, + }, + }, ]; From 51bbc02d5b00ce1b8fe8c91b4a7041e926bae9bd Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Wed, 13 Nov 2024 16:30:35 +0530 Subject: [PATCH 10/12] feat: now getting consent related fields from connection config from retl for GARL (#3877) --- .../recordTransform.js | 38 +++++++++++++++++-- .../router/data.ts | 4 +- .../router/record.ts | 2 + 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index 1c6284cd09..5866b66538 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -29,6 +29,8 @@ const processRecordEventArray = ( typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, operationType, ) => { let outputPayloads = {}; @@ -81,7 +83,10 @@ const processRecordEventArray = ( const toSendEvents = []; Object.values(outputPayloads).forEach((data) => { - const consentObj = populateConsentFromConfig(destination.Config, consentConfigMap); + const consentObj = populateConsentFromConfig( + { userDataConsent, personalizationConsent }, + consentConfigMap, + ); toSendEvents.push( responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), ); @@ -96,7 +101,14 @@ function preparepayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { audienceId, typeOfList, isHashRequired, userSchema } = config; + const { + audienceId, + typeOfList, + isHashRequired, + userSchema, + userDataConsent, + personalizationConsent, + } = config; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), @@ -117,6 +129,8 @@ function preparepayload(events, config) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'remove', ); } @@ -132,6 +146,8 @@ function preparepayload(events, config) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'add', ); } @@ -147,6 +163,8 @@ function preparepayload(events, config) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'add', ); } @@ -169,19 +187,29 @@ function preparepayload(events, config) { function processRecordInputsV0(groupedRecordInputs) { const { destination, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired, userSchema } = destination.Config; + const { + audienceId, + typeOfList, + isHashRequired, + userSchema, + userDataConsent, + personalizationConsent, + } = destination.Config; return preparepayload(groupedRecordInputs, { audienceId: getOperationAudienceId(audienceId, message), typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, }); } function processRecordInputsV1(groupedRecordInputs) { const { connection, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired } = connection.config.destination; + const { audienceId, typeOfList, isHashRequired, userDataConsent, personalizationConsent } = + connection.config.destination; const identifiers = message?.identifiers; let userSchema; @@ -202,6 +230,8 @@ function processRecordInputsV1(groupedRecordInputs) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, }); } diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts index 6878e81f0d..12d5c65f8f 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts @@ -884,8 +884,8 @@ export const data = [ listId: '7090784486', customerId: '7693729833', consent: { - adPersonalization: 'UNSPECIFIED', - adUserData: 'UNSPECIFIED', + adPersonalization: 'GRANTED', + adUserData: 'GRANTED', }, }, body: { diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts index 2661500b4d..b3f1095b1d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts @@ -75,6 +75,8 @@ const connection2: Connection = { isHashRequired: true, typeOfList: 'userID', audienceId: '7090784486', + personalizationConsent: 'GRANTED', + userDataConsent: 'GRANTED', }, }, }; From e6b5fb749dcf66036257a439ce994b9aa9eacebf Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Wed, 13 Nov 2024 17:58:38 +0530 Subject: [PATCH 11/12] chore: output of conversion is optional --- .../conversionStrategies/strategyV1ToV2.ts | 2 +- .../conversionStrategies/strategyV2ToV0.ts | 2 +- .../conversionStrategies/strategyV2ToV1.ts | 2 +- src/controllers/util/index.test.ts | 2 -- src/services/source/nativeIntegration.ts | 25 ++++++++++++++----- src/types/index.ts | 2 +- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts index b4f04ef858..7cf4e77808 100644 --- a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -35,7 +35,7 @@ export class StrategyV1ToV2 extends VersionConversionStrategy, conversionError }; + return { conversionError }; } }); } diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts index d651917096..52cade0d9d 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -10,7 +10,7 @@ export class StrategyV2ToV1 extends VersionConversionStrategy { implementationVersion: 'v0', input: [ { - output: {}, conversionError: new SyntaxError('Unexpected end of JSON input'), }, ], @@ -322,7 +321,6 @@ describe('adaptInputToVersion', () => { implementationVersion: 'v1', input: [ { - output: {}, conversionError: new SyntaxError('Unexpected end of JSON input'), }, ], diff --git a/src/services/source/nativeIntegration.ts b/src/services/source/nativeIntegration.ts index 58a6a19649..078716df96 100644 --- a/src/services/source/nativeIntegration.ts +++ b/src/services/source/nativeIntegration.ts @@ -53,12 +53,25 @@ export class NativeIntegrationSourceService implements SourceService { metaTO, ); } - const newSourceEvent = sourceEvent.output; - const { headers } = newSourceEvent; - delete newSourceEvent.headers; - const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = - await sourceHandler.process(newSourceEvent); - return SourcePostTransformationService.handleSuccessEventsSource(respEvents, { headers }); + + if (sourceEvent.output) { + const newSourceEvent = sourceEvent.output; + + const { headers } = newSourceEvent; + if (headers) { + delete newSourceEvent.headers; + } + + const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = + await sourceHandler.process(newSourceEvent); + return SourcePostTransformationService.handleSuccessEventsSource(respEvents, { + headers, + }); + } + return SourcePostTransformationService.handleFailureEventsSource( + new Error('Error post version converstion, converstion output is undefined'), + metaTO, + ); } catch (error: FixMe) { stats.increment('source_transform_errors', { source: sourceType, diff --git a/src/types/index.ts b/src/types/index.ts index ee225bb0c0..7c07f659df 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -374,7 +374,7 @@ type SourceInputV2 = { }; type SourceInputConversionResult = { - output: T; + output?: T; conversionError?: Error; }; From e90d2ad878f1350bc5b905a27772f9dc380ce5a5 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Wed, 13 Nov 2024 18:32:09 +0530 Subject: [PATCH 12/12] chore: added test cases for v1-v2 conversion errors --- src/controllers/util/index.test.ts | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index c5cefdeeeb..4559bccc52 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -402,6 +402,47 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + + it('should fail trying to convert input from v1 to v2 format when the request version is v1 and the implementation version is v2', () => { + const sourceType = 'someSourceType'; + const requestVersion = 'v1'; + + // Mock return value for getSourceVersionsMap + jest + .spyOn(ControllerUtility as any, 'getSourceVersionsMap') + .mockReturnValue(new Map([['someSourceType', 'v2']])); + + const input = [ + { + event: { + key: 'value', + query_parameters: { paramkey: ['paramvalue'] }, + largeNumber: BigInt(12345678901234567890n), + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: { key: 'value', largeNumber: BigInt(12345678901234567890n) }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + + const expected = { + implementationVersion: 'v2', + input: [ + { + conversionError: new TypeError('Do not know how to serialize a BigInt'), + }, + { + conversionError: new TypeError('Do not know how to serialize a BigInt'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); }); type timestampTestCases = {