diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 115cad4248..604ff1eaee 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -33,9 +33,19 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} + - name: Filter JS/TS Files + run: | + changed_files=$(echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' || true) + if [ -z "$changed_files" ]; then + echo "No JS/TS files to format or lint." + exit 0 + fi + - name: Run format Checks run: | - npx prettier ${{steps.files.outputs.added_modified}} --write + if [ -s changed_files.txt ]; then + npx prettier --write $(cat changed_files.txt) + fi - run: git diff --exit-code diff --git a/.gitignore b/.gitignore index 09c536ebb8..84421f49d9 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ dist # Stores VSCode versions used for testing VSCode extensions .vscode-test +.vscode # yarn v2 .yarn/cache @@ -133,9 +134,9 @@ dist # Others **/.DS_Store .dccache - +.python-version .idea # component test report test_reports/ -temp/ +temp/ \ No newline at end of file diff --git a/.husky/commit-msg b/.husky/commit-msg deleted file mode 100755 index 84dc58a421..0000000000 --- a/.husky/commit-msg +++ /dev/null @@ -1,2 +0,0 @@ - -npm run commit-msg diff --git a/.husky/pre-commit b/.husky/pre-commit deleted file mode 100755 index af964838e1..0000000000 --- a/.husky/pre-commit +++ /dev/null @@ -1,2 +0,0 @@ - -npm run pre-commit diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7591edec..bb1779fd3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,55 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21) + + +### Bug Fixes + +* braze subscription batch size ([#3897](https://github.com/rudderlabs/rudder-transformer/issues/3897)) ([ca71a31](https://github.com/rudderlabs/rudder-transformer/commit/ca71a318e4d8d098116fe539964b699254f58617)) +* stringifying session ID for airship ([#3896](https://github.com/rudderlabs/rudder-transformer/issues/3896)) ([bb0b9dc](https://github.com/rudderlabs/rudder-transformer/commit/bb0b9dc1e5a56e8141c6cb56e89835ba61ee7761)) + +## [1.85.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.84.0...v1.85.0) (2024-11-18) + + +### Features + +* added support to eu/us2 datacenter for gainsight px destination ([#3871](https://github.com/rudderlabs/rudder-transformer/issues/3871)) ([12ac3de](https://github.com/rudderlabs/rudder-transformer/commit/12ac3de6e7cc91a6cd52c33bc342f74bbaa8a631)) +* iterable EUDC ([#3828](https://github.com/rudderlabs/rudder-transformer/issues/3828)) ([1c134f8](https://github.com/rudderlabs/rudder-transformer/commit/1c134f84601aaea78581078137cb9955de576f9e)) +* iterable EUDC deleteUsers ([#3881](https://github.com/rudderlabs/rudder-transformer/issues/3881)) ([becb4fa](https://github.com/rudderlabs/rudder-transformer/commit/becb4fa54e9093ed69779f54c36864cb9d28d321)) +* moved userSchema to connection config in GARL vdmv2 ([#3870](https://github.com/rudderlabs/rudder-transformer/issues/3870)) ([640a11e](https://github.com/rudderlabs/rudder-transformer/commit/640a11eb3dca5735fed3ad9ad5bd058974b069d6)) +* now getting consent related fields from connection config from retl for GARL ([#3877](https://github.com/rudderlabs/rudder-transformer/issues/3877)) ([51bbc02](https://github.com/rudderlabs/rudder-transformer/commit/51bbc02d5b00ce1b8fe8c91b4a7041e926bae9bd)) +* onboard linkedin audience destination ([#3857](https://github.com/rudderlabs/rudder-transformer/issues/3857)) ([f3ff409](https://github.com/rudderlabs/rudder-transformer/commit/f3ff4092d455508dd3354ffb22d345fa97f4d1f2)) +* onboarding intercom v2 retl support ([#3843](https://github.com/rudderlabs/rudder-transformer/issues/3843)) ([3d7db73](https://github.com/rudderlabs/rudder-transformer/commit/3d7db7366e30df31c37cc473e344da82b49ed885)) +* sources v2 spec support along with adapters ([04c0694](https://github.com/rudderlabs/rudder-transformer/commit/04c069486bdd3c101906fa6c621e983090fcab25)) +* sources v2 spec support along with adapters ([#3810](https://github.com/rudderlabs/rudder-transformer/issues/3810)) ([c51cfbb](https://github.com/rudderlabs/rudder-transformer/commit/c51cfbb4664a8531dce23b2d06fe40997f95697e)) +* update pinterest_tag single product events with new mapping ([#3858](https://github.com/rudderlabs/rudder-transformer/issues/3858)) ([8520278](https://github.com/rudderlabs/rudder-transformer/commit/85202781de3464bd46fe910159d2b143cd4209e8)) + + +### Bug Fixes + +* adding logger for undefined source event ([#3879](https://github.com/rudderlabs/rudder-transformer/issues/3879)) ([79e5979](https://github.com/rudderlabs/rudder-transformer/commit/79e597907eee126b4187e4534b2aa2253d1431da)) +* adding uuid transformation for airship ([#3884](https://github.com/rudderlabs/rudder-transformer/issues/3884)) ([a80f874](https://github.com/rudderlabs/rudder-transformer/commit/a80f87486dc93b423e4fe6efbee6f4cb8330ba02)) +* handling invalid timestamp for adjust source ([#3866](https://github.com/rudderlabs/rudder-transformer/issues/3866)) ([d57f48e](https://github.com/rudderlabs/rudder-transformer/commit/d57f48e989d18d469bea0de94293bc685300945b)) +* revert gaec changes ([#3885](https://github.com/rudderlabs/rudder-transformer/issues/3885)) ([0aeaa39](https://github.com/rudderlabs/rudder-transformer/commit/0aeaa391b025fc68de6e3d63a6721f067c5be318)) + +## [1.84.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.2...v1.84.0) (2024-11-11) + + +### Features + +* gaec migration ([#3855](https://github.com/rudderlabs/rudder-transformer/issues/3855)) ([7a26459](https://github.com/rudderlabs/rudder-transformer/commit/7a264590b61d3d31d5559c8ac53fd572b40cddec)) +* **GARL:** support vdm next for GARL ([#3835](https://github.com/rudderlabs/rudder-transformer/issues/3835)) ([f4b38eb](https://github.com/rudderlabs/rudder-transformer/commit/f4b38eba3ca8dff602915853fda5cd7ca284bba3)) +* update on twitter_ads ([#3856](https://github.com/rudderlabs/rudder-transformer/issues/3856)) ([adc8976](https://github.com/rudderlabs/rudder-transformer/commit/adc8976990fa98c5b874472aee180cadfabb0088)) + + +### Bug Fixes + +* adding throttled status code for server unavailable error in salesforce ([#3862](https://github.com/rudderlabs/rudder-transformer/issues/3862)) ([fa93f09](https://github.com/rudderlabs/rudder-transformer/commit/fa93f0917d4f75fc197a6ea4c574d37faa0a3f77)) +* linkedin ads conversionValue object as well as price is not mandatory ([#3860](https://github.com/rudderlabs/rudder-transformer/issues/3860)) ([bfd7edc](https://github.com/rudderlabs/rudder-transformer/commit/bfd7edc5608c60a39644a1d4ad6e15e5dbcbea0e)) +* marketo bulk upload handle special chars ([#3859](https://github.com/rudderlabs/rudder-transformer/issues/3859)) ([f959a7d](https://github.com/rudderlabs/rudder-transformer/commit/f959a7dc2487dc7e36377f5f2e265014f692f476)) +* unsafe property getting set via set value library ([#3853](https://github.com/rudderlabs/rudder-transformer/issues/3853)) ([80d7b41](https://github.com/rudderlabs/rudder-transformer/commit/80d7b417be7a0e459de49caca25aba43ffdba337)) + ### [1.83.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.1...v1.83.2) (2024-11-05) diff --git a/package-lock.json b/package-lock.json index 0965688626..40033e278b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.85.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.85.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", @@ -19,7 +19,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.12", + "@rudderstack/integrations-lib": "^0.2.13", "@rudderstack/json-template-engine": "^0.18.0", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", @@ -73,7 +73,7 @@ "truncate-utf8-bytes": "^1.0.2", "ua-parser-js": "^1.0.37", "unset-value": "^2.0.1", - "uuid": "^9.0.0", + "uuid": "^9.0.1", "valid-url": "^1.0.9", "zod": "^3.22.4" }, @@ -6602,9 +6602,10 @@ } }, "node_modules/@rudderstack/integrations-lib": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.12.tgz", - "integrity": "sha512-xy+T9SHFkSeVDd4svGOyrTtIGljZ/l4qUh5o5EQWk3dTStzaV9mKnbXLsG62kEO3aTmCVg+VYr4OPwZY2+6rxQ==", + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.13.tgz", + "integrity": "sha512-MBI+OQpnYAuOzRlbGCnUX6oVfQsYA7daZ8z07WmqQYQtWFOfd2yFbaxKclu+R/a8W7+jBo4gvbW+ScEW6h+Mgg==", + "license": "MIT", "dependencies": { "axios": "^1.4.0", "axios-mock-adapter": "^1.22.0", @@ -21872,11 +21873,12 @@ }, "node_modules/uuid": { "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], - "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } diff --git a/package.json b/package.json index 65b7313e88..e0b17a4f47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.85.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -64,7 +64,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.12", + "@rudderstack/integrations-lib": "^0.2.13", "@rudderstack/json-template-engine": "^0.18.0", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", @@ -118,7 +118,7 @@ "truncate-utf8-bytes": "^1.0.2", "ua-parser-js": "^1.0.37", "unset-value": "^2.0.1", - "uuid": "^9.0.0", + "uuid": "^9.0.1", "valid-url": "^1.0.9", "zod": "^3.22.4" }, diff --git a/src/cdk/v2/destinations/linkedin_audience/config.ts b/src/cdk/v2/destinations/linkedin_audience/config.ts new file mode 100644 index 0000000000..86ea94425a --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/config.ts @@ -0,0 +1,10 @@ +export const SUPPORTED_EVENT_TYPE = 'record'; +export const ACTION_TYPES = ['insert', 'delete']; +export const BASE_ENDPOINT = 'https://api.linkedin.com/rest'; +export const USER_ENDPOINT = '/dmpSegments/audienceId/users'; +export const COMPANY_ENDPOINT = '/dmpSegments/audienceId/companies'; +export const FIELD_MAP = { + sha256Email: 'SHA256_EMAIL', + sha512Email: 'SHA512_EMAIL', + googleAid: 'GOOGLE_AID', +}; diff --git a/src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml b/src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml new file mode 100644 index 0000000000..f3f4ce0772 --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml @@ -0,0 +1,89 @@ +bindings: + - path: ./config + exportAll: true + - path: ./utils + exportAll: true + - name: defaultRequestConfig + path: ../../../../v0/util + +steps: + - name: validateInput + description: Validate input, if all the required fields are available or not. + template: | + const config = .connection.config.destination; + const secret = .metadata.secret; + let messageType = .message.type; + $.assertConfig(config.audienceId, "Audience Id is not present. Aborting"); + $.assertConfig(secret.accessToken, "Access Token is not present. Aborting"); + $.assertConfig(config.audienceType, "audienceType is not present. Aborting"); + $.assert(messageType, "Message Type is not present. Aborting message."); + $.assert(messageType.toLowerCase() === $.SUPPORTED_EVENT_TYPE, `Event type ${.message.type.toLowerCase()} is not supported. Aborting message.`); + $.assert(.message.fields, "`fields` is not present. Aborting message."); + $.assert(.message.identifiers, "`identifiers` is not present inside properties. Aborting message."); + $.assert($.containsAll([.message.action], $.ACTION_TYPES), "Unsupported action type. Aborting message.") + + - name: getConfigs + description: This step fetches the configs from different places and combines them. + template: | + const config = .connection.config.destination; + { + audienceType: config.audienceType, + audienceId: config.audienceId, + accessToken: .metadata.secret.accessToken, + isHashRequired: config.isHashRequired, + } + + - name: prepareUserTypeBasePayload + condition: $.outputs.getConfigs.audienceType === 'user' + steps: + - name: prepareUserIds + description: Prepare user ids for user audience type + template: | + const identifiers = $.outputs.getConfigs.isHashRequired === true ? + $.hashIdentifiers(.message.identifiers) : + .message.identifiers; + $.prepareUserIds(identifiers) + + - name: preparePayload + description: Prepare base payload for user audiences + template: | + const payload = { + 'elements': [ + { + 'action': $.generateActionType(.message.action), + 'userIds': $.outputs.prepareUserTypeBasePayload.prepareUserIds, + ....message.fields + } + ] + } + payload; + + - name: prepareCompanyTypeBasePayload + description: Prepare base payload for company audiences + condition: $.outputs.getConfigs.audienceType === 'company' + template: | + const payload = { + 'elements': [ + { + 'action': $.generateActionType(.message.action), + ....message.identifiers, + ....message.fields + } + ] + } + payload; + + - name: buildResponseForProcessTransformation + description: build response depending upon batch size + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = {...$.outputs.prepareUserTypeBasePayload, ...$.outputs.prepareCompanyTypeBasePayload}; + response.endpoint = $.generateEndpoint($.outputs.getConfigs.audienceType, $.outputs.getConfigs.audienceId); + response.headers = { + "Authorization": "Bearer " + $.outputs.getConfigs.accessToken, + "Content-Type": "application/json", + "X-RestLi-Method": "BATCH_CREATE", + "X-Restli-Protocol-Version": "2.0.0", + "LinkedIn-Version": "202409" + }; + response; diff --git a/src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml new file mode 100644 index 0000000000..fe16ab786a --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml @@ -0,0 +1,40 @@ +bindings: + - path: ./utils + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "message": .[], + "destination": ^ [idx].destination, + "metadata": ^ [idx].metadata + })[] + + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + $.batchResponseBuilder($.outputs.successfulEvents); + + - name: finalPayload + template: | + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/linkedin_audience/utils.ts b/src/cdk/v2/destinations/linkedin_audience/utils.ts new file mode 100644 index 0000000000..12f5a0572b --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/utils.ts @@ -0,0 +1,87 @@ +import lodash from 'lodash'; +import { hashToSha256 } from '@rudderstack/integrations-lib'; +import { createHash } from 'crypto'; +import { BASE_ENDPOINT, COMPANY_ENDPOINT, FIELD_MAP, USER_ENDPOINT } from './config'; + +export function hashIdentifiers(identifiers: string[]): Record { + const hashedIdentifiers = {}; + Object.keys(identifiers).forEach((key) => { + if (key === 'sha256Email') { + hashedIdentifiers[key] = hashToSha256(identifiers[key]); + } else if (key === 'sha512Email') { + hashedIdentifiers[key] = createHash('sha512').update(identifiers[key]).digest('hex'); + } else { + hashedIdentifiers[key] = identifiers[key]; + } + }); + return hashedIdentifiers; +} + +export function prepareUserIds( + identifiers: Record, +): { idType: string; idValue: string }[] { + const userIds: { idType: string; idValue: string }[] = []; + Object.keys(identifiers).forEach((key) => { + userIds.push({ idType: FIELD_MAP[key], idValue: identifiers[key] }); + }); + return userIds; +} + +export function generateEndpoint(audienceType: string, audienceId: string) { + if (audienceType === 'user') { + return BASE_ENDPOINT + USER_ENDPOINT.replace('audienceId', audienceId); + } + return BASE_ENDPOINT + COMPANY_ENDPOINT.replace('audienceId', audienceId); +} + +export function batchResponseBuilder(successfulEvents) { + const chunkOnActionType = lodash.groupBy( + successfulEvents, + (event) => event.message[0].body.JSON.elements[0].action, + ); + const result: any = []; + Object.keys(chunkOnActionType).forEach((actionType) => { + const firstEvent = chunkOnActionType[actionType][0]; + const { method, endpoint, headers, type, version } = firstEvent.message[0]; + const batchEvent = { + batchedRequest: { + body: { + JSON: { elements: firstEvent.message[0].body.JSON.elements }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version, + type, + method, + endpoint, + headers, + params: {}, + files: {}, + }, + metadata: [firstEvent.metadata], + batched: true, + statusCode: 200, + destination: firstEvent.destination, + }; + firstEvent.metadata = [firstEvent.metadata]; + chunkOnActionType[actionType].forEach((element, index) => { + if (index !== 0) { + batchEvent.batchedRequest.body.JSON.elements.push(element.message[0].body.JSON.elements[0]); + batchEvent.metadata.push(element.metadata); + } + }); + result.push(batchEvent); + }); + return result; +} + +export const generateActionType = (actionType: string): string => { + if (actionType === 'insert') { + return 'ADD'; + } + if (actionType === 'delete') { + return 'REMOVE'; + } + return actionType; +}; diff --git a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml index 64d391c888..aebb7b0667 100644 --- a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml +++ b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml @@ -11,6 +11,8 @@ bindings: path: ../../../../v0/util - name: validateEventName path: ../../../../v0/util + - path: '@rudderstack/integrations-lib' + steps: - name: checkIfProcessed condition: .message.statusCode @@ -67,9 +69,9 @@ steps: "event_id": $.getOneByPaths(., ^.destination.Config.deduplicationKey) ?? .messageId, "app_id": ^.destination.Config.appId, "advertiser_id": ^.destination.Config.advertiserId, - "partner_name": .properties.partnerName, - "device_carrier": .context.network.carrier, - "wifi": .context.network.wifi + "partner_name": .properties.partnerName ? $.convertToString(.properties.partnerName) : undefined, + "device_carrier": .properties.partnerName ? $.convertToString(.context.network.carrier) : undefined, + "wifi": .context.network.wifi ? Boolean(.context.network.wifi) : undefined }); $.outputs.apiVersion === {{$.API_VERSION.v5}} ? commonFields = commonFields{~["advertiser_id"]}; $.removeUndefinedValues(commonFields) @@ -107,7 +109,7 @@ steps: "client_user_agent": .context.userAgent, "external_id": {{{{$.getGenericPaths("userId")}}}}, "click_id": .properties.clickId, - "partner_id": .traits.partnerId ?? .context.traits.partnerId + "partner_id": .traits.partnerId ?? .context.traits.partnerId ? $.convertToString(.traits.partnerId ?? .context.traits.partnerId) : undefined }); !.destination.Config.sendExternalId ? userFields = userFields{~["external_id"]} : null; userFields = $.removeUndefinedAndNullAndEmptyValues(userFields); @@ -127,17 +129,17 @@ steps: template: | const customFields = .message.().({ "currency": .properties.currency, - "value": .properties.value !== undefined ? String(.properties.value) : - .properties.total !== undefined ? String(.properties.total) : - .properties.revenue !== undefined ? String(.properties.revenue) : undefined, + "value": .properties.value !== undefined ? $.convertToString(.properties.value) : + .properties.total !== undefined ? $.convertToString(.properties.total) : + .properties.revenue !== undefined ? $.convertToString(.properties.revenue) : undefined, "num_items": .properties.numOfItems && Number(.properties.numOfItems), "order_id": .properties.order_id, "search_string": .properties.query, "opt_out_type": .properties.optOutType, - "content_name": .properties.contentName, - "content_category": .properties.contentCategory, - "content_brand": .properties.contentBrand, - "np": .properties.np + "content_name": .properties.contentName ? $.convertToString(.properties.contentName) : undefined, + "content_category": .properties.contentCategory ? $.convertToString(.properties.contentCategory) : undefined, + "content_brand": .properties.contentBrand ? $.convertToString(.properties.contentBrand) : undefined, + "np": .properties.np ? $.convertToString(.properties.np) : undefined }); $.removeUndefinedValues(customFields) @@ -151,11 +153,11 @@ steps: "content_ids": products.(.product_id ?? .sku ?? .id)[], "contents": .message.properties@prop.products.({ "quantity": Number(.quantity ?? prop.quantity ?? 1), - "item_price": String(.price ?? prop.price), - "item_name": String(.name), - "id": .product_id ?? .sku, - "item_category": .category, - "item_brand": .brand + "item_price": $.convertToString(.price ?? prop.price), + "item_name": $.convertToString(.name), + "id": .product_id ?? .sku ? $.convertToString(.product_id ?? .sku) : undefined, + "item_category": .category ? $.convertToString(.category) : undefined, + "item_brand": .brand ? $.convertToString(.brand) : undefined })[] } else: @@ -167,7 +169,11 @@ steps: "content_ids": (props.product_id ?? props.sku ?? props.id)[], "contents": { "quantity": Number(props.quantity) || 1, - "item_price": String(props.price) + "item_price": props.price ? $.convertToString(props.price), + "item_name": props.name ? $.convertToString(props.name), + "id": props.product_id ?? props.sku ? $.convertToString(props.product_id ?? props.sku) : undefined, + "item_category": props.category ? $.convertToString(props.category) : undefined, + "item_brand": props.brand ? $.convertToString(props.brand) : undefined }[] }; - name: combineAllEcomFields diff --git a/src/controllers/__tests__/source.test.ts b/src/controllers/__tests__/source.test.ts index 565f39d559..72bee83282 100644 --- a/src/controllers/__tests__/source.test.ts +++ b/src/controllers/__tests__/source.test.ts @@ -6,6 +6,7 @@ import { applicationRoutes } from '../../routes'; import { NativeIntegrationSourceService } from '../../services/source/nativeIntegration'; import { ServiceSelector } from '../../helpers/serviceSelector'; import { ControllerUtility } from '../util/index'; +import { SourceInputConversionResult } from '../../types'; let server: any; const OLD_ENV = process.env; @@ -38,6 +39,19 @@ const getData = () => { return [{ event: { a: 'b1' } }, { event: { a: 'b2' } }]; }; +const getV2Data = () => { + return [ + { request: { body: '{"a": "b"}' }, source: { id: 1 } }, + { request: { body: '{"a": "b"}' }, source: { id: 1 } }, + ]; +}; + +const getConvertedData = () => { + return getData().map((eventInstance) => { + return { output: eventInstance } as SourceInputConversionResult; + }); +}; + describe('Source controller tests', () => { describe('V0 Source transform tests', () => { test('successful source transform', async () => { @@ -49,7 +63,7 @@ describe('Source controller tests', () => { mockSourceService.sourceTransformRoutine = jest .fn() .mockImplementation((i, s, v, requestMetadata) => { - expect(i).toEqual(getData()); + expect(i).toEqual(getConvertedData()); expect(s).toEqual(sourceType); expect(v).toEqual(version); return testOutput; @@ -66,7 +80,7 @@ describe('Source controller tests', () => { expect(s).toEqual(sourceType); expect(v).toEqual(version); expect(e).toEqual(getData()); - return { implementationVersion: version, input: e }; + return { implementationVersion: version, input: getConvertedData() }; }); const response = await request(server) @@ -139,7 +153,7 @@ describe('Source controller tests', () => { mockSourceService.sourceTransformRoutine = jest .fn() .mockImplementation((i, s, v, requestMetadata) => { - expect(i).toEqual(getData()); + expect(i).toEqual(getConvertedData()); expect(s).toEqual(sourceType); expect(v).toEqual(version); return testOutput; @@ -156,7 +170,7 @@ describe('Source controller tests', () => { expect(s).toEqual(sourceType); expect(v).toEqual(version); expect(e).toEqual(getData()); - return { implementationVersion: version, input: e }; + return { implementationVersion: version, input: getConvertedData() }; }); const response = await request(server) @@ -217,4 +231,93 @@ describe('Source controller tests', () => { expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); }); }); + + describe('V2 Source transform tests', () => { + test('successful source transform', async () => { + const sourceType = '__rudder_test__'; + const version = 'v2'; + const testOutput = [{ event: { a: 'b' }, source: { id: 'id' } }]; + + const mockSourceService = new NativeIntegrationSourceService(); + mockSourceService.sourceTransformRoutine = jest + .fn() + .mockImplementation((i, s, v, requestMetadata) => { + expect(i).toEqual(getConvertedData()); + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + return testOutput; + }); + const getNativeSourceServiceSpy = jest + .spyOn(ServiceSelector, 'getNativeSourceService') + .mockImplementation(() => { + return mockSourceService; + }); + + const adaptInputToVersionSpy = jest + .spyOn(ControllerUtility, 'adaptInputToVersion') + .mockImplementation((s, v, e) => { + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + expect(e).toEqual(getV2Data()); + return { implementationVersion: version, input: getConvertedData() }; + }); + + const response = await request(server) + .post('/v2/sources/__rudder_test__') + .set('Accept', 'application/json') + .send(getV2Data()); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(testOutput); + + expect(response.header['apiversion']).toEqual('2'); + + expect(getNativeSourceServiceSpy).toHaveBeenCalledTimes(1); + expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); + expect(mockSourceService.sourceTransformRoutine).toHaveBeenCalledTimes(1); + }); + + test('failing source transform', async () => { + const sourceType = '__rudder_test__'; + const version = 'v2'; + const mockSourceService = new NativeIntegrationSourceService(); + const getNativeSourceServiceSpy = jest + .spyOn(ServiceSelector, 'getNativeSourceService') + .mockImplementation(() => { + return mockSourceService; + }); + + const adaptInputToVersionSpy = jest + .spyOn(ControllerUtility, 'adaptInputToVersion') + .mockImplementation((s, v, e) => { + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + expect(e).toEqual(getV2Data()); + throw new Error('test error'); + }); + + const response = await request(server) + .post('/v2/sources/__rudder_test__') + .set('Accept', 'application/json') + .send(getV2Data()); + + const expectedResp = [ + { + error: 'test error', + statTags: { + errorCategory: 'transformation', + }, + statusCode: 500, + }, + ]; + + expect(response.status).toEqual(200); + expect(response.body).toEqual(expectedResp); + + expect(response.header['apiversion']).toEqual('2'); + + expect(getNativeSourceServiceSpy).toHaveBeenCalledTimes(1); + expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/src/controllers/source.ts b/src/controllers/source.ts index 230636f193..3d9fa4f4a4 100644 --- a/src/controllers/source.ts +++ b/src/controllers/source.ts @@ -11,7 +11,13 @@ export class SourceController { const requestMetadata = MiscService.getRequestMetadata(ctx); const events = ctx.request.body as object[]; const { version, source }: { version: string; source: string } = ctx.params; + const enrichedMetadata = { + ...requestMetadata, + source, + version, + }; const integrationService = ServiceSelector.getNativeSourceService(); + try { const { implementationVersion, input } = ControllerUtility.adaptInputToVersion( source, @@ -27,6 +33,7 @@ export class SourceController { ); ctx.body = resplist; } catch (err: any) { + logger.error(err?.message || 'error in source transformation', enrichedMetadata); const metaTO = integrationService.getTags(); const resp = SourcePostTransformationService.handleFailureEventsSource(err, metaTO); ctx.body = [resp]; diff --git a/src/controllers/util/conversionStrategies/abstractions.ts b/src/controllers/util/conversionStrategies/abstractions.ts new file mode 100644 index 0000000000..f25bc374a2 --- /dev/null +++ b/src/controllers/util/conversionStrategies/abstractions.ts @@ -0,0 +1,5 @@ +import { SourceInputConversionResult } from '../../../types'; + +export abstract class VersionConversionStrategy { + abstract convert(sourceEvents: I[]): SourceInputConversionResult[]; +} diff --git a/src/controllers/util/conversionStrategies/strategyDefault.ts b/src/controllers/util/conversionStrategies/strategyDefault.ts new file mode 100644 index 0000000000..44b9fbf312 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyDefault.ts @@ -0,0 +1,15 @@ +import { SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyDefault extends VersionConversionStrategy< + NonNullable, + NonNullable +> { + convert( + sourceEvents: NonNullable[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV0ToV1.ts b/src/controllers/util/conversionStrategies/strategyV0ToV1.ts new file mode 100644 index 0000000000..28f170c4dd --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV0ToV1.ts @@ -0,0 +1,11 @@ +import { SourceInput, SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV0ToV1 extends VersionConversionStrategy, SourceInput> { + convert(sourceEvents: NonNullable[]): SourceInputConversionResult[] { + // This should be deprecated along with v0-webhook-rudder-server deprecation + return sourceEvents.map((sourceEvent) => ({ + output: { event: sourceEvent, source: undefined } as SourceInput, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV0.ts b/src/controllers/util/conversionStrategies/strategyV1ToV0.ts new file mode 100644 index 0000000000..d0894099a5 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV1ToV0.ts @@ -0,0 +1,10 @@ +import { SourceInput, SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV1ToV0 extends VersionConversionStrategy> { + convert(sourceEvents: SourceInput[]): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent.event as NonNullable, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts new file mode 100644 index 0000000000..7cf4e77808 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -0,0 +1,42 @@ +import { + SourceInput, + SourceInputConversionResult, + SourceInputV2, + SourceRequestV2, +} from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV1ToV2 extends VersionConversionStrategy { + convert(sourceEvents: SourceInput[]): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => { + try { + const sourceEventParam = { ...sourceEvent }; + + let queryParameters: Record | undefined; + if (sourceEventParam.event && sourceEventParam.event.query_parameters) { + queryParameters = sourceEventParam.event.query_parameters; + delete sourceEventParam.event.query_parameters; + } + + const sourceRequest: SourceRequestV2 = { + body: JSON.stringify(sourceEventParam.event), + }; + if (queryParameters) { + sourceRequest.query_parameters = queryParameters; + } + + const sourceInputV2: SourceInputV2 = { + request: sourceRequest, + source: sourceEventParam.source, + }; + return { + output: sourceInputV2, + }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v1 to v2 spec'); + return { conversionError }; + } + }); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts new file mode 100644 index 0000000000..1145cf9763 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts @@ -0,0 +1,17 @@ +import { SourceInputConversionResult, SourceInputV2 } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV2ToV0 extends VersionConversionStrategy> { + convert(sourceEvents: SourceInputV2[]): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => { + try { + const v0Event = JSON.parse(sourceEvent.request.body); + return { output: v0Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v0 spec'); + return { conversionError }; + } + }); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts new file mode 100644 index 0000000000..52cade0d9d --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -0,0 +1,17 @@ +import { SourceInput, SourceInputConversionResult, SourceInputV2 } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV2ToV1 extends VersionConversionStrategy { + convert(sourceEvents: SourceInputV2[]): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => { + try { + const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; + return { output: v1Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v1 spec'); + return { conversionError }; + } + }); + } +} diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 6065920846..4559bccc52 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -19,9 +19,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: undefined, input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, ], }; @@ -40,9 +40,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, ], }; @@ -71,16 +71,22 @@ describe('adaptInputToVersion', () => { implementationVersion: 'v1', input: [ { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, ], }; @@ -100,9 +106,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v1', input: [ - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, ], }; @@ -131,9 +137,192 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should convert input from v2 to v0 format when the request version is v2 and the implementation version is v0', () => { + const sourceType = 'pipedream'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v0', + input: [ + { output: { key: 'value' } }, + { output: { key: 'value' } }, + { output: { key: 'value' } }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should fail trying to convert input from v2 to v0 format when the request version is v2 and the implementation version is v0', () => { + const sourceType = 'pipedream'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v0', + input: [ + { + conversionError: new SyntaxError('Unexpected end of JSON input'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { + const sourceType = 'webhook'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v1', + input: [ + { + output: { + event: { key: 'value' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + event: { key: 'value' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + event: { key: 'value' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should fail trying to convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { + const sourceType = 'webhook'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v1', + input: [ + { + conversionError: new SyntaxError('Unexpected end of JSON input'), + }, ], }; @@ -153,6 +342,107 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + + it('should convert input from v1 to v2 format when the request version is v1 and the implementation version is v2', () => { + const sourceType = 'someSourceType'; + const requestVersion = 'v1'; + + // Mock return value for getSourceVersionsMap + jest + .spyOn(ControllerUtility as any, 'getSourceVersionsMap') + .mockReturnValue(new Map([['someSourceType', 'v2']])); + + const input = [ + { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: { key: 'value' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: {}, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + + const expected = { + implementationVersion: 'v2', + input: [ + { + output: { + request: { + body: '{"key":"value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + request: { + body: '{"key":"value"}', + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + request: { + body: '{}', + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should fail trying to convert input from v1 to v2 format when the request version is v1 and the implementation version is v2', () => { + const sourceType = 'someSourceType'; + const requestVersion = 'v1'; + + // Mock return value for getSourceVersionsMap + jest + .spyOn(ControllerUtility as any, 'getSourceVersionsMap') + .mockReturnValue(new Map([['someSourceType', 'v2']])); + + const input = [ + { + event: { + key: 'value', + query_parameters: { paramkey: ['paramvalue'] }, + largeNumber: BigInt(12345678901234567890n), + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: { key: 'value', largeNumber: BigInt(12345678901234567890n) }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + + const expected = { + implementationVersion: 'v2', + input: [ + { + conversionError: new TypeError('Do not know how to serialize a BigInt'), + }, + { + conversionError: new TypeError('Do not know how to serialize a BigInt'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); }); type timestampTestCases = { diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index c5bf7ab358..ab2a0f5dc3 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -9,11 +9,12 @@ import { ProcessorTransformationRequest, RouterTransformationRequestData, RudderMessage, - SourceInput, + SourceInputConversionResult, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; import { EventType, MappedToDestinationKey } from '../../constants'; +import { versionConversionFactory } from './versionConversion'; export class ControllerUtility { private static sourceVersionMap: Map = new Map(); @@ -45,30 +46,19 @@ export class ControllerUtility { return this.sourceVersionMap; } - private static convertSourceInputv1Tov0(sourceEvents: SourceInput[]): NonNullable[] { - return sourceEvents.map((sourceEvent) => sourceEvent.event); - } - - private static convertSourceInputv0Tov1(sourceEvents: unknown[]): SourceInput[] { - return sourceEvents.map( - (sourceEvent) => ({ event: sourceEvent, source: undefined }) as SourceInput, - ); - } - public static adaptInputToVersion( sourceType: string, requestVersion: string, input: NonNullable[], - ): { implementationVersion: string; input: NonNullable[] } { + ): { implementationVersion: string; input: SourceInputConversionResult>[] } { const sourceToVersionMap = this.getSourceVersionsMap(); const implementationVersion = sourceToVersionMap.get(sourceType); - let updatedInput: NonNullable[] = input; - if (requestVersion === 'v0' && implementationVersion === 'v1') { - updatedInput = this.convertSourceInputv0Tov1(input); - } else if (requestVersion === 'v1' && implementationVersion === 'v0') { - updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); - } - return { implementationVersion, input: updatedInput }; + + const conversionStrategy = versionConversionFactory.getStrategy( + requestVersion, + implementationVersion, + ); + return { implementationVersion, input: conversionStrategy.convert(input) }; } private static getCompatibleStatusCode(status: number): number { diff --git a/src/controllers/util/versionConversion.ts b/src/controllers/util/versionConversion.ts new file mode 100644 index 0000000000..3058531f57 --- /dev/null +++ b/src/controllers/util/versionConversion.ts @@ -0,0 +1,65 @@ +import { VersionConversionStrategy } from './conversionStrategies/abstractions'; +import { StrategyDefault } from './conversionStrategies/strategyDefault'; +import { StrategyV0ToV1 } from './conversionStrategies/strategyV0ToV1'; +import { StrategyV1ToV0 } from './conversionStrategies/strategyV1ToV0'; +import { StrategyV1ToV2 } from './conversionStrategies/strategyV1ToV2'; +import { StrategyV2ToV0 } from './conversionStrategies/strategyV2ToV0'; +import { StrategyV2ToV1 } from './conversionStrategies/strategyV2ToV1'; + +export class VersionConversionFactory { + private strategyCache: Map> = new Map(); + + private getCase(requestVersion: string, implementationVersion: string) { + return `${String(requestVersion)}-to-${String(implementationVersion)}`; + } + + public getStrategy( + requestVersion: string, + implementationVersion: string, + ): VersionConversionStrategy { + const versionCase = this.getCase(requestVersion, implementationVersion); + + if (this.strategyCache.has(versionCase)) { + const cachedStrategy = this.strategyCache.get(versionCase); + if (cachedStrategy) { + return cachedStrategy; + } + } + + let strategy: VersionConversionStrategy; + + switch (versionCase) { + case 'v0-to-v1': + strategy = new StrategyV0ToV1(); + break; + + case 'v1-to-v0': + strategy = new StrategyV1ToV0(); + break; + + case 'v1-to-v2': + strategy = new StrategyV1ToV2(); + break; + + case 'v2-to-v0': + strategy = new StrategyV2ToV0(); + break; + + case 'v2-to-v1': + strategy = new StrategyV2ToV1(); + break; + + default: + strategy = new StrategyDefault(); + break; + } + + if (strategy) { + this.strategyCache[versionCase] = strategy; + } + + return strategy; + } +} + +export const versionConversionFactory = new VersionConversionFactory(); diff --git a/src/features.ts b/src/features.ts index 9f60d44483..4ff419a7fe 100644 --- a/src/features.ts +++ b/src/features.ts @@ -92,6 +92,7 @@ const defaultFeaturesConfig: FeaturesConfig = { HTTP: true, AMAZON_AUDIENCE: true, INTERCOM_V2: true, + LINKEDIN_AUDIENCE: true, }, regulations: [ 'BRAZE', diff --git a/src/interfaces/SourceService.ts b/src/interfaces/SourceService.ts index c7de8cfe8b..32a7125e7a 100644 --- a/src/interfaces/SourceService.ts +++ b/src/interfaces/SourceService.ts @@ -1,10 +1,14 @@ -import { MetaTransferObject, SourceTransformationResponse } from '../types/index'; +import { + MetaTransferObject, + SourceInputConversionResult, + SourceTransformationResponse, +} from '../types/index'; export interface SourceService { getTags(): MetaTransferObject; sourceTransformRoutine( - sourceEvents: NonNullable[], + sourceEvents: SourceInputConversionResult>[], sourceType: string, version: string, requestMetadata: NonNullable, diff --git a/src/services/source/__tests__/nativeIntegration.test.ts b/src/services/source/__tests__/nativeIntegration.test.ts index 2ef8129cdc..51bb37f5f1 100644 --- a/src/services/source/__tests__/nativeIntegration.test.ts +++ b/src/services/source/__tests__/nativeIntegration.test.ts @@ -44,7 +44,15 @@ describe('NativeIntegration Source Service', () => { }); const service = new NativeIntegrationSourceService(); - const resp = await service.sourceTransformRoutine(events, sourceType, version, requestMetadata); + const adapterConvertedEvents = events.map((eventInstance) => { + return { output: eventInstance }; + }); + const resp = await service.sourceTransformRoutine( + adapterConvertedEvents, + sourceType, + version, + requestMetadata, + ); expect(resp).toEqual(tresponse); @@ -81,7 +89,15 @@ describe('NativeIntegration Source Service', () => { jest.spyOn(stats, 'increment').mockImplementation(() => {}); const service = new NativeIntegrationSourceService(); - const resp = await service.sourceTransformRoutine(events, sourceType, version, requestMetadata); + const adapterConvertedEvents = events.map((eventInstance) => { + return { output: eventInstance }; + }); + const resp = await service.sourceTransformRoutine( + adapterConvertedEvents, + sourceType, + version, + requestMetadata, + ); expect(resp).toEqual(tresponse); diff --git a/src/services/source/nativeIntegration.ts b/src/services/source/nativeIntegration.ts index 5c89de7b92..078716df96 100644 --- a/src/services/source/nativeIntegration.ts +++ b/src/services/source/nativeIntegration.ts @@ -4,6 +4,7 @@ import { ErrorDetailer, MetaTransferObject, RudderMessage, + SourceInputConversionResult, SourceTransformationEvent, SourceTransformationResponse, } from '../../types/index'; @@ -28,7 +29,7 @@ export class NativeIntegrationSourceService implements SourceService { } public async sourceTransformRoutine( - sourceEvents: NonNullable[], + sourceEvents: SourceInputConversionResult>[], sourceType: string, version: string, // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -39,12 +40,38 @@ export class NativeIntegrationSourceService implements SourceService { const respList: SourceTransformationResponse[] = await Promise.all( sourceEvents.map(async (sourceEvent) => { try { - const newSourceEvent = sourceEvent; - const { headers } = newSourceEvent; - delete newSourceEvent.headers; - const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = - await sourceHandler.process(newSourceEvent); - return SourcePostTransformationService.handleSuccessEventsSource(respEvents, { headers }); + if (sourceEvent.conversionError) { + stats.increment('source_transform_errors', { + source: sourceType, + version, + }); + logger.debug(`Error during source Transform: ${sourceEvent.conversionError}`, { + ...logger.getLogMetadata(metaTO.errorDetails), + }); + return SourcePostTransformationService.handleFailureEventsSource( + sourceEvent.conversionError, + metaTO, + ); + } + + if (sourceEvent.output) { + const newSourceEvent = sourceEvent.output; + + const { headers } = newSourceEvent; + if (headers) { + delete newSourceEvent.headers; + } + + const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = + await sourceHandler.process(newSourceEvent); + return SourcePostTransformationService.handleSuccessEventsSource(respEvents, { + headers, + }); + } + return SourcePostTransformationService.handleFailureEventsSource( + new Error('Error post version converstion, converstion output is undefined'), + metaTO, + ); } catch (error: FixMe) { stats.increment('source_transform_errors', { source: sourceType, diff --git a/src/types/index.ts b/src/types/index.ts index 45ec7445c3..7c07f659df 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -352,9 +352,32 @@ type Source = { }; type SourceInput = { - event: NonNullable[]; + event: { + query_parameters?: any; + [key: string]: any; + }; + source?: Source; +}; + +type SourceRequestV2 = { + method?: string; + url?: string; + proto?: string; + body: string; + headers?: Record; + query_parameters?: Record; +}; + +type SourceInputV2 = { + request: SourceRequestV2; source?: Source; }; + +type SourceInputConversionResult = { + output?: T; + conversionError?: Error; +}; + export { ComparatorInput, DeliveryJobState, @@ -382,7 +405,10 @@ export { UserDeletionRequest, UserDeletionResponse, SourceInput, + SourceInputV2, + SourceRequestV2, Source, + SourceInputConversionResult, UserTransformationLibrary, UserTransformationResponse, UserTransformationServiceResponse, diff --git a/src/util/fetchDestinationHandlers.ts b/src/util/fetchDestinationHandlers.ts index 2661ef2e68..fa8cbb47c3 100644 --- a/src/util/fetchDestinationHandlers.ts +++ b/src/util/fetchDestinationHandlers.ts @@ -1,22 +1,18 @@ -import * as V0MarketoBulkUploadFileUpload from '../v0/destinations/marketo_bulk_upload/fileUpload'; -import * as V0MarketoBulkUploadPollStatus from '../v0/destinations/marketo_bulk_upload/poll'; -import * as V0MarketoBulkUploadJobStatus from '../v0/destinations/marketo_bulk_upload/fetchJobStatus'; - const fileUploadHandlers = { v0: { - marketo_bulk_upload: V0MarketoBulkUploadFileUpload, + marketo_bulk_upload: undefined, }, }; const pollStatusHandlers = { v0: { - marketo_bulk_upload: V0MarketoBulkUploadPollStatus, + marketo_bulk_upload: undefined, }, }; const jobStatusHandlers = { v0: { - marketo_bulk_upload: V0MarketoBulkUploadJobStatus, + marketo_bulk_upload: undefined, }, }; diff --git a/src/v0/destinations/airship/data/airshipTrackConfig.json b/src/v0/destinations/airship/data/airshipTrackConfig.json index 1f280f756b..4b09fdb943 100644 --- a/src/v0/destinations/airship/data/airshipTrackConfig.json +++ b/src/v0/destinations/airship/data/airshipTrackConfig.json @@ -22,7 +22,10 @@ { "destKey": "session_id", "sourceKeys": ["properties.sessionId", "context.sessionId"], - "required": false + "required": false, + "metadata": { + "type": "toString" + } }, { "destKey": "transaction", diff --git a/src/v0/destinations/airship/transform.js b/src/v0/destinations/airship/transform.js index 091c9b7f39..fcf18daa7e 100644 --- a/src/v0/destinations/airship/transform.js +++ b/src/v0/destinations/airship/transform.js @@ -22,11 +22,20 @@ const { extractCustomFields, isEmptyObject, simpleProcessRouterDest, + convertToUuid, } = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); const DEFAULT_ACCEPT_HEADER = 'application/vnd.urbanairship+json; version=3'; +const transformSessionId = (rawSessionId) => { + try { + return convertToUuid(rawSessionId); + } catch (error) { + throw new InstrumentationError(`Failed to transform session ID: ${error.message}`); + } +}; + const identifyResponseBuilder = (message, { Config }) => { const tagPayload = constructPayload(message, identifyMapping); const { apiKey, dataCenter } = Config; @@ -128,6 +137,11 @@ const trackResponseBuilder = async (message, { Config }) => { name = name.toLowerCase(); const payload = constructPayload(message, trackMapping); + + // ref : https://docs.airship.com/api/ua/#operation-api-custom-events-post + if (isDefinedAndNotNullAndNotEmpty(payload.session_id)) { + payload.session_id = transformSessionId(payload.session_id); + } let properties = {}; properties = extractCustomFields(message, properties, ['properties'], AIRSHIP_TRACK_EXCLUSION); if (!isEmptyObject(properties)) { diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 71052f8d77..985f2434d5 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -982,7 +982,7 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 25 subscription group + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates }); @@ -1104,8 +1104,8 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // First batch contains 20 subscription group - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group + expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates }); test('check success and failure scenarios both for processBatch', () => { diff --git a/src/v0/destinations/gainsight_px/config.js b/src/v0/destinations/gainsight_px/config.js index cc058f88d2..a5ced4f1a7 100644 --- a/src/v0/destinations/gainsight_px/config.js +++ b/src/v0/destinations/gainsight_px/config.js @@ -1,12 +1,27 @@ const { getMappingConfig } = require('../../util'); const BASE_ENDPOINT = 'https://api.aptrinsic.com/v1'; -const ENDPOINTS = { - USERS_ENDPOINT: `${BASE_ENDPOINT}/users`, - CUSTOM_EVENTS_ENDPOINT: `${BASE_ENDPOINT}/events/custom`, - ACCOUNTS_ENDPOINT: `${BASE_ENDPOINT}/accounts`, +const BASE_EU_ENDPOINT = 'https://api-eu.aptrinsic.com/v1'; +const BASE_US2_ENDPOINT = 'https://api-us2.aptrinsic.com/v1'; + +const getBaseEndpoint = (Config) => { + const { dataCenter } = Config; + switch (dataCenter) { + case 'EU': + return BASE_EU_ENDPOINT; + case 'US2': + return BASE_US2_ENDPOINT; + default: + return BASE_ENDPOINT; + } }; +const getUsersEndpoint = (Config) => `${getBaseEndpoint(Config)}/users`; + +const getCustomEventsEndpoint = (Config) => `${getBaseEndpoint(Config)}/events/custom`; + +const getAccountsEndpoint = (Config) => `${getBaseEndpoint(Config)}/accounts`; + const CONFIG_CATEGORIES = { IDENTIFY: { type: 'identify', name: 'GainsightPX_Identify' }, TRACK: { type: 'track', name: 'GainsightPX_Track' }, @@ -79,10 +94,16 @@ const ACCOUNT_EXCLUSION_FIELDS = [ ]; module.exports = { - ENDPOINTS, USER_EXCLUSION_FIELDS, ACCOUNT_EXCLUSION_FIELDS, identifyMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.IDENTIFY.name], trackMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK.name], groupMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.GROUP.name], + getUsersEndpoint, + getCustomEventsEndpoint, + getAccountsEndpoint, + BASE_ENDPOINT, + BASE_EU_ENDPOINT, + BASE_US2_ENDPOINT, + getBaseEndpoint, }; diff --git a/src/v0/destinations/gainsight_px/config.test.js b/src/v0/destinations/gainsight_px/config.test.js new file mode 100644 index 0000000000..825396d350 --- /dev/null +++ b/src/v0/destinations/gainsight_px/config.test.js @@ -0,0 +1,27 @@ +const { BASE_ENDPOINT, BASE_EU_ENDPOINT, BASE_US2_ENDPOINT, getBaseEndpoint } = require('./config'); + +describe('getBaseEndpoint method test', () => { + it('Should return BASE_ENDPOINT when destination.Config.dataCenter is not "EU" or "US2"', () => { + const Config = { + dataCenter: 'US', + }; + const result = getBaseEndpoint(Config); + expect(result).toBe(BASE_ENDPOINT); + }); + + it('Should return BASE_EU_ENDPOINT when destination.Config.dataCenter is "EU"', () => { + const Config = { + dataCenter: 'EU', + }; + const result = getBaseEndpoint(Config); + expect(result).toBe(BASE_EU_ENDPOINT); + }); + + it('Should return BASE_US2_ENDPOINT when destination.Config.dataCenter is "US2"', () => { + const Config = { + dataCenter: 'US2', + }; + const result = getBaseEndpoint(Config); + expect(result).toBe(BASE_US2_ENDPOINT); + }); +}); diff --git a/src/v0/destinations/gainsight_px/transform.js b/src/v0/destinations/gainsight_px/transform.js index 0911b76b6c..496099a6b4 100644 --- a/src/v0/destinations/gainsight_px/transform.js +++ b/src/v0/destinations/gainsight_px/transform.js @@ -27,12 +27,13 @@ const { formatEventProps, } = require('./util'); const { - ENDPOINTS, USER_EXCLUSION_FIELDS, ACCOUNT_EXCLUSION_FIELDS, trackMapping, groupMapping, identifyMapping, + getUsersEndpoint, + getCustomEventsEndpoint, } = require('./config'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -92,7 +93,7 @@ const identifyResponseBuilder = async (message, { Config }, metadata) => { if (isUserPresent) { // update user response.method = defaultPutRequestConfig.requestMethod; - response.endpoint = `${ENDPOINTS.USERS_ENDPOINT}/${userId}`; + response.endpoint = `${getUsersEndpoint(Config)}/${userId}`; response.body.JSON = removeUndefinedAndNullValues(payload); return response; } @@ -100,7 +101,7 @@ const identifyResponseBuilder = async (message, { Config }, metadata) => { // create new user payload.identifyId = userId; response.method = defaultPostRequestConfig.requestMethod; - response.endpoint = ENDPOINTS.USERS_ENDPOINT; + response.endpoint = getUsersEndpoint(Config); response.body.JSON = removeUndefinedAndNullValues(payload); return response; }; @@ -162,7 +163,7 @@ const newGroupResponseBuilder = async (message, { Config }, metadata) => { 'X-APTRINSIC-API-KEY': Config.apiKey, 'Content-Type': JSON_MIME_TYPE, }; - response.endpoint = `${ENDPOINTS.USERS_ENDPOINT}/${userId}`; + response.endpoint = `${getUsersEndpoint(Config)}/${userId}`; response.body.JSON = { accountId: groupId, }; @@ -230,7 +231,7 @@ const groupResponseBuilder = async (message, { Config }, metadata) => { 'X-APTRINSIC-API-KEY': Config.apiKey, 'Content-Type': JSON_MIME_TYPE, }; - response.endpoint = `${ENDPOINTS.USERS_ENDPOINT}/${userId}`; + response.endpoint = `${getUsersEndpoint(Config)}/${userId}`; response.body.JSON = { accountId: groupId, }; @@ -271,7 +272,7 @@ const trackResponseBuilder = (message, { Config }) => { 'X-APTRINSIC-API-KEY': Config.apiKey, 'Content-Type': JSON_MIME_TYPE, }; - response.endpoint = ENDPOINTS.CUSTOM_EVENTS_ENDPOINT; + response.endpoint = getCustomEventsEndpoint(Config); return response; }; diff --git a/src/v0/destinations/gainsight_px/util.js b/src/v0/destinations/gainsight_px/util.js index 7300189297..71d85438de 100644 --- a/src/v0/destinations/gainsight_px/util.js +++ b/src/v0/destinations/gainsight_px/util.js @@ -1,9 +1,9 @@ const { NetworkError } = require('@rudderstack/integrations-lib'); -const { ENDPOINTS } = require('./config'); const tags = require('../../util/tags'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { handleHttpRequest } = require('../../../adapters/network'); +const { getUsersEndpoint, getAccountsEndpoint } = require('./config'); const handleErrorResponse = (error, customErrMessage, expectedErrStatus, defaultStatus = 400) => { let destResp; @@ -38,10 +38,10 @@ const handleErrorResponse = (error, customErrMessage, expectedErrStatus, default * @returns */ const objectExists = async (id, Config, objectType, metadata) => { - let url = `${ENDPOINTS.USERS_ENDPOINT}/${id}`; + let url = `${getUsersEndpoint(Config)}/${id}`; if (objectType === 'account') { - url = `${ENDPOINTS.ACCOUNTS_ENDPOINT}/${id}`; + url = `${getAccountsEndpoint(Config)}/${id}`; } const { httpResponse: res } = await handleHttpRequest( 'get', @@ -70,7 +70,7 @@ const objectExists = async (id, Config, objectType, metadata) => { const createAccount = async (payload, Config, metadata) => { const { httpResponse: res } = await handleHttpRequest( 'post', - ENDPOINTS.ACCOUNTS_ENDPOINT, + getAccountsEndpoint(Config), payload, { headers: { @@ -96,7 +96,7 @@ const createAccount = async (payload, Config, metadata) => { const updateAccount = async (accountId, payload, Config, metadata) => { const { httpResponse: res } = await handleHttpRequest( 'put', - `${ENDPOINTS.ACCOUNTS_ENDPOINT}/${accountId}`, + `${getAccountsEndpoint(Config)}/${accountId}`, payload, { headers: { diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js index 497d4f294f..0badf49241 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js +++ b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js @@ -2,11 +2,7 @@ const get = require('get-value'); const { cloneDeep, isNumber } = require('lodash'); -const { - InstrumentationError, - ConfigurationError, - isDefinedAndNotNull, -} = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const isString = require('lodash/isString'); const { constructPayload, @@ -15,7 +11,6 @@ const { removeHyphens, simpleProcessRouterDest, getAccessToken, - isDefined, } = require('../../util'); const { trackMapping, BASE_ENDPOINT } = require('./config'); @@ -43,15 +38,6 @@ const responseBuilder = async (metadata, message, { Config }, payload) => { const { event } = message; const { subAccount } = Config; let { customerId, loginCustomerId } = Config; - const { configData } = Config; - - if (isDefinedAndNotNull(configData)) { - const configDetails = JSON.parse(configData); - customerId = configDetails.customerId; - if (isDefined(configDetails.loginCustomerId)) { - loginCustomerId = configDetails.loginCustomerId; - } - } if (isNumber(customerId)) { customerId = customerId.toString(); @@ -84,29 +70,18 @@ const responseBuilder = async (metadata, message, { Config }, payload) => { response.headers['login-customer-id'] = filteredLoginCustomerId; } - if (loginCustomerId) { - const filteredLoginCustomerId = removeHyphens(loginCustomerId); - response.headers['login-customer-id'] = filteredLoginCustomerId; - } - return response; }; const processTrackEvent = async (metadata, message, destination) => { - let flag = 0; + let flag = false; const { Config } = destination; const { event } = message; const { listOfConversions } = Config; - if (listOfConversions && listOfConversions.length > 0) { - if (typeof listOfConversions[0] === 'string') { - if (listOfConversions.includes(event)) { - flag = 1; - } - } else if (listOfConversions.some((i) => i.conversions === event)) { - flag = 1; - } + if (listOfConversions.some((i) => i.conversions === event)) { + flag = true; } - if (event === undefined || event === '' || flag === 0) { + if (event === undefined || event === '' || !flag) { throw new ConfigurationError( `Conversion named "${event}" was not specified in the RudderStack destination configuration`, ); diff --git a/src/v0/destinations/google_adwords_remarketing_lists/config.js b/src/v0/destinations/google_adwords_remarketing_lists/config.js index 0478a1b11b..1e943aee56 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/config.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/config.js @@ -7,6 +7,7 @@ const CONFIG_CATEGORIES = { AUDIENCE_LIST: { type: 'audienceList', name: 'offlineDataJobs' }, ADDRESSINFO: { type: 'addressInfo', name: 'addressInfo' }, }; +const ADDRESS_INFO_ATTRIBUTES = ['firstName', 'lastName', 'country', 'postalCode']; const attributeMapping = { email: 'hashedEmail', phone: 'hashedPhoneNumber', @@ -31,6 +32,7 @@ module.exports = { hashAttributes, offlineDataJobsMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.AUDIENCE_LIST.name], addressInfoMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.ADDRESSINFO.name], + ADDRESS_INFO_ATTRIBUTES, consentConfigMap, destType: 'google_adwords_remarketing_lists', }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index f8a2b0e586..5866b66538 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -11,7 +11,11 @@ const { isEventSentByVDMV2Flow, } = require('../../util'); const { populateConsentFromConfig } = require('../../util/googleUtils'); -const { populateIdentifiers, responseBuilder, getOperationAudienceId } = require('./util'); +const { + populateIdentifiersForRecordEvent, + responseBuilder, + getOperationAudienceId, +} = require('./util'); const { getErrorResponse, createFinalResponse } = require('../../util/recordUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); @@ -23,7 +27,10 @@ const processRecordEventArray = ( developerToken, audienceId, typeOfList, + userSchema, isHashRequired, + userDataConsent, + personalizationConsent, operationType, ) => { let outputPayloads = {}; @@ -36,10 +43,10 @@ const processRecordEventArray = ( metadata.push(record.metadata); }); - const userIdentifiersList = populateIdentifiers( + const userIdentifiersList = populateIdentifiersForRecordEvent( fieldsArray, - destination, typeOfList, + userSchema, isHashRequired, ); @@ -76,7 +83,10 @@ const processRecordEventArray = ( const toSendEvents = []; Object.values(outputPayloads).forEach((data) => { - const consentObj = populateConsentFromConfig(destination.Config, consentConfigMap); + const consentObj = populateConsentFromConfig( + { userDataConsent, personalizationConsent }, + consentConfigMap, + ); toSendEvents.push( responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), ); @@ -91,7 +101,14 @@ function preparepayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { audienceId, typeOfList, isHashRequired } = config; + const { + audienceId, + typeOfList, + isHashRequired, + userSchema, + userDataConsent, + personalizationConsent, + } = config; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), @@ -110,7 +127,10 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'remove', ); } @@ -124,7 +144,10 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'add', ); } @@ -138,7 +161,10 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'add', ); } @@ -161,18 +187,35 @@ function preparepayload(events, config) { function processRecordInputsV0(groupedRecordInputs) { const { destination, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired } = destination.Config; + const { + audienceId, + typeOfList, + isHashRequired, + userSchema, + userDataConsent, + personalizationConsent, + } = destination.Config; return preparepayload(groupedRecordInputs, { audienceId: getOperationAudienceId(audienceId, message), typeOfList, + userSchema, isHashRequired, + userDataConsent, + personalizationConsent, }); } function processRecordInputsV1(groupedRecordInputs) { - const { connection } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired } = connection.config.destination; + const { connection, message } = groupedRecordInputs[0]; + const { audienceId, typeOfList, isHashRequired, userDataConsent, personalizationConsent } = + connection.config.destination; + + const identifiers = message?.identifiers; + let userSchema; + if (identifiers) { + userSchema = Object.keys(identifiers); + } const events = groupedRecordInputs.map((record) => ({ ...record, @@ -185,7 +228,10 @@ function processRecordInputsV1(groupedRecordInputs) { return preparepayload(events, { audienceId, typeOfList, + userSchema, isHashRequired, + userDataConsent, + personalizationConsent, }); } diff --git a/src/v0/destinations/google_adwords_remarketing_lists/transform.js b/src/v0/destinations/google_adwords_remarketing_lists/transform.js index 299ab94846..4d173589e8 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/transform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/transform.js @@ -37,7 +37,7 @@ function extraKeysPresent(dictionary, keyList) { const createPayload = (message, destination) => { const { listData } = message.properties; const properties = ['add', 'remove']; - const { typeOfList, isHashRequired } = destination.Config; + const { typeOfList, userSchema, isHashRequired } = destination.Config; let outputPayloads = {}; const typeOfOperation = Object.keys(listData); @@ -45,8 +45,8 @@ const createPayload = (message, destination) => { if (properties.includes(key)) { const userIdentifiersList = populateIdentifiers( listData[key], - destination, typeOfList, + userSchema, isHashRequired, ); if (userIdentifiersList.length === 0) { diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.js b/src/v0/destinations/google_adwords_remarketing_lists/util.js index f4c33a9a6f..8e0aed0365 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.js @@ -18,6 +18,7 @@ const { TYPEOFLIST, BASE_ENDPOINT, hashAttributes, + ADDRESS_INFO_ATTRIBUTES, } = require('./config'); const hashEncrypt = (object) => { @@ -68,14 +69,13 @@ const responseBuilder = ( * Logics: Here we are creating an array with all the attributes provided in the add/remove array * inside listData. * @param {Array} attributeArray rudder event message properties listData add - * @param {object} Config rudder event destination * @param {string} typeOfList + * @param {Array} userSchema * @param {boolean} isHashRequired * @returns */ -const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequired) => { +const populateIdentifiers = (attributeArray, typeOfList, userSchema, isHashRequired) => { const userIdentifier = []; - const { userSchema } = Config; let attribute; if (TYPEOFLIST[typeOfList]) { attribute = TYPEOFLIST[typeOfList]; @@ -115,6 +115,40 @@ const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequi return userIdentifier; }; +const populateIdentifiersForRecordEvent = ( + identifiersArray, + typeOfList, + userSchema, + isHashRequired, +) => { + const userIdentifiers = []; + + if (isDefinedAndNotNullAndNotEmpty(identifiersArray)) { + // traversing through every element in the add array + identifiersArray.forEach((identifiers) => { + if (isHashRequired) { + hashEncrypt(identifiers); + } + if (TYPEOFLIST[typeOfList] && identifiers[TYPEOFLIST[typeOfList]]) { + userIdentifiers.push({ [TYPEOFLIST[typeOfList]]: identifiers[TYPEOFLIST[typeOfList]] }); + } else { + Object.entries(attributeMapping).forEach(([key, mappedKey]) => { + if (identifiers[key] && userSchema.includes(key)) + userIdentifiers.push({ [mappedKey]: identifiers[key] }); + }); + const addressInfo = constructPayload(identifiers, addressInfoMapping); + if ( + isDefinedAndNotNullAndNotEmpty(addressInfo) && + (userSchema.includes('addressInfo') || + userSchema.some((schema) => ADDRESS_INFO_ATTRIBUTES.includes(schema))) + ) + userIdentifiers.push({ addressInfo }); + } + }); + } + return userIdentifiers; +}; + const getOperationAudienceId = (audienceId, message) => { let operationAudienceId = audienceId; const mappedToDestination = get(message, MappedToDestinationKey); @@ -132,4 +166,5 @@ module.exports = { populateIdentifiers, responseBuilder, getOperationAudienceId, + populateIdentifiersForRecordEvent, }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js index 0b74b07b8e..a41c00f12f 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js @@ -199,8 +199,8 @@ describe('GARL utils test', () => { const { typeOfList, isHashRequired } = baseDestination.Config; const identifier = populateIdentifiers( attributeArray, - baseDestination, typeOfList, + baseDestination.Config.userSchema, isHashRequired, ); expect(identifier).toEqual(hashedArray); diff --git a/src/v0/destinations/intercom_v2/config.js b/src/v0/destinations/intercom_v2/config.js index c7cb43b093..5ff5566d2d 100644 --- a/src/v0/destinations/intercom_v2/config.js +++ b/src/v0/destinations/intercom_v2/config.js @@ -6,6 +6,12 @@ const ApiVersions = { v2: '2.10', }; +const RecordAction = { + INSERT: 'insert', + UPDATE: 'update', + DELETE: 'delete', +}; + const ConfigCategory = { IDENTIFY: { name: 'IntercomIdentifyConfig', @@ -25,4 +31,5 @@ module.exports = { ConfigCategory, MappingConfig, ApiVersions, + RecordAction, }; diff --git a/src/v0/destinations/intercom_v2/transform.js b/src/v0/destinations/intercom_v2/transform.js index 8d97e20bde..3f9457410f 100644 --- a/src/v0/destinations/intercom_v2/transform.js +++ b/src/v0/destinations/intercom_v2/transform.js @@ -1,4 +1,4 @@ -const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { handleRtTfSingleEventError, getSuccessRespEvents, @@ -17,13 +17,14 @@ const { addOrUpdateTagsToCompany, getStatusCode, getBaseEndpoint, + getRecordAction, } = require('./utils'); const { getName, filterCustomAttributes, addMetadataToPayload, } = require('../../../cdk/v2/destinations/intercom/utils'); -const { MappingConfig, ConfigCategory } = require('./config'); +const { MappingConfig, ConfigCategory, RecordAction } = require('./config'); const transformIdentifyPayload = (event) => { const { message, destination } = event; @@ -38,7 +39,7 @@ const transformIdentifyPayload = (event) => { } payload.name = getName(message); payload.custom_attributes = message.traits || message.context.traits || {}; - payload.custom_attributes = filterCustomAttributes(payload, 'user', destination); + payload.custom_attributes = filterCustomAttributes(payload, 'user', destination, message); return payload; }; @@ -66,7 +67,7 @@ const transformGroupPayload = (event) => { const category = ConfigCategory.GROUP; const payload = constructPayload(message, MappingConfig[category.name]); payload.custom_attributes = message.traits || message.context.traits || {}; - payload.custom_attributes = filterCustomAttributes(payload, 'company', destination); + payload.custom_attributes = filterCustomAttributes(payload, 'company', destination, message); return payload; }; @@ -131,6 +132,45 @@ const constructGroupResponse = async (event) => { return getResponse(method, endpoint, headers, finalPayload); }; +const constructRecordResponse = async (event) => { + const { message, destination, metadata } = event; + const { identifiers, fields } = message; + + let method = 'POST'; + let endpoint = `${getBaseEndpoint(destination)}/contacts`; + let payload = {}; + + const action = getRecordAction(message); + const contactId = await searchContact(event); + + if ((action === RecordAction.UPDATE || action === RecordAction.DELETE) && !contactId) { + throw new ConfigurationError('Contact is not present. Aborting.'); + } + + switch (action) { + case RecordAction.INSERT: + payload = { ...identifiers, ...fields }; + if (contactId) { + endpoint += `/${contactId}`; + payload = { ...fields }; + method = 'PUT'; + } + break; + case RecordAction.UPDATE: + endpoint += `/${contactId}`; + payload = { ...fields }; + method = 'PUT'; + break; + case RecordAction.DELETE: + endpoint += `/${contactId}`; + method = 'DELETE'; + break; + default: + throw new InstrumentationError(`action ${action} is not supported.`); + } + return getResponse(method, endpoint, getHeaders(metadata), payload); +}; + const processEvent = async (event) => { const { message } = event; const messageType = getEventType(message); @@ -145,6 +185,9 @@ const processEvent = async (event) => { case EventType.GROUP: response = await constructGroupResponse(event); break; + case EventType.RECORD: + response = constructRecordResponse(event); + break; default: throw new InstrumentationError(`message type ${messageType} is not supported.`); } diff --git a/src/v0/destinations/intercom_v2/utils.js b/src/v0/destinations/intercom_v2/utils.js index 69ea1385d9..df44b92e24 100644 --- a/src/v0/destinations/intercom_v2/utils.js +++ b/src/v0/destinations/intercom_v2/utils.js @@ -28,6 +28,8 @@ const { getAccessToken } = require('../../util'); const { ApiVersions, destType } = require('./config'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); +const getRecordAction = (message) => message?.action?.toLowerCase(); + /** * method to handle error during api call * ref docs: https://developers.intercom.com/docs/references/rest-api/errors/http-responses/ @@ -99,11 +101,25 @@ const getResponse = (method, endpoint, headers, payload) => { const searchContact = async (event) => { const { message, destination, metadata } = event; - const lookupField = getLookUpField(message); - let lookupFieldValue = getFieldValueFromMessage(message, lookupField); - if (!lookupFieldValue) { - lookupFieldValue = message?.context?.traits?.[lookupField]; + + const extractLookupFieldAndValue = () => { + const messageType = getEventType(message); + if (messageType === EventType.RECORD) { + const { identifiers } = message; + return Object.entries(identifiers || {})[0] || [null, null]; + } + const lookupField = getLookUpField(message); + const lookupFieldValue = + getFieldValueFromMessage(message, lookupField) || message?.context?.traits?.[lookupField]; + return [lookupField, lookupFieldValue]; + }; + + const [lookupField, lookupFieldValue] = extractLookupFieldAndValue(); + + if (!lookupField || !lookupFieldValue) { + throw new InstrumentationError('Missing lookup field or lookup field value for searchContact'); } + const data = JSON.stringify({ query: { operator: 'AND', @@ -329,4 +345,5 @@ module.exports = { attachContactToCompany, addOrUpdateTagsToCompany, getBaseEndpoint, + getRecordAction, }; diff --git a/src/v0/destinations/iterable/config.js b/src/v0/destinations/iterable/config.js index f74fdb4975..125367875f 100644 --- a/src/v0/destinations/iterable/config.js +++ b/src/v0/destinations/iterable/config.js @@ -1,42 +1,45 @@ const { getMappingConfig } = require('../../util'); -const BASE_URL = 'https://api.iterable.com/api/'; +const BASE_URL = { + USDC: 'https://api.iterable.com/api/', + EUDC: 'https://api.eu.iterable.com/api/', +}; const ConfigCategory = { IDENTIFY_BROWSER: { name: 'IterableRegisterBrowserTokenConfig', action: 'identifyBrowser', - endpoint: `${BASE_URL}users/registerBrowserToken`, + endpoint: `users/registerBrowserToken`, }, IDENTIFY_DEVICE: { name: 'IterableRegisterDeviceTokenConfig', action: 'identifyDevice', - endpoint: `${BASE_URL}users/registerDeviceToken`, + endpoint: `users/registerDeviceToken`, }, IDENTIFY: { name: 'IterableIdentifyConfig', action: 'identify', - endpoint: `${BASE_URL}users/update`, + endpoint: `users/update`, }, PAGE: { name: 'IterablePageConfig', action: 'page', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, SCREEN: { name: 'IterablePageConfig', action: 'screen', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, TRACK: { name: 'IterableTrackConfig', action: 'track', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, TRACK_PURCHASE: { name: 'IterableTrackPurchaseConfig', action: 'trackPurchase', - endpoint: `${BASE_URL}commerce/trackPurchase`, + endpoint: `commerce/trackPurchase`, }, PRODUCT: { name: 'IterableProductConfig', @@ -46,7 +49,7 @@ const ConfigCategory = { UPDATE_CART: { name: 'IterableProductConfig', action: 'updateCart', - endpoint: `${BASE_URL}commerce/updateCart`, + endpoint: `commerce/updateCart`, }, DEVICE: { name: 'IterableDeviceConfig', @@ -56,30 +59,33 @@ const ConfigCategory = { ALIAS: { name: 'IterableAliasConfig', action: 'alias', - endpoint: `${BASE_URL}users/updateEmail`, + endpoint: `users/updateEmail`, }, CATALOG: { name: 'IterableCatalogConfig', action: 'catalogs', - endpoint: `${BASE_URL}catalogs`, + endpoint: `catalogs`, }, }; const mappingConfig = getMappingConfig(ConfigCategory, __dirname); +// Function to construct endpoint based on the selected data center +const constructEndpoint = (dataCenter, category) => { + const baseUrl = BASE_URL[dataCenter] || BASE_URL.USDC; // Default to USDC if not found + return `${baseUrl}${category.endpoint}`; +}; + const IDENTIFY_MAX_BATCH_SIZE = 1000; const IDENTIFY_MAX_BODY_SIZE_IN_BYTES = 4000000; -const IDENTIFY_BATCH_ENDPOINT = 'https://api.iterable.com/api/users/bulkUpdate'; const TRACK_MAX_BATCH_SIZE = 8000; -const TRACK_BATCH_ENDPOINT = 'https://api.iterable.com/api/events/trackBulk'; module.exports = { mappingConfig, ConfigCategory, - TRACK_BATCH_ENDPOINT, + constructEndpoint, TRACK_MAX_BATCH_SIZE, IDENTIFY_MAX_BATCH_SIZE, - IDENTIFY_BATCH_ENDPOINT, IDENTIFY_MAX_BODY_SIZE_IN_BYTES, }; diff --git a/src/v0/destinations/iterable/deleteUsers.js b/src/v0/destinations/iterable/deleteUsers.js index 015a9de9a0..79c4c0affd 100644 --- a/src/v0/destinations/iterable/deleteUsers.js +++ b/src/v0/destinations/iterable/deleteUsers.js @@ -6,13 +6,14 @@ const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); const { executeCommonValidations } = require('../../util/regulation-api'); const tags = require('../../util/tags'); const { JSON_MIME_TYPE } = require('../../util/constant'); +const { constructEndpoint } = require('./config'); -// Ref-> https://developers.intercom.com/intercom-api-reference/v1.3/reference/permanently-delete-a-user +// Ref-> https://support.iterable.com/hc/en-us/articles/360032290032-Deleting-Users const userDeletionHandler = async (userAttributes, config) => { if (!config) { throw new ConfigurationError('Config for deletion not present'); } - const { apiKey } = config; + const { apiKey, dataCenter } = config; if (!apiKey) { throw new ConfigurationError('api key for deletion not present'); } @@ -26,7 +27,8 @@ const userDeletionHandler = async (userAttributes, config) => { const failedUserDeletions = []; await Promise.all( validUserIds.map(async (uId) => { - const url = `https://api.iterable.com/api/users/byUserId/${uId}`; + const endpointCategory = { endpoint: `users/byUserId/${uId}` }; + const url = constructEndpoint(dataCenter, endpointCategory); const requestOptions = { headers: { 'Content-Type': JSON_MIME_TYPE, diff --git a/src/v0/destinations/iterable/transform.js b/src/v0/destinations/iterable/transform.js index 207a8d1186..dd67deef69 100644 --- a/src/v0/destinations/iterable/transform.js +++ b/src/v0/destinations/iterable/transform.js @@ -14,6 +14,7 @@ const { filterEventsAndPrepareBatchRequests, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + getCategoryWithEndpoint, } = require('./util'); const { constructPayload, @@ -116,12 +117,11 @@ const responseBuilderForRegisterDeviceOrBrowserTokenEvents = (message, destinati /** * Function to find category value - * @param {*} messageType * @param {*} message * @returns */ -const getCategory = (messageType, message) => { - const eventType = messageType.toLowerCase(); +const getCategory = (message, dataCenter) => { + const eventType = message.type.toLowerCase(); switch (eventType) { case EventType.IDENTIFY: @@ -129,17 +129,17 @@ const getCategory = (messageType, message) => { get(message, MappedToDestinationKey) && getDestinationExternalIDInfoForRetl(message, 'ITERABLE').objectType !== 'users' ) { - return ConfigCategory.CATALOG; + return getCategoryWithEndpoint(ConfigCategory.CATALOG, dataCenter); } - return ConfigCategory.IDENTIFY; + return getCategoryWithEndpoint(ConfigCategory.IDENTIFY, dataCenter); case EventType.PAGE: - return ConfigCategory.PAGE; + return getCategoryWithEndpoint(ConfigCategory.PAGE, dataCenter); case EventType.SCREEN: - return ConfigCategory.SCREEN; + return getCategoryWithEndpoint(ConfigCategory.SCREEN, dataCenter); case EventType.TRACK: - return getCategoryUsingEventName(message); + return getCategoryUsingEventName(message, dataCenter); case EventType.ALIAS: - return ConfigCategory.ALIAS; + return getCategoryWithEndpoint(ConfigCategory.ALIAS, dataCenter); default: throw new InstrumentationError(`Message type ${eventType} not supported`); } @@ -150,8 +150,7 @@ const process = (event) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } - const messageType = message.type.toLowerCase(); - const category = getCategory(messageType, message); + const category = getCategory(message, destination.Config.dataCenter); const response = responseBuilder(message, category, destination); if (hasMultipleResponses(message, category, destination.Config)) { diff --git a/src/v0/destinations/iterable/util.js b/src/v0/destinations/iterable/util.js index 7c1509c2b7..b918600253 100644 --- a/src/v0/destinations/iterable/util.js +++ b/src/v0/destinations/iterable/util.js @@ -14,10 +14,9 @@ const { ConfigCategory, mappingConfig, TRACK_MAX_BATCH_SIZE, - TRACK_BATCH_ENDPOINT, IDENTIFY_MAX_BATCH_SIZE, - IDENTIFY_BATCH_ENDPOINT, IDENTIFY_MAX_BODY_SIZE_IN_BYTES, + constructEndpoint, } = require('./config'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { EventType, MappedToDestinationKey } = require('../../../constants'); @@ -88,12 +87,17 @@ const hasMultipleResponses = (message, category, config) => { return isIdentifyEvent && isIdentifyCategory && hasToken && hasRegisterDeviceOrBrowserKey; }; +const getCategoryWithEndpoint = (categoryConfig, dataCenter) => ({ + ...categoryConfig, + endpoint: constructEndpoint(dataCenter, categoryConfig), +}); + /** * Returns category value * @param {*} message * @returns */ -const getCategoryUsingEventName = (message) => { +const getCategoryUsingEventName = (message, dataCenter) => { let { event } = message; if (typeof event === 'string') { event = event.toLowerCase(); @@ -101,12 +105,12 @@ const getCategoryUsingEventName = (message) => { switch (event) { case 'order completed': - return ConfigCategory.TRACK_PURCHASE; + return getCategoryWithEndpoint(ConfigCategory.TRACK_PURCHASE, dataCenter); case 'product added': case 'product removed': - return ConfigCategory.UPDATE_CART; + return getCategoryWithEndpoint(ConfigCategory.UPDATE_CART, dataCenter); default: - return ConfigCategory.TRACK; + return getCategoryWithEndpoint(ConfigCategory.TRACK, dataCenter); } }; @@ -444,8 +448,8 @@ const processUpdateUserBatch = (chunk, registerDeviceOrBrowserTokenEvents) => { batchEventResponse.batchedRequest.body.JSON = { users: batch.users }; const { destination, metadata, nonBatchedRequests } = batch; - const { apiKey } = destination.Config; - + const { apiKey, dataCenter } = destination.Config; + const IDENTIFY_BATCH_ENDPOINT = constructEndpoint(dataCenter, { endpoint: 'users/bulkUpdate' }); const batchedResponse = combineBatchedAndNonBatchedEvents( apiKey, metadata, @@ -552,8 +556,8 @@ const processTrackBatch = (chunk) => { const metadata = []; const { destination } = chunk[0]; - const { apiKey } = destination.Config; - + const { apiKey, dataCenter } = destination.Config; + const TRACK_BATCH_ENDPOINT = constructEndpoint(dataCenter, { endpoint: 'events/trackBulk' }); chunk.forEach((event) => { metadata.push(event.metadata); events.push(get(event, `${MESSAGE_JSON_PATH}`)); @@ -653,12 +657,13 @@ const mapRegisterDeviceOrBrowserTokenEventsWithJobId = (events) => { */ const categorizeEvent = (event) => { const { message, metadata, destination, error } = event; + const { dataCenter } = destination.Config; if (error) { return { type: 'error', data: event }; } - if (message.endpoint === ConfigCategory.IDENTIFY.endpoint) { + if (message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY)) { return { type: 'updateUser', data: { message, metadata, destination } }; } @@ -667,8 +672,8 @@ const categorizeEvent = (event) => { } if ( - message.endpoint === ConfigCategory.IDENTIFY_BROWSER.endpoint || - message.endpoint === ConfigCategory.IDENTIFY_DEVICE.endpoint + message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY_BROWSER) || + message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY_DEVICE) ) { return { type: 'registerDeviceOrBrowser', data: { message, metadata, destination } }; } @@ -753,4 +758,5 @@ module.exports = { filterEventsAndPrepareBatchRequests, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + getCategoryWithEndpoint, }; diff --git a/src/v0/destinations/marketo_bulk_upload/config.js b/src/v0/destinations/marketo_bulk_upload/config.js deleted file mode 100644 index e3268711fe..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/config.js +++ /dev/null @@ -1,55 +0,0 @@ -const ABORTABLE_CODES = ['601', '603', '605', '609', '610']; -const RETRYABLE_CODES = ['713', '602', '604', '611']; -const THROTTLED_CODES = ['502', '606', '607', '608', '615']; - -const MARKETO_FILE_SIZE = 10485760; -const MARKETO_FILE_PATH = `${__dirname}/uploadFile/marketo_bulkupload.csv`; - -const FETCH_ACCESS_TOKEN = 'marketo_bulk_upload_access_token_fetching'; - -const POLL_ACTIVITY = 'marketo_bulk_upload_polling'; -const POLL_STATUS_ERR_MSG = 'Could not poll status'; - -const UPLOAD_FILE = 'marketo_bulk_upload_upload_file'; -const FILE_UPLOAD_ERR_MSG = 'Could not upload file'; - -const JOB_STATUS_ACTIVITY = 'marketo_bulk_upload_get_job_status'; -const FETCH_FAILURE_JOB_STATUS_ERR_MSG = 'Could not fetch failure job status'; -const FETCH_WARNING_JOB_STATUS_ERR_MSG = 'Could not fetch warning job status'; -const ACCESS_TOKEN_FETCH_ERR_MSG = 'Error during fetching access token'; - -const SCHEMA_DATA_TYPE_MAP = { - string: 'string', - number: 'number', - boolean: 'boolean', - undefined: 'undefined', - float: 'number', - text: 'string', - currency: 'string', - integer: 'number', - reference: 'string', - datetime: 'string', - date: 'string', - email: 'string', - phone: 'string', - url: 'string', - object: 'object', -}; - -module.exports = { - ABORTABLE_CODES, - RETRYABLE_CODES, - THROTTLED_CODES, - MARKETO_FILE_SIZE, - POLL_ACTIVITY, - UPLOAD_FILE, - JOB_STATUS_ACTIVITY, - MARKETO_FILE_PATH, - FETCH_ACCESS_TOKEN, - POLL_STATUS_ERR_MSG, - FILE_UPLOAD_ERR_MSG, - FETCH_FAILURE_JOB_STATUS_ERR_MSG, - FETCH_WARNING_JOB_STATUS_ERR_MSG, - ACCESS_TOKEN_FETCH_ERR_MSG, - SCHEMA_DATA_TYPE_MAP, -}; diff --git a/src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js b/src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js deleted file mode 100644 index db3b13eeb8..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js +++ /dev/null @@ -1,153 +0,0 @@ -/* eslint-disable no-restricted-syntax */ -/* eslint-disable no-prototype-builtins */ -const { PlatformError } = require('@rudderstack/integrations-lib'); -const { getAccessToken } = require('./util'); -const { handleHttpRequest } = require('../../../adapters/network'); -const stats = require('../../../util/stats'); -const { JSON_MIME_TYPE } = require('../../util/constant'); -const { - handleFetchJobStatusResponse, - getFieldSchemaMap, - checkEventStatusViaSchemaMatching, -} = require('./util'); -const { removeUndefinedValues } = require('../../util'); - -const getJobsStatus = async (event, type, accessToken) => { - const { config, importId } = event; - const { munchkinId } = config; - let url; - // Get status of each lead for failed leads - // DOC: https://developers.marketo.com/rest-api/bulk-import/bulk-lead-import/#failures - const requestOptions = { - headers: { - 'Content-Type': JSON_MIME_TYPE, - Authorization: `Bearer ${accessToken}`, - }, - }; - if (type === 'fail') { - url = `https://${munchkinId}.mktorest.com/bulk/v1/leads/batch/${importId}/failures.json`; - } else { - url = `https://${munchkinId}.mktorest.com/bulk/v1/leads/batch/${importId}/warnings.json`; - } - const startTime = Date.now(); - const { processedResponse: resp } = await handleHttpRequest('get', url, requestOptions, { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads/batch/', - requestMethod: 'GET', - module: 'router', - }); - const endTime = Date.now(); - const requestTime = endTime - startTime; - - stats.histogram('marketo_bulk_upload_fetch_job_time', requestTime); - - return handleFetchJobStatusResponse(resp, type); -}; - -/** - * Handles the response from the server based on the provided type. - * Retrieves the job status using the getJobsStatus function and processes the response data. - * Matches the response data with the data received from the server. - * Returns a response object containing the failed keys, failed reasons, warning keys, warning reasons, and succeeded keys. - * @param {Object} event - An object containing the input data and metadata. - * @param {string} type - A string indicating the type of job status to retrieve ("fail" or "warn"). - * @returns {Object} - A response object with the failed keys, failed reasons, warning keys, warning reasons, and succeeded keys. - */ -const responseHandler = async (event, type) => { - let FailedKeys = []; - const unsuccessfulJobIdsArr = []; - let successfulJobIdsArr = []; - let reasons = {}; - - const { config } = event; - const accessToken = await getAccessToken(config); - - /** - * { - "FailedKeys" : [jobID1,jobID3], - "FailedReasons" : { - "jobID1" : "failure-reason-1", - "jobID3" : "failure-reason-2", - }, - "WarningKeys" : [jobID2,jobID4], - "WarningReasons" : { - "jobID2" : "warning-reason-1", - "jobID4" : "warning-reason-2", - }, - "SucceededKeys" : [jobID5] -} - */ - - const jobStatus = - type === 'fail' - ? await getJobsStatus(event, 'fail', accessToken) - : await getJobsStatus(event, 'warn', accessToken); - const jobStatusArr = jobStatus.toString().split('\n'); // responseArr = ['field1,field2,Import Failure Reason', 'val1,val2,reason',...] - const { input, metadata } = event; - let headerArr; - if (metadata?.csvHeader) { - headerArr = metadata.csvHeader.split(','); - } else { - throw new PlatformError('No csvHeader in metadata'); - } - const startTime = Date.now(); - const data = {}; - const fieldSchemaMapping = await getFieldSchemaMap(accessToken, config.munchkinId); - const unsuccessfulJobInfo = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - const mismatchJobIdArray = Object.keys(unsuccessfulJobInfo); - const dataTypeMismatchKeys = mismatchJobIdArray.map((strJobId) => parseInt(strJobId, 10)); - reasons = { ...unsuccessfulJobInfo }; - - const filteredEvents = input.filter( - (item) => !dataTypeMismatchKeys.includes(item.metadata.job_id), - ); - // create a map of job_id and data sent from server - // {: ','} - filteredEvents.forEach((i) => { - const response = headerArr.map((fieldName) => Object.values(i)[0][fieldName]).join(','); - data[i.metadata.job_id] = response; - }); - - // match marketo response data with received data from server - for (const element of jobStatusArr) { - // split response by comma but ignore commas inside double quotes - const elemArr = element.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/); - // ref : - // https://developers.marketo.com/rest-api/bulk-import/bulk-custom-object-import/#:~:text=Now%20we%E2%80%99ll%20make%20Get%20Import%20Custom%20Object%20Failures%20endpoint%20call%20to%20get%20additional%20failure%20detail%3A - const reasonMessage = elemArr.pop(); // get the column named "Import Failure Reason" - for (const [key, val] of Object.entries(data)) { - // joining the parameter values sent from marketo match it with received data from server - if (val === `${elemArr.map((item) => item.replace(/"/g, '')).join(',')}`) { - // add job keys if warning/failure - if (!unsuccessfulJobIdsArr.includes(key)) { - unsuccessfulJobIdsArr.push(key); - } - reasons[key] = reasonMessage; - } - } - } - - FailedKeys = unsuccessfulJobIdsArr.map((strJobId) => parseInt(strJobId, 10)); - successfulJobIdsArr = Object.keys(data).filter((x) => !unsuccessfulJobIdsArr.includes(x)); - - const SucceededKeys = successfulJobIdsArr.map((strJobId) => parseInt(strJobId, 10)); - const endTime = Date.now(); - const requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_fetch_job_create_response_time', requestTime); - const response = { - statusCode: 200, - metadata: { - FailedKeys: [...dataTypeMismatchKeys, ...FailedKeys], - FailedReasons: reasons, - SucceededKeys, - }, - }; - return removeUndefinedValues(response); -}; - -const processJobStatus = async (event, type) => { - const resp = await responseHandler(event, type); - return resp; -}; -module.exports = { processJobStatus }; diff --git a/src/v0/destinations/marketo_bulk_upload/fileUpload.js b/src/v0/destinations/marketo_bulk_upload/fileUpload.js deleted file mode 100644 index b49a265fd5..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/fileUpload.js +++ /dev/null @@ -1,275 +0,0 @@ -/* eslint-disable no-plusplus */ -const FormData = require('form-data'); -const fs = require('fs'); -const { - NetworkError, - ConfigurationError, - RetryableError, - TransformationError, -} = require('@rudderstack/integrations-lib'); -const { - getAccessToken, - getMarketoFilePath, - handleFileUploadResponse, - getFieldSchemaMap, - hydrateStatusForServer, -} = require('./util'); -const { isHttpStatusSuccess } = require('../../util'); -const { MARKETO_FILE_SIZE, UPLOAD_FILE } = require('./config'); -const { - getHashFromArray, - removeUndefinedAndNullValues, - isDefinedAndNotNullAndNotEmpty, -} = require('../../util'); -const { handleHttpRequest } = require('../../../adapters/network'); -const { client } = require('../../../util/errorNotifier'); -const stats = require('../../../util/stats'); - -const fetchFieldSchemaNames = async (config, accessToken) => { - const fieldSchemaMapping = await getFieldSchemaMap(accessToken, config.munchkinId); - if (Object.keys(fieldSchemaMapping).length > 0) { - const fieldSchemaNames = Object.keys(fieldSchemaMapping); - return { fieldSchemaNames }; - } - throw new RetryableError('Failed to fetch Marketo Field Schema', 500, fieldSchemaMapping); -}; - -const getHeaderFields = (config, fieldSchemaNames) => { - const { columnFieldsMapping } = config; - - columnFieldsMapping.forEach((colField) => { - if (!fieldSchemaNames.includes(colField.to)) { - throw new ConfigurationError( - `The field ${colField.to} is not present in Marketo Field Schema. Aborting`, - ); - } - }); - const columnField = getHashFromArray(columnFieldsMapping, 'to', 'from', false); - return Object.keys(columnField); -}; -/** - * Processes input data to create a CSV file and returns the file data along with successful and unsuccessful job IDs. - * The file name is made unique with combination of UUID and current timestamp to avoid any overrides. It also has a - * maximum size limit of 10MB . The events that could be accomodated inside the file is marked as successful and the - * rest are marked as unsuccessful. Also the file is deleted when reading is complete. - * @param {Array} inputEvents - An array of input events. - * @param {Object} config - destination config - * @param {Array} headerArr - An array of header fields. - * @returns {Object} - An object containing the file stream, successful job IDs, and unsuccessful job IDs. - */ -const getFileData = async (inputEvents, config, headerArr) => { - const input = inputEvents; - const messageArr = []; - let startTime; - let endTime; - let requestTime; - startTime = Date.now(); - - input.forEach((i) => { - const inputData = i; - const jobId = inputData.metadata.job_id; - const data = {}; - data[jobId] = inputData.message; - messageArr.push(data); - }); - - if (isDefinedAndNotNullAndNotEmpty(config.deDuplicationField)) { - // dedup starts - // Time Complexity = O(n2) - const dedupMap = new Map(); - // iterating input and storing the occurences of messages - // with same dedup property received from config - // Example: dedup-property = email - // k (key) v (index of occurence in input) - // user@email [4,7,9] - // user2@email [2,3] - // user3@email [1] - input.forEach((element, index) => { - const indexAr = dedupMap.get(element.message[config.deDuplicationField]) || []; - indexAr.push(index); - dedupMap.set(element.message[config.deDuplicationField], indexAr); - return dedupMap; - }); - // 1. iterating dedupMap - // 2. storing the duplicate occurences in dupValues arr - // 3. iterating dupValues arr, and mapping each property on firstBorn - // 4. as dupValues arr is sorted hence the firstBorn will inherit properties of last occurence (most updated one) - // 5. store firstBorn to first occurence in input as it should get the highest priority - dedupMap.forEach((indexes) => { - let firstBorn = {}; - indexes.forEach((idx) => { - headerArr.forEach((headerStr) => { - // if duplicate item has defined property to offer we take it else old one remains - firstBorn[headerStr] = input[idx].message[headerStr] || firstBorn[headerStr]; - }); - }); - firstBorn = removeUndefinedAndNullValues(firstBorn); - input[indexes[0]].message = firstBorn; - }); - // dedup ends - } - - const csv = []; - csv.push(headerArr.toString()); - endTime = Date.now(); - requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_create_header_time', requestTime); - const unsuccessfulJobs = []; - const successfulJobs = []; - const MARKETO_FILE_PATH = getMarketoFilePath(); - startTime = Date.now(); - messageArr.forEach((row) => { - const csvSize = JSON.stringify(csv); // stringify and remove all "stringification" extra data - const response = headerArr - .map((fieldName) => JSON.stringify(Object.values(row)[0][fieldName], '')) - .join(','); - if (csvSize.length <= MARKETO_FILE_SIZE) { - csv.push(response); - successfulJobs.push(Object.keys(row)[0]); - } else { - unsuccessfulJobs.push(Object.keys(row)[0]); - } - }); - endTime = Date.now(); - requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_create_csvloop_time', requestTime); - const fileSize = Buffer.from(csv.join('\n')).length; - if (csv.length > 1) { - startTime = Date.now(); - fs.writeFileSync(MARKETO_FILE_PATH, csv.join('\n')); - const readStream = fs.readFileSync(MARKETO_FILE_PATH); - fs.unlinkSync(MARKETO_FILE_PATH); - endTime = Date.now(); - requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_create_file_time', requestTime); - stats.histogram('marketo_bulk_upload_upload_file_size', fileSize); - - return { readStream, successfulJobs, unsuccessfulJobs }; - } - return { successfulJobs, unsuccessfulJobs }; -}; - -const getImportID = async (input, config, accessToken, csvHeader) => { - let readStream; - let successfulJobs; - let unsuccessfulJobs; - try { - ({ readStream, successfulJobs, unsuccessfulJobs } = await getFileData( - input, - config, - csvHeader, - )); - } catch (err) { - client.notify(err, `Marketo File Upload: Error while creating file: ${err.message}`, { - config, - csvHeader, - }); - throw new TransformationError( - `Marketo File Upload: Error while creating file: ${err.message}`, - 500, - ); - } - - const formReq = new FormData(); - const { munchkinId, deDuplicationField } = config; - // create file for multipart form - if (readStream) { - formReq.append('format', 'csv'); - formReq.append('file', readStream, 'marketo_bulk_upload.csv'); - formReq.append('access_token', accessToken); - // Upload data received from server as files to marketo - // DOC: https://developers.marketo.com/rest-api/bulk-import/bulk-lead-import/#import_file - const requestOptions = { - headers: { - ...formReq.getHeaders(), - }, - }; - if (isDefinedAndNotNullAndNotEmpty(deDuplicationField)) { - requestOptions.params = { - lookupField: deDuplicationField, - }; - } - const startTime = Date.now(); - const { processedResponse: resp } = await handleHttpRequest( - 'post', - `https://${munchkinId}.mktorest.com/bulk/v1/leads.json`, - formReq, - requestOptions, - { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads.json', - requestMethod: 'POST', - module: 'router', - }, - ); - const endTime = Date.now(); - const requestTime = endTime - startTime; - stats.counter('marketo_bulk_upload_upload_file_succJobs', successfulJobs.length); - stats.counter('marketo_bulk_upload_upload_file_unsuccJobs', unsuccessfulJobs.length); - if (!isHttpStatusSuccess(resp.status)) { - throw new NetworkError( - `Unable to upload file due to error : ${JSON.stringify(resp.response)}`, - hydrateStatusForServer(resp.status, 'During uploading file'), - ); - } - return handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - } - return { importId: null, successfulJobs, unsuccessfulJobs }; -}; - -/** - * - * @param {*} input - * @param {*} config - * @returns returns the final response of fileUpload.js - */ -const responseHandler = async (input, config) => { - const accessToken = await getAccessToken(config); - /** - { - "importId" : , - "pollURL" : , - } - */ - const { fieldSchemaNames } = await fetchFieldSchemaNames(config, accessToken); - const headerForCsv = getHeaderFields(config, fieldSchemaNames); - if (Object.keys(headerForCsv).length === 0) { - throw new ConfigurationError( - 'Faulty configuration. Please map your traits to Marketo column fields', - ); - } - const { importId, successfulJobs, unsuccessfulJobs } = await getImportID( - input, - config, - accessToken, - headerForCsv, - ); - // if upload is successful - if (importId) { - const csvHeader = headerForCsv.toString(); - const metadata = { successfulJobs, unsuccessfulJobs, csvHeader }; - const response = { - statusCode: 200, - importId, - metadata, - }; - return response; - } - // if importId is returned null - stats.increment(UPLOAD_FILE, { - status: 500, - state: 'Retryable', - }); - return { - statusCode: 500, - FailedReason: '[Marketo File upload]: No import id received', - }; -}; -const processFileData = async (event) => { - const { input, config } = event; - const resp = await responseHandler(input, config); - return resp; -}; - -module.exports = { processFileData }; diff --git a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js b/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js deleted file mode 100644 index 13e1b3a09a..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js +++ /dev/null @@ -1,542 +0,0 @@ -const { - handleCommonErrorResponse, - handlePollResponse, - handleFileUploadResponse, - getAccessToken, - checkEventStatusViaSchemaMatching, -} = require('./util'); - -const { - AbortedError, - RetryableError, - NetworkError, - TransformationError, -} = require('@rudderstack/integrations-lib'); -const util = require('./util.js'); -const networkAdapter = require('../../../adapters/network'); -const { handleHttpRequest } = networkAdapter; - -// Mock the handleHttpRequest function -jest.mock('../../../adapters/network'); - -const successfulResponse = { - status: 200, - response: { - access_token: '', - token_type: 'bearer', - expires_in: 3600, - scope: 'dummy@scope.com', - success: true, - }, -}; - -const unsuccessfulResponse = { - status: 400, - response: '[ENOTFOUND] :: DNS lookup failed', -}; - -const emptyResponse = { - response: '', -}; - -const invalidClientErrorResponse = { - status: 401, - response: { - error: 'invalid_client', - error_description: 'Bad client credentials', - }, -}; - -describe('handleCommonErrorResponse', () => { - test('should throw AbortedError for abortable error codes', () => { - const resp = { - response: { - errors: [{ code: 1003, message: 'Aborted' }], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - AbortedError, - ); - }); - - test('should throw ThrottledError for throttled error codes', () => { - const resp = { - response: { - errors: [{ code: 615, message: 'Throttled' }], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - RetryableError, - ); - }); - - test('should throw RetryableError for other error codes', () => { - const resp = { - response: { - errors: [{ code: 2000, message: 'Retryable' }], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - RetryableError, - ); - }); - - test('should throw RetryableError by default', () => { - const resp = { - response: { - errors: [], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - RetryableError, - ); - }); -}); - -describe('handlePollResponse', () => { - // Tests that the function returns the response object if the polling operation was successful - it('should return the response object when the polling operation was successful', () => { - const pollStatus = { - response: { - success: true, - result: [ - { - batchId: '123', - status: 'Complete', - numOfLeadsProcessed: 2, - numOfRowsFailed: 1, - numOfRowsWithWarning: 0, - message: 'Import completed with errors, 2 records imported (2 members), 1 failed', - }, - ], - }, - }; - - const result = handlePollResponse(pollStatus); - - expect(result).toEqual(pollStatus.response); - }); - - // Tests that the function throws an AbortedError if the response contains an abortable error code - it('should throw an AbortedError when the response contains an abortable error code', () => { - const pollStatus = { - response: { - errors: [ - { - code: 1003, - message: 'Empty file', - }, - ], - }, - }; - - expect(() => handlePollResponse(pollStatus)).toThrow(AbortedError); - }); - - // Tests that the function throws a ThrottledError if the response contains a throttled error code - it('should throw a ThrottledError when the response contains a throttled error code', () => { - const pollStatus = { - response: { - errors: [ - { - code: 615, - message: 'Exceeded concurrent usage limit', - }, - ], - }, - }; - - expect(() => handlePollResponse(pollStatus)).toThrow(RetryableError); - }); - - // Tests that the function throws a RetryableError if the response contains an error code that is not abortable or throttled - it('should throw a RetryableError when the response contains an error code that is not abortable or throttled', () => { - const pollStatus = { - response: { - errors: [ - { - code: 601, - message: 'Unauthorized', - }, - ], - }, - }; - - expect(() => handlePollResponse(pollStatus)).toThrow(RetryableError); - }); - - // Tests that the function returns null if the polling operation was not successful - it('should return null when the polling operation was not successful', () => { - const pollStatus = { - response: { - success: false, - }, - }; - - const result = handlePollResponse(pollStatus); - - expect(result).toBeNull(); - }); -}); - -describe('handleFileUploadResponse', () => { - // Tests that the function returns an object with importId, successfulJobs, and unsuccessfulJobs when the response indicates a successful upload. - it('should return an object with importId, successfulJobs, and unsuccessfulJobs when the response indicates a successful upload', () => { - const resp = { - response: { - success: true, - result: [ - { - importId: '3404', - status: 'Queued', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - const result = handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - - expect(result).toEqual({ - importId: '3404', - successfulJobs: [], - unsuccessfulJobs: [], - }); - }); - - // Tests that the function throws a RetryableError when the response indicates an empty file. - it('should throw a RetryableError when the response indicates an empty file', () => { - const resp = { - response: { - errors: [ - { - code: '1003', - message: 'Empty File', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - expect(() => { - handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - }).toThrow(RetryableError); - }); - - // Tests that the function throws a RetryableError when the response indicates more than 10 concurrent uses. - it('should throw a RetryableError when the response indicates more than 10 concurrent uses', () => { - const resp = { - response: { - errors: [ - { - code: '615', - message: 'Concurrent Use Limit Exceeded', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - expect(() => { - handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - }).toThrow(RetryableError); - }); - - // Tests that the function throws a RetryableError when the response contains an error code between 1000 and 1077. - it('should throw a Aborted when the response contains an error code between 1000 and 1077', () => { - const resp = { - response: { - errors: [ - { - code: 1001, - message: 'Some Error', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - expect(() => { - handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - }).toThrow(AbortedError); - }); -}); - -describe('getAccessToken', () => { - beforeEach(() => { - handleHttpRequest.mockClear(); - }); - - it('should retrieve and return access token on successful response', async () => { - const url = - 'https://dummyMunchkinId.mktorest.com/identity/oauth/token?client_id=dummyClientId&client_secret=dummyClientSecret&grant_type=client_credentials'; - - handleHttpRequest.mockResolvedValueOnce({ - processedResponse: successfulResponse, - }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - const result = await getAccessToken(config); - expect(result).toBe(''); - expect(handleHttpRequest).toHaveBeenCalledTimes(1); - // Ensure your mock response structure is consistent with the actual behavior - expect(handleHttpRequest).toHaveBeenCalledWith('get', url, { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/identity/oauth/token', - feature: 'transformation', - module: 'router', - requestMethod: 'GET', - }); - }); - - it('should throw a NetworkError on unsuccessful HTTP status', async () => { - handleHttpRequest.mockResolvedValueOnce({ - processedResponse: unsuccessfulResponse, - }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(NetworkError); - }); - - it('should throw a RetryableError when expires_in is 0', async () => { - handleHttpRequest.mockResolvedValueOnce({ - processedResponse: { - ...successfulResponse, - response: { ...successfulResponse.response, expires_in: 0 }, - }, - }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(RetryableError); - }); - - it('should throw an AbortedError on unsuccessful response', async () => { - handleHttpRequest.mockResolvedValueOnce({ processedResponse: invalidClientErrorResponse }); - - const config = { - clientId: 'invalidClientID', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(NetworkError); - }); - - it('should throw transformation error response', async () => { - handleHttpRequest.mockResolvedValueOnce({ processedResponse: emptyResponse }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(TransformationError); - }); -}); - -describe('checkEventStatusViaSchemaMatching', () => { - // The function correctly identifies fields with expected data types. - it('if event data types match with expected data types we send no field as mismatch', () => { - const event = { - input: [ - { - message: { - email: 'value1', - id: 123, - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'integer', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({}); - }); - - // The function correctly identifies fields with unexpected data types. - it('if event data types do not match with expected data types we send that field as mismatch', () => { - const event = { - input: [ - { - message: { - email: 123, - city: '123', - islead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - city: 'number', - islead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({ - job1: 'invalid email', - }); - }); - - // The function correctly handles events with multiple fields. - it('For array of events the mismatch object fills up with each event errors', () => { - const event = { - input: [ - { - message: { - id: 'value1', - testCustomFieldScore: 123, - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - { - message: { - email: 'value2', - id: 456, - testCustomFieldScore: false, - }, - metadata: { - job_id: 'job2', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'email', - id: 'integer', - testCustomFieldScore: 'integer', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({ - job1: 'invalid id', - job2: 'invalid testCustomFieldScore', - }); - }); - - // The function correctly handles events with missing fields. - it('it is not mandatory to send all the fields present in schema', () => { - const event = { - input: [ - { - message: { - email: 'value1', - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'number', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({}); - }); - - // The function correctly handles events with additional fields. But this will not happen in our use case - it('for any field beyond schema fields will be mapped as invalid', () => { - const event = { - input: [ - { - message: { - email: 'value1', - id: 124, - isLead: true, - abc: 'value2', - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'number', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({ - job1: 'invalid abc', - }); - }); - - // The function correctly handles events with null values. - it('should ignore event properties with null values', () => { - const event = { - input: [ - { - message: { - email: 'value1', - id: null, - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'number', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({}); - }); -}); diff --git a/src/v0/destinations/marketo_bulk_upload/poll.js b/src/v0/destinations/marketo_bulk_upload/poll.js deleted file mode 100644 index f53347d6e5..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/poll.js +++ /dev/null @@ -1,126 +0,0 @@ -const { NetworkError } = require('@rudderstack/integrations-lib'); -const { removeUndefinedValues, isHttpStatusSuccess } = require('../../util'); -const { getAccessToken, handlePollResponse, hydrateStatusForServer } = require('./util'); -const { handleHttpRequest } = require('../../../adapters/network'); -const stats = require('../../../util/stats'); -const { JSON_MIME_TYPE } = require('../../util/constant'); -const { POLL_ACTIVITY } = require('./config'); - -const getPollStatus = async (event) => { - const accessToken = await getAccessToken(event.config); - const { munchkinId } = event.config; - - // To see the status of the import job polling is done - // DOC: https://developers.marketo.com/rest-api/bulk-import/bulk-lead-import/#polling_job_status - const requestOptions = { - headers: { - 'Content-Type': JSON_MIME_TYPE, - Authorization: `Bearer ${accessToken}`, - }, - }; - const pollUrl = `https://${munchkinId}.mktorest.com/bulk/v1/leads/batch/${event.importId}.json`; - const { processedResponse: pollStatus } = await handleHttpRequest( - 'get', - pollUrl, - requestOptions, - { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads/batch/importId.json', - requestMethod: 'GET', - module: 'router', - }, - ); - if (!isHttpStatusSuccess(pollStatus.status)) { - stats.counter(POLL_ACTIVITY, 1, { - status: pollStatus.status, - state: 'Retryable', - }); - throw new NetworkError( - `Could not poll status: due to error ${JSON.stringify(pollStatus.response)}`, - hydrateStatusForServer(pollStatus.status, 'During fetching poll status'), - ); - } - return handlePollResponse(pollStatus); -}; - -const responseHandler = async (event) => { - const pollResp = await getPollStatus(event); - // Server expects : - /** - * - * { - "Complete": true, - "statusCode": 200, - "hasFailed": true, - "InProgress": false, - "FailedJobURLs": "", // transformer URL - "HasWarning": false, - "WarningJobURLs": "", // transformer URL - } // Succesful Upload - { - "success": false, - "statusCode": 400, - "errorResponse": - } // Failed Upload - { - "success": false, - "Inprogress": true, - statusCode: 500, - } // Importing or Queue - - */ - if (pollResp) { - // As marketo lead import API or bulk API does not support record level error response we are considering - // file level errors only. - // ref: https://nation.marketo.com/t5/ideas/support-error-code-in-record-level-in-lead-bulk-api/idi-p/262191 - const { status, numOfRowsFailed, numOfRowsWithWarning, message } = pollResp.result[0]; - if (status === 'Complete') { - const response = { - Complete: true, - statusCode: 200, - InProgress: false, - hasFailed: numOfRowsFailed > 0, - FailedJobURLs: numOfRowsFailed > 0 ? '/getFailedJobs' : undefined, - HasWarning: numOfRowsWithWarning > 0, - WarningJobURLs: numOfRowsWithWarning > 0 ? '/getWarningJobs' : undefined, - }; - return removeUndefinedValues(response); - } - if (status === 'Importing' || status === 'Queued') { - return { - Complete: false, - statusCode: 500, - hasFailed: false, - InProgress: true, - HasWarning: false, - }; - } - if (status === 'Failed') { - return { - Complete: false, - statusCode: 500, - hasFailed: false, - InProgress: false, - HasWarning: false, - Error: message || 'Marketo Poll Status Failed', - }; - } - } - // when pollResp is null - return { - Complete: false, - statusCode: 500, - hasFailed: false, - InProgress: false, - HasWarning: false, - Error: 'No poll response received from Marketo', - }; -}; - -const processPolling = async (event) => { - const resp = await responseHandler(event); - return resp; -}; - -module.exports = { processPolling }; diff --git a/src/v0/destinations/marketo_bulk_upload/uploadFile/marketo_bulk_upload_example.csv b/src/v0/destinations/marketo_bulk_upload/uploadFile/marketo_bulk_upload_example.csv deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/v0/destinations/marketo_bulk_upload/util.js b/src/v0/destinations/marketo_bulk_upload/util.js deleted file mode 100644 index 033239b5e4..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/util.js +++ /dev/null @@ -1,436 +0,0 @@ -const { - AbortedError, - RetryableError, - NetworkError, - TransformationError, - isDefinedAndNotNull, -} = require('@rudderstack/integrations-lib'); -const { handleHttpRequest } = require('../../../adapters/network'); -const tags = require('../../util/tags'); -const { isHttpStatusSuccess, generateUUID } = require('../../util'); -const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); -const stats = require('../../../util/stats'); -const { - ABORTABLE_CODES, - THROTTLED_CODES, - POLL_ACTIVITY, - UPLOAD_FILE, - FETCH_ACCESS_TOKEN, - POLL_STATUS_ERR_MSG, - FILE_UPLOAD_ERR_MSG, - ACCESS_TOKEN_FETCH_ERR_MSG, - SCHEMA_DATA_TYPE_MAP, -} = require('./config'); -const logger = require('../../../logger'); - -const getMarketoFilePath = () => - `${__dirname}/uploadFile/${Date.now()}_marketo_bulk_upload_${generateUUID()}.csv`; - -// Server only aborts when status code is 400 -const hydrateStatusForServer = (statusCode, context) => { - const status = Number(statusCode); - if (Number.isNaN(status)) { - throw new TransformationError(`${context}: Couldn't parse status code ${statusCode}`); - } - if (status >= 400 && status <= 499) { - return 400; - } - return status; -}; - -/** - * Handles common error responses returned from API calls. - * Checks the error code and throws the appropriate error object based on the code. - * - * @param {object} resp - The response object containing the error information. - * @param {string} opErrorMessage - The error message to be used if the error code is not recognized. - * @param {string} opActivity - The activity name for tracking purposes. - * @throws {AbortedError} - If the error code is abortable. - * @throws {ThrottledError} - If the error code is within the range of throttled codes. - * @throws {RetryableError} - If the error code is neither abortable nor throttled. - * - * @example - * const resp = { - * response: { - * errors: [ - * { - * code: "1003", - * message: "Empty File" - * } - * ] - * } - * }; - * - * try { - * handleCommonErrorResponse(resp, "Error message", "Activity"); - * } catch (error) { - * console.log(error); - * } - */ -const handleCommonErrorResponse = (apiCallResult, opErrorMessage, opActivity) => { - // checking for invalid/expired token errors and evicting cache in that case - // rudderJobMetadata contains some destination info which is being used to evict the cache - if ( - apiCallResult.response?.errors && - apiCallResult.response?.errors?.length > 0 && - apiCallResult.response?.errors.some( - (errorObj) => errorObj.code === '601' || errorObj.code === '602', - ) - ) { - throw new RetryableError( - `[${opErrorMessage}]Error message: ${apiCallResult.response?.errors[0]?.message}`, - ); - } - if ( - apiCallResult.response?.errors?.length > 0 && - apiCallResult.response?.errors[0] && - ((apiCallResult.response?.errors[0]?.code >= 1000 && - apiCallResult.response?.errors[0]?.code <= 1077) || - ABORTABLE_CODES.includes(apiCallResult.response?.errors[0]?.code)) - ) { - // for empty file the code is 1003 and that should be retried - stats.increment(opActivity, { - status: 400, - state: 'Abortable', - }); - throw new AbortedError(apiCallResult.response?.errors[0]?.message || opErrorMessage, 400); - } else if (THROTTLED_CODES.includes(apiCallResult.response?.errors[0]?.code)) { - // for more than 10 concurrent uses the code is 615 and that should be retried - stats.increment(opActivity, { - status: 429, - state: 'Retryable', - }); - throw new RetryableError( - `[${opErrorMessage}]Error message: ${apiCallResult.response?.errors[0]?.message}`, - 500, - ); - } - // by default every thing will be retried - stats.increment(opActivity, { - status: 500, - state: 'Retryable', - }); - throw new RetryableError( - `[${opErrorMessage}]Error message: ${apiCallResult.response?.errors[0]?.message}`, - 500, - ); -}; - -const getAccessTokenURL = (config) => { - const { clientId, clientSecret, munchkinId } = config; - const url = `https://${munchkinId}.mktorest.com/identity/oauth/token?client_id=${clientId}&client_secret=${clientSecret}&grant_type=client_credentials`; - return url; -}; - -// Fetch access token from client id and client secret -// DOC: https://developers.marketo.com/rest-api/authentication/ -const getAccessToken = async (config) => { - const url = getAccessTokenURL(config); - const { processedResponse: accessTokenResponse } = await handleHttpRequest('get', url, { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/identity/oauth/token', - requestMethod: 'GET', - module: 'router', - }); - - // sample response : {response: '[ENOTFOUND] :: DNS lookup failed', status: 400} - if (!isHttpStatusSuccess(accessTokenResponse.status)) { - throw new NetworkError( - `Could not retrieve authorisation token due to error ${JSON.stringify(accessTokenResponse)}`, - hydrateStatusForServer(accessTokenResponse.status, FETCH_ACCESS_TOKEN), - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(accessTokenResponse.status), - }, - accessTokenResponse, - ); - } - if (accessTokenResponse.response?.success === false) { - handleCommonErrorResponse(accessTokenResponse, ACCESS_TOKEN_FETCH_ERR_MSG, FETCH_ACCESS_TOKEN); - } - - // when access token is present - if (accessTokenResponse.response.access_token) { - /* This scenario will handle the case when we get the following response - status: 200 - respnse: {"access_token":"","token_type":"bearer","expires_in":0,"scope":"dummy@scope.com"} - wherein "expires_in":0 denotes that we should refresh the accessToken but its not expired yet. - */ - if (accessTokenResponse.response?.expires_in === 0) { - throw new RetryableError( - `Request Failed for marketo_bulk_upload, Access Token Expired (Retryable).`, - 500, - ); - } - return accessTokenResponse.response.access_token; - } - throw new RetryableError( - `Could not retrieve authorisation token due to error ${JSON.stringify(accessTokenResponse)}`, - 500, - ); -}; - -/** - * Handles the response of a polling operation. - * Checks for any errors in the response and calls the `handleCommonErrorResponse` function to handle them. - * If the response is successful, increments the stats and returns the response. - * Otherwise, returns null. - * - * @param {object} pollStatus - The response object from the polling operation. - * @returns {object|null} - The response object if the polling operation was successful, otherwise null. - */ -const handlePollResponse = (pollStatus) => { - // DOC: https://developers.marketo.com/rest-api/error-codes/ - if (pollStatus.response.errors) { - /* Sample error response for poll is: - - { - "requestId": "e42b#14272d07d78", - "success": false, - "errors": [ - { - "code": "601", - "message": "Unauthorized" - } - ] - } - */ - handleCommonErrorResponse(pollStatus, POLL_STATUS_ERR_MSG, POLL_ACTIVITY); - } - - /* - Sample Successful Poll response structure: - { - "requestId":"8136#146daebc2ed", - "success":true, - "result":[ - { - "batchId":, - "status":"Complete", - "numOfLeadsProcessed":2, - "numOfRowsFailed":1, - "numOfRowsWithWarning":0, - "message":"Import completed with errors, 2 records imported (2 members), 1 failed" - } - ] - } - */ - if (pollStatus.response?.success) { - stats.counter(POLL_ACTIVITY, 1, { - status: 200, - state: 'Success', - }); - - if (pollStatus.response?.result?.length > 0) { - return pollStatus.response; - } - } - - return null; -}; - -const handleFetchJobStatusResponse = (resp, type) => { - const marketoResponse = resp.response; - const marketoReposnseStatus = resp.status; - - if (!isHttpStatusSuccess(marketoReposnseStatus)) { - logger.info('[Network Error]:Failed during fetching job status', { marketoResponse, type }); - throw new NetworkError( - `Unable to fetch job status: due to error ${JSON.stringify(marketoResponse)}`, - hydrateStatusForServer(marketoReposnseStatus, 'During fetching job status'), - ); - } - - if (marketoResponse?.success === false) { - logger.info('[Application Error]Failed during fetching job status', { marketoResponse, type }); - throw new RetryableError( - `Failure during fetching job status due to error : ${marketoResponse}`, - 500, - resp, - ); - } - - /* - successful response : - { - response: 'city, email,Import Failure ReasonChennai,s…a,Value for lookup field 'email' not found', - status: 200 - } - - */ - - return marketoResponse; -}; - -/** - * Handles the response received after a file upload request. - * Checks for errors in the response and throws appropriate error objects based on the error codes. - * If the response indicates a successful upload, extracts the importId and returns it along with other job details. - * - * @param {object} resp - The response object received after a file upload request. - * @param {array} successfulJobs - An array to store details of successful jobs. - * @param {array} unsuccessfulJobs - An array to store details of unsuccessful jobs. - * @param {number} requestTime - The time taken for the request in milliseconds. - * @returns {object} - An object containing the importId, successfulJobs, and unsuccessfulJobs. - */ -const handleFileUploadResponse = (resp, successfulJobs, unsuccessfulJobs, requestTime) => { - /* - For unsuccessful response - { - "requestId": "e42b#14272d07d78", - "success": false, - "errors": [ - { - "code": "1003", - "message": "Empty File" - } - ] - } - */ - if (resp.response?.errors) { - if (resp.response?.errors[0]?.code === '1003') { - stats.increment(UPLOAD_FILE, { - status: 500, - state: 'Retryable', - }); - throw new RetryableError( - `[${FILE_UPLOAD_ERR_MSG}]:Error Message ${resp.response.errors[0]?.message}`, - 500, - ); - } else { - handleCommonErrorResponse(resp, FILE_UPLOAD_ERR_MSG, UPLOAD_FILE); - } - } - - /** - * SuccessFul Upload Response : - { - "requestId": "d01f#15d672f8560", - "result": [ - { - "batchId": 3404, - "importId": "3404", - "status": "Queued" - } - ], - "success": true - } - */ - if ( - resp.response?.success && - resp.response?.result?.length > 0 && - resp.response?.result[0]?.importId - ) { - const { importId } = resp.response.result[0]; - stats.histogram('marketo_bulk_upload_upload_file_time', requestTime); - - stats.increment(UPLOAD_FILE, { - status: 200, - state: 'Success', - }); - return { importId, successfulJobs, unsuccessfulJobs }; - } - // if neither successful, nor the error message is appropriate sending importId as default null - return { importId: null, successfulJobs, unsuccessfulJobs }; -}; - -/** - * Retrieves the field schema mapping for a given access token and munchkin ID from the Marketo API. - * - * @param {string} accessToken - The access token used to authenticate the API request. - * @param {string} munchkinId - The munchkin ID of the Marketo instance. - * @returns {object} - The field schema mapping retrieved from the Marketo API. - */ -const getFieldSchemaMap = async (accessToken, munchkinId) => { - let fieldArr = []; - const fieldMap = {}; // map to store field name and data type - // ref: https://developers.marketo.com/rest-api/endpoint-reference/endpoint-index/#:~:text=Describe%20Lead2,leads/describe2.json - const { processedResponse: fieldSchemaMapping } = await handleHttpRequest( - 'get', - `https://${munchkinId}.mktorest.com/rest/v1/leads/describe2.json`, - { - params: { - access_token: accessToken, - }, - }, - { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads/describe2.json', - requestMethod: 'GET', - module: 'router', - }, - ); - if (fieldSchemaMapping.response.errors) { - handleCommonErrorResponse( - fieldSchemaMapping, - 'Error while fetching Marketo Field Schema', - 'FieldSchemaMapping', - ); - } - if ( - fieldSchemaMapping.response?.success && - fieldSchemaMapping.response?.result.length > 0 && - fieldSchemaMapping.response?.result[0] - ) { - fieldArr = - fieldSchemaMapping.response.result && Array.isArray(fieldSchemaMapping.response.result) - ? fieldSchemaMapping.response.result[0]?.fields - : []; - - fieldArr.forEach((field) => { - fieldMap[field?.name] = field?.dataType; - }); - } else { - throw new RetryableError( - `Failed to fetch Marketo Field Schema due to error ${JSON.stringify(fieldSchemaMapping)}`, - 500, - fieldSchemaMapping, - ); - } - return fieldMap; -}; - -/** - * Compares the data types of the fields in an event message with the expected data types defined in the field schema mapping. - * Identifies any mismatched fields and returns them as a map of job IDs and the corresponding invalid fields. - * - * @param {object} event - An object containing an `input` array of events. Each event has a `message` object with field-value pairs and a `metadata` object with a `job_id` property. - * @param {object} fieldSchemaMapping - An object containing the field schema mapping, which includes the expected data types for each field. - * @returns {object} - An object containing the job IDs as keys and the corresponding invalid fields as values. - */ -const checkEventStatusViaSchemaMatching = (event, fieldMap) => { - const mismatchedFields = {}; - const events = event.input; - events.forEach((ev) => { - const { message, metadata } = ev; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { job_id } = metadata; - - Object.entries(message).forEach(([paramName, paramValue]) => { - const expectedDataType = SCHEMA_DATA_TYPE_MAP[fieldMap[paramName]]; - const actualDataType = typeof paramValue; - - if ( - isDefinedAndNotNull(paramValue) && - !mismatchedFields[job_id] && - actualDataType !== expectedDataType - ) { - mismatchedFields[job_id] = `invalid ${paramName}`; - } - }); - }); - return mismatchedFields; -}; - -module.exports = { - checkEventStatusViaSchemaMatching, - handlePollResponse, - handleFetchJobStatusResponse, - handleFileUploadResponse, - handleCommonErrorResponse, - hydrateStatusForServer, - getAccessToken, - getMarketoFilePath, - getFieldSchemaMap, -}; diff --git a/src/v0/destinations/snowpipe_streaming/transform.js b/src/v0/destinations/snowpipe_streaming/transform.js index a71992949e..656cab5a95 100644 --- a/src/v0/destinations/snowpipe_streaming/transform.js +++ b/src/v0/destinations/snowpipe_streaming/transform.js @@ -1,8 +1,14 @@ -const transform = require('../snowflake/transform'); const { processWarehouseMessage } = require('../../../warehouse'); const provider = 'snowpipe_streaming'; +function getDataTypeOverride(key, val, options, jsonKey = false) { + if (key === 'violationErrors' || jsonKey) { + return 'json'; + } + return 'string'; +} + function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whIDResolve = event.request.query.whIDResolve === 'true' || false; @@ -13,7 +19,7 @@ function process(event) { whSchemaVersion, whStoreEvent, whIDResolve, - getDataTypeOverride: transform.getDataTypeOverride, + getDataTypeOverride, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, destJsonPaths, @@ -24,5 +30,5 @@ function process(event) { module.exports = { provider, process, - getDataTypeOverride: transform.getDataTypeOverride, -}; + getDataTypeOverride, +}; \ No newline at end of file diff --git a/src/v0/sources/adjust/transform.js b/src/v0/sources/adjust/transform.js index 9da90751b7..f68e87d476 100644 --- a/src/v0/sources/adjust/transform.js +++ b/src/v0/sources/adjust/transform.js @@ -7,6 +7,7 @@ const Message = require('../message'); const { CommonUtils } = require('../../../util/common'); const { excludedFieldList } = require('./config'); const { extractCustomFields, generateUUID } = require('../../util'); +const { convertToISODate } = require('./utils'); // ref : https://help.adjust.com/en/article/global-callbacks#general-recommended-placeholders // import mapping json using JSON.parse to preserve object key order @@ -43,11 +44,10 @@ const processEvent = (inputEvent) => { message.properties = { ...message.properties, ...customProperties }; if (formattedPayload.created_at) { - const ts = new Date(formattedPayload.created_at * 1000).toISOString(); + const ts = convertToISODate(formattedPayload.created_at); message.setProperty('originalTimestamp', ts); message.setProperty('timestamp', ts); } - // adjust does not has the concept of user but we need to set some random anonymousId in order to make the server accept the message message.anonymousId = generateUUID(); return message; diff --git a/src/v0/sources/adjust/utils.js b/src/v0/sources/adjust/utils.js new file mode 100644 index 0000000000..73ec696e34 --- /dev/null +++ b/src/v0/sources/adjust/utils.js @@ -0,0 +1,27 @@ +const { TransformationError } = require('@rudderstack/integrations-lib'); + +const convertToISODate = (rawTimestamp) => { + if (typeof rawTimestamp !== 'number' && typeof rawTimestamp !== 'string') { + throw new TransformationError( + `Invalid timestamp type: expected number or string, received ${typeof rawTimestamp}`, + ); + } + + const createdAt = Number(rawTimestamp); + + if (Number.isNaN(createdAt)) { + throw new TransformationError(`Failed to parse timestamp: "${rawTimestamp}"`); + } + + const date = new Date(createdAt * 1000); + + if (Number.isNaN(date.getTime())) { + throw new TransformationError(`Failed to create valid date for timestamp "${rawTimestamp}"`); + } + + return date.toISOString(); +}; + +module.exports = { + convertToISODate, +}; diff --git a/src/v0/sources/adjust/utils.test.js b/src/v0/sources/adjust/utils.test.js new file mode 100644 index 0000000000..f5a0caa832 --- /dev/null +++ b/src/v0/sources/adjust/utils.test.js @@ -0,0 +1,37 @@ +const { convertToISODate } = require('./utils'); +const { TransformationError } = require('@rudderstack/integrations-lib'); + +describe('convertToISODate', () => { + // Converts valid numeric timestamp to ISO date string + it('should return ISO date string when given a valid numeric timestamp', () => { + const timestamp = 1633072800; // Example timestamp for 2021-10-01T00:00:00.000Z + const result = convertToISODate(timestamp); + expect(result).toBe('2021-10-01T07:20:00.000Z'); + }); + + // Throws error for non-numeric string input + it('should throw TransformationError when given a non-numeric string', () => { + const invalidTimestamp = 'invalid'; + expect(() => convertToISODate(invalidTimestamp)).toThrow(TransformationError); + }); + + // Converts valid numeric string timestamp to ISO date string + it('should convert valid numeric string timestamp to ISO date string', () => { + const rawTimestamp = '1633072800'; // Corresponds to 2021-10-01T00:00:00.000Z + const result = convertToISODate(rawTimestamp); + expect(result).toBe('2021-10-01T07:20:00.000Z'); + }); + + // Throws error for non-number and non-string input + it('should throw error for non-number and non-string input', () => { + expect(() => convertToISODate({})).toThrow(TransformationError); + expect(() => convertToISODate([])).toThrow(TransformationError); + expect(() => convertToISODate(null)).toThrow(TransformationError); + expect(() => convertToISODate(undefined)).toThrow(TransformationError); + }); + + it('should throw error for timestamp that results in invalid date when multiplied', () => { + const hugeTimestamp = 999999999999999; // This will become invalid when multiplied by 1000 + expect(() => convertToISODate(hugeTimestamp)).toThrow(TransformationError); + }); +}); diff --git a/src/v0/sources/shopify/util.js b/src/v0/sources/shopify/util.js index 981832363e..b7e79e35a1 100644 --- a/src/v0/sources/shopify/util.js +++ b/src/v0/sources/shopify/util.js @@ -2,15 +2,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ const { v5 } = require('uuid'); const sha256 = require('sha256'); -const { TransformationError } = require('@rudderstack/integrations-lib'); +const { TransformationError, isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); const stats = require('../../../util/stats'); -const { - constructPayload, - extractCustomFields, - flattenJson, - generateUUID, - isDefinedAndNotNull, -} = require('../../util'); +const utils = require('../../util'); const { RedisDB } = require('../../../util/redis/redisConnector'); const { lineItemsMappingJSON, @@ -92,8 +86,8 @@ const getProductsListFromLineItems = (lineItems) => { } const products = []; lineItems.forEach((lineItem) => { - const product = constructPayload(lineItem, lineItemsMappingJSON); - extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS); + const product = utils.constructPayload(lineItem, lineItemsMappingJSON); + utils.extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS); product.variant = getVariantString(lineItem); products.push(product); }); @@ -103,14 +97,14 @@ const getProductsListFromLineItems = (lineItems) => { const createPropertiesForEcomEvent = (message) => { const { line_items: lineItems } = message; const productsList = getProductsListFromLineItems(lineItems); - const mappedPayload = constructPayload(message, productMappingJSON); - extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS); + const mappedPayload = utils.constructPayload(message, productMappingJSON); + utils.extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS); mappedPayload.products = productsList; return mappedPayload; }; const extractEmailFromPayload = (event) => { - const flattenedPayload = flattenJson(event); + const flattenedPayload = utils.flattenJson(event); let email; const regex_email = /\bemail\b/i; Object.entries(flattenedPayload).some(([key, value]) => { @@ -182,7 +176,7 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n return { anonymousId, sessionId }; } return { - anonymousId: isDefinedAndNotNull(anonymousId) ? anonymousId : generateUUID(), + anonymousId: isDefinedAndNotNull(anonymousId) ? anonymousId : utils.generateUUID(), sessionId, }; } @@ -281,4 +275,5 @@ module.exports = { checkAndUpdateCartItems, getHashLineItems, getDataFromRedis, + getVariantString, }; diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 1676498fdb..ad08be448e 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -16,6 +16,7 @@ const uaParser = require('ua-parser-js'); const moment = require('moment-timezone'); const sha256 = require('sha256'); const crypto = require('crypto'); +const { v5 } = require('uuid'); const { InstrumentationError, BaseError, @@ -2330,6 +2331,29 @@ const isEventSentByVDMV1Flow = (event) => event?.message?.context?.mappedToDesti const isEventSentByVDMV2Flow = (event) => event?.connection?.config?.destination?.schemaVersion === VDM_V2_SCHEMA_VERSION; + +const convertToUuid = (input) => { + const NAMESPACE = v5.DNS; + + if (!isDefinedAndNotNull(input)) { + throw new InstrumentationError('Input is undefined or null.'); + } + + try { + // Stringify and trim the input + const trimmedInput = String(input).trim(); + + // Check for empty input after trimming + if (!trimmedInput) { + throw new InstrumentationError('Input is empty or invalid.'); + } + // Generate and return UUID + return v5(trimmedInput, NAMESPACE); + } catch (error) { + const errorMessage = `Failed to transform input to uuid: ${error.message}`; + throw new InstrumentationError(errorMessage); + } +}; // ======================================================================== // EXPORTS // ======================================================================== @@ -2456,4 +2480,5 @@ module.exports = { getRelativePathFromURL, removeEmptyKey, isAxiosError, + convertToUuid, }; diff --git a/src/v0/util/index.test.js b/src/v0/util/index.test.js index 0b05b6f2d6..cfdfefddee 100644 --- a/src/v0/util/index.test.js +++ b/src/v0/util/index.test.js @@ -2,6 +2,7 @@ const { InstrumentationError } = require('@rudderstack/integrations-lib'); const utilities = require('.'); const { getFuncTestData } = require('../../../test/testHelper'); const { FilteredEventsError } = require('./errorTypes'); +const { v5 } = require('uuid'); const { hasCircularReference, flattenJson, @@ -11,6 +12,7 @@ const { groupRouterTransformEvents, isAxiosError, removeHyphens, + convertToUuid, } = require('./index'); const exp = require('constants'); @@ -985,3 +987,65 @@ describe('removeHyphens', () => { }); }); }); + +describe('convertToUuid', () => { + const NAMESPACE = v5.DNS; + + test('should generate UUID for valid string input', () => { + const input = 'testInput'; + const expectedUuid = '7ba1e88f-acf9-5528-9c1c-0c897ed80e1e'; + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('should generate UUID for valid numeric input', () => { + const input = 123456; + const expectedUuid = 'a52b2702-9bcf-5701-852a-2f4edc640fe1'; + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('should trim spaces and generate UUID', () => { + const input = ' testInput '; + const expectedUuid = '7ba1e88f-acf9-5528-9c1c-0c897ed80e1e'; + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('should throw an error for empty input', () => { + const input = ''; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is empty or invalid.'); + }); + + test('to throw an error for null input', () => { + const input = null; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is undefined or null'); + }); + + test('to throw an error for undefined input', () => { + const input = undefined; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is undefined or null'); + }); + + test('should throw an error for input that is whitespace only', () => { + const input = ' '; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is empty or invalid.'); + }); + + test('should handle long string input gracefully', () => { + const input = 'a'.repeat(1000); + const expectedUuid = v5(input, NAMESPACE); + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('any invalid input if stringified does not throw error', () => { + const input = {}; + const result = convertToUuid(input); + expect(result).toBe('672ca00c-37f4-5d71-b8c3-6ae0848080ec'); + }); +}); diff --git a/src/v1/sources/shopify/transform.js b/src/v1/sources/shopify/transform.js index dee5a14a9d..5ebf4a34fc 100644 --- a/src/v1/sources/shopify/transform.js +++ b/src/v1/sources/shopify/transform.js @@ -1,19 +1,27 @@ /* eslint-disable @typescript-eslint/naming-convention */ -const { processEventFromPixel } = require('./pixelTransform'); +const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform'); const { process: processWebhookEvents } = require('../../../v0/sources/shopify/transform'); +const { + process: processPixelWebhookEvents, +} = require('./webhookTransformations/serverSideTransform'); const process = async (inputEvent) => { const { event } = inputEvent; - // check on the source Config to identify the event is from the tracker-based (legacy) - // or the pixel-based (latest) implementation. + const { query_parameters } = event; + // check identify the event is from the web pixel based on the pixelEventLabel property. const { pixelEventLabel: pixelClientEventLabel } = event; if (pixelClientEventLabel) { // this is a event fired from the web pixel loaded on the browser // by the user interactions with the store. - const responseV2 = await processEventFromPixel(event); - return responseV2; + const pixelWebEventResponse = await processPixelWebEvents(event); + return pixelWebEventResponse; } - // this is for common logic for server-side events processing for both pixel and tracker apps. + if (query_parameters && query_parameters?.version?.[0] === 'pixel') { + // this is a server-side event from the webhook subscription made by the pixel app. + const pixelWebhookEventResponse = await processPixelWebhookEvents(event); + return pixelWebhookEventResponse; + } + // this is a server-side event from the webhook subscription made by the legacy tracker-based app. const response = await processWebhookEvents(event); return response; }; diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js new file mode 100644 index 0000000000..c31bc74bf1 --- /dev/null +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -0,0 +1,174 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +const lodash = require('lodash'); +const get = require('get-value'); +// const { RedisError } = require('@rudderstack/integrations-lib'); +const stats = require('../../../../util/stats'); +const { + getShopifyTopic, + // createPropertiesForEcomEvent, + extractEmailFromPayload, + getAnonymousIdAndSessionId, + // getHashLineItems, +} = require('../../../../v0/sources/shopify/util'); +// const logger = require('../../../logger'); +const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util'); +// const { RedisDB } = require('../../../util/redis/redisConnector'); +const Message = require('../../../../v0/sources/message'); +const { EventType } = require('../../../../constants'); +const { + INTEGERATION, + MAPPING_CATEGORIES, + IDENTIFY_TOPICS, + ECOM_TOPICS, + RUDDER_ECOM_MAP, + SUPPORTED_TRACK_EVENTS, + SHOPIFY_TRACK_MAP, + lineItemsMappingJSON, +} = require('../../../../v0/sources/shopify/config'); +const { + createPropertiesForEcomEventFromWebhook, + getProductsFromLineItems, +} = require('./serverSideUtlis'); + +const NO_OPERATION_SUCCESS = { + outputToSource: { + body: Buffer.from('OK').toString('base64'), + contentType: 'text/plain', + }, + statusCode: 200, +}; + +const identifyPayloadBuilder = (event) => { + const message = new Message(INTEGERATION); + message.setEventType(EventType.IDENTIFY); + message.setPropertiesV2(event, MAPPING_CATEGORIES[EventType.IDENTIFY]); + if (event.updated_at) { + // converting shopify updated_at timestamp to rudder timestamp format + message.setTimestamp(new Date(event.updated_at).toISOString()); + } + return message; +}; + +const ecomPayloadBuilder = (event, shopifyTopic) => { + const message = new Message(INTEGERATION); + message.setEventType(EventType.TRACK); + message.setEventName(RUDDER_ECOM_MAP[shopifyTopic]); + + const properties = createPropertiesForEcomEventFromWebhook(event); + message.properties = removeUndefinedAndNullValues(properties); + // Map Customer details if present + const customerDetails = get(event, 'customer'); + if (customerDetails) { + message.setPropertiesV2(customerDetails, MAPPING_CATEGORIES[EventType.IDENTIFY]); + } + if (event.updated_at) { + message.setTimestamp(new Date(event.updated_at).toISOString()); + } + if (event.customer) { + message.setPropertiesV2(event.customer, MAPPING_CATEGORIES[EventType.IDENTIFY]); + } + if (event.shipping_address) { + message.setProperty('traits.shippingAddress', event.shipping_address); + } + if (event.billing_address) { + message.setProperty('traits.billingAddress', event.billing_address); + } + if (!message.userId && event.user_id) { + message.setProperty('userId', event.user_id); + } + return message; +}; + +const trackPayloadBuilder = (event, shopifyTopic) => { + const message = new Message(INTEGERATION); + message.setEventType(EventType.TRACK); + message.setEventName(SHOPIFY_TRACK_MAP[shopifyTopic]); + // eslint-disable-next-line camelcase + const { line_items: lineItems } = event; + const productsList = getProductsFromLineItems(lineItems, lineItemsMappingJSON); + message.setProperty('properties.products', productsList); + return message; +}; + +const processEvent = async (inputEvent, metricMetadata) => { + let message; + const event = lodash.cloneDeep(inputEvent); + const shopifyTopic = getShopifyTopic(event); + delete event.query_parameters; + switch (shopifyTopic) { + case IDENTIFY_TOPICS.CUSTOMERS_CREATE: + case IDENTIFY_TOPICS.CUSTOMERS_UPDATE: + message = identifyPayloadBuilder(event); + break; + case ECOM_TOPICS.ORDERS_CREATE: + case ECOM_TOPICS.ORDERS_UPDATE: + case ECOM_TOPICS.CHECKOUTS_CREATE: + case ECOM_TOPICS.CHECKOUTS_UPDATE: + message = ecomPayloadBuilder(event, shopifyTopic); + break; + default: + if (!SUPPORTED_TRACK_EVENTS.includes(shopifyTopic)) { + stats.increment('invalid_shopify_event', { + writeKey: metricMetadata.writeKey, + source: metricMetadata.source, + shopifyTopic: metricMetadata.shopifyTopic, + }); + return NO_OPERATION_SUCCESS; + } + message = trackPayloadBuilder(event, shopifyTopic); + break; + } + + if (message.userId) { + message.userId = String(message.userId); + } + if (!get(message, 'traits.email')) { + const email = extractEmailFromPayload(event); + if (email) { + message.setProperty('traits.email', email); + } + } + if (message.type !== EventType.IDENTIFY) { + const { anonymousId } = await getAnonymousIdAndSessionId( + message, + { shopifyTopic, ...metricMetadata }, + null, + ); + if (isDefinedAndNotNull(anonymousId)) { + message.setProperty('anonymousId', anonymousId); + } + } + message.setProperty(`integrations.${INTEGERATION}`, true); + message.setProperty('context.library', { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }); + message.setProperty('context.topic', shopifyTopic); + // attaching cart, checkout and order tokens in context object + message.setProperty(`context.cart_token`, event.cart_token); + message.setProperty(`context.checkout_token`, event.checkout_token); + // raw shopify payload passed inside context object under shopifyDetails + message.setProperty('context.shopifyDetails', event); + if (shopifyTopic === 'orders_updated') { + message.setProperty(`context.order_token`, event.token); + } + message = removeUndefinedAndNullValues(message); + return message; +}; +const process = async (event) => { + const metricMetadata = { + writeKey: event.query_parameters?.writeKey?.[0], + source: 'SHOPIFY', + }; + const response = await processEvent(event, metricMetadata); + return response; +}; + +module.exports = { + process, + processEvent, + identifyPayloadBuilder, + ecomPayloadBuilder, + trackPayloadBuilder, +}; diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js new file mode 100644 index 0000000000..a611d1d8dc --- /dev/null +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -0,0 +1,112 @@ +const { + getProductsFromLineItems, + createPropertiesForEcomEventFromWebhook, +} = require('./serverSideUtlis'); + +const { constructPayload } = require('../../../../v0/util'); + +const { + lineItemsMappingJSON, + productMappingJSON, +} = require('../../../../v0/sources/shopify/config'); +const Message = require('../../../../v0/sources/message'); +jest.mock('../../../../v0/sources/message'); + +const LINEITEMS = [ + { + id: '41327142600817', + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + product_id: 7234590408818, + quantity: 1, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + line_price: '600.00', + price: '600.00', + applied_discounts: [], + properties: {}, + }, + { + id: 14234727743601, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Nitrogen', + price: '600.00', + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + sku: '', + title: 'The Collection Snowboard: Nitrogen', + total_discount: '0.00', + variant_id: 41327142600817, + vendor: 'Hydrogen Vendor', + }, +]; + +describe('serverSideUtils.js', () => { + beforeEach(() => { + Message.mockClear(); + }); + + describe('Test getProductsFromLineItems function', () => { + it('should return empty array if lineItems is empty', () => { + const lineItems = []; + const result = getProductsFromLineItems(lineItems, lineItemsMappingJSON); + expect(result).toEqual([]); + }); + + it('should return array of products', () => { + const mapping = {}; + const result = getProductsFromLineItems(LINEITEMS, lineItemsMappingJSON); + expect(result).toEqual([ + { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 }, + { + brand: 'Hydrogen Vendor', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + title: 'The Collection Snowboard: Nitrogen', + }, + ]); + }); + }); + + describe('Test createPropertiesForEcomEventFromWebhook function', () => { + it('should return empty array if lineItems is empty', () => { + const message = { + line_items: [], + type: 'track', + event: 'checkout created', + }; + const result = createPropertiesForEcomEventFromWebhook(message); + expect(result).toEqual([]); + }); + + it('should return array of products', () => { + const message = { + line_items: LINEITEMS, + type: 'track', + event: 'checkout updated', + }; + const result = createPropertiesForEcomEventFromWebhook(message); + expect(result).toEqual({ + products: [ + { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 }, + { + brand: 'Hydrogen Vendor', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + title: 'The Collection Snowboard: Nitrogen', + }, + ], + }); + }); + }); +}); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js new file mode 100644 index 0000000000..eed03de71f --- /dev/null +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -0,0 +1,45 @@ +const { constructPayload } = require('../../../../v0/util'); + +const { + lineItemsMappingJSON, + productMappingJSON, +} = require('../../../../v0/sources/shopify/config'); + +/** + * Returns an array of products from the lineItems array received from the webhook event + * @param {Array} lineItems + * @param {Object} mapping + * @returns {Array} products + */ +const getProductsFromLineItems = (lineItems, mapping) => { + if (!lineItems || lineItems.length === 0) { + return []; + } + const products = []; + lineItems.forEach((lineItem) => { + // const product = constructPayload(lineItem, lineItemsMappingJSON); + const product = constructPayload(lineItem, mapping); + products.push(product); + }); + return products; +}; + +/** + * Creates properties for the ecommerce webhook events received from the pixel based app + * @param {Object} message + * @returns {Object} properties + */ +const createPropertiesForEcomEventFromWebhook = (message) => { + const { line_items: lineItems } = message; + if (!lineItems || lineItems.length === 0) { + return []; + } + const mappedPayload = constructPayload(message, productMappingJSON); + mappedPayload.products = getProductsFromLineItems(lineItems, lineItemsMappingJSON); + return mappedPayload; +}; + +module.exports = { + createPropertiesForEcomEventFromWebhook, + getProductsFromLineItems, +}; diff --git a/src/v1/sources/shopify/pixelTransform.js b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js similarity index 94% rename from src/v1/sources/shopify/pixelTransform.js rename to src/v1/sources/shopify/webpixelTransformations/pixelTransform.js index e308f626b4..b1d1c8b2fa 100644 --- a/src/v1/sources/shopify/pixelTransform.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js @@ -2,10 +2,10 @@ // eslint-disable-next-line @typescript-eslint/naming-convention const _ = require('lodash'); const { isDefinedNotNullNotEmpty } = require('@rudderstack/integrations-lib'); -const stats = require('../../../util/stats'); -const logger = require('../../../logger'); -const { removeUndefinedAndNullValues } = require('../../../v0/util'); -const { RedisDB } = require('../../../util/redis/redisConnector'); +const stats = require('../../../../util/stats'); +const logger = require('../../../../logger'); +const { removeUndefinedAndNullValues } = require('../../../../v0/util'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); const { pageViewedEventBuilder, cartViewedEventBuilder, @@ -20,7 +20,7 @@ const { INTEGERATION, PIXEL_EVENT_TOPICS, pixelEventToCartTokenLocationMapping, -} = require('./config'); +} = require('../config'); const NO_OPERATION_SUCCESS = { outputToSource: { @@ -152,13 +152,13 @@ function processPixelEvent(inputEvent) { return message; } -const processEventFromPixel = async (event) => { +const processPixelWebEvents = async (event) => { const pixelEvent = processPixelEvent(event); return removeUndefinedAndNullValues(pixelEvent); }; module.exports = { - processEventFromPixel, + processPixelWebEvents, handleCartTokenRedisOperations, extractCartToken, }; diff --git a/src/v1/sources/shopify/pixelTransform.redisCartToken.test.js b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.redisCartToken.test.js similarity index 87% rename from src/v1/sources/shopify/pixelTransform.redisCartToken.test.js rename to src/v1/sources/shopify/webpixelTransformations/pixelTransform.redisCartToken.test.js index 8f54efc373..1e5cb94b19 100644 --- a/src/v1/sources/shopify/pixelTransform.redisCartToken.test.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.redisCartToken.test.js @@ -1,25 +1,25 @@ const { extractCartToken, handleCartTokenRedisOperations } = require('./pixelTransform'); -const { RedisDB } = require('../../../util/redis/redisConnector'); -const stats = require('../../../util/stats'); -const logger = require('../../../logger'); -const { pixelEventToCartTokenLocationMapping } = require('./config'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); +const stats = require('../../../../util/stats'); +const logger = require('../../../../logger'); +const { pixelEventToCartTokenLocationMapping } = require('../config'); -jest.mock('../../../util/redis/redisConnector', () => ({ +jest.mock('../../../../util/redis/redisConnector', () => ({ RedisDB: { setVal: jest.fn(), }, })); -jest.mock('../../../util/stats', () => ({ +jest.mock('../../../../util/stats', () => ({ increment: jest.fn(), })); -jest.mock('../../../logger', () => ({ +jest.mock('../../../../logger', () => ({ info: jest.fn(), error: jest.fn(), })); -jest.mock('./config', () => ({ +jest.mock('../config', () => ({ pixelEventToCartTokenLocationMapping: { cart_viewed: 'properties.cart_id' }, })); diff --git a/src/v1/sources/shopify/pixelUtils.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js similarity index 97% rename from src/v1/sources/shopify/pixelUtils.js rename to src/v1/sources/shopify/webpixelTransformations/pixelUtils.js index 9abef5c2f8..0c1007f311 100644 --- a/src/v1/sources/shopify/pixelUtils.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js @@ -1,6 +1,6 @@ /* eslint-disable no-param-reassign */ -const Message = require('../../../v0/sources/message'); -const { EventType } = require('../../../constants'); +const Message = require('../../../../v0/sources/message'); +const { EventType } = require('../../../../constants'); const { INTEGERATION, PIXEL_EVENT_MAPPING, @@ -10,7 +10,7 @@ const { productViewedEventMappingJSON, productToCartEventMappingJSON, checkoutStartedCompletedEventMappingJSON, -} = require('./config'); +} = require('../config'); function getNestedValue(object, path) { const keys = path.split('.'); diff --git a/src/v1/sources/shopify/pixelUtils.test.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js similarity index 99% rename from src/v1/sources/shopify/pixelUtils.test.js rename to src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js index 4bff8eada4..e8f53a5f15 100644 --- a/src/v1/sources/shopify/pixelUtils.test.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js @@ -8,10 +8,9 @@ const { checkoutStepEventBuilder, searchEventBuilder, } = require('./pixelUtils'); -const { EventType } = require('../../../constants'); -const Message = require('../../../v0/sources/message'); +const Message = require('../../../../v0/sources/message'); jest.mock('ioredis', () => require('../../../../test/__mocks__/redis')); -jest.mock('../../../v0/sources/message'); +jest.mock('../../../../v0/sources/message'); describe('utilV2.js', () => { beforeEach(() => { diff --git a/test/__tests__/data/marketo_bulk_upload_fileUpload_input.json b/test/__tests__/data/marketo_bulk_upload_fileUpload_input.json deleted file mode 100644 index 737cd36ec3..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_fileUpload_input.json +++ /dev/null @@ -1,272 +0,0 @@ -[ - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "b", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "b", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "wrongClientId", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "a", - "clientId": "b", - "clientSecret": "forThrottle", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "b", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin1", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin2", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin3", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin4", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_fileUpload_output.json b/test/__tests__/data/marketo_bulk_upload_fileUpload_output.json deleted file mode 100644 index 0ea94284ae..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_fileUpload_output.json +++ /dev/null @@ -1,63 +0,0 @@ -[ - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 400, - "error": "The field Email is not present in Marketo Field Schema. Aborting", - "metadata": null - }, - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: undefined", - "metadata": null - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: There are 10 imports currently being processed. Please try again later", - "metadata": null - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: Empty file", - "metadata": null - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: Any other error", - "metadata": null - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_input.json b/test/__tests__/data/marketo_bulk_upload_input.json deleted file mode 100644 index ce48c8e7fe..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_input.json +++ /dev/null @@ -1,270 +0,0 @@ -[ - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - }, - { - "message": { - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - }, - { - "message": { - "type": "track", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - }, - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "1" - }, - { - "to": "email__c", - "from": "email1" - }, - { - "to": "plan__c", - "from": "plan1" - } - ] - } - } - }, - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email1" - }, - { - "to": "plan__c", - "from": "plan1" - } - ] - } - } - }, - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": 1 - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_jobStatus_input.json b/test/__tests__/data/marketo_bulk_upload_jobStatus_input.json deleted file mode 100644 index b36363e0db..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_jobStatus_input.json +++ /dev/null @@ -1,102 +0,0 @@ -[ - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 12345, - "input": [ - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 2 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb", - "phone": "99" - }, - "metadata": { - "job_id": 4 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 3 - } - } - ], - "config": { - "clientId": "b", - "clientSecret": "c", - "munchkinId": "a", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "metadata": {} - } - } - }, - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 12345, - "input": [ - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 2 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb", - "phone": "99" - }, - "metadata": { - "job_id": 4 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 3 - } - } - ], - "config": { - "clientId": "b", - "clientSecret": "c", - "munchkinId": "testMunchkin3", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "metadata": {} - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_jobStatus_output.json b/test/__tests__/data/marketo_bulk_upload_jobStatus_output.json deleted file mode 100644 index 60628f6b3f..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_jobStatus_output.json +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "type": "warn", - "data": [ - { - "statusCode": 400, - "error": "No csvHeader in metadata" - }, - { - "statusCode": 400, - "error": "Unable to fetch job status: due to error \"\"" - } - ] - }, - { - "type": "fail", - "data": [ - { - "statusCode": 400, - "error": "No csvHeader in metadata" - }, - { - "statusCode": 400, - "error": "Unable to fetch job status: due to error \"\"" - } - ] - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_output.json b/test/__tests__/data/marketo_bulk_upload_output.json deleted file mode 100644 index 9911a7e831..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_output.json +++ /dev/null @@ -1,89 +0,0 @@ -[ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "statusCode": 400, - "error": "Event type is required", - "statTags": { - "destination": "marketo_bulk_upload", - "stage": "transform", - "scope": "exception" - } - }, - { - "statusCode": 400, - "error": "Event type track is not supported", - "statTags": { - "destination": "marketo_bulk_upload", - "stage": "transform", - "scope": "exception" - } - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": { - "name__c": "Carlo Lombard" - }, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": { - "name__c": "Carlo Lombard", - "plan__c": 1 - }, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_poll_input.json b/test/__tests__/data/marketo_bulk_upload_poll_input.json deleted file mode 100644 index f5457bd79c..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_poll_input.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 1234, - "config": { - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "from": "email", - "to": "Email" - } - ], - "munchkinId": "a" - } - } - } - }, - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 1234, - "config": { - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "from": "email", - "to": "Email" - } - ], - "munchkinId": "testMunchkin4" - } - } - } - }, - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 1234, - "config": { - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "from": "email", - "to": "Email" - } - ], - "munchkinId": "testMunchkin500" - } - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_poll_output.json b/test/__tests__/data/marketo_bulk_upload_poll_output.json deleted file mode 100644 index 92e312072e..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_poll_output.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "Complete": true, - "statusCode": 200, - "hasFailed": false, - "InProgress": false, - "HasWarning": false - }, - { - "statusCode": 400, - "error": "Any 400 error" - }, - { - "statusCode": 400, - "error": "[Could not poll status]Error message: Any 500 error" - } -] diff --git a/test/__tests__/marketo_bulk_upload.test.js b/test/__tests__/marketo_bulk_upload.test.js deleted file mode 100644 index 6cf4d559b9..0000000000 --- a/test/__tests__/marketo_bulk_upload.test.js +++ /dev/null @@ -1,127 +0,0 @@ -const fs = require("fs"); -const path = require("path"); -const vRouter = require("../../src/legacy/router"); - -const version = "v0"; -const integration = "marketo_bulk_upload"; -const transformer = require(`../../src/${version}/destinations/${integration}/transform`); - -jest.mock("axios"); -let reqTransformBody; -let respTransformBody; -let respFileUploadBody; -let reqFileUploadBody; -let reqPollBody; -let respPollBody; -let reqJobStatusBody; -let respJobStatusBody; - -try { - reqTransformBody = JSON.parse( - fs.readFileSync(path.resolve(__dirname, `./data/${integration}_input.json`)) - ); - respTransformBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_output.json`) - ) - ); - reqFileUploadBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_fileUpload_input.json`) - ) - ); - respFileUploadBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_fileUpload_output.json`) - ) - ); - reqPollBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_poll_input.json`) - ) - ); - respPollBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_poll_output.json`) - ) - ); - reqJobStatusBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_jobStatus_input.json`) - ) - ); - respJobStatusBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_jobStatus_output.json`) - ) - ); -} catch (error) { - throw new Error("Could not read files." + error); -} - -describe(`${integration} Tests`, () => { - describe("Transformer.js", () => { - reqTransformBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await transformer.process(input); - expect(output).toEqual(respTransformBody[index]); - } catch (error) { - expect(error.message).toEqual(respTransformBody[index].error); - } - }); - }); - }); - - describe("fileUpload.js", () => { - reqFileUploadBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.fileUpload(input); - expect(output).toEqual(respFileUploadBody[index]); - } catch (error) { - expect(error.message).toEqual(respFileUploadBody[index].error); - } - }); - }); - }); - - describe("poll.js", () => { - reqPollBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.pollStatus(input); - expect(output).toEqual(respPollBody[index]); - } catch (error) { - expect(error.message).toEqual(respPollBody[index].error); - } - }); - }); - }); - - describe("fetchJobStatus.js for warn", () => { - reqJobStatusBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.getJobStatus(input, "warn"); - expect(output).toEqual(respJobStatusBody[0].data[index]); - } catch (error) { - expect(error.message).toEqual(respJobStatusBody[0].data[index].error); - } - }); - }); - }); - - describe("fetchJobStatus.js for fail", () => { - reqJobStatusBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.getJobStatus(input, "fail"); - expect(output).toEqual(respJobStatusBody[1].data[index]); - } catch (error) { - expect(error.message).toEqual(respJobStatusBody[1].data[index].error); - } - }); - }); - }); -}); diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 4d564199e5..789b612586 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -38,9 +38,6 @@ const integrations = [ "gcs_datalake", ]; -const integration = (index ) => { - return integrations[index]; -} const transformers = integrations.map(integration => require(`../../src/${version}/destinations/${integration}/transform`) ); @@ -70,7 +67,7 @@ describe("event types", () => { const i = input("track"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("track", integration(index))); + expect(received).toMatchObject(output("track", integrations[index])); }); }); }); @@ -81,7 +78,7 @@ describe("event types", () => { // also verfies priority order between traits and context.traits transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("identify", integration(index))); + expect(received).toMatchObject(output("identify", integrations[index])); }); }); }); @@ -91,7 +88,7 @@ describe("event types", () => { const i = input("page"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -100,7 +97,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("page", integration(index))); + expect(received).toMatchObject(output("page", integrations[index])); }); }); }); @@ -110,7 +107,7 @@ describe("event types", () => { const i = input("screen"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); it("should take name from properties if top-level name is missing", () => { @@ -119,7 +116,7 @@ describe("event types", () => { delete i.message.name; transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("screen", integration(index))); + expect(received).toMatchObject(output("screen", integrations[index])); }); }); }); @@ -129,7 +126,7 @@ describe("event types", () => { const i = input("alias"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("alias", integration(index))); + expect(received).toMatchObject(output("alias", integrations[index])); }); }); }); @@ -139,7 +136,7 @@ describe("event types", () => { const i = input("extract"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toMatchObject(output("extract", integration(index))); + expect(received).toMatchObject(output("extract", integrations[index])); }); }); }); @@ -158,7 +155,7 @@ describe("column & table names", () => { const received = transformer.process(i); const provider = - (integration(index) === "snowflake" || integration(index) == "snowpipe_streaming") ? "snowflake" : "default"; + (integrations[index] === "snowflake" || integrations[index] == "snowpipe_streaming") ? "snowflake" : "default"; expect(received[1].metadata.columns).toMatchObject( names.output.columns[provider] @@ -191,7 +188,7 @@ describe("column & table names", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1" @@ -215,7 +212,7 @@ describe("column & table names", () => { //KEY should be trimmed to 63 return; } - if (integration(index) === "snowflake" || integration(index) === "snowpipe_streaming") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(received[1].metadata).toHaveProperty( "table", "A_1_A_2_A_3_A_4_A_5_B_1_B_2_B_3_B_4_B_5_C_1_C_2_C_3_C_4_C_5_D_1_D_2_D_3_D_4_D_5_E_1_E_2_E_3_E_4_E_5_F_1_F_2_F_3_F_4_F_5_G_1_G_2" @@ -238,7 +235,7 @@ describe("column & table names", () => { ); return; } - if (integration(index) === "s3_datalake" || integration(index) === "gcs_datalake" || integration(index) === "azure_datalake") { + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" @@ -321,7 +318,7 @@ describe("conflict between rudder set props and user set props", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { let sampleRudderPropKey = "id"; - if (integration(index) === "snowflake" || integration(index) === "snowpipe_streaming") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { sampleRudderPropKey = "ID"; } @@ -359,7 +356,7 @@ describe("handle reserved words", () => { const propsKey = propsKeyMap[evType]; transformers.forEach((transformer, index) => { const reserverdKeywordsMap = - reservedANSIKeywordsMap[integration(index).toUpperCase()]; + reservedANSIKeywordsMap[integrations[index].toUpperCase()]; i.message[propsKey] = Object.assign( i.message[propsKey] || {}, @@ -369,7 +366,7 @@ describe("handle reserved words", () => { const received = transformer.process(i); const out = - evType === "track" || (evType === "identify" && integration(index) !== 'snowpipe_streaming') + evType === "track" || (evType === "identify" && integrations[index] !== 'snowpipe_streaming') ? received[1] : received[0]; @@ -382,7 +379,7 @@ describe("handle reserved words", () => { } else { k = snakeCasedKey; } - if (integration(index) === "snowflake" || integration(index) === "snowpipe_streaming") { + if (integrations[index] === "snowflake" || integrations[index] === "snowpipe_streaming") { expect(out.metadata.columns).toHaveProperty(k); } else { expect(out.metadata.columns).toHaveProperty(k.toLowerCase()); @@ -466,24 +463,24 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("new_ip"); } @@ -501,23 +498,23 @@ describe("context ip", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[0].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); if (received[1]) { expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toBe("string"); expect( received[1].data[ - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ] ).toEqual("requested_ip"); } @@ -538,10 +535,10 @@ describe("remove rudder property if rudder property is null", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "context_ip") + integrationCasedString(integrations[index], "context_ip") ); }); }); @@ -556,29 +553,29 @@ describe("remove any property if event is object ", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "event_text") + integrationCasedString(integrations[index], "event_text") ); }); i.message.channel = { channel: "android" }; transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "channel") + integrationCasedString(integrations[index], "channel") ); }); }); @@ -594,13 +591,13 @@ describe("store full rudder event", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); const columnName = integrationCasedString( - integration(index), + integrations[index], "rudder_event" ); expect(received[0].metadata.columns).toHaveProperty(columnName); expect(received[0].metadata.columns[columnName]).toEqual( - fullEventColumnTypeByProvider[integration(index)] + fullEventColumnTypeByProvider[integrations[index]] ); expect(received[0].data[columnName]).toEqual(JSON.stringify(i.message)); @@ -640,7 +637,7 @@ describe("rudder reserved columns", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); checkProps.forEach(k => { - k = integrationCasedString(integration(index), k); + k = integrationCasedString(integrations[index], k); expect(received[0].metadata.columns).not.toHaveProperty(k); expect(received[0].data).not.toHaveProperty(k); if (received[1]) { @@ -662,15 +659,15 @@ describe("id column datatype for users table", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("int"); - if (integration(index) === 'snowpipe_streaming') { + if (integrations[index] === 'snowpipe_streaming') { return } expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); }); @@ -682,15 +679,15 @@ describe("id column datatype for users table", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "user_id") + integrationCasedString(integrations[index], "user_id") ] ).toEqual("float"); - if (integration(index) === 'snowpipe_streaming') { + if (integrations[index] === 'snowpipe_streaming') { return } expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("float"); }); @@ -709,22 +706,22 @@ describe("handle leading underscores in properties", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "_timestamp") + integrationCasedString(integrations[index], "_timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp") + integrationCasedString(integrations[index], "__timestamp") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "__timestamp_new") + integrationCasedString(integrations[index], "__timestamp_new") ); }); }); @@ -738,22 +735,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -766,22 +763,22 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); }); }); @@ -794,17 +791,17 @@ describe("handle recordId from cloud sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[0].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toEqual(i.message.messageId); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("string"); }); @@ -820,28 +817,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -856,28 +853,28 @@ describe("handle recordId from cloud sources", () => { const received = transformer.process(i); expect( received[0].metadata.columns[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toEqual("string"); expect( received[0].data[ - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ] ).toBe("42"); expect( received[1].metadata.columns[ - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ] ).toEqual("int"); expect( - received[1].data[integrationCasedString(integration(index), "id")] + received[1].data[integrationCasedString(integrations[index], "id")] ).toBe(42); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "record_id") + integrationCasedString(integrations[index], "record_id") ); }); }); @@ -917,10 +914,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).not.toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -945,10 +942,10 @@ describe("handle level three nested events from sources", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[1].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); expect(received[1].data).toHaveProperty( - integrationCasedString(integration(index), "n_0_n_1_n_2_n_3_prop_3") + integrationCasedString(integrations[index], "n_0_n_1_n_2_n_3_prop_3") ); }); }); @@ -958,7 +955,7 @@ describe("Handle no of columns in an event", () => { it("should throw an error if no of columns are more than 200", () => { const i = input("track"); transformers - .filter((transformer, index) => integration(index) !== "s3_datalake" && integration(index) !== "gcs_datalake" && integration(index) !== "azure_datalake") + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") .forEach((transformer, index) => { i.message.properties = largeNoOfColumnsevent; expect(() => transformer.process(i)).toThrow( @@ -986,13 +983,13 @@ describe("Add auto generated messageId for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "id") + integrationCasedString(integrations[index], "id") ); expect( - received[0].data[integrationCasedString(integration(index), "id")] + received[0].data[integrationCasedString(integrations[index], "id")] ).toMatch(/auto-.*/); }); }); @@ -1009,10 +1006,10 @@ describe("Add receivedAt for events missing it", () => { transformers.forEach((transformer, index) => { const received = transformer.process(i); expect(received[0].metadata.columns).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); expect(received[0].data).toHaveProperty( - integrationCasedString(integration(index), "received_at") + integrationCasedString(integrations[index], "received_at") ); }); }); @@ -1025,12 +1022,12 @@ describe("Integration options", () => { const i = opInput("track"); transformers.forEach((transformer, index) => { const {jsonPaths} = i.destination.Config; - if (integration(index) === "postgres") { + if (integrations[index] === "postgres") { delete i.destination.Config.jsonPaths; } const received = transformer.process(i); i.destination.Config.jsonPaths = jsonPaths; - expect(received).toEqual(opOutput("track", integration(index))); + expect(received).toEqual(opOutput("track", integrations[index])); }); }); }); @@ -1040,7 +1037,7 @@ describe("Integration options", () => { const i = opInput("users"); transformers.forEach((transformer, index) => { const received = transformer.process(i); - expect(received).toEqual(opOutput("users", integration(index))); + expect(received).toEqual(opOutput("users", integrations[index])); }); }); }); @@ -1101,18 +1098,18 @@ describe("Integration options", () => { for (const testCase of testCases) { transformers.forEach((transformer, index) => { - it(`new ${testCase.eventType} for ${integration(index)}`, () => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) - it(`legacy ${testCase.eventType} for ${integration(index)}`, () => { + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); const input = _.cloneDeep(config.input); const received = transformer.process(input); - expect(received).toEqual(output(testCase.eventType, config, integration(index))); + expect(received).toEqual(output(testCase.eventType, config, integrations[index])); }) }); } @@ -1250,7 +1247,7 @@ describe("Destination config", () => { transformers.forEach((transformer, index) => { const received = transformer.process(scenario.event); - if (integration(index) === 'snowpipe_streaming' && scenario.event.message.type == 'identify') { + if (integrations[index] === 'snowpipe_streaming' && scenario.event.message.type == 'identify') { expect(received).toHaveLength(1); } else { expect(received).toHaveLength(scenario.expected.length); @@ -1298,7 +1295,7 @@ describe("Destination config", () => { } const output = transformer.process(event); let events = []; - if (integration(index) === 'snowpipe_streaming') { + if (integrations[index] === 'snowpipe_streaming') { events = [output[0]]; } else { events = [output[0], output[1]]; @@ -1311,10 +1308,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data[integrationCasedString(integration(index), trait)]).toEqual(value); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data[integrationCasedString(integrations[index], trait)]).toEqual(value); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1342,13 +1339,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v_2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v_3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v_2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v_3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v_2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v_3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v_2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v_3`)); }); }); }); @@ -1382,7 +1379,7 @@ describe("Destination config", () => { } const received = transformer.process(event); let events = []; - if (integration(index) === 'snowpipe_streaming') { + if (integrations[index] === 'snowpipe_streaming') { events = [received[0]]; } else { events = [received[0], received[1]]; @@ -1396,10 +1393,10 @@ describe("Destination config", () => { }; events.forEach(event => { Object.entries(traitsToCheck).forEach(([trait, value]) => { - expect(event.data).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.data[integrationCasedString(integration(index), `context_traits_${trait}`)]).toEqual(value); - expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integration(index), trait)); - expect(event.metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_traits_${trait}`)); + expect(event.data).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.data[integrationCasedString(integrations[index], `context_traits_${trait}`)]).toEqual(value); + expect(event.metadata.columns).not.toHaveProperty(integrationCasedString(integrations[index], trait)); + expect(event.metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_traits_${trait}`)); }); }); }); @@ -1425,13 +1422,13 @@ describe("Destination config", () => { } } const output = transformer.process(event); - expect(output[0].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[0].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); - expect(output[1].data[integrationCasedString(integration(index), `event`)]).toEqual('button_clicked_v2'); - expect(output[1].data[integrationCasedString(integration(index), `context_attribute_v3`)]).toEqual('some-value'); - expect(output[1].metadata.table).toEqual(integrationCasedString(integration(index), 'button_clicked_v2')); - expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integration(index), `context_attribute_v3`)); + expect(output[0].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[0].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[0].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); + expect(output[1].data[integrationCasedString(integrations[index], `event`)]).toEqual('button_clicked_v2'); + expect(output[1].data[integrationCasedString(integrations[index], `context_attribute_v3`)]).toEqual('some-value'); + expect(output[1].metadata.table).toEqual(integrationCasedString(integrations[index], 'button_clicked_v2')); + expect(output[1].metadata.columns).toHaveProperty(integrationCasedString(integrations[index], `context_attribute_v3`)); }); }); }); @@ -1638,8 +1635,8 @@ describe("context traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/context_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); } @@ -1731,8 +1728,8 @@ describe("group traits", () => { expect(Object.keys(received[0].data).join()).not.toMatch(/group_traits/g); } for (const column of t.expectedColumns) { - expect(received[0].metadata.columns[integrationCasedString(integration(index), column)]).toEqual(t.expectedMetadata); - expect(received[0].data[integrationCasedString(integration(index), column)]).toEqual(t.expectedData); + expect(received[0].metadata.columns[integrationCasedString(integrations[index], column)]).toEqual(t.expectedMetadata); + expect(received[0].data[integrationCasedString(integrations[index], column)]).toEqual(t.expectedData); } }); }); diff --git a/test/apitests/data_scenarios/cdk_v2/failure.json b/test/apitests/data_scenarios/cdk_v2/failure.json index 1635a3f0db..154d24481d 100644 --- a/test/apitests/data_scenarios/cdk_v2/failure.json +++ b/test/apitests/data_scenarios/cdk_v2/failure.json @@ -556,6 +556,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -679,6 +683,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/cdk_v2/success.json b/test/apitests/data_scenarios/cdk_v2/success.json index ced7433a28..88f430dd7c 100644 --- a/test/apitests/data_scenarios/cdk_v2/success.json +++ b/test/apitests/data_scenarios/cdk_v2/success.json @@ -556,6 +556,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -634,6 +638,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -712,6 +720,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json index 3ce7c15091..3deb7d4b8b 100644 --- a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json +++ b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json @@ -388,6 +388,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -466,6 +470,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -544,6 +552,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json index 0e467c26d0..a2652855d5 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json @@ -388,6 +388,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -466,6 +470,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_success.json b/test/apitests/data_scenarios/destination/proc/multiplex_success.json index 66b6c870a9..ba4d5266f3 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_success.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_success.json @@ -207,6 +207,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -285,6 +289,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/router/failure_test.json b/test/apitests/data_scenarios/destination/router/failure_test.json index 9e36da50cb..197456f66a 100644 --- a/test/apitests/data_scenarios/destination/router/failure_test.json +++ b/test/apitests/data_scenarios/destination/router/failure_test.json @@ -754,6 +754,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -781,6 +785,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/service.api.test.ts b/test/apitests/service.api.test.ts index 9c1d96e7fe..2ad1f323ac 100644 --- a/test/apitests/service.api.test.ts +++ b/test/apitests/service.api.test.ts @@ -78,6 +78,13 @@ describe('features tests', () => { const supportTransformerProxyV1 = JSON.parse(response.text).supportTransformerProxyV1; expect(typeof supportTransformerProxyV1).toBe('boolean'); }); + + test('features upgradedToSourceTransformV2 to be boolean', async () => { + const response = await request(server).get('/features'); + expect(response.status).toEqual(200); + const upgradedToSourceTransformV2 = JSON.parse(response.text).upgradedToSourceTransformV2; + expect(typeof upgradedToSourceTransformV2).toBe('boolean'); + }); }); describe('Api tests with a mock source/destination', () => { diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts index daed7c9e1f..baad6813df 100644 --- a/test/integrations/component.test.ts +++ b/test/integrations/component.test.ts @@ -41,6 +41,7 @@ command .option('-i, --index ', 'Enter Test index') .option('-g, --generate ', 'Enter "true" If you want to generate network file') .option('-id, --id ', 'Enter unique "Id" of the test case you want to run') + .option('-s, --source ', 'Enter Source Name') .parse(); const opts = command.opts(); diff --git a/test/integrations/destinations/active_campaign/router/data.ts b/test/integrations/destinations/active_campaign/router/data.ts index f65a65d9bc..a73140c161 100644 --- a/test/integrations/destinations/active_campaign/router/data.ts +++ b/test/integrations/destinations/active_campaign/router/data.ts @@ -258,7 +258,7 @@ export const data = [ path: 'path', title: 'title', search: 'search', - tab_url: 'https://simple-tenet.github.io/rudderstack-sample-site/', + tab_url: 'https://abc.com/sample-site/', referrer: 'referrer', initial_referrer: '$direct', referring_domain: '', @@ -288,7 +288,7 @@ export const data = [ path: 'path', title: 'title', search: 'search', - tab_url: 'https://simple-tenet.github.io/rudderstack-sample-site/', + tab_url: 'https://abc.com/sample-site/', referrer: 'referrer', initial_referrer: '$direct', referring_domain: '', diff --git a/test/integrations/destinations/airship/processor/data.ts b/test/integrations/destinations/airship/processor/data.ts index 3a6c5394cb..3c6d827ce6 100644 --- a/test/integrations/destinations/airship/processor/data.ts +++ b/test/integrations/destinations/airship/processor/data.ts @@ -2296,7 +2296,7 @@ export const data = [ }, { name: 'airship', - description: 'Test 22', + description: 'Test 22 : session id from Web SDK gets converted to v5 uuid format', feature: 'processor', module: 'destination', version: 'v0', @@ -2321,7 +2321,7 @@ export const data = [ ip: '0.0.0.0', os: { name: '', version: '' }, screen: { density: 2 }, - sessionId: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + sessionId: '1731403898', }, type: 'track', messageId: '84e26acc-56a5-4835-8233-591137fca468', @@ -2365,7 +2365,268 @@ export const data = [ user: { named_user_id: 'testuserId1' }, body: { name: 'product_clicked', - session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + session_id: 'd5627eac-795d-5005-9bb4-2c7c0af6cab0', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'airship', + description: 'Test 23 : session id from mobile SDK gets converted to v5 uuid format', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { email: 'testone@gmail.com', firstName: 'test', lastName: 'one' }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + sessionId: 1731403898, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + anonymousId: '123456', + event: 'Product Clicked', + userId: 'testuserId1', + properties: {}, + integrations: { All: true }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'ffdf', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/custom-events', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + 'X-UA-Appkey': 'ffdf', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + user: { named_user_id: 'testuserId1' }, + body: { + name: 'product_clicked', + session_id: 'd5627eac-795d-5005-9bb4-2c7c0af6cab0', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'airship', + description: 'Test 24 : session id null gets ignored', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { email: 'testone@gmail.com', firstName: 'test', lastName: 'one' }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + anonymousId: '123456', + event: 'Product Clicked', + userId: 'testuserId1', + properties: { + sessionId: null, + }, + integrations: { All: true }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'ffdf', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/custom-events', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + 'X-UA-Appkey': 'ffdf', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + user: { named_user_id: 'testuserId1' }, + body: { + name: 'product_clicked', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'airship', + description: 'Test 24 : session id undefined gets ignored', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { email: 'testone@gmail.com', firstName: 'test', lastName: 'one' }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + anonymousId: '123456', + event: 'Product Clicked', + userId: 'testuserId1', + properties: { + sessionId: undefined, + }, + integrations: { All: true }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'ffdf', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/custom-events', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + 'X-UA-Appkey': 'ffdf', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + user: { named_user_id: 'testuserId1' }, + body: { + name: 'product_clicked', }, }, JSON_ARRAY: {}, diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts index fcdb6f15ca..1d20e887e9 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts @@ -1720,194 +1720,4 @@ export const data = [ }, }, }, - { - name: 'google_adwords_enhanced_conversions', - description: 'Success test with configDetails', - feature: 'processor', - module: 'destination', - version: 'v0', - input: { - request: { - body: [ - { - metadata: { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - }, - destination: { - Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - listOfConversions: ['Page View', 'Product Added'], - authStatus: 'active', - configData: '{"customerId": "1234567890", "loginCustomerId": ""}', - }, - }, - message: { - channel: 'web', - context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.0', - }, - traits: { - phone: '912382193', - firstName: 'John', - lastName: 'Gomes', - city: 'London', - state: 'UK', - countryCode: 'us', - streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', - }, - library: { - name: 'RudderLabs JavaScript SDK', - version: '1.0.0', - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - locale: 'en-US', - ip: '0.0.0.0', - os: { - name: '', - version: '', - }, - screen: { - density: 2, - }, - }, - event: 'Page View', - type: 'track', - messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', - originalTimestamp: '2019-10-14T11:15:18.299Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - properties: { - gclid: 'gclid1234', - conversionDateTime: '2022-01-01 12:32:45-08:00', - adjustedValue: '10', - currency: 'INR', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - partialFailure: true, - campaignId: '1', - templateId: '0', - order_id: 10000, - total: 1000, - products: [ - { - product_id: '507f1f77bcf86cd799439011', - sku: '45790-32', - name: 'Monopoly: 3rd Edition', - price: '19', - position: '1', - category: 'cars', - url: 'https://www.example.com/product/path', - image_url: 'https://www.example.com/product/path.jpg', - quantity: '2', - }, - { - product_id: '507f1f77bcf86cd7994390112', - sku: '45790-322', - name: 'Monopoly: 3rd Edition2', - price: '192', - quantity: 22, - position: '12', - category: 'Cars2', - url: 'https://www.example.com/product/path2', - image_url: 'https://www.example.com/product/path.jpg2', - }, - ], - }, - integrations: { - All: true, - }, - name: 'ApplicationLoaded', - sentAt: '2019-10-14T11:15:53.296Z', - }, - }, - ], - }, - }, - output: { - response: { - status: 200, - body: [ - { - output: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - }, - params: { - event: 'Page View', - customerId: '1234567890', - }, - body: { - JSON: { - conversionAdjustments: [ - { - gclidDateTimePair: { - gclid: 'gclid1234', - conversionDateTime: '2022-01-01 12:32:45-08:00', - }, - restatementValue: { - adjustedValue: 10, - currencyCode: 'INR', - }, - orderId: '10000', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - userIdentifiers: [ - { - hashedPhoneNumber: - '04387707e6cbed8c4538c81cc570ed9252d579469f36c273839b26d784e4bdbe', - }, - { - addressInfo: { - hashedFirstName: - 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', - hashedLastName: - '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - state: 'UK', - city: 'London', - countryCode: 'us', - hashedStreetAddress: - '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', - }, - }, - ], - adjustmentType: 'ENHANCEMENT', - }, - ], - partialFailure: true, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - userId: '', - }, - metadata: { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - }, - statusCode: 200, - }, - ], - }, - }, - }, ]; diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts index fe0acf7964..5ac05b5a53 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts @@ -1,5 +1,3 @@ -import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_enhanced_conversions/config'; - const events = [ { metadata: { @@ -413,100 +411,6 @@ const events = [ sentAt: '2019-10-14T11:15:53.296Z', }, }, - { - metadata: { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - jobId: 6, - userId: 'u1', - }, - destination: { - Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', - customerId: '1234567890', - subAccount: true, - listOfConversions: [{ conversions: 'Page View' }, { conversions: 'Product Added' }], - authStatus: 'active', - }, - }, - message: { - channel: 'web', - context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.0', - }, - traits: { - phone: '912382193', - firstName: 'John', - lastName: 'Gomes', - city: 'London', - state: 'UK', - streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', - }, - library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - locale: 'en-US', - ip: '0.0.0.0', - os: { name: '', version: '' }, - screen: { density: 2 }, - customerID: {}, - subaccountID: 11, - }, - event: 'Page View', - type: 'track', - messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', - originalTimestamp: '2019-10-14T11:15:18.299Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - properties: { - gclid: 'gclid1234', - conversionDateTime: '2022-01-01 12:32:45-08:00', - adjustedValue: '10', - currency: 'INR', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - partialFailure: true, - campaignId: '1', - templateId: '0', - order_id: 10000, - total: 1000, - products: [ - { - product_id: '507f1f77bcf86cd799439011', - sku: '45790-32', - name: 'Monopoly: 3rd Edition', - price: '19', - position: '1', - category: 'cars', - url: 'https://www.example.com/product/path', - image_url: 'https://www.example.com/product/path.jpg', - quantity: '2', - }, - { - product_id: '507f1f77bcf86cd7994390112', - sku: '45790-322', - name: 'Monopoly: 3rd Edition2', - price: '192', - quantity: 22, - position: '12', - category: 'Cars2', - url: 'https://www.example.com/product/path2', - image_url: 'https://www.example.com/product/path.jpg2', - }, - ], - }, - integrations: { All: true }, - name: 'ApplicationLoaded', - sentAt: '2019-10-14T11:15:53.296Z', - }, - }, { metadata: { secret: { @@ -1012,99 +916,6 @@ export const data = [ module: 'destination', }, }, - { - batched: false, - batchedRequest: { - body: { - FORM: {}, - JSON: { - conversionAdjustments: [ - { - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - adjustmentType: 'ENHANCEMENT', - gclidDateTimePair: { - conversionDateTime: '2022-01-01 12:32:45-08:00', - gclid: 'gclid1234', - }, - orderId: '10000', - restatementValue: { - adjustedValue: 10, - currencyCode: 'INR', - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - userIdentifiers: [ - { - hashedPhoneNumber: - '04387707e6cbed8c4538c81cc570ed9252d579469f36c273839b26d784e4bdbe', - }, - { - addressInfo: { - city: 'London', - hashedFirstName: - 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', - hashedLastName: - '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - hashedStreetAddress: - '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', - state: 'UK', - }, - }, - ], - }, - ], - partialFailure: true, - }, - JSON_ARRAY: {}, - XML: {}, - }, - endpoint: - 'https://googleads.googleapis.com/v17/customers/1234567890:uploadConversionAdjustments', - files: {}, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - 'login-customer-id': '65656565', - }, - method: 'POST', - params: { - customerId: '1234567890', - event: 'Page View', - }, - type: 'REST', - version: '1', - }, - destination: { - Config: { - authStatus: 'active', - configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', - customerId: '1234567890', - listOfConversions: [ - { - conversions: 'Page View', - }, - { - conversions: 'Product Added', - }, - ], - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - subAccount: true, - }, - }, - metadata: [ - { - jobId: 6, - secret: { - access_token: 'abcd1234', - developer_token: 'ijkl91011', - refresh_token: 'efgh5678', - }, - userId: 'u1', - }, - ], - statusCode: 200, - }, { batched: false, destination: { diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts index a5e28996b1..12d5c65f8f 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts @@ -1,5 +1,9 @@ import { rETLAudienceRouterRequest } from './audience'; -import { rETLRecordRouterRequest } from './record'; +import { + rETLRecordRouterRequest, + rETLRecordRouterRequestVDMv2General, + rETLRecordRouterRequestVDMv2UserId, +} from './record'; import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_remarketing_lists/config'; export const data = [ @@ -732,4 +736,221 @@ export const data = [ }, }, }, + { + name: 'google_adwords_remarketing_lists record event tests VDMv2 General typeOfList', + description: 'Test 2', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordRouterRequestVDMv2General, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/7693729833/offlineUserDataJobs`, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + }, + params: { + listId: '7090784486', + customerId: '7693729833', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + body: { + JSON: { + operations: [ + { + create: { + userIdentifiers: [ + { + hashedEmail: + 'd3142c8f9c9129484daf28df80cc5c955791efed5e69afabb603bc8cb9ffd419', + }, + { + hashedPhoneNumber: + '8846dcb6ab2d73a0e67dbd569fa17cec2d9d391e5b05d1dd42919bc21ae82c45', + }, + { + addressInfo: { + hashedFirstName: + '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', + hashedLastName: + 'dcf000c2386fb76d22cefc0d118a8511bb75999019cd373df52044bccd1bd251', + countryCode: 'US', + postalCode: '1245', + }, + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + jobId: 1, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + DestinationDefinition: { + Config: {}, + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + }, + Enabled: true, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Transformations: [], + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + }, + ], + }, + }, + }, + }, + { + name: 'google_adwords_remarketing_lists record event tests VDMv2 UserId typeOfList', + description: 'Test 3', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordRouterRequestVDMv2UserId, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/7693729833/offlineUserDataJobs`, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + }, + params: { + listId: '7090784486', + customerId: '7693729833', + consent: { + adPersonalization: 'GRANTED', + adUserData: 'GRANTED', + }, + }, + body: { + JSON: { + operations: [ + { + create: { + userIdentifiers: [ + { + thirdPartyUserId: 'useri1234', + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + DestinationDefinition: { + Config: {}, + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + }, + Enabled: true, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Transformations: [], + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts index de76aae17c..b3f1095b1d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts @@ -1,4 +1,5 @@ -import { Destination, RouterTransformationRequest } from '../../../../../src/types'; +import { Connection, Destination, RouterTransformationRequest } from '../../../../../src/types'; +import { VDM_V2_SCHEMA_VERSION } from '../../../../../src/v0/util/constant'; import { generateGoogleOAuthMetadata } from '../../../testUtils'; const destination: Destination = { @@ -27,6 +28,59 @@ const destination: Destination = { IsProcessorEnabled: true, }; +const destination2: Destination = { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Enabled: true, + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + DestinationDefinition: { + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Config: {}, + }, + Transformations: [], + IsConnectionEnabled: true, + IsProcessorEnabled: true, +}; + +const connection1: Connection = { + sourceId: '2MUWghI7u85n91dd1qzGyswpZan', + destinationId: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + isHashRequired: true, + typeOfList: 'General', + audienceId: '7090784486', + }, + }, +}; + +const connection2: Connection = { + sourceId: '2MUWghI7u85n91dd1qzGyswpZan', + destinationId: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + isHashRequired: true, + typeOfList: 'userID', + audienceId: '7090784486', + personalizationConsent: 'GRANTED', + userDataConsent: 'GRANTED', + }, + }, +}; + export const rETLRecordRouterRequest: RouterTransformationRequest = { input: [ { @@ -153,6 +207,71 @@ export const rETLRecordRouterRequest: RouterTransformationRequest = { destType: 'google_adwords_remarketing_lists', }; +export const rETLRecordRouterRequestVDMv2General: RouterTransformationRequest = { + input: [ + { + destination: destination2, + connection: connection1, + message: { + action: 'insert', + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test@abc.com', + phone: '@09876543210', + firstName: 'test', + lastName: 'rudderlabs', + country: 'US', + postalCode: '1245', + }, + type: 'record', + }, + metadata: generateGoogleOAuthMetadata(1), + }, + ], + destType: 'google_adwords_remarketing_lists', +}; + +export const rETLRecordRouterRequestVDMv2UserId: RouterTransformationRequest = { + input: [ + { + destination: destination2, + connection: connection2, + message: { + action: 'insert', + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test@abc.com', + phone: '@09876543210', + firstName: 'test', + lastName: 'rudderlabs', + country: 'US', + postalCode: '1245', + thirdPartyUserId: 'useri1234', + }, + type: 'record', + }, + metadata: generateGoogleOAuthMetadata(2), + }, + ], + destType: 'google_adwords_remarketing_lists', +}; + module.exports = { rETLRecordRouterRequest, + rETLRecordRouterRequestVDMv2General, + rETLRecordRouterRequestVDMv2UserId, }; diff --git a/test/integrations/destinations/intercom_v2/network.ts b/test/integrations/destinations/intercom_v2/network.ts index 26ff3c38ee..e4cae04d07 100644 --- a/test/integrations/destinations/intercom_v2/network.ts +++ b/test/integrations/destinations/intercom_v2/network.ts @@ -746,6 +746,108 @@ const deliveryCallsData = [ }, }, }, + { + httpReq: { + method: 'post', + url: 'https://api.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'email', operator: '=', value: 'test-rETL-available@gmail.com' }], + }, + }, + headers, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 0, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 0, + }, + data: [ + { + type: 'contact', + id: 'retl-available-contact-id', + workspace_id: 'rudderWorkspace', + external_id: 'detach-company-user-id', + role: 'user', + email: 'test-rETL-available@gmail.com', + }, + ], + }, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'email', operator: '=', value: 'test-rETL-unavailable@gmail.com' }], + }, + }, + headers, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 0, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 0, + }, + data: [], + }, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.au.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'external_id', operator: '=', value: 'known-user-id-1' }], + }, + }, + headers, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 0, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 0, + }, + data: [ + { + type: 'contact', + id: 'contact-id-by-intercom-known-user-id-1', + workspace_id: 'rudderWorkspace', + external_id: 'user-id-1', + role: 'user', + email: 'test@rudderlabs.com', + }, + ], + }, + }, + }, ]; export const networkCallsData = [...deliveryCallsData]; diff --git a/test/integrations/destinations/intercom_v2/router/data.ts b/test/integrations/destinations/intercom_v2/router/data.ts index 7656914059..75f5ba6ae7 100644 --- a/test/integrations/destinations/intercom_v2/router/data.ts +++ b/test/integrations/destinations/intercom_v2/router/data.ts @@ -17,6 +17,7 @@ import { userTraits, } from '../common'; import { RouterTestData } from '../../../testTypes'; +import { rETLRecordV2RouterRequest } from './rETL'; const routerRequest1: RouterTransformationRequest = { input: [ @@ -222,6 +223,26 @@ const routerRequest3: RouterTransformationRequest = { }, metadata: generateMetadata(3), }, + { + destination: destinationApiServerAU, + message: { + userId: 'known-user-id-1', + channel, + context: { + traits: { ...userTraits, external_id: 'known-user-id-1' }, + }, + type: 'identify', + integrations: { + All: true, + Intercom: { + lookup: 'external_id', + }, + }, + originalTimestamp, + timestamp, + }, + metadata: generateMetadata(4), + }, ], destType: 'intercom_v2', }; @@ -735,6 +756,38 @@ export const data: RouterTestData[] = [ metadata: [generateMetadata(3)], statusCode: 400, }, + { + batched: false, + batchedRequest: { + body: { + JSON: { + email: 'test@rudderlabs.com', + external_id: 'known-user-id-1', + name: 'John Snow', + owner_id: 13, + phone: '+91 9999999999', + custom_attributes: { + address: 'california usa', + age: 23, + }, + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: + 'https://api.au.intercom.io/contacts/contact-id-by-intercom-known-user-id-1', + files: {}, + headers, + method: 'PUT', + params: {}, + type: 'REST', + version: '1', + }, + destination: destinationApiServerAU, + metadata: [generateMetadata(4)], + statusCode: 200, + }, ], }, }, @@ -880,4 +933,150 @@ export const data: RouterTestData[] = [ }, }, }, + { + id: 'INTERCOM-V2-router-test-6', + scenario: 'Framework', + successCriteria: 'Some events should be transformed successfully and some should fail for rETL', + name: 'intercom_v2', + description: 'INTERCOM V2 rETL tests', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordV2RouterRequest, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: false, + batchedRequest: { + body: { + JSON: { + email: 'test-rETL-unavailable@gmail.com', + external_id: 'rEtl_external_id', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts', + files: {}, + headers, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(1)], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + JSON: { + external_id: 'rEtl_external_id', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts/retl-available-contact-id', + files: {}, + headers, + method: 'PUT', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(2)], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + JSON: {}, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts/retl-available-contact-id', + files: {}, + headers, + method: 'DELETE', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(3)], + statusCode: 200, + }, + { + batched: false, + error: 'Contact is not present. Aborting.', + statTags: { + ...RouterInstrumentationErrorStatTags, + errorType: 'configuration', + }, + destination, + metadata: [generateMetadata(4)], + statusCode: 400, + }, + { + batched: false, + batchedRequest: { + body: { + JSON: { + external_id: 'rEtl_external_id', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts/retl-available-contact-id', + files: {}, + headers, + method: 'PUT', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(5)], + statusCode: 200, + }, + { + batched: false, + error: 'action dummyaction is not supported.', + statTags: { + ...RouterInstrumentationErrorStatTags, + }, + destination, + metadata: [generateMetadata(6)], + statusCode: 400, + }, + { + batched: false, + error: 'Missing lookup field or lookup field value for searchContact', + statTags: { + ...RouterInstrumentationErrorStatTags, + }, + destination, + metadata: [generateMetadata(7)], + statusCode: 400, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/intercom_v2/router/rETL.ts b/test/integrations/destinations/intercom_v2/router/rETL.ts new file mode 100644 index 0000000000..0a36b8cfa6 --- /dev/null +++ b/test/integrations/destinations/intercom_v2/router/rETL.ts @@ -0,0 +1,182 @@ +import { RouterTransformationRequest } from '../../../../../src/types'; +import { destination } from '../common'; +import { generateMetadata } from '../../../testUtils'; + +export const rETLRecordV2RouterRequest: RouterTransformationRequest = { + input: [ + { + destination, + message: { + type: 'record', + action: 'insert', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-unavailable@gmail.com', + }, + }, + metadata: generateMetadata(1), + }, + { + destination, + message: { + type: 'record', + action: 'update', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(2), + }, + { + destination, + message: { + type: 'record', + action: 'delete', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '3', + rudderId: '3', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(3), + }, + { + destination, + message: { + type: 'record', + action: 'update', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-unavailable@gmail.com', + }, + }, + metadata: generateMetadata(4), + }, + { + destination, + message: { + type: 'record', + action: 'insert', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(5), + }, + { + destination, + message: { + type: 'record', + action: 'dummyAction', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(6), + }, + { + destination, + message: { + type: 'record', + action: 'insert', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: {}, + }, + metadata: generateMetadata(7), + }, + ], + destType: 'intercom_v2', +}; diff --git a/test/integrations/destinations/iterable/deleteUsers/data.ts b/test/integrations/destinations/iterable/deleteUsers/data.ts index 79d801f4ee..9e7eab1ee1 100644 --- a/test/integrations/destinations/iterable/deleteUsers/data.ts +++ b/test/integrations/destinations/iterable/deleteUsers/data.ts @@ -183,4 +183,40 @@ export const data = [ }, }, }, + { + name: destType, + description: 'Test 5: should pass when dataCenter is selected as EUDC', + feature: 'userDeletion', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destType: destType.toUpperCase(), + userAttributes: [ + { + userId: 'rudder7', + }, + ], + config: { + apiKey: 'dummyApiKey', + dataCenter: 'EUDC', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 200, + status: 'successful', + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/network.ts b/test/integrations/destinations/iterable/network.ts index 39544b2647..1cf26dfd4f 100644 --- a/test/integrations/destinations/iterable/network.ts +++ b/test/integrations/destinations/iterable/network.ts @@ -105,5 +105,22 @@ const deleteNwData = [ status: 200, }, }, + { + httpReq: { + method: 'delete', + url: 'https://api.eu.iterable.com/api/users/byUserId/rudder7', + headers: { + api_key: 'dummyApiKey', + }, + }, + httpRes: { + data: { + msg: 'All users associated with rudder7 were successfully deleted', + code: 'Success', + params: null, + }, + status: 200, + }, + }, ]; export const networkCallsData = [...deleteNwData]; diff --git a/test/integrations/destinations/iterable/processor/aliasTestData.ts b/test/integrations/destinations/iterable/processor/aliasTestData.ts index cac43767bb..1ee4134859 100644 --- a/test/integrations/destinations/iterable/processor/aliasTestData.ts +++ b/test/integrations/destinations/iterable/processor/aliasTestData.ts @@ -1,4 +1,8 @@ -import { generateMetadata, transformResultBuilder } from './../../../testUtils'; +import { + generateMetadata, + overrideDestination, + transformResultBuilder, +} from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -15,6 +19,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -94,4 +99,56 @@ export const aliasTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-alias-test-1', + name: 'iterable', + description: 'Alias call with dataCenter as EUDC', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update email payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId: 'anonId', + userId: 'new@email.com', + previousId: 'old@email.com', + name: 'ApplicationLoaded', + context: {}, + properties, + type: 'alias', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: 'https://api.eu.iterable.com/api/users/updateEmail', + JSON: { + currentEmail: 'old@email.com', + newEmail: 'new@email.com', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/identifyTestData.ts b/test/integrations/destinations/iterable/processor/identifyTestData.ts index d05f87a11f..21d294e232 100644 --- a/test/integrations/destinations/iterable/processor/identifyTestData.ts +++ b/test/integrations/destinations/iterable/processor/identifyTestData.ts @@ -2,6 +2,7 @@ import { generateMetadata, transformResultBuilder, generateIndentifyPayload, + overrideDestination, } from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -19,6 +20,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -55,6 +57,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const updateUserEndpoint = 'https://api.iterable.com/api/users/update'; +const updateUserEndpointEUDC = 'https://api.eu.iterable.com/api/users/update'; export const identifyTestData: ProcessorTestData[] = [ { @@ -404,4 +407,58 @@ export const identifyTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-identify-test-7', + name: 'iterable', + description: 'Indentify call to update user in iterable with EUDC dataCenter', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update user payload with all user traits and updateUserEndpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + context: { + traits: user1Traits, + }, + traits: user1Traits, + type: 'identify', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: updateUserEndpointEUDC, + JSON: { + email: user1Traits.email, + userId: anonymousId, + dataFields: user1Traits, + preferUserId: false, + mergeNestedObjects: true, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/pageScreenTestData.ts b/test/integrations/destinations/iterable/processor/pageScreenTestData.ts index 074d6b56df..a27cf9fe3b 100644 --- a/test/integrations/destinations/iterable/processor/pageScreenTestData.ts +++ b/test/integrations/destinations/iterable/processor/pageScreenTestData.ts @@ -1,4 +1,8 @@ -import { generateMetadata, transformResultBuilder } from './../../../testUtils'; +import { + generateMetadata, + overrideDestination, + transformResultBuilder, +} from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -15,6 +19,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -43,6 +48,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const pageEndpoint = 'https://api.iterable.com/api/events/track'; +const pageEndpointEUDC = 'https://api.eu.iterable.com/api/events/track'; export const pageScreenTestData: ProcessorTestData[] = [ { @@ -406,4 +412,61 @@ export const pageScreenTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-page-test-4', + name: 'iterable', + description: 'Page call with dataCenter as EUDC', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain endpoint as pageEndpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + name: 'ApplicationLoaded', + context: { + traits: { + email: 'sayan@gmail.com', + }, + }, + properties, + type: 'page', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: pageEndpointEUDC, + JSON: { + userId: anonymousId, + dataFields: properties, + email: 'sayan@gmail.com', + createdAt: 1598631966468, + eventName: 'ApplicationLoaded page', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/trackTestData.ts b/test/integrations/destinations/iterable/processor/trackTestData.ts index 296275ad77..2b7d2a9c47 100644 --- a/test/integrations/destinations/iterable/processor/trackTestData.ts +++ b/test/integrations/destinations/iterable/processor/trackTestData.ts @@ -1,6 +1,7 @@ import { generateMetadata, generateTrackPayload, + overrideDestination, transformResultBuilder, } from './../../../testUtils'; import { Destination } from '../../../../../src/types'; @@ -19,6 +20,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -126,6 +128,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const endpoint = 'https://api.iterable.com/api/events/track'; +const endpointEUDC = 'https://api.eu.iterable.com/api/events/track'; const updateCartEndpoint = 'https://api.iterable.com/api/commerce/updateCart'; const trackPurchaseEndpoint = 'https://api.iterable.com/api/commerce/trackPurchase'; @@ -714,4 +717,56 @@ export const trackTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-track-test-9', + name: 'iterable', + description: 'Track call to add event with user with EUDC dataCenter', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain event properties, event name and endpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + event: 'Email Opened', + type: 'track', + context: {}, + properties, + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: endpointEUDC, + JSON: { + userId: 'anonId', + createdAt: 1598631966468, + eventName: 'Email Opened', + dataFields: properties, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/router/data.ts b/test/integrations/destinations/iterable/router/data.ts index 09eedc8eb8..1917c078eb 100644 --- a/test/integrations/destinations/iterable/router/data.ts +++ b/test/integrations/destinations/iterable/router/data.ts @@ -247,6 +247,7 @@ export const data = [ destination: { Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -308,6 +309,7 @@ export const data = [ destConfig: { defaultConfig: [ 'apiKey', + 'dataCenter', 'mapToSingleEvent', 'trackAllPages', 'trackCategorisedPages', @@ -339,6 +341,7 @@ export const data = [ }, Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: true, trackAllPages: false, trackCategorisedPages: true, @@ -414,6 +417,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -442,6 +446,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -472,6 +477,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -623,6 +629,7 @@ export const data = [ destination: { Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -686,6 +693,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -732,6 +740,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -765,6 +774,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -821,6 +831,7 @@ export const data = [ destConfig: { defaultConfig: [ 'apiKey', + 'dataCenter', 'mapToSingleEvent', 'trackAllPages', 'trackCategorisedPages', @@ -852,6 +863,7 @@ export const data = [ }, Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: true, trackAllPages: false, trackCategorisedPages: true, @@ -867,4 +879,147 @@ export const data = [ }, }, }, + { + name: 'iterable', + description: 'Simple identify call with EUDC dataCenter', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + receivedAt: '2022-09-27T11:12:59.080Z', + sentAt: '2022-09-27T11:13:03.777Z', + messageId: '9ad41366-8060-4c9f-b181-f6bea67d5469', + originalTimestamp: '2022-09-27T11:13:03.777Z', + traits: { ruchira: 'donaldbaker@ellis.com', new_field2: 'GB' }, + channel: 'sources', + rudderId: '3d51640c-ab09-42c1-b7b2-db6ab433b35e', + context: { + sources: { + version: 'feat.SupportForTrack', + job_run_id: 'ccpdlajh6cfi19mr1vs0', + task_run_id: 'ccpdlajh6cfi19mr1vsg', + batch_id: '4917ad78-280b-40d2-a30d-119434152a0f', + job_id: '2FLKJDcTdjPHQpq7pUjB34dQ5w6/Syncher', + task_id: 'rows_100', + }, + mappedToDestination: 'true', + externalId: [ + { id: 'Tiffany', type: 'ITERABLE-test-ruchira', identifierType: 'itemId' }, + ], + }, + timestamp: '2022-09-27T11:12:59.079Z', + type: 'identify', + userId: 'Tiffany', + recordId: '10', + request_ip: '10.1.86.248', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + { + message: { + receivedAt: '2022-09-27T11:12:59.080Z', + sentAt: '2022-09-27T11:13:03.777Z', + messageId: '9ad41366-8060-4c9f-b181-f6bea67d5469', + originalTimestamp: '2022-09-27T11:13:03.777Z', + traits: { ruchira: 'abc@ellis.com', new_field2: 'GB1' }, + channel: 'sources', + rudderId: '3d51640c-ab09-42c1-b7b2-db6ab433b35e', + context: { + sources: { + version: 'feat.SupportForTrack', + job_run_id: 'ccpdlajh6cfi19mr1vs0', + task_run_id: 'ccpdlajh6cfi19mr1vsg', + batch_id: '4917ad78-280b-40d2-a30d-119434152a0f', + job_id: '2FLKJDcTdjPHQpq7pUjB34dQ5w6/Syncher', + task_id: 'rows_100', + }, + mappedToDestination: 'true', + externalId: [ + { id: 'ABC', type: 'ITERABLE-test-ruchira', identifierType: 'itemId' }, + ], + }, + timestamp: '2022-09-27T11:12:59.079Z', + type: 'identify', + userId: 'Tiffany', + recordId: '10', + request_ip: '10.1.86.248', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + ], + destType: 'iterable', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.eu.iterable.com/api/catalogs/test-ruchira/items', + headers: { + 'Content-Type': 'application/json', + api_key: '583af2f8-15ba-49c0-8511-76383e7de07e', + }, + params: {}, + body: { + JSON: { + documents: { + Tiffany: { ruchira: 'donaldbaker@ellis.com', new_field2: 'GB' }, + ABC: { ruchira: 'abc@ellis.com', new_field2: 'GB1' }, + }, + replaceUploadedFieldsOnly: true, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { jobId: 2, userId: 'u1' }, + { jobId: 2, userId: 'u1' }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/linkedin_audience/processor/business.ts b/test/integrations/destinations/linkedin_audience/processor/business.ts new file mode 100644 index 0000000000..28cb6a9a97 --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/processor/business.ts @@ -0,0 +1,520 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, generateRecordPayload } from '../../../testUtils'; + +export const businessTestData: ProcessorTestData[] = [ + { + id: 'linkedin_audience-business-test-1', + name: 'linkedin_audience', + description: 'Record call : non string values provided as email', + scenario: 'Business', + successCriteria: 'should fail with 400 status code and error message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 12345, + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'The "string" argument must be of type string. Received type number (12345): Workflow: procWorkflow, Step: prepareUserTypeBasePayload, ChildStep: prepareUserIds, OriginalError: The "string" argument must be of type string. Received type number (12345)', + metadata: generateMetadata(1), + statTags: { + destType: 'LINKEDIN_AUDIENCE', + destinationId: 'default-destinationId', + errorCategory: 'transformation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 500, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-business-test-2', + name: 'linkedin_audience', + description: 'Record call : Valid event without any field mappings', + scenario: 'Business', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: {}, + identifiers: { + sha256Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-business-test-2', + name: 'linkedin_audience', + description: 'Record call : customer provided hashed value and isHashRequired is false', + scenario: 'Business', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: {}, + identifiers: { + sha256Email: '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + sha512Email: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: false, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + { + idType: 'SHA512_EMAIL', + idValue: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-business-test-2', + name: 'linkedin_audience', + description: 'Record call : event with company audience details', + scenario: 'Business', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + city: 'Dhaka', + state: 'Dhaka', + industries: 'Information Technology', + postalCode: '123456', + }, + identifiers: { + companyName: 'Rudderstack', + organizationUrn: 'urn:li:organization:456', + companyWebsiteDomain: 'rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'company', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'city', + to: 'city', + }, + { + from: 'state', + to: 'state', + }, + { + from: 'domain', + to: 'industries', + }, + { + from: 'psCode', + to: 'postalCode', + }, + ], + identifierMappings: [ + { + from: 'name', + to: 'companyName', + }, + { + from: 'urn', + to: 'organizationUrn', + }, + { + from: 'Website Domain', + to: 'companyWebsiteDomain', + }, + ], + isHashRequired: false, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + city: 'Dhaka', + companyName: 'Rudderstack', + companyWebsiteDomain: 'rudderstack.com', + industries: 'Information Technology', + organizationUrn: 'urn:li:organization:456', + postalCode: '123456', + state: 'Dhaka', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/companies', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_audience/processor/data.ts b/test/integrations/destinations/linkedin_audience/processor/data.ts new file mode 100644 index 0000000000..233a9dcf86 --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/processor/data.ts @@ -0,0 +1,3 @@ +import { businessTestData } from './business'; +import { validationTestData } from './validation'; +export const data = [...validationTestData, ...businessTestData]; diff --git a/test/integrations/destinations/linkedin_audience/processor/validation.ts b/test/integrations/destinations/linkedin_audience/processor/validation.ts new file mode 100644 index 0000000000..3ad37b2f4d --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/processor/validation.ts @@ -0,0 +1,396 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, generateRecordPayload } from '../../../testUtils'; + +export const validationTestData: ProcessorTestData[] = [ + { + id: 'linkedin_audience-validation-test-1', + name: 'linkedin_audience', + description: 'Record call : event is valid with all required elements', + scenario: 'Validation', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + company: 'Rudderlabs', + country: 'Dhaka', + firstName: 'Test', + lastName: 'User', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + { + idType: 'SHA512_EMAIL', + idValue: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-validation-test-2', + name: 'linkedin_audience', + description: 'Record call : event is not valid with all required elements', + scenario: 'Validation', + successCriteria: 'should fail with 400 status code and error message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Audience Id is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Audience Id is not present. Aborting', + metadata: generateMetadata(1), + statTags: { + destType: 'LINKEDIN_AUDIENCE', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-validation-test-3', + name: 'linkedin_audience', + description: 'Record call : isHashRequired is not provided', + scenario: 'Validation', + successCriteria: + 'should succeed with 200 status code and transformed message with provided values of identifiers', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 1234, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + company: 'Rudderlabs', + country: 'Dhaka', + firstName: 'Test', + lastName: 'User', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'random@rudderstack.com', + }, + { + idType: 'SHA512_EMAIL', + idValue: 'random@rudderstack.com', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/1234/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_audience/router/data.ts b/test/integrations/destinations/linkedin_audience/router/data.ts new file mode 100644 index 0000000000..c76d3e84c6 --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/router/data.ts @@ -0,0 +1,384 @@ +import { generateMetadata, generateRecordPayload } from '../../../testUtils'; + +export const data = [ + { + name: 'linkedin_audience', + description: 'Test 0', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + { + message: generateRecordPayload({ + fields: {}, + identifiers: { + sha256Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(2), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 12345, + }, + action: 'insert', + }), + metadata: generateMetadata(3), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + destType: 'linkedin_audience', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: true, + batchedRequest: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + company: 'Rudderlabs', + country: 'Dhaka', + firstName: 'Test', + lastName: 'User', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + { + idType: 'SHA512_EMAIL', + idValue: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + ], + }, + { + action: 'ADD', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + DisplayName: 'Linkedin Audience', + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + }, + Enabled: true, + ID: '123', + Name: 'Linkedin Audience', + Transformations: [], + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + }, + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 1, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 2, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + ], + statusCode: 200, + }, + { + batched: false, + destination: { + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + DisplayName: 'Linkedin Audience', + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + }, + Enabled: true, + ID: '123', + Name: 'Linkedin Audience', + Transformations: [], + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + }, + error: 'The "string" argument must be of type string. Received type number (12345)', + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 3, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + ], + statTags: { + destType: 'LINKEDIN_AUDIENCE', + destinationId: 'default-destinationId', + errorCategory: 'transformation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 500, + }, + ], + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/pinterest_tag/processor/data.ts b/test/integrations/destinations/pinterest_tag/processor/data.ts index b856d247d7..4982444346 100644 --- a/test/integrations/destinations/pinterest_tag/processor/data.ts +++ b/test/integrations/destinations/pinterest_tag/processor/data.ts @@ -482,7 +482,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [{ quantity: 2, item_price: '25' }], + contents: [{ id: '123', quantity: 2, item_price: '25' }], }, }, JSON_ARRAY: {}, @@ -2405,7 +2405,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 0, content_ids: ['1234'], - contents: [{ quantity: 1, item_price: 'undefined' }], + contents: [{ id: '1234', quantity: 1 }], }, }, JSON_ARRAY: {}, @@ -2666,7 +2666,7 @@ export const data = [ advertiser_id: '123456', app_id: '429047995', custom_data: { - contents: [{ item_price: 'undefined', quantity: 1 }], + contents: [{ quantity: 1 }], currency: 'USD', num_items: 0, order_id: '50314b8e9bcf000000000000', @@ -3486,7 +3486,7 @@ export const data = [ timestamp: '2020-08-14T05:30:30.118Z', properties: { tax: 2, - total: 27.5, + total: [27.5, 123], coupon: 'hasbros', revenue: 48, currency: 'USD', @@ -3562,11 +3562,10 @@ export const data = [ contents: [ { quantity: 1, - item_price: 'undefined', }, ], currency: 'USD', - value: '27.5', + value: '[27.5,123]', order_id: '50314b8e9bcf000000000000', }, event_name: 'custom event', diff --git a/test/integrations/destinations/pinterest_tag/router/data.ts b/test/integrations/destinations/pinterest_tag/router/data.ts index c9ab29a45a..4049f7663a 100644 --- a/test/integrations/destinations/pinterest_tag/router/data.ts +++ b/test/integrations/destinations/pinterest_tag/router/data.ts @@ -815,7 +815,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [{ quantity: 2, item_price: '25' }], + contents: [{ id: '123', quantity: 2, item_price: '25' }], }, }, { diff --git a/test/integrations/destinations/pinterest_tag/step/data.ts b/test/integrations/destinations/pinterest_tag/step/data.ts index b607e3c9fa..71f12c735c 100644 --- a/test/integrations/destinations/pinterest_tag/step/data.ts +++ b/test/integrations/destinations/pinterest_tag/step/data.ts @@ -468,7 +468,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [{ quantity: 2, item_price: '25' }], + contents: [{ id: '123', quantity: 2, item_price: '25' }], }, }, JSON_ARRAY: {}, @@ -2420,7 +2420,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 0, content_ids: ['1234'], - contents: [{ quantity: 1, item_price: 'undefined' }], + contents: [{ id: '1234', quantity: 1 }], }, }, JSON_ARRAY: {}, @@ -2685,7 +2685,7 @@ export const data = [ advertiser_id: '123456', app_id: '429047995', custom_data: { - contents: [{ item_price: 'undefined', quantity: 1 }], + contents: [{ quantity: 1 }], currency: 'USD', num_items: 0, order_id: '50314b8e9bcf000000000000', @@ -3606,7 +3606,6 @@ export const data = [ contents: [ { quantity: 1, - item_price: 'undefined', }, ], }, diff --git a/test/integrations/sources/adjust/data.ts b/test/integrations/sources/adjust/data.ts index e57feb45d4..107bb444c4 100644 --- a/test/integrations/sources/adjust/data.ts +++ b/test/integrations/sources/adjust/data.ts @@ -125,4 +125,57 @@ export const data = [ defaultMockFns(); }, }, + { + name: 'adjust', + description: 'Simple track call with wrong created at', + module: 'source', + version: 'v0', + skipGo: 'FIXME', + input: { + request: { + body: [ + { + id: 'adjust', + query_parameters: { + gps_adid: ['38400000-8cf0-11bd-b23e-10b96e40000d'], + adid: ['18546f6171f67e29d1cb983322ad1329'], + tracker_token: ['abc'], + custom: ['custom'], + tracker_name: ['dummy'], + created_at: ['test'], + event_name: ['Click'], + }, + updated_at: '2023-02-10T12:16:07.251Z', + created_at: 'test', + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Failed to parse timestamp: "test"', + statTags: { + destinationId: 'Non determinable', + errorCategory: 'transformation', + implementation: 'native', + module: 'source', + workspaceId: 'Non determinable', + }, + statusCode: 400, + }, + ], + }, + }, + mockFns: () => { + defaultMockFns(); + }, + }, ]; diff --git a/test/integrations/sources/shopify/data.ts b/test/integrations/sources/shopify/data.ts index a2b27cbbcc..d4498e089c 100644 --- a/test/integrations/sources/shopify/data.ts +++ b/test/integrations/sources/shopify/data.ts @@ -1,8 +1,10 @@ -import { skip } from 'node:test'; import { pixelCheckoutEventsTestScenarios } from './pixelTestScenarios/CheckoutEventsTests'; import { pixelCheckoutStepsScenarios } from './pixelTestScenarios/CheckoutStepsTests'; import { pixelEventsTestScenarios } from './pixelTestScenarios/ProductEventsTests'; -import { v1ServerSideEventsScenarios } from './v1ServerSideEventsTests'; +import { checkoutEventsTestScenarios } from './webhookTestScenarios/CheckoutEventsTests'; +import { genericTrackTestScenarios } from './webhookTestScenarios/GenericTrackTests'; +import { identityTestScenarios } from './webhookTestScenarios/IdentifyTests'; +import { mockFns } from './mocks'; const serverSideEventsScenarios = [ { @@ -1422,6 +1424,7 @@ const serverSideEventsScenarios = [ verifiedEmail: true, }, type: 'track', + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', userId: '115310627314723950', }, ], @@ -1430,13 +1433,16 @@ const serverSideEventsScenarios = [ ], }, }, + mockFns, }, ]; export const data = [ + ...serverSideEventsScenarios, + ...checkoutEventsTestScenarios, + ...genericTrackTestScenarios, + ...identityTestScenarios, ...pixelCheckoutEventsTestScenarios, ...pixelCheckoutStepsScenarios, ...pixelEventsTestScenarios, - ...serverSideEventsScenarios, - ...v1ServerSideEventsScenarios, ]; diff --git a/test/integrations/sources/shopify/mocks.ts b/test/integrations/sources/shopify/mocks.ts new file mode 100644 index 0000000000..e1895e7812 --- /dev/null +++ b/test/integrations/sources/shopify/mocks.ts @@ -0,0 +1,5 @@ +import utils from '../../../../src/v0/util'; + +export const mockFns = (_) => { + jest.spyOn(utils, 'generateUUID').mockReturnValue('5d3e2cb6-4011-5c9c-b7ee-11bc1e905097'); +}; diff --git a/test/integrations/sources/shopify/v1ServerSideEventsTests.ts b/test/integrations/sources/shopify/v1ServerSideEventsTests.ts deleted file mode 100644 index 2c323cb370..0000000000 --- a/test/integrations/sources/shopify/v1ServerSideEventsTests.ts +++ /dev/null @@ -1,596 +0,0 @@ -// This file contains the test scenarios for the server-side events from the Shopify GraphQL API for -// the v1 transformation flow -import utils from '../../../../src/v0/util'; -const defaultMockFns = () => { - jest.spyOn(utils, 'generateUUID').mockReturnValue('5d3e2cb6-4011-5c9c-b7ee-11bc1e905097'); -}; -import { dummySourceConfig } from './constants'; - -export const v1ServerSideEventsScenarios = [ - { - name: 'shopify', - description: 'Track Call -> Checkout Updated event', - module: 'source', - version: 'v1', - input: { - request: { - body: [ - { - event: { - id: 35374569160817, - token: 'e89d4437003b6b8480f8bc7f8036a659', - cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - email: 'testuser101@gmail.com', - gateway: null, - buyer_accepts_marketing: false, - buyer_accepts_sms_marketing: false, - sms_marketing_phone: null, - created_at: '2024-09-16T03:50:15+00:00', - updated_at: '2024-09-17T03:29:02-04:00', - landing_site: '/', - note: '', - note_attributes: [], - referring_site: '', - shipping_lines: [ - { - code: 'Standard', - price: '6.90', - original_shop_price: '6.90', - original_shop_markup: '0.00', - source: 'shopify', - title: 'Standard', - presentment_title: 'Standard', - phone: null, - tax_lines: [], - custom_tax_lines: null, - markup: '0.00', - carrier_identifier: null, - carrier_service_id: null, - api_client_id: '580111', - delivery_option_group: { - token: '26492692a443ee35c30eb82073bacaa8', - type: 'one_time_purchase', - }, - delivery_expectation_range: null, - delivery_expectation_type: null, - id: null, - requested_fulfillment_service_id: null, - delivery_category: null, - validation_context: null, - applied_discounts: [], - }, - ], - shipping_address: { - first_name: 'testuser', - address1: 'oakwood bridge', - phone: null, - city: 'KLF', - zip: '85003', - province: 'Arizona', - country: 'United States', - last_name: 'dummy', - address2: 'Hedgetown', - company: null, - latitude: null, - longitude: null, - name: 'testuser dummy', - country_code: 'US', - province_code: 'AZ', - }, - taxes_included: false, - total_weight: 0, - currency: 'USD', - completed_at: null, - phone: null, - customer_locale: 'en-US', - line_items: [ - { - key: '41327143059569', - fulfillment_service: 'manual', - gift_card: false, - grams: 0, - presentment_title: 'The Multi-location Snowboard', - presentment_variant_title: '', - product_id: 7234590638193, - quantity: 1, - requires_shipping: true, - sku: '', - tax_lines: [], - taxable: true, - title: 'The Multi-location Snowboard', - variant_id: 41327143059569, - variant_title: '', - variant_price: '729.95', - vendor: 'pixel-testing-rs', - unit_price_measurement: { - measured_type: null, - quantity_value: null, - quantity_unit: null, - reference_value: null, - reference_unit: null, - }, - compare_at_price: null, - line_price: '729.95', - price: '729.95', - applied_discounts: [], - destination_location_id: null, - user_id: null, - rank: null, - origin_location_id: null, - properties: {}, - }, - ], - name: '#35374569160817', - abandoned_checkout_url: - 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', - discount_codes: [], - tax_lines: [], - presentment_currency: 'USD', - source_name: 'web', - total_line_items_price: '729.95', - total_tax: '0.00', - total_discounts: '0.00', - subtotal_price: '729.95', - total_price: '736.85', - total_duties: '0.00', - device_id: null, - user_id: null, - location_id: null, - source_identifier: null, - source_url: null, - source: null, - closed_at: null, - customer: { - id: 7188389789809, - email: 'testuser101@gmail.com', - accepts_marketing: false, - created_at: null, - updated_at: null, - first_name: 'testuser', - last_name: 'dummy', - orders_count: 0, - state: 'disabled', - total_spent: '0.00', - last_order_id: null, - note: null, - verified_email: true, - multipass_identifier: null, - tax_exempt: false, - phone: null, - tags: '', - currency: 'USD', - accepts_marketing_updated_at: null, - admin_graphql_api_id: 'gid://shopify/Customer/7188389789809', - default_address: { - id: null, - customer_id: 7188389789809, - first_name: 'testuser', - last_name: 'dummy', - company: null, - address1: 'oakwood bridge', - address2: 'Hedgetown', - city: 'KLF', - province: 'Arizona', - country: 'United States', - zip: '85003', - phone: null, - name: 'testuser dummy', - province_code: 'AZ', - country_code: 'US', - country_name: 'United States', - default: true, - }, - last_order_name: null, - marketing_opt_in_level: null, - }, - query_parameters: { - topic: ['checkouts_update'], - writeKey: ['2l9QoM7KRMJLMcYhXNUVDT0Mqbd'], - }, - }, - source: dummySourceConfig, - }, - ], - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - }, - pathSuffix: '', - }, - output: { - response: { - status: 200, - body: [ - { - output: { - batch: [ - { - context: { - library: { - name: 'RudderStack Shopify Cloud', - version: '1.0.0', - }, - integration: { - name: 'SHOPIFY', - }, - topic: 'checkouts_update', - cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - }, - integrations: { - SHOPIFY: true, - }, - type: 'track', - event: 'Checkout Updated', - properties: { - order_id: 35374569160817, - value: '736.85', - tax: '0.00', - currency: 'USD', - token: 'e89d4437003b6b8480f8bc7f8036a659', - cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - email: 'testuser101@gmail.com', - buyer_accepts_marketing: false, - buyer_accepts_sms_marketing: false, - created_at: '2024-09-16T03:50:15+00:00', - updated_at: '2024-09-17T03:29:02-04:00', - landing_site: '/', - note: '', - note_attributes: [], - referring_site: '', - shipping_lines: [ - { - code: 'Standard', - price: '6.90', - original_shop_price: '6.90', - original_shop_markup: '0.00', - source: 'shopify', - title: 'Standard', - presentment_title: 'Standard', - phone: null, - tax_lines: [], - custom_tax_lines: null, - markup: '0.00', - carrier_identifier: null, - carrier_service_id: null, - api_client_id: '580111', - delivery_option_group: { - token: '26492692a443ee35c30eb82073bacaa8', - type: 'one_time_purchase', - }, - delivery_expectation_range: null, - delivery_expectation_type: null, - id: null, - requested_fulfillment_service_id: null, - delivery_category: null, - validation_context: null, - applied_discounts: [], - }, - ], - taxes_included: false, - total_weight: 0, - customer_locale: 'en-US', - name: '#35374569160817', - abandoned_checkout_url: - 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', - discount_codes: [], - tax_lines: [], - presentment_currency: 'USD', - source_name: 'web', - total_line_items_price: '729.95', - total_discounts: '0.00', - subtotal_price: '729.95', - total_duties: '0.00', - products: [ - { - product_id: 7234590638193, - price: '729.95', - brand: 'pixel-testing-rs', - quantity: 1, - key: '41327143059569', - fulfillment_service: 'manual', - gift_card: false, - grams: 0, - presentment_title: 'The Multi-location Snowboard', - presentment_variant_title: '', - requires_shipping: true, - tax_lines: [], - taxable: true, - title: 'The Multi-location Snowboard', - unit_price_measurement: { - measured_type: null, - quantity_value: null, - quantity_unit: null, - reference_value: null, - reference_unit: null, - }, - compare_at_price: null, - line_price: '729.95', - applied_discounts: [], - destination_location_id: null, - user_id: null, - rank: null, - origin_location_id: null, - properties: {}, - variant: '41327143059569 729.95 ', - }, - ], - }, - userId: '7188389789809', - traits: { - email: 'testuser101@gmail.com', - firstName: 'testuser', - lastName: 'dummy', - address: { - id: null, - customer_id: 7188389789809, - first_name: 'testuser', - last_name: 'dummy', - company: null, - address1: 'oakwood bridge', - address2: 'Hedgetown', - city: 'KLF', - province: 'Arizona', - country: 'United States', - zip: '85003', - phone: null, - name: 'testuser dummy', - province_code: 'AZ', - country_code: 'US', - country_name: 'United States', - default: true, - }, - acceptsMarketing: false, - orderCount: 0, - state: 'disabled', - totalSpent: '0.00', - verifiedEmail: true, - taxExempt: false, - tags: '', - currency: 'USD', - adminGraphqlApiId: 'gid://shopify/Customer/7188389789809', - shippingAddress: { - first_name: 'testuser', - address1: 'oakwood bridge', - phone: null, - city: 'KLF', - zip: '85003', - province: 'Arizona', - country: 'United States', - last_name: 'dummy', - address2: 'Hedgetown', - company: null, - latitude: null, - longitude: null, - name: 'testuser dummy', - country_code: 'US', - province_code: 'AZ', - }, - }, - timestamp: '2024-09-17T07:29:02.000Z', - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', - }, - ], - }, - }, - ], - }, - }, - mockFns: () => { - defaultMockFns(); - }, - }, - { - name: 'shopify', - description: 'Track Call -> Cart Update event', - module: 'source', - version: 'v1', - input: { - request: { - body: [ - { - event: { - id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - line_items: [ - { - id: 41327143059569, - properties: null, - quantity: 3, - variant_id: 41327143059569, - key: '41327143059569:90562f18109e0e6484b0c297e7981b30', - discounted_price: '729.95', - discounts: [], - gift_card: false, - grams: 0, - line_price: '2189.85', - original_line_price: '2189.85', - original_price: '729.95', - price: '729.95', - product_id: 7234590638193, - sku: '', - taxable: true, - title: 'The Multi-location Snowboard', - total_discount: '0.00', - vendor: 'pixel-testing-rs', - discounted_price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - original_line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - total_discount_set: { - shop_money: { - amount: '0.0', - currency_code: 'USD', - }, - presentment_money: { - amount: '0.0', - currency_code: 'USD', - }, - }, - }, - ], - note: '', - updated_at: '2024-09-17T08:15:13.280Z', - created_at: '2024-09-16T03:50:15.478Z', - query_parameters: { - topic: ['carts_update'], - writeKey: ['2l9QoM7KRMJLMcYhXNUVDT0Mqbd'], - }, - }, - source: dummySourceConfig, - }, - ], - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - }, - pathSuffix: '', - }, - output: { - response: { - status: 200, - body: [ - { - output: { - batch: [ - { - context: { - library: { - name: 'RudderStack Shopify Cloud', - version: '1.0.0', - }, - integration: { - name: 'SHOPIFY', - }, - topic: 'carts_update', - }, - integrations: { - SHOPIFY: true, - }, - type: 'track', - event: 'Cart Update', - properties: { - id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - note: '', - updated_at: '2024-09-17T08:15:13.280Z', - created_at: '2024-09-16T03:50:15.478Z', - products: [ - { - product_id: 7234590638193, - price: '729.95', - brand: 'pixel-testing-rs', - quantity: 3, - id: 41327143059569, - properties: null, - key: '41327143059569:90562f18109e0e6484b0c297e7981b30', - discounted_price: '729.95', - discounts: [], - gift_card: false, - grams: 0, - line_price: '2189.85', - original_line_price: '2189.85', - original_price: '729.95', - taxable: true, - title: 'The Multi-location Snowboard', - total_discount: '0.00', - discounted_price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - original_line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - total_discount_set: { - shop_money: { - amount: '0.0', - currency_code: 'USD', - }, - presentment_money: { - amount: '0.0', - currency_code: 'USD', - }, - }, - variant: '41327143059569 ', - }, - ], - }, - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', - }, - ], - }, - }, - ], - }, - }, - mockFns: () => { - defaultMockFns(); - }, - }, -]; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts new file mode 100644 index 0000000000..ade496efb7 --- /dev/null +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -0,0 +1,1687 @@ +// This file contains the test scenarios for the server-side events from the Shopify GraphQL API for +// the v1 transformation flow +import { mockFns } from '../mocks'; +import { dummySourceConfig } from '../constants'; + +export const checkoutEventsTestScenarios = [ + { + id: 'c001', + name: 'shopify', + description: 'Track Call -> Checkout Started event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 35550298931313, + token: '84ad78572dae52a8cbea7d55371afe89', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + email: null, + gateway: null, + buyer_accepts_marketing: false, + buyer_accepts_sms_marketing: false, + sms_marketing_phone: null, + created_at: '2024-11-06T02:22:00+00:00', + updated_at: '2024-11-05T21:22:02-05:00', + landing_site: '/', + note: '', + note_attributes: [], + referring_site: '', + shipping_lines: [], + shipping_address: [], + taxes_included: false, + total_weight: 0, + currency: 'USD', + completed_at: null, + phone: null, + customer_locale: 'en-US', + line_items: [ + { + key: '41327142600817', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + presentment_variant_title: '', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '600.00', + price: '600.00', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35550298931313', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ/recover?key=0385163be3875d3a2117e982d9cc3517&locale=en-US', + discount_codes: [], + tax_lines: [], + presentment_currency: 'USD', + source_name: 'web', + total_line_items_price: '600.00', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '600.00', + total_price: '600.00', + total_duties: '0.00', + device_id: null, + user_id: null, + location_id: null, + source_identifier: null, + source_url: null, + source: null, + closed_at: null, + query_parameters: { + topic: ['checkouts_create'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'checkouts_create', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + shopifyDetails: { + id: 35550298931313, + token: '84ad78572dae52a8cbea7d55371afe89', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + email: null, + gateway: null, + buyer_accepts_marketing: false, + buyer_accepts_sms_marketing: false, + sms_marketing_phone: null, + created_at: '2024-11-06T02:22:00+00:00', + updated_at: '2024-11-05T21:22:02-05:00', + landing_site: '/', + note: '', + note_attributes: [], + referring_site: '', + shipping_lines: [], + shipping_address: [], + taxes_included: false, + total_weight: 0, + currency: 'USD', + completed_at: null, + phone: null, + customer_locale: 'en-US', + line_items: [ + { + key: '41327142600817', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + presentment_variant_title: '', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '600.00', + price: '600.00', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35550298931313', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ/recover?key=0385163be3875d3a2117e982d9cc3517&locale=en-US', + discount_codes: [], + tax_lines: [], + presentment_currency: 'USD', + source_name: 'web', + total_line_items_price: '600.00', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '600.00', + total_price: '600.00', + total_duties: '0.00', + device_id: null, + user_id: null, + location_id: null, + source_identifier: null, + source_url: null, + source: null, + closed_at: null, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Checkout Started', + properties: { + order_id: 35550298931313, + value: '600.00', + tax: '0.00', + currency: 'USD', + products: [ + { + product_id: 7234590408817, + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + timestamp: '2024-11-06T02:22:02.000Z', + traits: { + shippingAddress: [], + }, + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c002', + name: 'shopify', + description: 'Track Call -> Checkout Updated event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + query_parameters: { + topic: ['checkouts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + id: 35374569160817, + token: 'e89d4437003b6b8480f8bc7f8036a659', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + email: 'testuser101@gmail.com', + created_at: '2024-09-16T03:50:1500:00', + updated_at: '2024-09-17T03:29:02-04:00', + note: '', + note_attributes: [], + shipping_address: { + first_name: 'testuser', + address1: 'oakwood bridge', + phone: null, + city: 'KLF', + zip: '85003', + province: 'Arizona', + country: 'United States', + last_name: 'dummy', + address2: 'Hedgetown', + company: null, + latitude: null, + longitude: null, + name: 'testuser dummy', + country_code: 'US', + province_code: 'AZ', + }, + total_weight: 0, + currency: 'USD', + customer_locale: 'en-US', + line_items: [ + { + key: '41327143059569', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Multi-location Snowboard', + presentment_variant_title: '', + product_id: 7234590638193, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Multi-location Snowboard', + variant_id: 41327143059569, + variant_title: '', + variant_price: '729.95', + vendor: 'pixel-testing-rs', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '729.95', + price: '729.95', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35374569160817', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', + presentment_currency: 'USD', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '729.95', + total_price: '736.85', + total_duties: '0.00', + customer: { + id: 7188389789809, + email: 'testuser101@gmail.com', + accepts_marketing: false, + created_at: null, + updated_at: null, + first_name: 'testuser', + last_name: 'dummy', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + tags: '', + currency: 'USD', + accepts_marketing_updated_at: null, + admin_graphql_api_id: 'gid://shopify/Customer/7188389789809', + default_address: { + id: null, + customer_id: 7188389789809, + first_name: 'testuser', + last_name: 'dummy', + company: null, + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + province: 'Arizona', + country: 'United States', + zip: '85003', + phone: null, + name: 'testuser dummy', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: true, + }, + last_order_name: null, + marketing_opt_in_level: null, + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + context: { + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + integration: { + name: 'SHOPIFY', + }, + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + shopifyDetails: { + id: 35374569160817, + token: 'e89d4437003b6b8480f8bc7f8036a659', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + email: 'testuser101@gmail.com', + created_at: '2024-09-16T03:50:1500:00', + updated_at: '2024-09-17T03:29:02-04:00', + note: '', + note_attributes: [], + shipping_address: { + first_name: 'testuser', + address1: 'oakwood bridge', + phone: null, + city: 'KLF', + zip: '85003', + province: 'Arizona', + country: 'United States', + last_name: 'dummy', + address2: 'Hedgetown', + company: null, + latitude: null, + longitude: null, + name: 'testuser dummy', + country_code: 'US', + province_code: 'AZ', + }, + total_weight: 0, + currency: 'USD', + customer_locale: 'en-US', + line_items: [ + { + key: '41327143059569', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Multi-location Snowboard', + presentment_variant_title: '', + product_id: 7234590638193, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Multi-location Snowboard', + variant_id: 41327143059569, + variant_title: '', + variant_price: '729.95', + vendor: 'pixel-testing-rs', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '729.95', + price: '729.95', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35374569160817', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', + presentment_currency: 'USD', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '729.95', + total_price: '736.85', + total_duties: '0.00', + customer: { + id: 7188389789809, + email: 'testuser101@gmail.com', + accepts_marketing: false, + created_at: null, + updated_at: null, + first_name: 'testuser', + last_name: 'dummy', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + tags: '', + currency: 'USD', + accepts_marketing_updated_at: null, + admin_graphql_api_id: 'gid://shopify/Customer/7188389789809', + default_address: { + id: null, + customer_id: 7188389789809, + first_name: 'testuser', + last_name: 'dummy', + company: null, + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + province: 'Arizona', + country: 'United States', + zip: '85003', + phone: null, + name: 'testuser dummy', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: true, + }, + last_order_name: null, + marketing_opt_in_level: null, + }, + }, + topic: 'checkouts_update', + }, + event: 'Checkout Updated', + integrations: { + SHOPIFY: true, + }, + properties: { + currency: 'USD', + order_id: 35374569160817, + products: [ + { + brand: 'pixel-testing-rs', + price: '729.95', + product_id: 7234590638193, + quantity: 1, + }, + ], + tax: '0.00', + value: '736.85', + }, + timestamp: '2024-09-17T07:29:02.000Z', + traits: { + acceptsMarketing: false, + address: { + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + company: null, + country: 'United States', + country_code: 'US', + country_name: 'United States', + customer_id: 7188389789809, + default: true, + first_name: 'testuser', + id: null, + last_name: 'dummy', + name: 'testuser dummy', + phone: null, + province: 'Arizona', + province_code: 'AZ', + zip: '85003', + }, + adminGraphqlApiId: 'gid://shopify/Customer/7188389789809', + currency: 'USD', + email: 'testuser101@gmail.com', + firstName: 'testuser', + lastName: 'dummy', + orderCount: 0, + shippingAddress: { + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + company: null, + country: 'United States', + country_code: 'US', + first_name: 'testuser', + last_name: 'dummy', + latitude: null, + longitude: null, + name: 'testuser dummy', + phone: null, + province: 'Arizona', + province_code: 'AZ', + zip: '85003', + }, + state: 'disabled', + tags: '', + taxExempt: false, + totalSpent: '0.00', + verifiedEmail: true, + }, + type: 'track', + userId: '7188389789809', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c003', + name: 'shopify', + description: 'Track Call -> Order Updated event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_discounts: '0.00', + current_total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + current_total_price: '600.00', + current_total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_tax: '0.00', + current_total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + customer_locale: 'en-US', + discount_codes: [], + email: 'henry@wfls.com', + estimated_taxes: false, + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + number: 17, + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + presentment_currency: 'USD', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + tax_lines: [], + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_line_items_price: '600.00', + total_line_items_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_price: '600.00', + total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_shipping_price_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_tax: '0.00', + total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + payment_terms: null, + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + query_parameters: { + topic: ['orders_updated'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'orders_updated', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + shopifyDetails: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_discounts: '0.00', + current_total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + current_total_price: '600.00', + current_total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_tax: '0.00', + current_total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + customer_locale: 'en-US', + discount_codes: [], + email: 'henry@wfls.com', + estimated_taxes: false, + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + number: 17, + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + presentment_currency: 'USD', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + tax_lines: [], + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_line_items_price: '600.00', + total_line_items_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_price: '600.00', + total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_shipping_price_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_tax: '0.00', + total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + payment_terms: null, + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + }, + order_token: '676613a0027fc8240e16d67fdc9f5ac8', + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Order Updated', + properties: { + order_id: 5778367414385, + value: '600.00', + tax: '0.00', + currency: 'USD', + products: [ + { + product_id: 7234590408817, + title: 'The Collection Snowboard: Hydrogen', + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + userId: '7358220173425', + traits: { + email: 'henry@wfls.com', + firstName: 'yodi', + lastName: 'waffles', + address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + state: 'disabled', + verifiedEmail: true, + taxExempt: false, + tags: '', + currency: 'USD', + taxExemptions: [], + adminGraphqlApiId: 'gid://shopify/Customer/7358220173425', + shippingAddress: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + billingAddress: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + }, + timestamp: '2024-11-06T02:54:50.000Z', + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c004', + name: 'shopify', + description: 'Track Call -> Order Created event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_discounts: '0.00', + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + name: '#1017', + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + phone: null, + presentment_currency: 'USD', + subtotal_price: '600.00', + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + phone: null, + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + query_parameters: { + topic: ['orders_create'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnZkCHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'orders_create', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + shopifyDetails: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_discounts: '0.00', + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + name: '#1017', + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + phone: null, + presentment_currency: 'USD', + subtotal_price: '600.00', + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + phone: null, + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Order Created', + properties: { + order_id: 5778367414385, + value: '600.00', + tax: '0.00', + currency: 'USD', + products: [ + { + product_id: 7234590408817, + title: 'The Collection Snowboard: Hydrogen', + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + userId: '7358220173425', + traits: { + email: 'henry@wfls.com', + firstName: 'yodi', + lastName: 'waffles', + address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + state: 'disabled', + currency: 'USD', + taxExemptions: [], + adminGraphqlApiId: 'gid://shopify/Customer/7358220173425', + shippingAddress: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + billingAddress: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + }, + timestamp: '2024-11-06T02:54:50.000Z', + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d1) => ({ ...d1, mockFns })); diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts new file mode 100644 index 0000000000..f04fd7e08e --- /dev/null +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -0,0 +1,557 @@ +// This file contains the test scenarios for the server-side events from the Shopify GraphQL API for +// the v1 transformation flow +import { mockFns } from '../mocks'; +import { dummySourceConfig } from '../constants'; + +export const genericTrackTestScenarios = [ + { + id: 'c005', + name: 'shopify', + description: 'Track Call -> Cart Update event with no line items from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + line_items: [], + note: '', + updated_at: '2024-09-17T08:15:13.280Z', + created_at: '2024-09-16T03:50:15.478Z', + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + context: { + integration: { + name: 'SHOPIFY', + }, + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + shopifyDetails: { + created_at: '2024-09-16T03:50:15.478Z', + id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + line_items: [], + note: '', + token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + updated_at: '2024-09-17T08:15:13.280Z', + }, + topic: 'carts_update', + }, + event: 'Cart Update', + integrations: { + SHOPIFY: true, + }, + properties: { + products: [], + }, + type: 'track', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c006', + name: 'shopify', + description: 'Track Call -> Unsupported event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 35550298931313, + query_parameters: { + topic: ['unsupported_event'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + outputToSource: { + body: 'T0s=', + contentType: 'text/plain', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'c007', + name: 'shopify', + description: 'Track Call -> generic event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_additional_fees_set: null, + current_total_discounts: '0.00', + current_total_duties_set: null, + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + order_number: 1017, + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + po_number: null, + presentment_currency: 'USD', + processed_at: '2024-11-05T21:54:48-05:00', + reference: '4d92cf60cc24a1bd95929e17ead9845f', + referring_site: '', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + query_parameters: { + topic: ['orders_paid'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnZkCHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'orders_paid', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + shopifyDetails: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_additional_fees_set: null, + current_total_discounts: '0.00', + current_total_duties_set: null, + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + order_number: 1017, + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + po_number: null, + presentment_currency: 'USD', + processed_at: '2024-11-05T21:54:48-05:00', + reference: '4d92cf60cc24a1bd95929e17ead9845f', + referring_site: '', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Order Paid', + properties: { + products: [ + { + product_id: 7234590408817, + title: 'The Collection Snowboard: Hydrogen', + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + traits: { + email: 'henry@wfls.com', + }, + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d2) => ({ ...d2, mockFns })); diff --git a/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts new file mode 100644 index 0000000000..b03f5635b6 --- /dev/null +++ b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts @@ -0,0 +1,256 @@ +import { mockFns } from '../mocks'; +import { dummySourceConfig } from '../constants'; + +export const identityTestScenarios = [ + { + id: 'c008', + name: 'shopify', + description: 'Identify Call -> Customer update event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + tags: '', + last_order_name: null, + currency: 'USD', + phone: null, + addresses: [ + { + id: 8715246895217, + customer_id: 7358220173425, + first_name: 'yodi', + last_name: 'waffles', + company: null, + address1: 'Yuma Proving Ground', + address2: 'suite 001', + city: 'Yuma Proving Ground', + province: 'Arizona', + country: 'United States', + zip: '85365', + phone: null, + name: 'yodi waffles', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: false, + }, + ], + tax_exemptions: [], + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + query_parameters: { + topic: ['customers_update'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'customers_update', + shopifyDetails: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + tags: '', + last_order_name: null, + currency: 'USD', + phone: null, + addresses: [ + { + id: 8715246895217, + customer_id: 7358220173425, + first_name: 'yodi', + last_name: 'waffles', + company: null, + address1: 'Yuma Proving Ground', + address2: 'suite 001', + city: 'Yuma Proving Ground', + province: 'Arizona', + country: 'United States', + zip: '85365', + phone: null, + name: 'yodi waffles', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: false, + }, + ], + tax_exemptions: [], + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'identify', + userId: '7358220173425', + traits: { + email: 'henry@wfls.com', + firstName: 'yodi', + lastName: 'waffles', + addressList: [ + { + id: 8715246895217, + customer_id: 7358220173425, + first_name: 'yodi', + last_name: 'waffles', + company: null, + address1: 'Yuma Proving Ground', + address2: 'suite 001', + city: 'Yuma Proving Ground', + province: 'Arizona', + country: 'United States', + zip: '85365', + phone: null, + name: 'yodi waffles', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: false, + }, + ], + address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + orderCount: 0, + state: 'disabled', + totalSpent: '0.00', + verifiedEmail: true, + taxExempt: false, + tags: '', + currency: 'USD', + taxExemptions: [], + adminGraphqlApiId: 'gid://shopify/Customer/7358220173425', + }, + timestamp: '2024-11-06T02:54:49.000Z', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d3) => ({ ...d3, mockFns })); diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 4eda20a901..7e6e6b9acb 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -237,6 +237,30 @@ export const generateTrackPayload: any = (parametersOverride: any) => { return removeUndefinedAndNullValues(payload); }; +export const generateRecordPayload: any = (parametersOverride: any) => { + const payload = { + type: 'record', + action: parametersOverride.action || 'insert', + fields: parametersOverride.fields || {}, + channel: 'sources', + context: { + sources: { + job_id: 'randomJobId', + version: 'local', + job_run_id: 'jobRunId', + task_run_id: 'taskRunId', + }, + }, + recordId: '3', + rudderId: 'randomRudderId', + messageId: 'randomMessageId', + receivedAt: '2024-11-08T10:30:41.618+05:30', + request_ip: '[::1]', + identifiers: parametersOverride.identifiers || {}, + }; + return removeUndefinedAndNullValues(payload); +}; + export const generateSimplifiedTrackPayload: any = (parametersOverride: any) => { return removeUndefinedAndNullValues({ type: 'track',