diff --git a/.eslintignore b/.eslintignore index ce54730f4b..b1bbed4416 100644 --- a/.eslintignore +++ b/.eslintignore @@ -21,3 +21,8 @@ src/v0/destinations/personalize/scripts/ test/integrations/destinations/testTypes.d.ts *.config*.js scripts/skipPrepareScript.js +*.yaml +*.yml +.eslintignore +.prettierignore +*.json \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json index 556470697d..144b90e348 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -19,7 +19,8 @@ "parserOptions": { "ecmaVersion": 12, "sourceType": "module", - "project": "./tsconfig.json" + "project": "./tsconfig.json", + "extraFileExtensions": [".yaml"] }, "rules": { "unicorn/filename-case": [ diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 885ecf4fd6..69f68c1676 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -23,6 +23,15 @@ on: type: string build_type: type: string + use_merge_sha: + type: boolean + default: false + skip_tests: + type: boolean + default: false + description: if this option is true, we would skip tests while building docker image + workflow_url: + type: string secrets: DOCKERHUB_PROD_TOKEN: required: true @@ -31,27 +40,74 @@ env: DOCKERHUB_USERNAME: rudderlabs jobs: + get_sha: + runs-on: ubuntu-latest + name: Get SHA information + outputs: + sha: ${{steps.getSHA.outputs.SHA}} + steps: + - name: Checkout SHA + id: getSHA + run: | + if ${{inputs.use_merge_sha}} == true; then + sha=$(echo ${{github.sha}}) + else + sha=$(echo ${{ github.event.pull_request.head.sha }}) + fi + echo "SHA: $sha" + echo "SHA=$sha" >> $GITHUB_OUTPUT + + get_changed_files: + runs-on: ubuntu-latest + name: Get Changed files + outputs: + should_execute_tests: ${{ steps.processing.outputs.should_execute_tests }} + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 1 + - id: files + uses: Ana06/get-changed-files@v1.2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + format: 'json' + - id: processing + run: | + readarray -t modified_files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.modified }}')" + echo "Modified files: $modified_files" + found=false + for modified_file in "${modified_files[@]}"; do + if [[ "$modified_file" == "Dockerfile" || "$modified_file" == "docker-compose.yml" || "$modified_file" == "Dockerfile" || "$modified_file" == "Dockerfile-ut-func" ]]; then + found=true + break + fi + done + echo "Match Found: $found" + echo "::set-output name=should_execute_tests::$found" + build-transformer-image-arm64: name: Build Transformer Docker Image ARM64 runs-on: [self-hosted, Linux, ARM64] + needs: [get_sha, get_changed_files] steps: - name: Checkout uses: actions/checkout@v4.1.1 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.6.1 - name: Login to DockerHub - uses: docker/login-action@v3.2.0 + uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - name: Build Docker Image - uses: docker/build-push-action@v6.4.1 + uses: docker/build-push-action@v6.5.0 with: context: . file: ${{ inputs.dockerfile }} @@ -62,12 +118,13 @@ jobs: # cache-to: type=gha,mode=max - name: Run Tests + if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }} run: | docker run ${{ inputs.build_tag }} npm run test:js:ci docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images - uses: docker/build-push-action@v6.4.1 + uses: docker/build-push-action@v6.5.0 with: context: . file: ${{ inputs.dockerfile }} @@ -85,24 +142,25 @@ jobs: build-transformer-image-amd64: name: Build Transformer Docker Image AMD64 runs-on: [self-hosted, Linux, X64] + needs: [get_sha, get_changed_files] steps: - name: Checkout uses: actions/checkout@v4.1.1 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ needs.get_sha.outputs.sha }} fetch-depth: 1 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.6.1 - name: Login to DockerHub - uses: docker/login-action@v3.2.0 + uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - name: Build Docker Image - uses: docker/build-push-action@v6.4.1 + uses: docker/build-push-action@v6.5.0 with: context: . file: ${{ inputs.dockerfile }} @@ -113,12 +171,13 @@ jobs: # cache-to: type=gha,mode=max - name: Run Tests + if: ${{ inputs.skip_tests != true || needs.get_changed_files.outputs.should_execute_tests == true }} run: | docker run ${{ inputs.build_tag }} npm run test:js:ci docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images - uses: docker/build-push-action@v6.4.1 + uses: docker/build-push-action@v6.5.0 with: context: . file: ${{ inputs.dockerfile }} @@ -140,10 +199,10 @@ jobs: steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.6.1 - name: Login to DockerHub - uses: docker/login-action@v3.2.0 + uses: docker/login-action@v3.3.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 755eb24397..b1da46898a 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -13,10 +13,22 @@ concurrency: cancel-in-progress: true jobs: + get_workflow_url: + runs-on: ubuntu-latest + steps: + - id: get_url + run: | + curl -s https://api.github.com/repos/${{ github.repository }}/actions/workflows/${{ github.workflow }}/runs/${{ github.run_id }} | jq -r .html_url >> workflow_url.txt + echo "::set-output name=workflow_url::$(cat workflow_url.txt)" + outputs: + url: ${{ steps.get_url.outputs.workflow_url }} + coverage: name: Code Coverage runs-on: ubuntu-latest - + needs: [get_workflow_url] + outputs: + tests_run_outcome: ${{steps.run_tests.outcome}} steps: - name: Checkout uses: actions/checkout@v4.1.1 @@ -33,6 +45,8 @@ jobs: run: npm ci - name: Run Tests + id: run_tests + continue-on-error: true run: | # Supress logging in tests LOG_LEVEL=100 npm run test:js:ci @@ -70,3 +84,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.PAT }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + notify: + name: slack notification on failure + needs: [get_workflow_url, coverage] + if: needs.coverage.outputs.tests_run_outcome == 'failure' || failure() + uses: ./.github/workflows/slack-notify.yml + with: + workflow_url: ${{ needs.get_workflow_url.outputs.url }} + should_notify: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}} + secrets: inherit diff --git a/.github/workflows/prepare-for-dev-deploy.yml b/.github/workflows/prepare-for-dev-deploy.yml index 66020feef0..a1fdda8ccd 100644 --- a/.github/workflows/prepare-for-dev-deploy.yml +++ b/.github/workflows/prepare-for-dev-deploy.yml @@ -60,6 +60,7 @@ jobs: dockerfile: Dockerfile load_target: development push_target: production + use_merge_sha: true secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/prepare-for-prod-dt-deploy.yml b/.github/workflows/prepare-for-prod-dt-deploy.yml index 4a4d656e78..8e589f5b20 100644 --- a/.github/workflows/prepare-for-prod-dt-deploy.yml +++ b/.github/workflows/prepare-for-prod-dt-deploy.yml @@ -57,6 +57,8 @@ jobs: load_target: development push_target: production build_type: dt + use_merge_sha: true + skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}} secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/prepare-for-prod-ut-deploy.yml b/.github/workflows/prepare-for-prod-ut-deploy.yml index a5afed6dee..468307b94a 100644 --- a/.github/workflows/prepare-for-prod-ut-deploy.yml +++ b/.github/workflows/prepare-for-prod-ut-deploy.yml @@ -60,6 +60,8 @@ jobs: load_target: development push_target: production build_type: ut + use_merge_sha: true + skip_tests: ${{startsWith(github.event.pull_request.head.ref, 'hotfix-release/')}} secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/prepare-for-staging-deploy.yml b/.github/workflows/prepare-for-staging-deploy.yml index 4a4bf44f75..60a6238aa8 100644 --- a/.github/workflows/prepare-for-staging-deploy.yml +++ b/.github/workflows/prepare-for-staging-deploy.yml @@ -52,6 +52,7 @@ jobs: dockerfile: Dockerfile load_target: development push_target: production + use_merge_sha: true secrets: DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }} diff --git a/.github/workflows/slack-notify.yml b/.github/workflows/slack-notify.yml new file mode 100644 index 0000000000..bb5cac82f1 --- /dev/null +++ b/.github/workflows/slack-notify.yml @@ -0,0 +1,51 @@ +name: Notify workflow failure + +on: + workflow_call: + inputs: + should_notify: + type: boolean + default: true + workflow_url: + type: string + required: true + +jobs: + notify: + runs-on: ubuntu-latest + if: ${{ inputs.should_notify }} + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - name: notify + uses: slackapi/slack-github-action@v1.25.0 + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + PROJECT_NAME: 'Rudder Transformer' + with: + channel-id: ${{ secrets.SLACK_INTEGRATION_DEV_CHANNEL_ID }} + payload: | + { + "text": "**\nCC: ", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":siren2: prod release tests - Failed :siren2:" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*<${{inputs.workflow_url}}|failed workflow>*\nCC: " + } + } + ] + } diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index a03037d4ad..0d389f0e55 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -28,13 +28,29 @@ jobs: - name: Install Dependencies run: npm ci - - name: Run Lint Checks + - id: files + uses: Ana06/get-changed-files@v1.2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run format Checks + run: | + npx prettier ${{steps.files.outputs.added_modified}} --write + + - run: git diff --exit-code + + - name: Formatting Error message + if: ${{ failure() }} + run: | + echo 'prettier formatting failure. Ensure you run `npm run format` and commit the files.' + + - name: Run eslint Checks run: | - npm run lint + npx eslint ${{steps.files.outputs.added_modified}} --fix - run: git diff --exit-code - - name: Error message + - name: Eslint Error message if: ${{ failure() }} run: | - echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.' + echo 'Eslint failure. Ensure you run `npm run lint:fix` and commit the files.' diff --git a/.husky/commit-msg b/.husky/commit-msg index 9db017095e..84dc58a421 100755 --- a/.husky/commit-msg +++ b/.husky/commit-msg @@ -1,4 +1,2 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" npm run commit-msg diff --git a/.husky/pre-commit b/.husky/pre-commit index d4a43dd13e..af964838e1 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,2 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" npm run pre-commit diff --git a/.prettierignore b/.prettierignore index 99747b29bb..ac5f1fc409 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,3 +8,5 @@ test/**/*.js src/util/lodash-es-core.js src/util/url-search-params.min.js dist +.eslintignore +.prettierignore diff --git a/.vscode/settings.json b/.vscode/settings.json index 13c49c08f1..8d723306a9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,6 +13,10 @@ "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true }, + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true + }, "[yaml]": { "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e4a636b4a..dbdea91711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.76.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.1...v1.76.0) (2024-08-20) + + +### Features + +* klaviyo onboard unsubscribe profile support ([#3646](https://github.com/rudderlabs/rudder-transformer/issues/3646)) ([474f2bd](https://github.com/rudderlabs/rudder-transformer/commit/474f2bddc58e1962206e39d92514827f29f84c83)) +* onboard sfmc with vdm for rETL ([#3655](https://github.com/rudderlabs/rudder-transformer/issues/3655)) ([d987d1f](https://github.com/rudderlabs/rudder-transformer/commit/d987d1fc9afb9e1dc7482b2fe1458573f0f2699e)) +* onboard smartly destination ([#3660](https://github.com/rudderlabs/rudder-transformer/issues/3660)) ([474a36e](https://github.com/rudderlabs/rudder-transformer/commit/474a36ec385abf9ff83596b062d4d8e4c24469b8)) +* add bloomreach retl support ([#3619](https://github.com/rudderlabs/rudder-transformer/issues/3619)) ([6b1a23a](https://github.com/rudderlabs/rudder-transformer/commit/6b1a23af845084d6f2f5fd14656e4a1d11a7e34b)) + + +### Bug Fixes + +* add alias support in case alias details are present ([#3579](https://github.com/rudderlabs/rudder-transformer/issues/3579)) ([cb67262](https://github.com/rudderlabs/rudder-transformer/commit/cb672628b312f20ea0fcc27a60ec8ab5692f8b06)) +* attentive tag bugsnag issue ([#3663](https://github.com/rudderlabs/rudder-transformer/issues/3663)) ([866dbf3](https://github.com/rudderlabs/rudder-transformer/commit/866dbf3e81754e71ff8ac08b258b359ec5cc6889)) +* fixing facebook utils ([#3664](https://github.com/rudderlabs/rudder-transformer/issues/3664)) ([1a61675](https://github.com/rudderlabs/rudder-transformer/commit/1a6167584a5780ab50beda13cc5ef6bf4e283e38)) +* reserved properties for braze ([#3573](https://github.com/rudderlabs/rudder-transformer/issues/3573)) ([413e9ce](https://github.com/rudderlabs/rudder-transformer/commit/413e9ce56f8f6569bbeb188bff4f43d400ea71b1)) +* source transformation integration test generation ([#3645](https://github.com/rudderlabs/rudder-transformer/issues/3645)) ([23196ec](https://github.com/rudderlabs/rudder-transformer/commit/23196ec42acf35f314e1953f339f6acbb72edd70)) + ### [1.75.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.75.0...v1.75.1) (2024-08-14) diff --git a/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json b/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json index 3bb8d60503..24bb4546b4 100644 --- a/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json +++ b/go/webhook/testcases/testdata/testcases/close_crm/group_creation.json @@ -34,7 +34,8 @@ ] }, "previous_data": {} - } + }, + "source": {} }, "headers": { "Content-Type": "application/json" diff --git a/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json b/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json index 34a36a8da8..3b8a2ce578 100644 --- a/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json +++ b/go/webhook/testcases/testdata/testcases/close_crm/lead_deletion.json @@ -43,7 +43,8 @@ "updated_by": "user_123", "created_by": "user_123" } - } + }, + "source": {} }, "headers": { "Content-Type": "application/json" diff --git a/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json b/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json index 38aeba6468..99cdfe061c 100644 --- a/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json +++ b/go/webhook/testcases/testdata/testcases/close_crm/lead_update.json @@ -61,7 +61,8 @@ "object_type": "opportunity", "lead_id": "lead_123" }, - "subscription_id": "whsub_123" + "subscription_id": "whsub_123", + "source": {} }, "headers": { "Content-Type": "application/json" diff --git a/go/webhook/testcases/testdata/testcases/cordial/multiple_object_input_event_with_batched_payload.json b/go/webhook/testcases/testdata/testcases/cordial/multiple_object_input_event_with_batched_payload.json index 6ab438b077..1217608454 100644 --- a/go/webhook/testcases/testdata/testcases/cordial/multiple_object_input_event_with_batched_payload.json +++ b/go/webhook/testcases/testdata/testcases/cordial/multiple_object_input_event_with_batched_payload.json @@ -64,7 +64,8 @@ "product_name": ["wtp ab"], "product_group": ["women"] } - } + }, + "source": {} }, { "contact": { @@ -126,7 +127,8 @@ "product_name": ["wtp ab"], "product_group": ["women"] } - } + }, + "source": {} } ], "headers": { diff --git a/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_no_cid.json b/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_no_cid.json index eac08aea16..5b7f898cb3 100644 --- a/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_no_cid.json +++ b/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_no_cid.json @@ -49,7 +49,8 @@ "title": "Khaki Shirt", "test_key": "value" } - } + }, + "source": {} }, "headers": { "Content-Type": "application/json" diff --git a/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_normal_channel_and_action.json b/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_normal_channel_and_action.json index b7e4cae319..72c76816ec 100644 --- a/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_normal_channel_and_action.json +++ b/go/webhook/testcases/testdata/testcases/cordial/simple_single_object_input_event_with_normal_channel_and_action.json @@ -44,7 +44,8 @@ "title": "Khaki Shirt", "test_key": "value" } - } + }, + "source": {} }, "headers": { "Content-Type": "application/json" diff --git a/go/webhook/testcases/testdata/testcases/mailjet/mail_jet_when_no_email_is_present.json b/go/webhook/testcases/testdata/testcases/mailjet/mail_jet_when_no_email_is_present.json index 301cf0a780..67935f116f 100644 --- a/go/webhook/testcases/testdata/testcases/mailjet/mail_jet_when_no_email_is_present.json +++ b/go/webhook/testcases/testdata/testcases/mailjet/mail_jet_when_no_email_is_present.json @@ -59,5 +59,6 @@ } ], "errQueue": [] - } + }, + "skip": "FIXME" } diff --git a/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json b/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json index 76f72961ca..58d4513db2 100644 --- a/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json +++ b/go/webhook/testcases/testdata/testcases/moengage/batch_of_events.json @@ -376,8 +376,8 @@ "shipping": 4, "value": 31.98 }, - "receivedAt": "2024-03-03T04:48:29.000Z", - "request_ip": "192.0.2.30", + "receivedAt": "2020-10-16T13:40:12.792+05:30", + "request_ip": "[::1]", "sentAt": "2020-10-16T08:10:12.783Z", "timestamp": "2020-10-16T13:40:12.791+05:30", "type": "track", diff --git a/go/webhook/testcases/testdata/testcases/segment/test_0.json b/go/webhook/testcases/testdata/testcases/segment/test_0.json index d3103126c9..868300e20e 100644 --- a/go/webhook/testcases/testdata/testcases/segment/test_0.json +++ b/go/webhook/testcases/testdata/testcases/segment/test_0.json @@ -246,5 +246,6 @@ } ], "errQueue": [] - } + }, + "skip": "NoAnonID error" } diff --git a/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json b/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json index 5e176d4e5b..8439ce36e0 100644 --- a/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json +++ b/go/webhook/testcases/testdata/testcases/shopify/invalid_topic.json @@ -17,7 +17,7 @@ "output": { "response": { "status": 400, - "body": "Invalid topic in query_parameters" + "body": "Invalid topic in query_parameters\n" }, "queue": [], "errQueue": [ diff --git a/go/webhook/testcases/testdata/testcases/shopify/no_query_parameters.json b/go/webhook/testcases/testdata/testcases/shopify/no_query_parameters.json index 6466693059..1c69d854fd 100644 --- a/go/webhook/testcases/testdata/testcases/shopify/no_query_parameters.json +++ b/go/webhook/testcases/testdata/testcases/shopify/no_query_parameters.json @@ -12,7 +12,7 @@ "output": { "response": { "status": 400, - "body": "Query_parameters is missing" + "body": "Query_parameters is missing\n" }, "queue": [], "errQueue": [{}] diff --git a/go/webhook/testcases/testdata/testcases/shopify/topic_not_found.json b/go/webhook/testcases/testdata/testcases/shopify/topic_not_found.json index bd0e37ab98..5b97996d17 100644 --- a/go/webhook/testcases/testdata/testcases/shopify/topic_not_found.json +++ b/go/webhook/testcases/testdata/testcases/shopify/topic_not_found.json @@ -18,7 +18,7 @@ "output": { "response": { "status": 400, - "body": "Topic not found" + "body": "Topic not found\n" }, "queue": [], "errQueue": [ diff --git a/package-lock.json b/package-lock.json index d16e52424a..3e1ab9730f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,16 @@ { "name": "rudder-transformer", - "version": "1.75.1", + "version": "1.76.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.75.1", + "version": "1.76.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", - "@aws-sdk/client-personalize": "^3.470.0", + "@aws-sdk/client-personalize": "^3.616.0", "@aws-sdk/client-s3": "^3.474.0", "@aws-sdk/credential-providers": "^3.391.0", "@aws-sdk/lib-storage": "^3.474.0", @@ -104,7 +104,7 @@ "eslint-plugin-unicorn": "^46.0.1", "glob": "^10.3.3", "http-terminator": "^3.2.0", - "husky": "^8.0.3", + "husky": "^9.1.1", "jest": "^29.5.0", "jest-sonar": "^0.2.16", "jest-when": "^3.5.2", @@ -694,53 +694,1124 @@ } }, "node_modules/@aws-sdk/client-personalize": { - "version": "3.485.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-personalize/-/client-personalize-3.485.0.tgz", - "integrity": "sha512-HWv+HAKtbfje8QH8jE3B+nvjDDsc8E7hhc16128lX4tc1M9sPfC2qfw4R2hZ9YZxAYSEWryRuOMcp8YQylwpzg==", + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-personalize/-/client-personalize-3.616.0.tgz", + "integrity": "sha512-IrlnpVGJKK+aTkNT5q/1UjJULjHRS9eGmfXLXMZmGk3MsyZBAzxPlW8/eT/dxBwBieuLmHpysPGnerduZCwggw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.616.0", + "@aws-sdk/client-sts": "3.616.0", + "@aws-sdk/core": "3.616.0", + "@aws-sdk/credential-provider-node": "3.616.0", + "@aws-sdk/middleware-host-header": "3.616.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.616.0", + "@aws-sdk/middleware-user-agent": "3.616.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.7", + "@smithy/fetch-http-handler": "^3.2.2", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.4", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.10", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/smithy-client": "^3.1.8", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.10", + "@smithy/util-defaults-mode-node": "^3.0.10", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/client-sso": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.616.0.tgz", + "integrity": "sha512-hwW0u1f8U4dSloAe61/eupUiGd5Q13B72BuzGxvRk0cIpYX/2m0KBG8DDl7jW1b2QQ+CflTLpG2XUf2+vRJxGA==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.616.0", + "@aws-sdk/middleware-host-header": "3.616.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.616.0", + "@aws-sdk/middleware-user-agent": "3.616.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.7", + "@smithy/fetch-http-handler": "^3.2.2", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.4", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.10", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/smithy-client": "^3.1.8", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.10", + "@smithy/util-defaults-mode-node": "^3.0.10", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.616.0.tgz", + "integrity": "sha512-YY1hpYS/G1uRGjQf88dL8VLHkP/IjGxKeXdhy+JnzMdCkAWl3V9j0fEALw40NZe0x79gr6R2KUOUH/IKYQfUmg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.616.0", + "@aws-sdk/credential-provider-node": "3.616.0", + "@aws-sdk/middleware-host-header": "3.616.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.616.0", + "@aws-sdk/middleware-user-agent": "3.616.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.7", + "@smithy/fetch-http-handler": "^3.2.2", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.4", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.10", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/smithy-client": "^3.1.8", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.10", + "@smithy/util-defaults-mode-node": "^3.0.10", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.616.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/client-sts": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.616.0.tgz", + "integrity": "sha512-FP7i7hS5FpReqnysQP1ukQF1OUWy8lkomaOnbu15H415YUrfCp947SIx6+BItjmx+esKxPkEjh/fbCVzw2D6hQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.616.0", + "@aws-sdk/core": "3.616.0", + "@aws-sdk/credential-provider-node": "3.616.0", + "@aws-sdk/middleware-host-header": "3.616.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.616.0", + "@aws-sdk/middleware-user-agent": "3.616.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.7", + "@smithy/fetch-http-handler": "^3.2.2", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.4", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.10", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/smithy-client": "^3.1.8", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.10", + "@smithy/util-defaults-mode-node": "^3.0.10", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/core": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.616.0.tgz", + "integrity": "sha512-O/urkh2kECs/IqZIVZxyeyHZ7OR2ZWhLNK7btsVQBQvJKrEspLrk/Fp20Qfg5JDerQfBN83ZbyRXLJOOucdZpw==", + "dependencies": { + "@smithy/core": "^2.2.7", + "@smithy/protocol-http": "^4.0.4", + "@smithy/signature-v4": "^4.0.0", + "@smithy/smithy-client": "^3.1.8", + "@smithy/types": "^3.3.0", + "fast-xml-parser": "4.2.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.609.0.tgz", + "integrity": "sha512-v69ZCWcec2iuV9vLVJMa6fAb5xwkzN4jYIT8yjo2c4Ia/j976Q+TPf35Pnz5My48Xr94EFcaBazrWedF+kwfuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.616.0.tgz", + "integrity": "sha512-1rgCkr7XvEMBl7qWCo5BKu3yAxJs71dRaZ55Xnjte/0ZHH6Oc93ZrHzyYy6UH6t0nZrH+FAuw7Yko2YtDDwDeg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.2", + "@smithy/node-http-handler": "^3.1.3", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/smithy-client": "^3.1.8", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.616.0.tgz", + "integrity": "sha512-5gQdMr9cca3xV7FF2SxpxWGH2t6+t4o+XBGiwsHm8muEjf4nUmw7Ij863x25Tjt2viPYV0UStczSb5Sihp7bkA==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.609.0", + "@aws-sdk/credential-provider-http": "3.616.0", + "@aws-sdk/credential-provider-process": "3.614.0", + "@aws-sdk/credential-provider-sso": "3.616.0", + "@aws-sdk/credential-provider-web-identity": "3.609.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.616.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.616.0.tgz", + "integrity": "sha512-Se+u6DAxjDPjKE3vX1X2uxjkWgGq69BTo0uTB0vDUiWwBVgh16s9BsBhSAlKEH1CCbbJHvOg4YdTrzjwzqyClg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.609.0", + "@aws-sdk/credential-provider-http": "3.616.0", + "@aws-sdk/credential-provider-ini": "3.616.0", + "@aws-sdk/credential-provider-process": "3.614.0", + "@aws-sdk/credential-provider-sso": "3.616.0", + "@aws-sdk/credential-provider-web-identity": "3.609.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.614.0.tgz", + "integrity": "sha512-Q0SI0sTRwi8iNODLs5+bbv8vgz8Qy2QdxbCHnPk/6Cx6LMf7i3dqmWquFbspqFRd8QiqxStrblwxrUYZi09tkA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.616.0.tgz", + "integrity": "sha512-3rsWs9GBi8Z8Gps5ROwqguxtw+J6OIg1vawZMLRNMqqZoBvbOToe9wEnpid8ylU+27+oG8uibJNlNuRyXApUjw==", + "dependencies": { + "@aws-sdk/client-sso": "3.616.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.609.0.tgz", + "integrity": "sha512-U+PG8NhlYYF45zbr1km3ROtBMYqyyj/oK8NRp++UHHeuavgrP+4wJ4wQnlEaKvJBjevfo3+dlIBcaeQ7NYejWg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.609.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.616.0.tgz", + "integrity": "sha512-mhNfHuGhCDZwYCABebaOvTgOM44UCZZRq2cBpgPZLVKP0ydAv5aFHXv01goexxXHqgHoEGx0uXWxlw0s2EpFDg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.0.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.616.0.tgz", + "integrity": "sha512-LQKAcrZRrR9EGez4fdCIVjdn0Ot2HMN12ChnoMGEU6oIxnQ2aSC7iASFFCV39IYfeMh7iSCPj7Wopqw8rAouzg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.0.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.616.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.616.0.tgz", + "integrity": "sha512-iMcAb4E+Z3vuEcrDsG6T2OBNiqWAquwahP9qepHqfmnmJqHr1mSHtXDYTGBNid31+621sUQmneUQ+fagpGAe4w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@smithy/protocol-http": "^4.0.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.614.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/util-endpoints": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.614.0.tgz", + "integrity": "sha512-wK2cdrXHH4oz4IomV/yrGkftU9A+ITB6nFL+rxxyO78is2ifHJpFdV4aqk4LSkXYPi6CXWNru/Dqc7yiKXgJPw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/core": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.8.tgz", + "integrity": "sha512-1Y0XX0Ucyg0LWTfTVLWpmvSRtFRniykUl3dQ0os1sTd03mKDudR6mVyX+2ak1phwPXx2aEWMAAdW52JNi0mc3A==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.11", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/smithy-client": "^3.1.9", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/credential-provider-imds": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.4.tgz", + "integrity": "sha512-NKyH01m97Xa5xf3pB2QOF3lnuE8RIK0hTVNU5zvZAwZU8uspYO4DHQVlK+Y5gwSrujTfHvbfd1D9UFJAc0iYKQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/fetch-http-handler": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.2.tgz", + "integrity": "sha512-3LaWlBZObyGrOOd7e5MlacnAKEwFBmAeiW/TOj2eR9475Vnq30uS2510+tnKbxrGjROfNdOhQqGo5j3sqLT6bA==", + "dependencies": { + "@smithy/protocol-http": "^4.0.4", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/middleware-content-length": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.4.tgz", + "integrity": "sha512-wySGje/KfhsnF8YSh9hP16pZcl3C+X6zRsvSfItQGvCyte92LliilU3SD0nR7kTlxnAJwxY8vE/k4Eoezj847Q==", + "dependencies": { + "@smithy/protocol-http": "^4.0.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/middleware-endpoint": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.5.tgz", + "integrity": "sha512-V4acqqrh5tDxUEGVTOgf2lYMZqPQsoGntCrjrJZEeBzEzDry2d2vcI1QCXhGltXPPY+BMc6eksZMguA9fIY8vA==", + "dependencies": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/middleware-retry": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.11.tgz", + "integrity": "sha512-/TIRWmhwMpv99JCGuMhJPnH7ggk/Lah7s/uNDyr7faF02BxNsyD/fz9Tw7pgCf9tYOKgjimm2Qml1Aq1pbkt6g==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.0.4", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.1.9", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.485.0", - "@aws-sdk/core": "3.485.0", - "@aws-sdk/credential-provider-node": "3.485.0", - "@aws-sdk/middleware-host-header": "3.485.0", - "@aws-sdk/middleware-logger": "3.485.0", - "@aws-sdk/middleware-recursion-detection": "3.485.0", - "@aws-sdk/middleware-signing": "3.485.0", - "@aws-sdk/middleware-user-agent": "3.485.0", - "@aws-sdk/region-config-resolver": "3.485.0", - "@aws-sdk/types": "3.485.0", - "@aws-sdk/util-endpoints": "3.485.0", - "@aws-sdk/util-user-agent-browser": "3.485.0", - "@aws-sdk/util-user-agent-node": "3.485.0", - "@smithy/config-resolver": "^2.0.23", - "@smithy/core": "^1.2.2", - "@smithy/fetch-http-handler": "^2.3.2", - "@smithy/hash-node": "^2.0.18", - "@smithy/invalid-dependency": "^2.0.16", - "@smithy/middleware-content-length": "^2.0.18", - "@smithy/middleware-endpoint": "^2.3.0", - "@smithy/middleware-retry": "^2.0.26", - "@smithy/middleware-serde": "^2.0.16", - "@smithy/middleware-stack": "^2.0.10", - "@smithy/node-config-provider": "^2.1.9", - "@smithy/node-http-handler": "^2.2.2", - "@smithy/protocol-http": "^3.0.12", - "@smithy/smithy-client": "^2.2.1", - "@smithy/types": "^2.8.0", - "@smithy/url-parser": "^2.0.16", - "@smithy/util-base64": "^2.0.1", - "@smithy/util-body-length-browser": "^2.0.1", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.24", - "@smithy/util-defaults-mode-node": "^2.0.32", - "@smithy/util-endpoints": "^1.0.8", - "@smithy/util-retry": "^2.0.9", - "@smithy/util-utf8": "^2.0.2", - "tslib": "^2.5.0" + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/node-http-handler": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.3.tgz", + "integrity": "sha512-UiKZm8KHb/JeOPzHZtRUfyaRDO1KPKPpsd7iplhiwVGOeVdkiVJ5bVe7+NhWREMOKomrDIDdSZyglvMothLg0Q==", + "dependencies": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.0.4", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/protocol-http": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.4.tgz", + "integrity": "sha512-fAA2O4EFyNRyYdFLVIv5xMMeRb+3fRKc/Rt2flh5k831vLvUmNFXcydeg7V3UeEhGURJI4c1asmGJBjvmF6j8Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "dependencies": { + "@smithy/types": "^3.3.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/signature-v4": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.0.0.tgz", + "integrity": "sha512-ervYjQ+ZvmNG51Ui77IOTPri7nOyo8Kembzt9uwwlmtXJPmFXvslOahbA1blvAVs7G0KlYMiOBog1rAt7RVXxg==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/smithy-client": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.9.tgz", + "integrity": "sha512-My2RaInZ4gSwJUPMaiLR/Nk82+c4LlvqpXA+n7lonGYgCZq23Tg+/xFhgmiejJ6XPElYJysTPyV90vKyp17+1g==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.0.4", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.11.tgz", + "integrity": "sha512-O3s9DGb3bmRvEKmT8RwvSWK4A9r6svfd+MnJB+UMi9ZcCkAnoRtliulOnGF0qCMkKF9mwk2tkopBBstalPY/vg==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.9", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.11.tgz", + "integrity": "sha512-qd4a9qtyOa/WY14aHHOkMafhh9z8D2QTwlcBoXMTPnEwtcY+xpe1JyFm9vya7VsB8hHsfn3XodEtwqREiu4ygQ==", + "dependencies": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.1.4", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.9", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "dependencies": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-stream": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.1.tgz", + "integrity": "sha512-EhRnVvl3AhoHAT2rGQ5o+oSDRM/BUSMPLZZdRJZLcNVUsFAjOs4vHaPdNQivTSzRcFxf5DA4gtO46WWU2zimaw==", + "dependencies": { + "@smithy/fetch-http-handler": "^3.2.2", + "@smithy/node-http-handler": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-personalize/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@aws-sdk/client-s3": { @@ -11824,15 +12895,15 @@ } }, "node_modules/husky": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", - "integrity": "sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==", + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.1.tgz", + "integrity": "sha512-fCqlqLXcBnXa/TJXmT93/A36tJsjdJkibQ1MuIiFyCCYUlpYpIaj2mv1w+3KR6Rzu1IC3slFTje5f6DUp2A2rg==", "dev": true, "bin": { - "husky": "lib/bin.js" + "husky": "bin.js" }, "engines": { - "node": ">=14" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/typicode" diff --git a/package.json b/package.json index 2dad49bba6..9a0d1faccb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.75.1", + "version": "1.76.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -55,7 +55,7 @@ }, "dependencies": { "@amplitude/ua-parser-js": "0.7.24", - "@aws-sdk/client-personalize": "^3.470.0", + "@aws-sdk/client-personalize": "^3.616.0", "@aws-sdk/client-s3": "^3.474.0", "@aws-sdk/credential-providers": "^3.391.0", "@aws-sdk/lib-storage": "^3.474.0", @@ -149,7 +149,7 @@ "eslint-plugin-unicorn": "^46.0.1", "glob": "^10.3.3", "http-terminator": "^3.2.0", - "husky": "^8.0.3", + "husky": "^9.1.1", "jest": "^29.5.0", "jest-sonar": "^0.2.16", "jest-when": "^3.5.2", diff --git a/src/cdk/v2/destinations/bloomreach_catalog/config.ts b/src/cdk/v2/destinations/bloomreach_catalog/config.ts new file mode 100644 index 0000000000..8b469c3cf9 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/config.ts @@ -0,0 +1,31 @@ +export const MAX_PAYLOAD_SIZE = 10000000; +export const MAX_ITEMS = 5000; + +// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-update-catalog-item +export const getCreateBulkCatalogItemEndpoint = ( + apiBaseUrl: string, + projectToken: string, + catalogId: string, +): string => `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items`; + +// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-partial-update-catalog-item +export const getUpdateBulkCatalogItemEndpoint = ( + apiBaseUrl: string, + projectToken: string, + catalogId: string, +): string => + `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items/partial-update`; + +// ref:- https://documentation.bloomreach.com/engagement/reference/bulk-delete-catalog-items +export const getDeleteBulkCatalogItemEndpoint = ( + apiBaseUrl: string, + projectToken: string, + catalogId: string, +): string => + `${apiBaseUrl}/data/v2/projects/${projectToken}/catalogs/${catalogId}/items/bulk-delete`; + +export const CatalogAction = { + INSERT: 'insert', + UPDATE: 'update', + DELETE: 'delete', +}; diff --git a/src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml b/src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml new file mode 100644 index 0000000000..55809350eb --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/rtWorkflow.yaml @@ -0,0 +1,42 @@ +bindings: + - name: EventType + path: ../../../../constants + - name: processRecordInputs + path: ./transformRecord + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - name: InstrumentationError + path: '@rudderstack/integrations-lib' + +steps: + - name: validateConfig + template: | + const config = ^[0].destination.Config + $.assertConfig(config.apiBaseUrl, "API Base URL is not present. Aborting"); + $.assertConfig(config.apiKey, "API Key is not present . Aborting"); + $.assertConfig(config.apiSecret, "API Secret is not present. Aborting"); + $.assertConfig(config.projectToken, "Project Token is not present. Aborting"); + $.assertConfig(config.catalogID, "Catalog Id is not present. Aborting"); + + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: processRecordEvents + template: | + $.processRecordInputs(^.{.message.type === $.EventType.RECORD}[], ^[0].destination) + + - name: failOtherEvents + template: | + const otherEvents = ^.{.message.type !== $.EventType.RECORD}[] + let failedEvents = otherEvents.map( + function(event) { + const error = new $.InstrumentationError("Event type " + event.message.type + " is not supported"); + $.handleRtTfSingleEventError(event, error, {}) + } + ) + failedEvents ?? [] + + - name: finalPayload + template: | + [...$.outputs.processRecordEvents, ...$.outputs.failOtherEvents] diff --git a/src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts b/src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts new file mode 100644 index 0000000000..68277448d0 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/transformRecord.ts @@ -0,0 +1,93 @@ +import { InstrumentationError } from '@rudderstack/integrations-lib'; +import { CatalogAction } from './config'; +import { batchResponseBuilder } from './utils'; + +import { handleRtTfSingleEventError, isEmptyObject } from '../../../../v0/util'; + +const prepareCatalogInsertOrUpdatePayload = (fields: any): any => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { item_id, ...properties } = fields; + return { item_id, properties }; +}; + +const processEvent = (event: any) => { + const { message } = event; + const { fields, action } = message; + const response = { + action, + payload: null, + }; + if (isEmptyObject(fields)) { + throw new InstrumentationError('`fields` cannot be empty'); + } + if (!fields.item_id) { + throw new InstrumentationError('`item_id` cannot be empty'); + } + if (action === CatalogAction.INSERT || action === CatalogAction.UPDATE) { + response.payload = prepareCatalogInsertOrUpdatePayload(fields); + } else if (action === CatalogAction.DELETE) { + response.payload = fields.item_id; + } else { + throw new InstrumentationError( + `Invalid action type ${action}. You can only add, update or remove items from the catalog`, + ); + } + return response; +}; + +const getEventChunks = ( + input: any, + insertItemRespList: any[], + updateItemRespList: any[], + deleteItemRespList: any[], +) => { + switch (input.response.action) { + case CatalogAction.INSERT: + insertItemRespList.push({ payload: input.response.payload, metadata: input.metadata }); + break; + case CatalogAction.UPDATE: + updateItemRespList.push({ payload: input.response.payload, metadata: input.metadata }); + break; + case CatalogAction.DELETE: + deleteItemRespList.push({ payload: input.response.payload, metadata: input.metadata }); + break; + default: + throw new InstrumentationError(`Invalid action type ${input.response.action}`); + } +}; + +export const processRecordInputs = (inputs: any[], destination: any) => { + const insertItemRespList: any[] = []; + const updateItemRespList: any[] = []; + const deleteItemRespList: any[] = []; + const batchErrorRespList: any[] = []; + + if (!inputs || inputs.length === 0) { + return []; + } + + inputs.forEach((input) => { + try { + getEventChunks( + { + response: processEvent(input), + metadata: input.metadata, + }, + insertItemRespList, + updateItemRespList, + deleteItemRespList, + ); + } catch (error) { + const errRespEvent = handleRtTfSingleEventError(input, error, {}); + batchErrorRespList.push(errRespEvent); + } + }); + + const batchSuccessfulRespList = batchResponseBuilder( + insertItemRespList, + updateItemRespList, + deleteItemRespList, + destination, + ); + return [...batchSuccessfulRespList, ...batchErrorRespList]; +}; diff --git a/src/cdk/v2/destinations/bloomreach_catalog/utils.ts b/src/cdk/v2/destinations/bloomreach_catalog/utils.ts new file mode 100644 index 0000000000..0e74ce9379 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach_catalog/utils.ts @@ -0,0 +1,147 @@ +import { BatchUtils } from '@rudderstack/workflow-engine'; +import { base64Convertor } from '@rudderstack/integrations-lib'; +import { + getCreateBulkCatalogItemEndpoint, + getDeleteBulkCatalogItemEndpoint, + getUpdateBulkCatalogItemEndpoint, + MAX_ITEMS, + MAX_PAYLOAD_SIZE, +} from './config'; + +const buildBatchedRequest = ( + payload: string, + method: string, + endpoint: string, + headers: any, + metadata: any, + destination: any, +) => ({ + batchedRequest: { + body: { + JSON: {}, + JSON_ARRAY: { batch: payload }, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method, + endpoint, + headers, + params: {}, + files: {}, + }, + metadata, + batched: true, + statusCode: 200, + destination, +}); + +const getHeaders = (destination: any) => ({ + 'Content-Type': 'application/json', + Authorization: `Basic ${base64Convertor(`${destination.Config.apiKey}:${destination.Config.apiSecret}`)}`, +}); + +// returns merged metadata for a batch +const getMergedMetadata = (batch: any[]) => batch.map((input) => input.metadata); + +// returns merged payload for a batch +const getMergedEvents = (batch: any[]) => batch.map((input) => input.payload); + +// builds final batched response for insert action records +const insertItemBatchResponseBuilder = (insertItemRespList: any[], destination: any) => { + const insertItemBatchedResponse: any[] = []; + + const method = 'PUT'; + const endpoint = getCreateBulkCatalogItemEndpoint( + destination.Config.apiBaseUrl, + destination.Config.projectToken, + destination.Config.catalogID, + ); + const headers = getHeaders(destination); + + const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(insertItemRespList, { + maxSizeInBytes: MAX_PAYLOAD_SIZE, + maxItems: MAX_ITEMS, + }); + batchesOfEvents.items.forEach((batch: any) => { + const mergedPayload = JSON.stringify(getMergedEvents(batch)); + const mergedMetadata = getMergedMetadata(batch); + insertItemBatchedResponse.push( + buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination), + ); + }); + return insertItemBatchedResponse; +}; + +// builds final batched response for update action records +const updateItemBatchResponseBuilder = (updateItemRespList: any[], destination: any) => { + const updateItemBatchedResponse: any[] = []; + + const method = 'POST'; + const endpoint = getUpdateBulkCatalogItemEndpoint( + destination.Config.apiBaseUrl, + destination.Config.projectToken, + destination.Config.catalogID, + ); + const headers = getHeaders(destination); + + const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(updateItemRespList, { + maxSizeInBytes: MAX_PAYLOAD_SIZE, + maxItems: MAX_ITEMS, + }); + batchesOfEvents.items.forEach((batch: any) => { + const mergedPayload = JSON.stringify(getMergedEvents(batch)); + const mergedMetadata = getMergedMetadata(batch); + updateItemBatchedResponse.push( + buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination), + ); + }); + return updateItemBatchedResponse; +}; + +// builds final batched response for delete action records +const deleteItemBatchResponseBuilder = (deleteItemRespList: any[], destination: any) => { + const deleteItemBatchedResponse: any[] = []; + + const method = 'DELETE'; + const endpoint = getDeleteBulkCatalogItemEndpoint( + destination.Config.apiBaseUrl, + destination.Config.projectToken, + destination.Config.catalogID, + ); + const headers = getHeaders(destination); + + const batchesOfEvents = BatchUtils.chunkArrayBySizeAndLength(deleteItemRespList, { + maxSizeInBytes: MAX_PAYLOAD_SIZE, + maxItems: MAX_ITEMS, + }); + batchesOfEvents.items.forEach((batch: any) => { + const mergedPayload = JSON.stringify(getMergedEvents(batch)); + const mergedMetadata = getMergedMetadata(batch); + deleteItemBatchedResponse.push( + buildBatchedRequest(mergedPayload, method, endpoint, headers, mergedMetadata, destination), + ); + }); + return deleteItemBatchedResponse; +}; + +// returns final batched response +export const batchResponseBuilder = ( + insertItemRespList: any, + updateItemRespList: any, + deleteItemRespList: any, + destination: any, +) => { + const response: any[] = []; + if (insertItemRespList.length > 0) { + response.push(...insertItemBatchResponseBuilder(insertItemRespList, destination)); + } + if (updateItemRespList.length > 0) { + response.push(...updateItemBatchResponseBuilder(updateItemRespList, destination)); + } + if (deleteItemRespList.length > 0) { + response.push(...deleteItemBatchResponseBuilder(deleteItemRespList, destination)); + } + return response; +}; diff --git a/src/cdk/v2/destinations/smartly/config.js b/src/cdk/v2/destinations/smartly/config.js new file mode 100644 index 0000000000..5083fde5fe --- /dev/null +++ b/src/cdk/v2/destinations/smartly/config.js @@ -0,0 +1,21 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +const ConfigCategories = { + TRACK: { + type: 'track', + name: 'trackMapping', + }, +}; + +const mappingConfig = getMappingConfig(ConfigCategories, __dirname); +const singleEventEndpoint = 'https://s2s.smartly.io/events'; +const batchEndpoint = 'https://s2s.smartly.io/events/batch'; + +module.exports = { + ConfigCategories, + mappingConfig, + singleEventEndpoint, + batchEndpoint, + TRACK_CONFIG: mappingConfig[ConfigCategories.TRACK.name], + MAX_BATCH_SIZE: 1000, +}; diff --git a/src/cdk/v2/destinations/smartly/data/trackMapping.json b/src/cdk/v2/destinations/smartly/data/trackMapping.json new file mode 100644 index 0000000000..55ba437f12 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/data/trackMapping.json @@ -0,0 +1,76 @@ +[ + { + "destKey": "value", + "sourceKeys": [ + "properties.total", + "properties.value", + "properties.revenue", + { + "operation": "multiplication", + "args": [ + { + "sourceKeys": "properties.price" + }, + { + "sourceKeys": "properties.quantity", + "default": 1 + } + ] + } + ], + "metadata": { + "type": "toNumber" + }, + "required": false + }, + { + "sourceKeys": ["properties.conversions", "properties.products.length"], + "required": false, + "metadata": { + "defaultValue": "1" + }, + "destKey": "conversions" + }, + { + "sourceKeys": ["properties.adUnitId", "properties.ad_unit_id"], + "required": true, + "destKey": "ad_unit_id", + "metadata": { + "type": "toString" + } + }, + { + "sourceKeys": ["properties.platform"], + "required": true, + "destKey": "platform" + }, + { + "sourceKeys": ["properties.adInteractionTime", "properties.ad_interaction_time"], + "required": true, + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "ad_interaction_time" + }, + { + "sourceKeys": ["properties.installTime"], + "required": false, + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "installTime" + }, + { + "sourceKeys": ["originalTimestamp", "timestamp"], + "required": false, + "metadata": { + "type": "secondTimestamp" + }, + "destKey": "event_time" + }, + { + "sourceKeys": ["properties.currency"], + "required": false, + "destKey": "value_currency" + } +] diff --git a/src/cdk/v2/destinations/smartly/procWorkflow.yaml b/src/cdk/v2/destinations/smartly/procWorkflow.yaml new file mode 100644 index 0000000000..b69df0dd09 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/procWorkflow.yaml @@ -0,0 +1,31 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: constructPayload + path: ../../../../v0/util + - path: ./config + - path: ./utils +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.apiToken, "API Token is not present. Aborting"); + - name: preparePayload + template: | + const payload = $.removeUndefinedAndNullValues($.constructPayload(.message, $.TRACK_CONFIG)); + $.verifyAdInteractionTime(payload.ad_interaction_time); + $.context.payloadList = $.getPayloads(.message.event, .destination.Config, payload) + - name: buildResponse + template: | + const response = $.buildResponseList($.context.payloadList) + response diff --git a/src/cdk/v2/destinations/smartly/rtWorkflow.yaml b/src/cdk/v2/destinations/smartly/rtWorkflow.yaml new file mode 100644 index 0000000000..4d3afdb6d0 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/rtWorkflow.yaml @@ -0,0 +1,35 @@ +bindings: + - path: ./config + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "output": .body.JSON, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + $.batchResponseBuilder($.outputs.successfulEvents); + + - name: finalPayload + template: | + [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] diff --git a/src/cdk/v2/destinations/smartly/utils.js b/src/cdk/v2/destinations/smartly/utils.js new file mode 100644 index 0000000000..7d53ed0d27 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/utils.js @@ -0,0 +1,108 @@ +const { BatchUtils } = require('@rudderstack/workflow-engine'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const moment = require('moment'); +const config = require('./config'); +const { + getHashFromArrayWithDuplicate, + defaultRequestConfig, + isDefinedAndNotNull, +} = require('../../../../v0/util'); + +// docs reference : https://support.smartly.io/hc/en-us/articles/4406049685788-S2S-integration-API-description#01H8HBXZF6WSKSYBW1C6NY8A88 + +/** + * This function generates an array of payload objects, each with the event property set + * to different values associated with the given event name according to eventsMapping + * @param {*} event + * @param {*} eventsMapping + * @param {*} payload + * @returns + */ +const getPayloads = (event, Config, payload) => { + if (!isDefinedAndNotNull(event) || typeof event !== 'string') { + throw new InstrumentationError('Event is not defined or is not String'); + } + const eventsMap = getHashFromArrayWithDuplicate(Config.eventsMapping); + // eventsMap = hashmap {"prop1":["val1","val2"],"prop2":["val2"]} + const eventList = Array.isArray(eventsMap[event.toLowerCase()]) + ? eventsMap[event.toLowerCase()] + : Array.from(eventsMap[event.toLowerCase()] || [event]); + + const payloadLists = eventList.map((ev) => ({ ...payload, event_name: ev })); + return payloadLists; +}; + +// ad_interaction_time must be within one year in the future and three years in the past from the current date +// Example : "1735680000" +const verifyAdInteractionTime = (adInteractionTime) => { + if (isDefinedAndNotNull(adInteractionTime)) { + const now = moment(); + const threeYearAgo = now.clone().subtract(3, 'year'); + const oneYearFromNow = now.clone().add(1, 'year'); + const inputMoment = moment(adInteractionTime * 1000); // Convert to milliseconds + if (!inputMoment.isAfter(threeYearAgo) || !inputMoment.isBefore(oneYearFromNow)) { + throw new InstrumentationError( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + } + } +}; + +const buildResponseList = (payloadList) => + payloadList.map((payload) => { + const response = defaultRequestConfig(); + response.body.JSON = payload; + response.endpoint = config.singleEventEndpoint; + response.method = 'POST'; + return response; + }); + +const batchBuilder = (batch, destination) => ({ + batchedRequest: { + body: { + JSON: { events: batch.map((event) => event.output) }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: config.batchEndpoint, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${destination.Config.apiToken}`, + }, + params: {}, + files: {}, + }, + metadata: batch + .map((event) => event.metadata) + .filter((metadata, index, self) => self.findIndex((m) => m.jobId === metadata.jobId) === index), // handling jobId duplication for multiplexed events + batched: true, + statusCode: 200, + destination: batch[0].destination, +}); + +/** + * This fucntions make chunk of successful events based on MAX_BATCH_SIZE + * and then build the response for each chunk to be returned as object of an array + * @param {*} events + * @returns + */ +const batchResponseBuilder = (events) => { + if (events.length === 0) { + return []; + } + const { destination } = events[0]; + const batches = BatchUtils.chunkArrayBySizeAndLength(events, { maxItems: config.MAX_BATCH_SIZE }); + + const response = []; + batches.items.forEach((batch) => { + const batchedResponse = batchBuilder(batch, destination); + response.push(batchedResponse); + }); + return response; +}; + +module.exports = { batchResponseBuilder, getPayloads, buildResponseList, verifyAdInteractionTime }; diff --git a/src/cdk/v2/destinations/smartly/utils.test.js b/src/cdk/v2/destinations/smartly/utils.test.js new file mode 100644 index 0000000000..0ad73f5369 --- /dev/null +++ b/src/cdk/v2/destinations/smartly/utils.test.js @@ -0,0 +1,59 @@ +const moment = require('moment'); +const { verifyAdInteractionTime } = require('./utils'); + +describe('verifyAdInteractionTime', () => { + it('should pass when adInteractionTime is 2 years in the past (UNIX timestamp)', () => { + // 2 years ago from now + const adInteractionTime = moment().subtract(2, 'years').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).not.toThrow(); + }); + + it('should pass when adInteractionTime is 10 months in the future (UNIX timestamp)', () => { + // 10 months in the future from now + const adInteractionTime = moment().add(10, 'months').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).not.toThrow(); + }); + + it('should fail when adInteractionTime is 4 years in the past (UNIX timestamp)', () => { + // 4 years ago from now + const adInteractionTime = moment().subtract(4, 'years').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should fail when adInteractionTime is 2 years in the future (UNIX timestamp)', () => { + // 2 years in the future from now + const adInteractionTime = moment().add(2, 'years').unix(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should pass when adInteractionTime is exactly 1 year in the future (UTC date string)', () => { + // Exactly 1 year in the future from now + const adInteractionTime = moment.utc().add(1, 'year').toISOString(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow(); + }); + + it('should fail when adInteractionTime is 4 years in the past (UTC date string)', () => { + // 4 years ago from now + const adInteractionTime = moment.utc().subtract(4, 'years').toISOString(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should fail when adInteractionTime is 2 years in the future (UTC date string)', () => { + // 2 years in the future from now + const adInteractionTime = moment.utc().add(2, 'years').toISOString(); + expect(() => verifyAdInteractionTime(adInteractionTime)).toThrow( + 'ad_interaction_time must be within one year in the future and three years in the past.', + ); + }); + + it('should not throw an error if adInteractionTime is null or undefined', () => { + expect(() => verifyAdInteractionTime(null)).not.toThrow(); + expect(() => verifyAdInteractionTime(undefined)).not.toThrow(); + }); +}); diff --git a/src/features.json b/src/features.json index 94e36a2416..a0261683a8 100644 --- a/src/features.json +++ b/src/features.json @@ -76,7 +76,9 @@ "WUNDERKIND": true, "CLICKSEND": true, "ZOHO": true, - "CORDIAL": true + "CORDIAL": true, + "BLOOMREACH_CATALOG": true, + "SMARTLY": true }, "regulations": [ "BRAZE", diff --git a/src/routerUtils.js b/src/routerUtils.js index ff9dd4b6f8..081070d78a 100644 --- a/src/routerUtils.js +++ b/src/routerUtils.js @@ -22,6 +22,7 @@ const userTransformHandler = () => { async function sendToDestination(destination, payload) { let parsedResponse; logger.info('Request recieved for destination', destination); + const resp = await proxyRequest(payload); if (resp.success) { diff --git a/src/services/comparator.ts b/src/services/comparator.ts index 0e28339797..1eb67cd597 100644 --- a/src/services/comparator.ts +++ b/src/services/comparator.ts @@ -33,6 +33,7 @@ export class ComparatorService implements DestinationService { public init(): void { this.primaryService.init(); + this.secondaryService.init(); } diff --git a/src/v0/destinations/attentive_tag/transform.js b/src/v0/destinations/attentive_tag/transform.js index fa05eb6c21..5522c078b0 100644 --- a/src/v0/destinations/attentive_tag/transform.js +++ b/src/v0/destinations/attentive_tag/transform.js @@ -16,7 +16,7 @@ const { const { getDestinationItemProperties, getExternalIdentifiersMapping, - getPropertiesKeyValidation, + arePropertiesValid, validateTimestamp, } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -137,9 +137,9 @@ const trackResponseBuilder = (message, { Config }) => { payload = constructPayload(message, mappingConfig[ConfigCategory.TRACK.name]); endpoint = ConfigCategory.TRACK.endpoint; payload.type = get(message, 'event'); - if (!getPropertiesKeyValidation(payload)) { + if (!arePropertiesValid(payload.properties)) { throw new InstrumentationError( - '[Attentive Tag]:The event name contains characters which is not allowed', + '[Attentive Tag]:The properties contains characters which is not allowed', ); } } diff --git a/src/v0/destinations/attentive_tag/util.js b/src/v0/destinations/attentive_tag/util.js index abecf76542..8e73f572fc 100644 --- a/src/v0/destinations/attentive_tag/util.js +++ b/src/v0/destinations/attentive_tag/util.js @@ -14,18 +14,23 @@ const { mappingConfig, ConfigCategory } = require('./config'); * The keys should not contain any of the values inside the validationArray. * STEP 1: Storing keys in the array. * Checking for the non-valid characters inside the keys of properties. + * ref: https://docs.attentivemobile.com/openapi/reference/tag/Custom-Events/ * @param {*} payload * @returns */ -const getPropertiesKeyValidation = (payload) => { +const arePropertiesValid = (properties) => { + if (!isDefinedAndNotNullAndNotEmpty(properties)) { + return true; + } + if (typeof properties !== 'object') { + return false; + } const validationArray = [`'`, `"`, `{`, `}`, `[`, `]`, ',', `,`]; - if (payload.properties) { - const keys = Object.keys(payload.properties); - for (const key of keys) { - for (const validationChar of validationArray) { - if (key.includes(validationChar)) { - return false; - } + const keys = Object.keys(properties); + for (const key of keys) { + for (const validationChar of validationArray) { + if (key.includes(validationChar)) { + return false; } } } @@ -136,6 +141,6 @@ const getDestinationItemProperties = (message, isItemsRequired) => { module.exports = { getDestinationItemProperties, getExternalIdentifiersMapping, - getPropertiesKeyValidation, + arePropertiesValid, validateTimestamp, }; diff --git a/src/v0/destinations/attentive_tag/util.test.js b/src/v0/destinations/attentive_tag/util.test.js new file mode 100644 index 0000000000..59fdc9f361 --- /dev/null +++ b/src/v0/destinations/attentive_tag/util.test.js @@ -0,0 +1,51 @@ +const { arePropertiesValid } = require('./util'); + +describe('arePropertiesValid', () => { + // returns true for valid properties object with no special characters in keys + it('should return true when properties object has no special characters in keys', () => { + const properties = { key1: 'value1', key2: 'value2' }; + const result = arePropertiesValid(properties); + expect(result).toBe(true); + }); + + // returns true for null properties input + it('should return true when properties input is null', () => { + const properties = null; + const result = arePropertiesValid(properties); + expect(result).toBe(true); + }); + + // returns false for properties object with keys containing special characters + it('should return false for properties object with keys containing special characters', () => { + const properties = { + key1: 'value1', + 'key,2': 'value2', + key3: 'value3', + }; + expect(arePropertiesValid(properties)).toBe(false); + }); + + // returns true for empty properties object + it('should return true for empty properties object', () => { + const properties = {}; + expect(arePropertiesValid(properties)).toBe(true); + }); + + // returns true for undefined properties input + it('should return true for undefined properties input', () => { + const result = arePropertiesValid(undefined); + expect(result).toBe(true); + }); + + // returns true for empty string properties input + it('should return true for empty string properties input', () => { + const result = arePropertiesValid(''); + expect(result).toBe(true); + }); + + // returns false for empty string properties input + it('should return false for non object properties input', () => { + const result = arePropertiesValid('1234'); + expect(result).toBe(false); + }); +}); diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 8199aae70b..5ec48d29f1 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -1,6 +1,12 @@ const _ = require('lodash'); const { handleHttpRequest } = require('../../../adapters/network'); -const { BrazeDedupUtility, addAppId, getPurchaseObjs, setAliasObject } = require('./util'); +const { + BrazeDedupUtility, + addAppId, + getPurchaseObjs, + setAliasObject, + handleReservedProperties, +} = require('./util'); const { processBatch } = require('./util'); const { removeUndefinedAndNullValues, @@ -1670,3 +1676,64 @@ describe('setAliasObject function', () => { }); }); }); + +describe('handleReservedProperties', () => { + // Removes 'time' and 'event_name' keys from the input object + it('should remove "time" and "event_name" keys when they are present in the input object', () => { + const props = { time: '2023-10-01T00:00:00Z', event_name: 'test_event', other_key: 'value' }; + const result = handleReservedProperties(props); + expect(result).toEqual({ other_key: 'value' }); + }); + + // Input object is empty + it('should return an empty object when the input object is empty', () => { + const props = {}; + const result = handleReservedProperties(props); + expect(result).toEqual({}); + }); + + // Works correctly with an object that has no reserved keys + it('should remove reserved keys when present in the input object', () => { + const props = { time_stamp: '2023-10-01T00:00:00Z', event: 'test_event', other_key: 'value' }; + const result = handleReservedProperties(props); + expect(result).toEqual({ + time_stamp: '2023-10-01T00:00:00Z', + event: 'test_event', + other_key: 'value', + }); + }); + + // Input object is null or undefined + it('should return an empty object when input object is null', () => { + const props = null; + const result = handleReservedProperties(props); + expect(result).toEqual({}); + }); + + // Handles non-object inputs gracefully + it('should return an empty object when a non-object input is provided', () => { + const props = 'not an object'; + try { + handleReservedProperties(props); + } catch (e) { + expect(e.message).toBe('Invalid event properties'); + } + }); + + // Input object has only reserved keys + it('should remove "time" and "event_name" keys when they are present in the input object', () => { + const props = { time: '2023-10-01T00:00:00Z', event_name: 'test_event', other_key: 'value' }; + const result = handleReservedProperties(props); + expect(result).toEqual({ other_key: 'value' }); + }); + + // Works with objects having special characters in keys + it('should not remove special characters keys when they are present in the input object', () => { + const props = { 'special!@#$%^&*()_+-={}[]|\\;:\'",.<>?/`~': 'value', other_key: 'value' }; + const result = handleReservedProperties(props); + expect(result).toEqual({ + other_key: 'value', + 'special!@#$%^&*()_+-={}[]|\\;:\'",.<>?/`~': 'value', + }); + }); +}); diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js index 155f32c145..09fb0205c5 100644 --- a/src/v0/destinations/braze/transform.js +++ b/src/v0/destinations/braze/transform.js @@ -15,6 +15,7 @@ const { setAliasObject, collectStatsForAliasFailure, collectStatsForAliasMissConfigurations, + handleReservedProperties, } = require('./util'); const tags = require('../../util/tags'); const { EventType, MappedToDestinationKey } = require('../../../constants'); @@ -29,6 +30,7 @@ const { simpleProcessRouterDest, isNewStatusCodesAccepted, getDestinationExternalID, + getIntegrationsObj, } = require('../../util'); const { ConfigCategory, @@ -280,17 +282,6 @@ function processTrackWithUserAttributes( throw new InstrumentationError('No attributes found to update the user profile'); } -function handleReservedProperties(props) { - // remove reserved keys from custom event properties - // https://www.appboy.com/documentation/Platform_Wide/#reserved-keys - const reserved = ['time', 'product_id', 'quantity', 'event_name', 'price', 'currency']; - - reserved.forEach((element) => { - delete props[element]; - }); - return props; -} - function addMandatoryEventProperties(payload, message) { payload.name = message.event; payload.time = message.timestamp; @@ -332,13 +323,6 @@ function processTrackEvent(messageType, message, destination, mappingJson, proce eventName.toLowerCase() === 'order completed' ) { const purchaseObjs = getPurchaseObjs(message, destination.Config); - - // del used properties - delete properties.products; - delete properties.currency; - - const payload = { properties }; - setExternalIdOrAliasObject(payload, message); return buildResponse( message, destination, @@ -515,10 +499,14 @@ async function process(event, processParams = { userStore: new Map() }, reqMetad if (mappedToDestination) { adduserIdFromExternalId(message); } + + const integrationsObj = getIntegrationsObj(message, 'BRAZE'); + const isAliasPresent = isDefinedAndNotNull(integrationsObj?.alias); + const brazeExternalID = getDestinationExternalID(message, 'brazeExternalId') || message.userId; - if (message.anonymousId && brazeExternalID) { - await processIdentify(event); + if ((message.anonymousId || isAliasPresent) && brazeExternalID) { + await processIdentify({ message, destination }); } else { collectStatsForAliasMissConfigurations(destination.ID); } diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 7f4afaf6c1..e5df75b562 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -714,6 +714,16 @@ const collectStatsForAliasMissConfigurations = (destinationId) => { stats.increment('braze_alias_missconfigured_count', { destination_id: destinationId }); }; +function handleReservedProperties(props) { + if (typeof props !== 'object') { + throw new InstrumentationError('Invalid event properties'); + } + // remove reserved keys from custom event properties + const reserved = ['time', 'event_name']; + + return _.omit(props, reserved); +} + module.exports = { BrazeDedupUtility, CustomAttributeOperationUtil, @@ -728,4 +738,5 @@ module.exports = { addMandatoryPurchaseProperties, collectStatsForAliasFailure, collectStatsForAliasMissConfigurations, + handleReservedProperties, }; diff --git a/src/v0/destinations/klaviyo/batchUtil.js b/src/v0/destinations/klaviyo/batchUtil.js index 3e99e03deb..0351bd2e2f 100644 --- a/src/v0/destinations/klaviyo/batchUtil.js +++ b/src/v0/destinations/klaviyo/batchUtil.js @@ -25,16 +25,17 @@ const groupSubscribeResponsesUsingListIdV2 = (subscribeResponseList) => { /** * This function takes susbscription as input and batches them into a single request body * @param {subscription} - * subscription= {listId, subscriptionProfileList} + * subscription= {listId, subscriptionProfileList, operation} + * subscription.operation could be either subscribe or unsubscribe */ const generateBatchedSubscriptionRequest = (subscription, destination) => { const subscriptionPayloadResponse = defaultRequestConfig(); // fetching listId from first event as listId is same for all the events const profiles = []; // list of profiles to be subscribed - const { listId, subscriptionProfileList } = subscription; + const { listId, subscriptionProfileList, operation } = subscription; subscriptionProfileList.forEach((profileList) => profiles.push(...profileList)); - subscriptionPayloadResponse.body.JSON = getSubscriptionPayload(listId, profiles); - subscriptionPayloadResponse.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + subscriptionPayloadResponse.body.JSON = getSubscriptionPayload(listId, profiles, operation); + subscriptionPayloadResponse.endpoint = `${BASE_ENDPOINT}/api/${operation === 'subscribe' ? 'profile-subscription-bulk-create-jobs' : 'profile-subscription-bulk-delete-jobs'}`; subscriptionPayloadResponse.headers = { Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, 'Content-Type': JSON_MIME_TYPE, @@ -90,12 +91,12 @@ const populateArrWithRespectiveProfileData = ( * ex: * profileSubscriptionAndMetadataArr = [ { - subscription: { subscriptionProfileList, listId1 }, + subscription: { subscriptionProfileList, listId1, operation }, metadataList1, profiles: [respectiveProfiles for above metadata] }, { - subscription: { subscriptionProfile List With No Profiles, listId2 }, + subscription: { subscriptionProfile List With No Profiles, listId2, operation }, metadataList2, }, { @@ -107,10 +108,7 @@ const populateArrWithRespectiveProfileData = ( * @param {*} destination * @returns */ -const buildRequestsForProfileSubscriptionAndMetadataArr = ( - profileSubscriptionAndMetadataArr, - destination, -) => { +const buildProfileAndSubscriptionRequests = (profileSubscriptionAndMetadataArr, destination) => { const finalResponseList = []; profileSubscriptionAndMetadataArr.forEach((profileSubscriptionData) => { const batchedRequest = []; @@ -118,7 +116,7 @@ const buildRequestsForProfileSubscriptionAndMetadataArr = ( if (profileSubscriptionData.profiles?.length > 0) { batchedRequest.push(...getProfileRequests(profileSubscriptionData.profiles, destination)); } - + // following condition ensures if no subscriptions are present we won't build subscription payload if (profileSubscriptionData.subscription?.subscriptionProfileList?.length > 0) { batchedRequest.push( generateBatchedSubscriptionRequest(profileSubscriptionData.subscription, destination), @@ -132,46 +130,88 @@ const buildRequestsForProfileSubscriptionAndMetadataArr = ( return finalResponseList; }; -const batchRequestV2 = (subscribeRespList, profileRespList, destination) => { - const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeRespList); - let profileSubscriptionAndMetadataArr = []; - const metaDataIndexMap = new Map(); +/** + * This function updates the profileSubscriptionAndMetadataArr array with the subscription requests + * @param {*} subscribeStatusList + * @param {*} profilesList + * @param {*} operation + * @param {*} profileSubscriptionAndMetadataArr + * @param {*} metaDataIndexMap + */ +const updateArrWithSubscriptions = ( + subscribeStatusList, + profilesList, + operation, + profileSubscriptionAndMetadataArr, + metaDataIndexMap, +) => { + const subscribeEventGroups = groupSubscribeResponsesUsingListIdV2(subscribeStatusList); + Object.keys(subscribeEventGroups).forEach((listId) => { // eventChunks = [[e1,e2,e3,..batchSize],[e1,e2,e3,..batchSize]..] const eventChunks = lodash.chunk(subscribeEventGroups[listId], MAX_BATCH_SIZE); - eventChunks.forEach((chunk, index) => { + eventChunks.forEach((chunk) => { // get subscriptionProfiles for the chunk const subscriptionProfileList = chunk.map((event) => event.payload?.profile); // get metadata for this chunk const metadataList = chunk.map((event) => event.metadata); // get list of jobIds from the above metdadata const jobIdList = metadataList.map((metadata) => metadata.jobId); + // using length as index + const index = profileSubscriptionAndMetadataArr.length; // push the jobId: index to metadataIndex mapping which let us know the metadata respective payload index position in batched request jobIdList.forEach((jobId) => { metaDataIndexMap.set(jobId, index); }); profileSubscriptionAndMetadataArr.push({ - subscription: { subscriptionProfileList, listId }, + subscription: { subscriptionProfileList, listId, operation }, metadataList, profiles: [], }); }); }); - profileSubscriptionAndMetadataArr = populateArrWithRespectiveProfileData( +}; + +/** + * This function performs batching for the subscription and unsubscription requests and attaches respective profile request as well if present + * @param {*} subscribeList + * @param {*} unsubscribeList + * @param {*} profilesList + * @param {*} destination + * @returns + */ +const batchRequestV2 = (subscribeList, unsubscribeList, profilesList, destination) => { + const profileSubscriptionAndMetadataArr = []; + const metaDataIndexMap = new Map(); + updateArrWithSubscriptions( + subscribeList, + profilesList, + 'subscribe', profileSubscriptionAndMetadataArr, metaDataIndexMap, - profileRespList, + ); + updateArrWithSubscriptions( + unsubscribeList, + profilesList, + 'unsubscribe', + profileSubscriptionAndMetadataArr, + metaDataIndexMap, + ); + const subscriptionsAndProfileArr = populateArrWithRespectiveProfileData( + profileSubscriptionAndMetadataArr, + metaDataIndexMap, + profilesList, ); /* Till this point I have a profileSubscriptionAndMetadataArr containing the the events in one object for which batching has to happen in following format [ { - subscription: { subscriptionProfileList, listId1 }, + subscription: { subscriptionProfileList, listId1, operation }, metadataList1, profiles: [respectiveProfiles for above metadata] }, { - subscription: { subscriptionProfile List With No Profiles, listId2 }, + subscription: { subscriptionProfile List With No Profiles, listId2, operation }, metadataList2, }, { @@ -180,14 +220,11 @@ const batchRequestV2 = (subscribeRespList, profileRespList, destination) => { } ] */ - return buildRequestsForProfileSubscriptionAndMetadataArr( - profileSubscriptionAndMetadataArr, - destination, - ); + return buildProfileAndSubscriptionRequests(subscriptionsAndProfileArr, destination); /* for identify calls with batching batched with identify with no batching - we will sonctruct O/P as: + we will construct O/P as: [ - [2 calls for identifywith batching], + [2 calls for identify with batching], [1 call identify calls with batching] ] */ diff --git a/src/v0/destinations/klaviyo/batchUtil.test.js b/src/v0/destinations/klaviyo/batchUtil.test.js index af1afd8670..9c04a402ca 100644 --- a/src/v0/destinations/klaviyo/batchUtil.test.js +++ b/src/v0/destinations/klaviyo/batchUtil.test.js @@ -1,3 +1,4 @@ +const { OperatorType } = require('@rudderstack/json-template-engine'); const { groupSubscribeResponsesUsingListIdV2, populateArrWithRespectiveProfileData, @@ -94,6 +95,7 @@ describe('generateBatchedSubscriptionRequest', () => { const subscription = { listId: 'test-list-id', subscriptionProfileList: [[{ id: 'profile1' }, { id: 'profile2' }], [{ id: 'profile3' }]], + operation: 'subscribe', }; const destination = { Config: { @@ -144,6 +146,7 @@ describe('generateBatchedSubscriptionRequest', () => { const subscription = { listId: 'test-list-id', subscriptionProfileList: [], + operation: 'subscribe', }; const destination = { Config: { diff --git a/src/v0/destinations/klaviyo/config.js b/src/v0/destinations/klaviyo/config.js index 54216852f7..9f907330df 100644 --- a/src/v0/destinations/klaviyo/config.js +++ b/src/v0/destinations/klaviyo/config.js @@ -19,6 +19,11 @@ const CONFIG_CATEGORIES = { VIEWED_PRODUCT: { name: 'ViewedProduct' }, ADDED_TO_CART: { name: 'AddedToCart' }, ITEMS: { name: 'Items' }, + SUBSCRIBE: { name: 'KlaviyoProfileV2', apiUrl: '/api/profile-subscription-bulk-create-jobs' }, + UNSUBSCRIBE: { + name: 'KlaviyoProfileV2', + apiUrl: '/api/profile-subscription-bulk-delete-jobs', + }, }; const ecomExclusionKeys = [ 'name', diff --git a/src/v0/destinations/klaviyo/transformV2.js b/src/v0/destinations/klaviyo/transformV2.js index ad98d2f559..6d04cb8644 100644 --- a/src/v0/destinations/klaviyo/transformV2.js +++ b/src/v0/destinations/klaviyo/transformV2.js @@ -7,9 +7,9 @@ const { EventType, MappedToDestinationKey } = require('../../../constants'); const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config'); const { constructProfile, - subscribeUserToListV2, + subscribeOrUnsubscribeUserToListV2, buildRequest, - buildSubscriptionRequest, + buildSubscriptionOrUnsubscriptionPayload, getTrackRequests, fetchTransformedEvents, addSubscribeFlagToTraits, @@ -24,6 +24,7 @@ const { adduserIdFromExternalId, groupEventsByType, flattenJson, + isDefinedAndNotNull, } = require('../../util'); /** @@ -49,9 +50,17 @@ const identifyRequestHandler = (message, category, destination) => { } const payload = removeUndefinedAndNullValues(constructProfile(message, destination, true)); const response = { profile: payload }; - // check if user wants to subscribe profile or not and listId is present or not - if (traitsInfo?.properties?.subscribe && (traitsInfo.properties?.listId || listId)) { - response.subscription = subscribeUserToListV2(message, traitsInfo, destination); + // check if user wants to subscribe/unsubscribe profile or do nothing and listId is present or not + if ( + isDefinedAndNotNull(traitsInfo?.properties?.subscribe) && + (traitsInfo.properties?.listId || listId) + ) { + response.subscription = subscribeOrUnsubscribeUserToListV2( + message, + traitsInfo, + destination, + traitsInfo.properties.subscribe ? 'subscribe' : 'unsubscribe', + ); } return response; }; @@ -93,7 +102,7 @@ const trackOrScreenRequestHandler = (message, category, destination) => { }; /** - * Main handlerfunc for group request add/subscribe users to the list based on property sent + * Main handlerfunc for group request add/subscribe to or remove/delete users to the list based on property sent * DOCS:https://developers.klaviyo.com/en/reference/subscribe_profiles * @param {*} message * @param {*} category @@ -105,11 +114,17 @@ const groupRequestHandler = (message, category, destination) => { throw new InstrumentationError('groupId is a required field for group events'); } const traitsInfo = getFieldValueFromMessage(message, 'traits'); - if (!traitsInfo?.subscribe) { - throw new InstrumentationError('Subscribe flag should be true for group call'); + if (!isDefinedAndNotNull(traitsInfo?.subscribe)) { + throw new InstrumentationError('Subscribe flag should be included in group call'); } - // throwing error for subscribe flag - return { subscription: subscribeUserToListV2(message, traitsInfo, destination) }; + return { + subscription: subscribeOrUnsubscribeUserToListV2( + message, + traitsInfo, + destination, + traitsInfo.subscribe ? 'subscribe' : 'unsubscribe', + ), + }; }; const processEvent = (event) => { @@ -152,9 +167,7 @@ const processV2 = (event) => { respList.push(buildRequest(response.profile, destination, CONFIG_CATEGORIES.IDENTIFYV2)); } if (response.subscription) { - respList.push( - buildSubscriptionRequest(response.subscription, destination, CONFIG_CATEGORIES.TRACKV2), - ); + respList.push(buildSubscriptionOrUnsubscriptionPayload(response.subscription, destination)); } if (response.event) { respList.push(buildRequest(response.event, destination, CONFIG_CATEGORIES.TRACKV2)); @@ -163,9 +176,19 @@ const processV2 = (event) => { }; // This function separates subscribe, proifle and event responses from process () and other responses in chunks -const getEventChunks = (input, subscribeRespList, profileRespList, eventRespList) => { +const getEventChunks = ( + input, + subscribeRespList, + profileRespList, + eventRespList, + unsubscriptionList, +) => { if (input.payload.subscription) { - subscribeRespList.push({ payload: input.payload.subscription, metadata: input.metadata }); + if (input.payload.subscription.operation === 'subscribe') { + subscribeRespList.push({ payload: input.payload.subscription, metadata: input.metadata }); + } else { + unsubscriptionList.push({ payload: input.payload.subscription, metadata: input.metadata }); + } } if (input.payload.profile) { profileRespList.push({ payload: input.payload.profile, metadata: input.metadata }); @@ -179,6 +202,7 @@ const processRouter = (inputs, reqMetadata) => { const batchResponseList = []; const batchErrorRespList = []; const subscribeRespList = []; + const unsubscriptionList = []; const profileRespList = []; const eventRespList = []; const { destination } = inputs[0]; @@ -197,6 +221,7 @@ const processRouter = (inputs, reqMetadata) => { subscribeRespList, profileRespList, eventRespList, + unsubscriptionList, ); } } catch (error) { @@ -204,7 +229,12 @@ const processRouter = (inputs, reqMetadata) => { batchErrorRespList.push(errRespEvent); } }); - const batchedResponseList = batchRequestV2(subscribeRespList, profileRespList, destination); + const batchedResponseList = batchRequestV2( + subscribeRespList, + unsubscriptionList, + profileRespList, + destination, + ); const trackRespList = getTrackRequests(eventRespList, destination); batchResponseList.push(...trackRespList, ...batchedResponseList); diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 7b2b011d43..4421764d95 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -454,42 +454,62 @@ const constructProfile = (message, destination, isIdentifyCall) => { return { data }; }; +/** This function update profile with consents for subscribing to email and/or phone + * @param {*} profileAttributes + * @param {*} subscribeConsent + * @param {*} email + * @param {*} phone + */ +const updateProfileWithConsents = (profileAttributes, subscribeConsent, email, phone) => { + let consent = subscribeConsent; + if (!Array.isArray(consent)) { + consent = [consent]; + } + if (consent.includes('email') && email) { + set(profileAttributes, 'subscriptions.email.marketing.consent', 'SUBSCRIBED'); + } + if (consent.includes('sms') && phone) { + set(profileAttributes, 'subscriptions.sms.marketing.consent', 'SUBSCRIBED'); + } +}; /** * This function is used for creating profile response for subscribing users to a particular list for V2 * DOCS: https://developers.klaviyo.com/en/reference/subscribe_profiles + * Return an object with listId, profiles and operation */ -const subscribeUserToListV2 = (message, traitsInfo, destination) => { +const subscribeOrUnsubscribeUserToListV2 = (message, traitsInfo, destination, operation) => { // listId from message properties are preferred over Config listId const { consent } = destination.Config; let { listId } = destination.Config; - let subscribeConsent = traitsInfo.consent || traitsInfo.properties?.consent || consent; + const subscribeConsent = traitsInfo.consent || traitsInfo.properties?.consent || consent; const email = getFieldValueFromMessage(message, 'email'); const phone = getFieldValueFromMessage(message, 'phone'); const profileAttributes = { email, phone_number: phone, }; - if (subscribeConsent) { - if (!Array.isArray(subscribeConsent)) { - subscribeConsent = [subscribeConsent]; - } - if (subscribeConsent.includes('email') && email) { - set(profileAttributes, 'subscriptions.email.marketing.consent', 'SUBSCRIBED'); - } - if (subscribeConsent.includes('sms') && phone) { - set(profileAttributes, 'subscriptions.sms.marketing.consent', 'SUBSCRIBED'); - } + + // used only for subscription and not for unsubscription + if (operation === 'subscribe' && subscribeConsent) { + updateProfileWithConsents(profileAttributes, subscribeConsent, email, phone); } const profile = removeUndefinedAndNullValues({ type: 'profile', - id: getDestinationExternalID(message, 'klaviyo-profileId'), + id: + operation === 'subscribe' + ? getDestinationExternalID(message, 'klaviyo-profileId') + : undefined, // id is not applicable for unsubscription attributes: removeUndefinedAndNullValues(profileAttributes), }); if (!email && !phone && profile.id) { - throw new InstrumentationError( - 'Profile Id, Email or/and Phone are required to subscribe to a list', - ); + if (operation === 'subscribe') { + throw new InstrumentationError( + 'Profile Id, Email or/and Phone are required to subscribe to a list', + ); + } else { + throw new InstrumentationError('Email or/and Phone are required to unsubscribe from a list'); + } } // fetch list id from message if (traitsInfo?.properties?.listId) { @@ -499,17 +519,20 @@ const subscribeUserToListV2 = (message, traitsInfo, destination) => { if (message.type === 'group') { listId = message.groupId; } - - return { listId, profile: [profile] }; + return { listId, profile: [profile], operation }; }; /** * This Create a subscription payload to subscribe profile(s) to list listId * @param {*} listId * @param {*} profiles + * @param {*} operation can be either subscribe or unsubscribe */ -const getSubscriptionPayload = (listId, profiles) => ({ +const getSubscriptionPayload = (listId, profiles, operation) => ({ data: { - type: 'profile-subscription-bulk-create-job', + type: + operation === 'subscribe' + ? 'profile-subscription-bulk-create-job' + : 'profile-subscription-bulk-delete-job', attributes: { profiles: { data: profiles } }, relationships: { list: { @@ -523,14 +546,15 @@ const getSubscriptionPayload = (listId, profiles) => ({ }); /** - * This function accepts subscriptions object and builds a request for it + * This function accepts subscriptions/ unsubscription object and builds a request for it * @param {*} subscription * @param {*} destination + * @param {*} operation can be either subscription or unsubscription * @returns defaultRequestConfig */ -const buildSubscriptionRequest = (subscription, destination) => { +const buildSubscriptionOrUnsubscriptionPayload = (subscription, destination) => { const response = defaultRequestConfig(); - response.endpoint = `${BASE_ENDPOINT}/api/profile-subscription-bulk-create-jobs`; + response.endpoint = `${BASE_ENDPOINT}${CONFIG_CATEGORIES[subscription.operation.toUpperCase()].apiUrl}`; response.method = defaultPostRequestConfig.requestMethod; response.headers = { Authorization: `Klaviyo-API-Key ${destination.Config.privateApiKey}`, @@ -538,7 +562,11 @@ const buildSubscriptionRequest = (subscription, destination) => { 'Content-Type': JSON_MIME_TYPE, revision, }; - response.body.JSON = getSubscriptionPayload(subscription.listId, subscription.profile); + response.body.JSON = getSubscriptionPayload( + subscription.listId, + subscription.profile, + subscription.operation, + ); return response; }; @@ -602,10 +630,10 @@ module.exports = { profileUpdateResponseBuilder, getIdFromNewOrExistingProfile, constructProfile, - subscribeUserToListV2, + subscribeOrUnsubscribeUserToListV2, getProfileMetadataAndMetadataFields, buildRequest, - buildSubscriptionRequest, + buildSubscriptionOrUnsubscriptionPayload, getTrackRequests, fetchTransformedEvents, addSubscribeFlagToTraits, diff --git a/src/v0/destinations/personalize/scripts/personalize_cleanup.sh b/src/v0/destinations/personalize/scripts/personalize_cleanup.sh new file mode 100644 index 0000000000..5bf6426fa6 --- /dev/null +++ b/src/v0/destinations/personalize/scripts/personalize_cleanup.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Set the AWS region +AWS_REGION="us-east-1" + +# Function to wait for dataset deletion +wait_for_deletion() { + local resource_arn=$1 + local resource_type=$2 + local status="" + + echo "Waiting for $resource_type $resource_arn to be deleted..." + + while true; do + if [ "$resource_type" == "dataset" ]; then + status=$(aws personalize describe-dataset --dataset-arn $resource_arn --region $AWS_REGION --query "dataset.status" --output text 2>/dev/null) + fi + + if [ -z "$status" ]; then + echo "$resource_type $resource_arn has been deleted." + break + else + echo "$resource_type $resource_arn status: $status. Waiting..." + sleep 10 + fi + done +} + +# Delete dataset groups +for dataset_group_arn in $(aws personalize list-dataset-groups --region $AWS_REGION --query "datasetGroups[*].datasetGroupArn" --output text); do + + echo "Processing dataset group: $dataset_group_arn" + + # List and delete all event trackers in the dataset group + for event_tracker_arn in $(aws personalize list-event-trackers --dataset-group-arn $dataset_group_arn --region $AWS_REGION --query "eventTrackers[*].eventTrackerArn" --output text); do + echo "Deleting event tracker $event_tracker_arn" + aws personalize delete-event-tracker --event-tracker-arn $event_tracker_arn --region $AWS_REGION + done + + # List and delete all solutions in the dataset group + for solution_arn in $(aws personalize list-solutions --dataset-group-arn $dataset_group_arn --region $AWS_REGION --query "solutions[*].solutionArn" --output text); do + + # List and delete all campaigns for the solution + for campaign_arn in $(aws personalize list-campaigns --solution-arn $solution_arn --region $AWS_REGION --query "campaigns[*].campaignArn" --output text); do + echo "Deleting campaign $campaign_arn" + aws personalize delete-campaign --campaign-arn $campaign_arn --region $AWS_REGION + done + + echo "Deleting solution $solution_arn" + aws personalize delete-solution --solution-arn $solution_arn --region $AWS_REGION + done + + # List and delete all datasets in the dataset group + for dataset_arn in $(aws personalize list-datasets --dataset-group-arn $dataset_group_arn --region $AWS_REGION --query "datasets[*].datasetArn" --output text); do + echo "Deleting dataset $dataset_arn" + aws personalize delete-dataset --dataset-arn $dataset_arn --region $AWS_REGION + wait_for_deletion $dataset_arn "dataset" + done + + # Finally, delete the dataset group + echo "Deleting dataset group $dataset_group_arn" + aws personalize delete-dataset-group --dataset-group-arn $dataset_group_arn --region $AWS_REGION + wait_for_deletion $dataset_group_arn "dataset_group" +done + +echo "All datasets, event trackers, solutions, campaigns, and dataset groups have been deleted." + diff --git a/src/v0/destinations/sfmc/config.js b/src/v0/destinations/sfmc/config.js index 1b1f5c323b..1c89c04112 100644 --- a/src/v0/destinations/sfmc/config.js +++ b/src/v0/destinations/sfmc/config.js @@ -7,6 +7,10 @@ const ENDPOINTS = { EVENT: 'rest.marketingcloudapis.com/interaction/v1/events', }; +const ACCESS_TOKEN_CACHE_TTL = process.env.SFMC_ACCESS_TOKEN_CACHE_TTL + ? parseInt(process.env.SFMC_ACCESS_TOKEN_CACHE_TTL, 10) + : 1000; + const CONFIG_CATEGORIES = { IDENTIFY: { type: 'identify', @@ -24,4 +28,5 @@ module.exports = { ENDPOINTS, MAPPING_CONFIG, CONFIG_CATEGORIES, + ACCESS_TOKEN_CACHE_TTL, }; diff --git a/src/v0/destinations/sfmc/transform.js b/src/v0/destinations/sfmc/transform.js index a433179f9c..d20a9ed40d 100644 --- a/src/v0/destinations/sfmc/transform.js +++ b/src/v0/destinations/sfmc/transform.js @@ -6,10 +6,19 @@ const { InstrumentationError, isDefinedAndNotNull, isEmpty, + MappedToDestinationKey, + GENERIC_TRUE_VALUES, + PlatformError, } = require('@rudderstack/integrations-lib'); +const get = require('get-value'); const { EventType } = require('../../../constants'); const { handleHttpRequest } = require('../../../adapters/network'); -const { CONFIG_CATEGORIES, MAPPING_CONFIG, ENDPOINTS } = require('./config'); +const { + CONFIG_CATEGORIES, + MAPPING_CONFIG, + ENDPOINTS, + ACCESS_TOKEN_CACHE_TTL, +} = require('./config'); const { removeUndefinedAndNullValues, getFieldValueFromMessage, @@ -21,12 +30,15 @@ const { toTitleCase, getHashFromArray, simpleProcessRouterDest, + getDestinationExternalIDInfoForRetl, } = require('../../util'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); const { isHttpStatusSuccess } = require('../../util'); const tags = require('../../util/tags'); const { JSON_MIME_TYPE } = require('../../util/constant'); +const Cache = require('../../util/cache'); +const accessTokenCache = new Cache(ACCESS_TOKEN_CACHE_TTL); const CONTACT_KEY_KEY = 'Contact Key'; // DOC: https://developer.salesforce.com/docs/atlas.en-us.mc-app-development.meta/mc-app-development/access-token-s2s.htm @@ -271,11 +283,41 @@ const responseBuilderSimple = async ({ message, destination, metadata }, categor throw new ConfigurationError(`Event type '${category.type}' not supported`); }; +const retlResponseBuilder = async (message, destination, metadata) => { + const { clientId, clientSecret, subDomain, externalKey } = destination.Config; + const token = await accessTokenCache.get(metadata.destinationId, () => + getToken(clientId, clientSecret, subDomain, metadata), + ); + const { destinationExternalId, objectType, identifierType } = getDestinationExternalIDInfoForRetl( + message, + 'SFMC', + ); + if (objectType?.toLowerCase() === 'data extension') { + const response = defaultRequestConfig(); + response.method = defaultPutRequestConfig.requestMethod; + response.endpoint = `https://${subDomain}.${ENDPOINTS.INSERT_CONTACTS}${externalKey}/rows/${identifierType}:${destinationExternalId}`; + response.headers = { + 'Content-Type': JSON_MIME_TYPE, + Authorization: `Bearer ${token}`, + }; + response.body.JSON = { + values: { + ...message.traits, + }, + }; + return response; + } + throw new PlatformError('Unsupported object type for rETL use case'); +}; + const processEvent = async ({ message, destination, metadata }) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } - + const mappedToDestination = get(message, MappedToDestinationKey); + if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) { + return retlResponseBuilder(message, destination, metadata); + } const messageType = message.type.toLowerCase(); let category; // only accept track and identify calls diff --git a/src/v0/util/facebookUtils/index.js b/src/v0/util/facebookUtils/index.js index 7462320cca..fd94db442b 100644 --- a/src/v0/util/facebookUtils/index.js +++ b/src/v0/util/facebookUtils/index.js @@ -148,11 +148,13 @@ const getContentType = (message, defaultValue, categoryToContent, destinationNam } let { category } = properties || {}; - if (!category) { - const { products } = properties; - if (products && products.length > 0 && Array.isArray(products) && isObject(products[0])) { - category = products[0].category; - } + if ( + !category && + Array.isArray(properties?.products) && + properties?.products.length > 0 && + isObject(properties?.products[0]) + ) { + category = properties?.products[0].category; } if (Array.isArray(categoryToContent) && category) { diff --git a/src/v0/util/facebookUtils/index.test.js b/src/v0/util/facebookUtils/index.test.js index 1a2de4ed12..90588c627b 100644 --- a/src/v0/util/facebookUtils/index.test.js +++ b/src/v0/util/facebookUtils/index.test.js @@ -639,6 +639,21 @@ describe('getContentType', () => { expect(result).toBe(defaultValue); }); + + it('should return default value when no product array or categoryToContent is provided', () => { + const message = { + properties: { + revenue: 1234, + }, + }; + const defaultValue = 'product'; + const categoryToContent = []; + const destinationName = 'fb_pixel'; + + const result = getContentType(message, defaultValue, categoryToContent, destinationName); + + expect(result).toBe(defaultValue); + }); }); describe('isHtmlFormat', () => { diff --git a/src/v1/destinations/bloomreach_catalog/networkHandler.js b/src/v1/destinations/bloomreach_catalog/networkHandler.js new file mode 100644 index 0000000000..1fb987b840 --- /dev/null +++ b/src/v1/destinations/bloomreach_catalog/networkHandler.js @@ -0,0 +1,85 @@ +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { proxyRequest, prepareProxyRequest } = require('../../../adapters/network'); +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); +const tags = require('../../../v0/util/tags'); + +// Catalog response +// [ +// { +// "errors": { +// "properties": [ +// "Fields [field1, field2] are not properly defined." +// ] +// }, +// "queued": false, +// "success": false +// }, +// { +// "success" : "True", +// "queued" : "True", +// }, +// ] +const checkIfEventIsAbortableAndExtractErrorMessage = (element) => { + if (element.success) { + return { isAbortable: false, errorMsg: '' }; + } + + const errorMsg = Object.values(element.errors || {}) + .flat() + .join(', '); + + return { isAbortable: true, errorMsg }; +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + + const message = '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully'; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + response.forEach((event, idx) => { + const proxyOutput = { + statusCode: 200, + error: 'success', + metadata: rudderJobMetadata[idx], + }; + // update status of partial event if abortable + const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage(event); + if (isAbortable) { + proxyOutput.error = errorMsg; + proxyOutput.statusCode = 400; + } + responseWithIndividualEvents.push(proxyOutput); + }); + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + throw new TransformerProxyError( + `BLOOMREACH_CATALOG: Error encountered in transformer proxy V1`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); +}; +function networkHandler() { + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.prepareProxy = prepareProxyRequest; + this.responseHandler = responseHandler; +} +module.exports = { networkHandler }; diff --git a/test/integrations/destinations/attentive_tag/processor/data.ts b/test/integrations/destinations/attentive_tag/processor/data.ts index 2e602610e0..137a9aee9a 100644 --- a/test/integrations/destinations/attentive_tag/processor/data.ts +++ b/test/integrations/destinations/attentive_tag/processor/data.ts @@ -1458,7 +1458,7 @@ export const data = [ body: [ { statusCode: 400, - error: '[Attentive Tag]:The event name contains characters which is not allowed', + error: '[Attentive Tag]:The properties contains characters which is not allowed', statTags: { destType: 'ATTENTIVE_TAG', errorCategory: 'dataValidation', diff --git a/test/integrations/destinations/bloomreach_catalog/common.ts b/test/integrations/destinations/bloomreach_catalog/common.ts new file mode 100644 index 0000000000..2b4266837b --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/common.ts @@ -0,0 +1,81 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'bloomreach_catalog'; +const destTypeInUpperCase = 'BLOOMREACH_CATALOG'; +const displayName = 'bloomreach catalog'; +const channel = 'web'; +const destination: Destination = { + Config: { + apiBaseUrl: 'https://demoapp-api.bloomreach.com', + apiKey: 'test-api-key', + apiSecret: 'test-api-secret', + projectToken: 'test-project-token', + catalogID: 'test-catalog-id', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const insertEndpoint = + 'https://demoapp-api.bloomreach.com/data/v2/projects/test-project-token/catalogs/test-catalog-id/items'; +const updateEndpoint = + 'https://demoapp-api.bloomreach.com/data/v2/projects/test-project-token/catalogs/test-catalog-id/items/partial-update'; +const deleteEndpoint = + 'https://demoapp-api.bloomreach.com/data/v2/projects/test-project-token/catalogs/test-catalog-id/items/bulk-delete'; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const proxyV1RetryableErrorStatTags = { + ...RouterInstrumentationErrorStatTags, + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hcGkta2V5OnRlc3QtYXBpLXNlY3JldA==', +}; + +const sampleContext = { + destinationFields: 'item_id, title, status, unprinted', + mappedToDestination: 'true', +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + headers, + proxyV1RetryableErrorStatTags, + insertEndpoint, + updateEndpoint, + deleteEndpoint, + sampleContext, +}; diff --git a/test/integrations/destinations/bloomreach_catalog/dataDelivery/data.ts b/test/integrations/destinations/bloomreach_catalog/dataDelivery/data.ts new file mode 100644 index 0000000000..f8cccd04ed --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/dataDelivery/data.ts @@ -0,0 +1,197 @@ +import { ProxyV1TestData } from '../../../testTypes'; +import { destType, headers, updateEndpoint } from '../common'; +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; + +export const data: ProxyV1TestData[] = [ + { + id: 'bloomreach_catalog_v1_business_scenario_1', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid record request - where the destination responds with 200 with error for request 2 in a batch', + successCriteria: 'Should return 200 with partial failures within the response payload', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id","properties":{"unprinted":"1"}},{"item_id":"test-item-id-faulty","properties":{"unprinted1":"1"}}]', + }, + endpoint: updateEndpoint, + }, + [generateMetadata(1), generateMetadata(2)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: [ + { + success: true, + queued: true, + }, + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: 'Fields [unprinted1] are not properly defined.', + }, + ], + }, + }, + }, + }, + }, + { + id: 'bloomreach_catalog_v1_business_scenario_2', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid rETL request - where the destination responds with 200 without any error', + successCriteria: 'Should return 200 with no error with destination response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id-1","properties":{"unprinted":"1"}},{"item_id":"test-item-id-2","properties":{"unprinted":"2"}}]', + }, + endpoint: updateEndpoint, + }, + [generateMetadata(3), generateMetadata(4)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: [ + { + success: true, + queued: true, + }, + { + success: true, + queued: true, + }, + ], + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(3), + error: 'success', + }, + { + statusCode: 200, + metadata: generateMetadata(4), + error: 'success', + }, + ], + }, + }, + }, + }, + }, + { + id: 'bloomreach_catalog_v1_business_scenario_3', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid rETL request - where the destination responds with 200 with error', + successCriteria: 'Should return 400 with error message', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: {}, + JSON_ARRAY: { + batch: '[{"item_id":"test-item-id-faulty","properties":{"unprinted1":"1"}}]', + }, + endpoint: updateEndpoint, + }, + [generateMetadata(5)], + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH_CATALOG Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: [ + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + }, + response: [ + { + statusCode: 400, + metadata: generateMetadata(5), + error: 'Fields [unprinted1] are not properly defined.', + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach_catalog/mocks.ts b/test/integrations/destinations/bloomreach_catalog/mocks.ts new file mode 100644 index 0000000000..bb07c0ea72 --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/mocks.ts @@ -0,0 +1,5 @@ +import * as config from '../../../../src/cdk/v2/destinations/bloomreach_catalog/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_ITEMS', 2 as typeof config.MAX_ITEMS); +}; diff --git a/test/integrations/destinations/bloomreach_catalog/network.ts b/test/integrations/destinations/bloomreach_catalog/network.ts new file mode 100644 index 0000000000..b8ae078498 --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/network.ts @@ -0,0 +1,108 @@ +import { destType, headers, updateEndpoint } from './common'; + +export const networkCallsData = [ + { + httpReq: { + url: updateEndpoint, + data: [ + { + item_id: 'test-item-id', + properties: { + unprinted: '1', + }, + }, + { + item_id: 'test-item-id-faulty', + properties: { + unprinted1: '1', + }, + }, + ], + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: [ + { + success: true, + queued: true, + }, + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: updateEndpoint, + data: [ + { + item_id: 'test-item-id-1', + properties: { + unprinted: '1', + }, + }, + { + item_id: 'test-item-id-2', + properties: { + unprinted: '2', + }, + }, + ], + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: [ + { + success: true, + queued: true, + }, + { + success: true, + queued: true, + }, + ], + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: updateEndpoint, + data: [ + { + item_id: 'test-item-id-faulty', + properties: { + unprinted1: '1', + }, + }, + ], + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: [ + { + success: false, + queued: false, + errors: { + properties: ['Fields [unprinted1] are not properly defined.'], + }, + }, + ], + status: 200, + statusText: 'Ok', + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach_catalog/router/data.ts b/test/integrations/destinations/bloomreach_catalog/router/data.ts new file mode 100644 index 0000000000..68ab422444 --- /dev/null +++ b/test/integrations/destinations/bloomreach_catalog/router/data.ts @@ -0,0 +1,328 @@ +import { generateMetadata } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; +import { + destType, + destination, + headers, + RouterInstrumentationErrorStatTags, + insertEndpoint, + updateEndpoint, + deleteEndpoint, + sampleContext, +} from '../common'; + +const routerRequest = { + input: [ + { + message: { + type: 'record', + action: 'insert', + fields: { + item_id: 'test-item-id', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '1', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'record', + action: 'insert', + fields: { + item_id: 'test-item-id-7', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + test_empty: '', + test_null: null, + test_empty_array: [], + }, + channel: 'sources', + context: sampleContext, + recordId: '2', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'record', + action: 'update', + fields: { + item_id: 'test-item-id', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 3, + }, + channel: 'sources', + context: sampleContext, + recordId: '3', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'record', + action: 'update', + fields: { + item_id: 'test-item-id', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 2, + }, + channel: 'sources', + context: sampleContext, + recordId: '4', + }, + metadata: generateMetadata(4), + destination, + }, + { + message: { + type: 'record', + action: 'delete', + fields: { + item_id: 'test-item-id-1', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '5', + }, + metadata: generateMetadata(5), + destination, + }, + { + message: { + type: 'record', + action: 'delete', + fields: { + item_id: 'test-item-id-2', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '6', + }, + metadata: generateMetadata(6), + destination, + }, + { + message: { + type: 'record', + action: 'delete', + fields: { + item_id: 'test-item-id-3', + title: 'Hardcover Monthbooks', + status: 'up to date', + unprinted: 1, + }, + channel: 'sources', + context: sampleContext, + recordId: '7', + }, + metadata: generateMetadata(7), + destination, + }, + { + message: { + type: 'record', + action: 'insert', + fields: {}, + channel: 'sources', + context: sampleContext, + recordId: '8', + }, + metadata: generateMetadata(8), + destination, + }, + { + message: { + type: 'record', + action: 'dummy-action', + fields: { + item_id: 'test-item-id', + }, + channel: 'sources', + context: sampleContext, + recordId: '9', + }, + metadata: generateMetadata(9), + destination, + }, + { + message: { + type: 'record', + action: 'insert', + fields: { + item_id: null, + }, + channel: 'sources', + context: sampleContext, + recordId: '10', + }, + metadata: generateMetadata(10), + destination, + }, + ], + destType, +}; + +export const data = [ + { + id: 'bloomreach-catalog-router-test-1', + name: destType, + description: 'Basic Router Test to test record payloads', + scenario: 'Framework', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'PUT', + endpoint: insertEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":1}},{"item_id":"test-item-id-7","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":1,"test_empty":"","test_null":null,"test_empty_array":[]}}]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: updateEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: + '[{"item_id":"test-item-id","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":3}},{"item_id":"test-item-id","properties":{"title":"Hardcover Monthbooks","status":"up to date","unprinted":2}}]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(3), generateMetadata(4)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: deleteEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: '["test-item-id-1","test-item-id-2"]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(5), generateMetadata(6)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'DELETE', + endpoint: deleteEndpoint, + headers, + params: {}, + body: { + JSON: {}, + JSON_ARRAY: { + batch: '["test-item-id-3"]', + }, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(7)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(8)], + batched: false, + statusCode: 400, + error: '`fields` cannot be empty', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(9)], + batched: false, + statusCode: 400, + error: + 'Invalid action type dummy-action. You can only add, update or remove items from the catalog', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(10)], + batched: false, + statusCode: 400, + error: '`item_id` cannot be empty', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/braze/processor/data.ts b/test/integrations/destinations/braze/processor/data.ts index 644bebace1..240206791e 100644 --- a/test/integrations/destinations/braze/processor/data.ts +++ b/test/integrations/destinations/braze/processor/data.ts @@ -309,6 +309,7 @@ export const data = [ name: 'braze revenue test', time: '2020-01-24T11:59:02.403+05:30', properties: { + currency: 'USD', revenue: 50, }, external_id: 'mickeyMouse', @@ -437,6 +438,7 @@ export const data = [ time: '2020-01-24T11:59:02.403+05:30', properties: { revenue: 50, + currency: 'USD', }, external_id: 'mickeyMouse', }, @@ -772,6 +774,7 @@ export const data = [ properties: { currency: 'USD', revenue: 50, + event_name: 'braze revenue test 2', }, receivedAt: '2020-01-24T11:59:02.403+05:30', request_ip: '[::1]:53710', @@ -823,6 +826,7 @@ export const data = [ time: '2020-01-24T11:59:02.403+05:30', properties: { revenue: 50, + currency: 'USD', }, _update_existing_only: false, user_alias: { @@ -993,6 +997,7 @@ export const data = [ affiliation: 'Google Store', checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', coupon: 'hasbros', + currency: 'USD', discount: 2.5, order_id: '50314b8e9bcf000000000000', products: [ @@ -1976,6 +1981,7 @@ export const data = [ properties: { mergeObjectsUpdateOperation: false, revenue: 50, + currency: 'USD', }, external_id: 'finalUserTestCA', }, @@ -2192,6 +2198,7 @@ export const data = [ properties: { mergeObjectsUpdateOperation: false, revenue: 50, + currency: 'USD', }, external_id: 'finalUserTestCA', }, @@ -3398,6 +3405,7 @@ export const data = [ time: '2020-01-24T11:59:02.403+05:30', properties: { revenue: 50, + currency: 'USD', }, external_id: 'mickeyMouse', app_id: '123', @@ -3985,6 +3993,7 @@ export const data = [ subtotal: 22.5, tax: 2, total: 27.5, + currency: 'USD', }, _update_existing_only: false, user_alias: { @@ -4036,7 +4045,6 @@ export const data = [ Transformations: [], }, message: { - anonymousId: 'e6ab2c5e-2cda-44a9-a962-e2f67df78bca', channel: 'web', context: { traits: { diff --git a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts index dcd7fbc38e..da7769b110 100644 --- a/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/groupTestDataV2.ts @@ -32,7 +32,7 @@ const headers = { revision: '2024-06-15', }; -const endpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs'; +const subscriptionEndpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs'; const commonOutputSubscriptionProps = { profiles: { @@ -50,6 +50,19 @@ const commonOutputSubscriptionProps = { ], }, }; +const commonOutputUnsubscriptionProps = { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 678 900', + }, + }, + ], + }, +}; const subscriptionRelations = { list: { @@ -59,12 +72,13 @@ const subscriptionRelations = { }, }, }; +const unsubscriptionEndpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-delete-jobs'; export const groupTestData: ProcessorTestData[] = [ { id: 'klaviyo-group-test-1', name: 'klaviyo', - description: 'Simple group call', + description: 'Simple group call for subscription', scenario: 'Business', successCriteria: 'Response should contain only group payload and status code should be 200, for the group payload a subscription payload should be present in the final payload with email and phone', @@ -109,7 +123,67 @@ export const groupTestData: ProcessorTestData[] = [ relationships: subscriptionRelations, }, }, - endpoint: endpoint, + endpoint: subscriptionEndpoint, + headers: headers, + method: 'POST', + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'klaviyo-group-test-1', + name: 'klaviyo', + description: 'Simple group call for subscription', + scenario: 'Business', + successCriteria: + 'Response should contain only group payload and status code should be 200, for the group payload a unsubscription payload should be present in the final payload with email and phone', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: generateSimplifiedGroupPayload({ + userId: 'user123', + groupId: 'group_list_id', + traits: { + subscribe: false, + }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: ['email'], + }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }), + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + JSON: { + data: { + type: 'profile-subscription-bulk-delete-job', + attributes: commonOutputUnsubscriptionProps, + relationships: subscriptionRelations, + }, + }, + endpoint: unsubscriptionEndpoint, headers: headers, method: 'POST', userId: '', @@ -122,7 +196,7 @@ export const groupTestData: ProcessorTestData[] = [ }, }, { - id: 'klaviyo-group-test-2', + id: 'klaviyo-group-test-3', name: 'klaviyo', description: 'Simple group call without groupId', scenario: 'Business', diff --git a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts index 80ae918af0..612bfe88f8 100644 --- a/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/identifyTestDataV2.ts @@ -87,6 +87,19 @@ const commonOutputSubscriptionProps = { ], }, }; +const commonOutputUnsubscriptionProps = { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'test@rudderstack.com', + phone_number: '+12 345 578 900', + }, + }, + ], + }, +}; const subscriptionRelations = { list: { data: { @@ -109,6 +122,7 @@ const sentAt = '2021-01-03T17:02:53.195Z'; const originalTimestamp = '2021-01-03T17:02:53.193Z'; const userProfileCommonEndpoint = 'https://a.klaviyo.com/api/profile-import'; const subscribeEndpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs'; +const unsubscribeEndpoint = 'https://a.klaviyo.com/api/profile-subscription-bulk-delete-jobs'; export const identifyData: ProcessorTestData[] = [ { @@ -206,7 +220,99 @@ export const identifyData: ProcessorTestData[] = [ id: 'klaviyo-identify-150624-test-2', name: 'klaviyo', description: - '150624 -> Profile without subscribing the user and get klaviyo id from externalId', + '150624 -> Identify call with flattenProperties enabled in destination config and unsubscribe', + scenario: 'Business', + successCriteria: + 'The profile response should contain the flattened properties of the friend object and one request object for subscribe', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { flattenProperties: true }), + message: generateSimplifiedIdentifyPayload({ + sentAt, + userId, + context: { + traits: { + ...commonTraits, + properties: { ...commonTraits.properties, subscribe: false }, + friend: { + names: { + first: 'Alice', + last: 'Smith', + }, + age: 25, + }, + }, + }, + anonymousId, + originalTimestamp, + }), + metadata: generateMetadata(2), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile', + attributes: { + ...commonOutputUserProps, + properties: { + ...commonOutputUserProps.properties, + 'friend.age': 25, + 'friend.names.first': 'Alice', + 'friend.names.last': 'Smith', + }, + }, + meta: { + patch_properties: {}, + }, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + { + output: transformResultBuilder({ + userId: '', + method: 'POST', + endpoint: unsubscribeEndpoint, + headers: commonOutputHeaders, + JSON: { + data: { + type: 'profile-subscription-bulk-delete-job', + attributes: commonOutputUnsubscriptionProps, + relationships: subscriptionRelations, + }, + }, + }), + statusCode: 200, + metadata: generateMetadata(2), + }, + ], + }, + }, + }, + { + id: 'klaviyo-identify-150624-test-3', + name: 'klaviyo', + description: + '150624 -> Profile without subscribing/unsubscribing the user and get klaviyo id from externalId', scenario: 'Business', successCriteria: 'Response should contain only profile update payload and status code should be 200 as subscribe is set to false in the payload', @@ -234,7 +340,7 @@ export const identifyData: ProcessorTestData[] = [ appendList2: 'New Value 2', unappendList1: 'Old Value 1', unappendList2: 'Old Value 2', - properties: { ...commonTraits.properties, subscribe: false }, + properties: { ...commonTraits.properties, subscribe: undefined }, }, }, integrations: { @@ -292,7 +398,7 @@ export const identifyData: ProcessorTestData[] = [ }, }, { - id: 'klaviyo-identify-150624-test-5', + id: 'klaviyo-identify-150624-test-4', name: 'klaviyo', description: '150624 -> Identify call with enforceEmailAsPrimary enabled in destination config', scenario: 'Business', diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index 714560fdfd..5a0a06fad1 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -1260,4 +1260,788 @@ export const dataV2: RouterTestData[] = [ }, }, }, + { + id: 'klaviyo-router-150624-test-5', + name: 'klaviyo', + description: '150624 -> Only Identify calls with some subcribe and some unsubscribe operation', + scenario: 'Framework', + successCriteria: + 'All the subscription events with same listId should be batched and same for unsubscribe as well.', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + // user 1 idenitfy call with anonymousId and subscription as true + channel: 'web', + traits: { + email: 'testklaviyo1@rs.com', + firstname: 'Test Klaviyo 1', + properties: { + subscribe: true, + listId: 'configListId', + consent: ['email'], + }, + }, + context: {}, + anonymousId: 'anonTestKlaviyo1', + type: 'identify', + userId: 'testKlaviyo1', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1, 'testKlaviyo1'), + destination, + }, + { + message: { + // user 2 idenitfy call with no anonymousId and subscription as true + channel: 'web', + traits: { + email: 'testklaviyo2@rs.com', + firstname: 'Test Klaviyo 2', + properties: { + subscribe: true, + listId: 'configListId', + consent: ['email'], + }, + }, + context: {}, + type: 'identify', + userId: 'testKlaviyo2', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(2, 'testKlaviyo2'), + destination, + }, + { + message: { + // user 3 idenitfy call with no anonymousId and subscription as false + channel: 'web', + traits: { + email: 'testklaviyo3@rs.com', + firstname: 'Test Klaviyo 3', + properties: { + subscribe: false, + listId: 'configListId', + consent: ['email'], + }, + }, + context: {}, + type: 'identify', + userId: 'testKlaviyo3', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(3, 'testKlaviyo3'), + destination, + }, + { + message: { + // user 4 idenitfy call with anonymousId and subscription as false + channel: 'web', + traits: { + email: 'testklaviyo4@rs.com', + firstname: 'Test Klaviyo 4', + properties: { + subscribe: false, + listId: 'configListId', + consent: ['email'], + }, + }, + context: {}, + type: 'identify', + anonymousId: 'anon id 4', + userId: 'testKlaviyo4', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(4, 'testKlaviyo4'), + destination, + }, + ], + destType: 'klaviyo', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + // 2 identify calls and one batched subscription request for user 1 and user 2 + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo1', + email: 'testklaviyo1@rs.com', + first_name: 'Test Klaviyo 1', + anonymous_id: 'anonTestKlaviyo1', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo2', + email: 'testklaviyo2@rs.com', + first_name: 'Test Klaviyo 2', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-create-jobs', + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-create-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'testklaviyo1@rs.com', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + { + type: 'profile', + attributes: { + email: 'testklaviyo2@rs.com', + subscriptions: { + email: { marketing: { consent: 'SUBSCRIBED' } }, + }, + }, + }, + ], + }, + }, + relationships: { + list: { + data: { + type: 'list', + id: 'configListId', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(1, 'testKlaviyo1'), generateMetadata(2, 'testKlaviyo2')], + batched: true, + statusCode: 200, + destination, + }, + { + // 2 identify calls and one batched unsubscription request for user 3 and user 4 + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo3', + email: 'testklaviyo3@rs.com', + first_name: 'Test Klaviyo 3', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: userProfileCommonEndpoint, + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo4', + email: 'testklaviyo4@rs.com', + first_name: 'Test Klaviyo 4', + anonymous_id: 'anon id 4', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-delete-jobs', + headers, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-delete-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'testklaviyo3@rs.com', + }, + }, + { + type: 'profile', + attributes: { + email: 'testklaviyo4@rs.com', + }, + }, + ], + }, + }, + relationships: { + list: { + data: { + type: 'list', + id: 'configListId', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(3, 'testKlaviyo3'), generateMetadata(4, 'testKlaviyo4')], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, + { + id: 'klaviyo-router-150624-test-6', + name: 'klaviyo', + description: + '150624 -> Router tests to have some anonymous track event, some identify events with unsubscription and some identified track event', + scenario: 'Framework', + successCriteria: + 'All the unsubscription events under same message type should be batched and respective profile requests should also be placed in same batched request', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + // user 1 track call with userId and anonymousId + channel: 'web', + context: { + traits: { + email: 'testklaviyo1@email.com', + firstname: 'Test Klaviyo 1', + }, + }, + type: 'track', + anonymousId: 'anonTestKlaviyo1', + userId: 'testKlaviyo1', + event: 'purchase', + properties: { + price: '12', + }, + }, + metadata: generateMetadata(1, 'testKlaviyo1'), + destination, + }, + { + message: { + // Anonymous Tracking -> user 2 track call with anonymousId only + channel: 'web', + context: { + traits: {}, + }, + type: 'track', + anonymousId: 'anonTestKlaviyo2', + event: 'viewed product', + properties: { + price: '120', + }, + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + // user 2 idenitfy call with anonymousId and subscription + channel: 'web', + traits: { + email: 'testklaviyo2@rs.com', + firstname: 'Test Klaviyo 2', + properties: { + subscribe: false, + listId: 'configListId', + consent: ['email'], + }, + }, + context: {}, + anonymousId: 'anonTestKlaviyo2', + type: 'identify', + userId: 'testKlaviyo2', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(3, 'testKlaviyo2'), + destination, + }, + { + message: { + // user 2 track call with email only + channel: 'web', + context: { + traits: { + email: 'testklaviyo2@email.com', + firstname: 'Test Klaviyo 2', + }, + }, + type: 'track', + userId: 'testKlaviyo2', + event: 'purchase', + properties: { + price: '120', + }, + }, + metadata: generateMetadata(4, 'testKlaviyo2'), + destination, + }, + { + message: { + // for user 3 identify call without anonymousId and subscriptiontraits: + channel: 'web', + traits: { + email: 'testklaviyo3@rs.com', + firstname: 'Test Klaviyo 3', + properties: { + subscribe: false, + listId: 'configListId', + consent: ['email', 'sms'], + }, + }, + context: {}, + type: 'identify', + userId: 'testKlaviyo3', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(5, 'testKlaviyo3'), + destination, + }, + ], + destType: 'klaviyo', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + // user 1 track call with userId and anonymousId + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'event', + attributes: { + properties: { + price: '12', + }, + profile: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo1', + anonymous_id: 'anonTestKlaviyo1', + email: 'testklaviyo1@email.com', + first_name: 'Test Klaviyo 1', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'purchase', + }, + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1, 'testKlaviyo1')], + batched: false, + statusCode: 200, + destination, + }, + { + // anonn event for user 2 + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'event', + attributes: { + properties: { + price: '120', + }, + profile: { + data: { + type: 'profile', + attributes: { + anonymous_id: 'anonTestKlaviyo2', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'viewed product', + }, + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(2)], + batched: false, + statusCode: 200, + destination, + }, + { + // identify call for user 2 and user 3 with subscription + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo2', + anonymous_id: 'anonTestKlaviyo2', + email: 'testklaviyo2@rs.com', + first_name: 'Test Klaviyo 2', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-import', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo3', + email: 'testklaviyo3@rs.com', + first_name: 'Test Klaviyo 3', + properties: {}, + }, + meta: { + patch_properties: {}, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/profile-subscription-bulk-delete-jobs', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + 'Content-Type': 'application/json', + Accept: 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'profile-subscription-bulk-delete-job', + attributes: { + profiles: { + data: [ + { + type: 'profile', + attributes: { + email: 'testklaviyo2@rs.com', + }, + }, + { + type: 'profile', + attributes: { + email: 'testklaviyo3@rs.com', + }, + }, + ], + }, + }, + relationships: { + list: { + data: { + type: 'list', + id: 'configListId', + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [generateMetadata(3, 'testKlaviyo2'), generateMetadata(5, 'testKlaviyo3')], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://a.klaviyo.com/api/events', + headers: { + Authorization: 'Klaviyo-API-Key dummyPrivateApiKey', + Accept: 'application/json', + 'Content-Type': 'application/json', + revision: '2024-06-15', + }, + params: {}, + body: { + JSON: { + data: { + type: 'event', + attributes: { + properties: { + price: '120', + }, + profile: { + data: { + type: 'profile', + attributes: { + external_id: 'testKlaviyo2', + email: 'testklaviyo2@email.com', + first_name: 'Test Klaviyo 2', + properties: {}, + meta: { + patch_properties: {}, + }, + }, + }, + }, + metric: { + data: { + type: 'metric', + attributes: { + name: 'purchase', + }, + }, + }, + }, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4, 'testKlaviyo2')], + batched: false, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/sfmc/processor/data.ts b/test/integrations/destinations/sfmc/processor/data.ts index 883032d223..e8d9375e43 100644 --- a/test/integrations/destinations/sfmc/processor/data.ts +++ b/test/integrations/destinations/sfmc/processor/data.ts @@ -2216,4 +2216,182 @@ export const data = [ }, }, }, + { + name: 'sfmc', + description: 'success scenario for rETL use case', + feature: 'processor', + id: 'sfmcRetlTestCase-1', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + key2: 'value2', + key3: 'value3', + key4: 'value4', + }, + userId: 'someRandomEmail@test.com', + channel: 'sources', + context: { + sources: { + job_id: '2kbW13URkJ6jfeo5SbFcC7ecP6d', + version: 'v1.53.1', + job_run_id: 'cqtl6pfqskjtoh6t24i0', + task_run_id: 'cqtl6pfqskjtoh6t24ig', + }, + externalId: [ + { + id: 'someRandomEmail@test.com', + type: 'SFMC-data extension', + identifierType: 'key1', + }, + ], + mappedToDestination: 'true', + }, + recordId: '3', + rudderId: 'c5741aa5-b038-4079-99ec-e4169eb0d9e2', + messageId: '95a1b214-03d9-4824-8ada-bc6ef2398100', + }, + destination: { + ID: '1pYpzzvcn7AQ2W9GGIAZSsN6Mfq', + Name: 'SFMC', + Config: { + clientId: 'dummyClientId', + clientSecret: 'dummyClientSecret', + subDomain: 'vcn7AQ2W9GGIAZSsN6Mfq', + createOrUpdateContacts: false, + externalKey: 'externalKey', + }, + Enabled: true, + Transformations: [], + }, + metadata: { + destinationId: 'destId', + jobId: 'jobid1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + destinationId: 'destId', + jobId: 'jobid1', + }, + output: { + body: { + FORM: {}, + JSON: { + values: { + key2: 'value2', + key3: 'value3', + key4: 'value4', + }, + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: + 'https://vcn7AQ2W9GGIAZSsN6Mfq.rest.marketingcloudapis.com/hub/v1/dataevents/key:externalKey/rows/key1:someRandomEmail@test.com', + files: {}, + headers: { + Authorization: 'Bearer yourAuthToken', + 'Content-Type': 'application/json', + }, + method: 'PUT', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'sfmc', + description: 'failure scenario for rETL use case when wrong object type is used', + feature: 'processor', + id: 'sfmcRetlTestCase-2', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + traits: { + key2: 'value2', + key3: 'value3', + key4: 'value4', + }, + userId: 'someRandomEmail@test.com', + channel: 'sources', + context: { + externalId: [ + { + id: 'someRandomEmail@test.com', + type: 'SFMC-contacts', + identifierType: 'key1', + }, + ], + mappedToDestination: 'true', + }, + }, + destination: { + ID: '1pYpzzvcn7AQ2W9GGIAZSsN6Mfq', + Name: 'SFMC', + Config: { + clientId: 'dummyClientId', + clientSecret: 'dummyClientSecret', + subDomain: 'vcn7AQ2W9GGIAZSsN6Mfq', + createOrUpdateContacts: false, + externalKey: 'externalKey', + }, + Enabled: true, + Transformations: [], + }, + metadata: { + destinationId: 'destId', + jobId: 'jobid1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Unsupported object type for rETL use case', + metadata: { + destinationId: 'destId', + jobId: 'jobid1', + }, + statTags: { + destType: 'SFMC', + destinationId: 'destId', + errorCategory: 'platform', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/smartly/commonConfig.ts b/test/integrations/destinations/smartly/commonConfig.ts new file mode 100644 index 0000000000..f5b0a6f4d4 --- /dev/null +++ b/test/integrations/destinations/smartly/commonConfig.ts @@ -0,0 +1,40 @@ +export const destination = { + ID: 'random_id', + Name: 'smartly', + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiToken: 'testAuthToken', + eventsMapping: [ + { + from: 'product list viewed', + to: 'event1', + }, + { + from: 'product list viewed', + to: 'event2', + }, + ], + }, +}; + +export const routerInstrumentationErrorStatTags = { + destType: 'SMARTLY', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', +}; +export const processInstrumentationErrorStatTags = { + destType: 'SMARTLY', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'dummyDestId', +}; diff --git a/test/integrations/destinations/smartly/mocks.ts b/test/integrations/destinations/smartly/mocks.ts new file mode 100644 index 0000000000..78773d8853 --- /dev/null +++ b/test/integrations/destinations/smartly/mocks.ts @@ -0,0 +1,6 @@ +import config from '../../../../src/cdk/v2/destinations/smartly/config'; + +export const defaultMockFns = () => { + jest.useFakeTimers().setSystemTime(new Date('2024-02-01')); + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 2); +}; diff --git a/test/integrations/destinations/smartly/processor/data.ts b/test/integrations/destinations/smartly/processor/data.ts new file mode 100644 index 0000000000..a94f6b220f --- /dev/null +++ b/test/integrations/destinations/smartly/processor/data.ts @@ -0,0 +1,9 @@ +import { trackTestData } from './track'; +import { validationFailures } from './validation'; + +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2024-02-01')); +}; + +export const data = [...trackTestData, ...validationFailures].map((d) => ({ ...d, mockFns })); diff --git a/test/integrations/destinations/smartly/processor/track.ts b/test/integrations/destinations/smartly/processor/track.ts new file mode 100644 index 0000000000..944327fce3 --- /dev/null +++ b/test/integrations/destinations/smartly/processor/track.ts @@ -0,0 +1,71 @@ +import { destination } from '../commonConfig'; + +export const trackTestData = [ + { + name: 'smartly', + description: 'Test 0', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + event: 'Add to cart', + properties: { + platform: 'meta', + ad_unit_id: '228287', + ad_interaction_time: 1735680000, + email: 'eventIdn01@sample.com', + }, + type: 'track', + userId: 'eventIdn01', + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events', + headers: {}, + params: {}, + body: { + JSON: { + platform: 'meta', + ad_unit_id: '228287', + ad_interaction_time: 1735680000, + conversions: '1', + event_name: 'Add to cart', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/smartly/processor/validation.ts b/test/integrations/destinations/smartly/processor/validation.ts new file mode 100644 index 0000000000..996afc34b3 --- /dev/null +++ b/test/integrations/destinations/smartly/processor/validation.ts @@ -0,0 +1,174 @@ +import { processInstrumentationErrorStatTags, destination } from '../commonConfig'; + +export const validationFailures = [ + { + id: 'Smartly-validation-test-1', + name: 'smartly', + description: 'Required field anonymousId not present', + scenario: 'Framework', + successCriteria: 'Transformationn Error for anonymousId not present', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + event: 'product purchased', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'john123', + properties: { + products: [{}], + ad_unit_id: '22123387', + ad_interaction_time: '1690867200', + }, + integrations: { + All: true, + }, + originalTimestamp: '2021-01-25T15:32:56.409Z', + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Missing required value from ["properties.platform"]: Workflow: procWorkflow, Step: preparePayload, ChildStep: undefined, OriginalError: Missing required value from ["properties.platform"]', + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + statTags: processInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'Smartly-test-2', + name: 'smartly', + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Transformationn Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + event_name: 'purchase', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'john67', + channel: 'mobile', + rudderId: 'b7b24f86-cccx-46d8-b2b4-ccaxxx80239c', + messageId: 'dummy_msg_id', + properties: { + platform: 'snapchat', + ad_unit_id: '2653387', + ad_interaction_time: '1690867200', + }, + anonymousId: 'anon_123', + integrations: { + All: true, + }, + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + statTags: processInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'Smartly-test-3', + name: 'smartly', + description: 'Event name not defined', + scenario: 'Framework', + successCriteria: 'Transformationn Error for Undefined Event', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + sentAt: '2021-01-25T16:12:02.048Z', + userId: 'john67', + channel: 'mobile', + rudderId: 'b7b24f86-cccx-46d8-b2b4-ccaxxx80239c', + messageId: 'dummy_msg_id', + properties: { + platform: 'snapchat', + ad_unit_id: '2653387', + ad_interaction_time: 1675094400, + }, + anonymousId: 'anon_123', + integrations: { + All: true, + }, + }, + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Event is not defined or is not String: Workflow: procWorkflow, Step: preparePayload, ChildStep: undefined, OriginalError: Event is not defined or is not String', + metadata: { + destinationId: 'dummyDestId', + jobId: '1', + }, + statTags: processInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/smartly/router/data.ts b/test/integrations/destinations/smartly/router/data.ts new file mode 100644 index 0000000000..7c2d74e6f0 --- /dev/null +++ b/test/integrations/destinations/smartly/router/data.ts @@ -0,0 +1,329 @@ +import { destination, routerInstrumentationErrorStatTags } from '../commonConfig'; +import { defaultMockFns } from '../mocks'; + +export const data = [ + { + name: 'smartly', + id: 'Test 0 - router', + description: 'Track call with multiplexing and batching', + scenario: 'Framework+Buisness', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'track', + event: 'product list viewed', + properties: { + platform: 'meta', + conversions: 1, + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 2, userId: 'u2' }, + destination, + }, + { + message: { + type: 'track', + event: 'add to cart', + properties: { + conversions: 3, + platform: 'snapchat', + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 3, userId: 'u3' }, + destination, + }, + ], + destType: 'smartly', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + body: { + JSON: { + events: [ + { + conversions: 1, + ad_unit_id: '221187', + platform: 'meta', + ad_interaction_time: 1690867200, + event_name: 'event1', + }, + { + conversions: 1, + ad_unit_id: '221187', + platform: 'meta', + ad_interaction_time: 1690867200, + event_name: 'event2', + }, + { + conversions: 3, + ad_unit_id: '77187', + platform: 'snapchat', + ad_interaction_time: 1690867200, + event_name: 'add to cart', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events/batch', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer testAuthToken`, + }, + params: {}, + files: {}, + }, + metadata: [ + { + jobId: 2, + userId: 'u2', + }, + { + jobId: 3, + userId: 'u3', + }, + ], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + mockFns: () => { + jest.useFakeTimers().setSystemTime(new Date('2024-02-01')); + }, + }, + { + name: 'smartly', + id: 'Test 1 - router', + description: 'Batch calls with 4 succesfull events including multiplexing and 2 failed events', + scenario: 'Framework+Buisness', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'track', + event: 'product list viewed', + properties: { + platform: 'meta', + conversions: 1, + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 11, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + event: 'purchase', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + platform: 'snapchat', + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 13, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + platform: 'snapchat', + ad_unit_id: '12387', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 14, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + event: 'random event', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 15, userId: 'u1' }, + destination, + }, + { + message: { + type: 'track', + event: 'add to cart', + userId: 'testuserId1', + integrations: { All: true }, + properties: { + conversions: 3, + platform: 'tiktok', + ad_unit_id: '789187', + ad_interaction_time: 1690867200, + }, + }, + metadata: { jobId: 16, userId: 'u1' }, + destination, + }, + ], + destType: 'smartly', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: false, + destination, + error: 'Event is not defined or is not String', + metadata: [{ jobId: 14, userId: 'u1' }], + statTags: routerInstrumentationErrorStatTags, + statusCode: 400, + }, + { + batched: false, + destination, + error: 'Missing required value from ["properties.platform"]', + metadata: [{ jobId: 15, userId: 'u1' }], + statTags: routerInstrumentationErrorStatTags, + statusCode: 400, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events/batch', + params: {}, + body: { + FORM: {}, + JSON: { + events: [ + { + platform: 'meta', + conversions: 1, + event_name: 'event1', + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + { + platform: 'meta', + conversions: 1, + event_name: 'event2', + ad_unit_id: '221187', + ad_interaction_time: 1690867200, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer testAuthToken`, + }, + files: {}, + }, + metadata: [{ jobId: 11, userId: 'u1' }], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://s2s.smartly.io/events/batch', + params: {}, + body: { + FORM: {}, + JSON: { + events: [ + { + conversions: 3, + event_name: 'purchase', + platform: 'snapchat', + ad_unit_id: '77187', + ad_interaction_time: 1690867200, + }, + { + conversions: 3, + event_name: 'add to cart', + platform: 'tiktok', + ad_unit_id: '789187', + ad_interaction_time: 1690867200, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer testAuthToken`, + }, + files: {}, + }, + metadata: [ + { jobId: 13, userId: 'u1' }, + { jobId: 16, userId: 'u1' }, + ], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/sources/auth0/data.ts b/test/integrations/sources/auth0/data.ts index daedf1b75d..b012887bc4 100644 --- a/test/integrations/sources/auth0/data.ts +++ b/test/integrations/sources/auth0/data.ts @@ -1508,6 +1508,7 @@ export const data = [ description: 'empty batch', module: 'source', version: 'v0', + skipGo: 'Created this case manually', input: { request: { body: [], diff --git a/test/integrations/sources/iterable/data.ts b/test/integrations/sources/iterable/data.ts index b6fda071e4..8912c83434 100644 --- a/test/integrations/sources/iterable/data.ts +++ b/test/integrations/sources/iterable/data.ts @@ -2216,4 +2216,7 @@ export const data = [ }, }, }, -]; +].map((tc) => ({ + ...tc, + overrideReceivedAt: true, +})); diff --git a/test/integrations/sources/mailjet/data.ts b/test/integrations/sources/mailjet/data.ts index f7f56182c8..2a8f3eaf46 100644 --- a/test/integrations/sources/mailjet/data.ts +++ b/test/integrations/sources/mailjet/data.ts @@ -236,6 +236,7 @@ export const data = [ description: 'MailJet when no email is present', module: 'source', version: 'v0', + skipGo: 'FIXME', input: { request: { body: [ diff --git a/test/integrations/sources/moengage/data.ts b/test/integrations/sources/moengage/data.ts index e8160ae08b..c307959121 100644 --- a/test/integrations/sources/moengage/data.ts +++ b/test/integrations/sources/moengage/data.ts @@ -264,6 +264,8 @@ const data = [ description: 'Batch of events', module: 'source', version: 'v0', + overrideReceivedAt: true, + overrideRequestIP: true, input: { request: { body: [ diff --git a/test/integrations/sources/segment/data.ts b/test/integrations/sources/segment/data.ts index d16176e8f1..8c66abd252 100644 --- a/test/integrations/sources/segment/data.ts +++ b/test/integrations/sources/segment/data.ts @@ -11,6 +11,7 @@ export const data: SrcTestCaseData[] = [ description: 'test-0', module: 'source', version: 'v0', + skipGo: 'NoAnonID error', input: { request: { body: [ diff --git a/test/integrations/testTypes.ts b/test/integrations/testTypes.ts index 3ccc792caf..3c5cf60600 100644 --- a/test/integrations/testTypes.ts +++ b/test/integrations/testTypes.ts @@ -52,6 +52,8 @@ export interface TestCaseData { input: inputType; output: outputType; mock?: mockType[]; + overrideReceivedAt?: string; + overrideRequestIP?: string; mockFns?: (mockAdapter: MockAdapter) => {}; } diff --git a/test/scripts/generateJson.ts b/test/scripts/generateJson.ts index a4a254f8f3..388c81477d 100644 --- a/test/scripts/generateJson.ts +++ b/test/scripts/generateJson.ts @@ -61,6 +61,9 @@ function getErrorResponse(outputResponse?: responseType) { const errorResponse = bodyKeys .map((statusKey) => get(outputResponse, statusKey)) .find(isDefinedAndNotNull); + if (errorResponse) { + return errorResponse + '\n'; + } return errorResponse; } @@ -72,6 +75,9 @@ function getSourceRequestBody(testCase: any, version?: string) { if (version === 'v0') { return bodyElement; } + if (Array.isArray(bodyElement?.event)) { + return bodyElement.event.map((e) => ({ ...e, source: bodyElement.source })); + } return { ...bodyElement.event, source: bodyElement.source }; } @@ -158,11 +164,17 @@ function generateSources(outputFolder: string, options: OptionValues) { goTest.skip = testCase.skipGo; } - goTest.output.queue.forEach((queueItem) => { + goTest.output.queue.forEach((queueItem, i) => { queueItem['receivedAt'] = - testCase.output.response?.body?.[0]?.output?.batch?.[0]?.receivedAt ?? - '2024-03-03T04:48:29.000Z'; - queueItem['request_ip'] = '192.0.2.30'; + testCase?.overrideReceivedAt && + testCase.output.response?.body?.[0]?.output?.batch?.[i]?.receivedAt + ? testCase.output.response?.body?.[0]?.output?.batch?.[i]?.receivedAt + : '2024-03-03T04:48:29.000Z'; + queueItem['request_ip'] = + testCase?.overrideRequestIP && + testCase.output.response?.body?.[0]?.output?.batch?.[i]?.request_ip + ? testCase.output.response?.body?.[0]?.output?.batch?.[i]?.request_ip + : '192.0.2.30'; if (!queueItem['messageId']) { queueItem['messageId'] = '00000000-0000-0000-0000-000000000000'; }