diff --git a/.gitignore b/.gitignore index 6689cb02..8a02dabf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ transcend.yml transcend-privacy-requests-cache.json cron-identifiers.csv manual-enrichment-identifiers.csv +preference-management-upload-receipts.json build/ privacy-request-upload-receipts/ diff --git a/.pnp.cjs b/.pnp.cjs index e7071bb5..b8422964 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -32,13 +32,14 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@transcend-io/handlebars-utils", "npm:1.1.0"],\ ["@transcend-io/internationalization", "npm:1.6.0"],\ ["@transcend-io/persisted-state", "npm:1.0.4"],\ - ["@transcend-io/privacy-types", "npm:4.94.0"],\ + ["@transcend-io/privacy-types", "npm:4.98.0"],\ ["@transcend-io/secret-value", "npm:1.2.0"],\ ["@transcend-io/type-utils", "npm:1.5.0"],\ ["@types/bluebird", "npm:3.5.38"],\ ["@types/chai", "npm:4.3.4"],\ ["@types/cli-progress", "npm:3.11.0"],\ ["@types/colors", "npm:1.2.1"],\ + ["@types/deep-equal-in-any-order", "npm:1.0.1"],\ ["@types/fuzzysearch", "npm:1.0.0"],\ ["@types/global-agent", "npm:2.1.1"],\ ["@types/inquirer", "npm:7.3.3"],\ @@ -59,6 +60,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["cli-progress", "npm:3.11.2"],\ ["colors", "npm:1.4.0"],\ ["csv-parse", "npm:4.9.1"],\ + ["deep-equal-in-any-order", "npm:1.1.20"],\ ["depcheck", "npm:1.4.3"],\ ["eslint", "npm:8.38.0"],\ ["eslint-config-airbnb-base", "virtual:05f71a130d973c89ca25c909f694e525ea840a597fa145c0b76e367faaae056c000c13c8dc936d4a8d6e8c6aa1f1290e6e296ad4666bf503fd491f764698380a#npm:15.0.0"],\ @@ -682,13 +684,14 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@transcend-io/handlebars-utils", "npm:1.1.0"],\ ["@transcend-io/internationalization", "npm:1.6.0"],\ ["@transcend-io/persisted-state", "npm:1.0.4"],\ - ["@transcend-io/privacy-types", "npm:4.94.0"],\ + ["@transcend-io/privacy-types", "npm:4.98.0"],\ ["@transcend-io/secret-value", "npm:1.2.0"],\ ["@transcend-io/type-utils", "npm:1.5.0"],\ ["@types/bluebird", "npm:3.5.38"],\ ["@types/chai", "npm:4.3.4"],\ ["@types/cli-progress", "npm:3.11.0"],\ ["@types/colors", "npm:1.2.1"],\ + ["@types/deep-equal-in-any-order", "npm:1.0.1"],\ ["@types/fuzzysearch", "npm:1.0.0"],\ ["@types/global-agent", "npm:2.1.1"],\ ["@types/inquirer", "npm:7.3.3"],\ @@ -709,6 +712,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["cli-progress", "npm:3.11.2"],\ ["colors", "npm:1.4.0"],\ ["csv-parse", "npm:4.9.1"],\ + ["deep-equal-in-any-order", "npm:1.1.20"],\ ["depcheck", "npm:1.4.3"],\ ["eslint", "npm:8.38.0"],\ ["eslint-config-airbnb-base", "virtual:05f71a130d973c89ca25c909f694e525ea840a597fa145c0b76e367faaae056c000c13c8dc936d4a8d6e8c6aa1f1290e6e296ad4666bf503fd491f764698380a#npm:15.0.0"],\ @@ -781,10 +785,10 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { }]\ ]],\ ["@transcend-io/privacy-types", [\ - ["npm:4.94.0", {\ - "packageLocation": "./.yarn/cache/@transcend-io-privacy-types-npm-4.94.0-c5ce6b7558-8cc1a8dd54.zip/node_modules/@transcend-io/privacy-types/",\ + ["npm:4.98.0", {\ + "packageLocation": "./.yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip/node_modules/@transcend-io/privacy-types/",\ "packageDependencies": [\ - ["@transcend-io/privacy-types", "npm:4.94.0"],\ + ["@transcend-io/privacy-types", "npm:4.98.0"],\ ["@transcend-io/type-utils", "npm:1.0.5"],\ ["fp-ts", "npm:2.16.1"],\ ["io-ts", "virtual:a57afaf9d13087a7202de8c93ac4854c9e2828bad7709250829ec4c7bc9dc95ecc2858c25612aa1774c986aedc232c76957076a1da3156fd2ab63ae5551b086f#npm:2.2.21"]\ @@ -929,6 +933,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@types/deep-equal-in-any-order", [\ + ["npm:1.0.1", {\ + "packageLocation": "./.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip/node_modules/@types/deep-equal-in-any-order/",\ + "packageDependencies": [\ + ["@types/deep-equal-in-any-order", "npm:1.0.1"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@types/emscripten", [\ ["npm:1.39.6", {\ "packageLocation": "./.yarn/cache/@types-emscripten-npm-1.39.6-c9c4021365-437f2f9cdf.zip/node_modules/@types/emscripten/",\ @@ -2617,6 +2630,17 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["deep-equal-in-any-order", [\ + ["npm:1.1.20", {\ + "packageLocation": "./.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip/node_modules/deep-equal-in-any-order/",\ + "packageDependencies": [\ + ["deep-equal-in-any-order", "npm:1.1.20"],\ + ["lodash.mapvalues", "npm:4.6.0"],\ + ["sort-any", "npm:2.0.0"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["deep-is", [\ ["npm:0.1.4", {\ "packageLocation": "./.yarn/cache/deep-is-npm-0.1.4-88938b5a67-edb65dd0d7.zip/node_modules/deep-is/",\ @@ -5137,6 +5161,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["lodash.mapvalues", [\ + ["npm:4.6.0", {\ + "packageLocation": "./.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip/node_modules/lodash.mapvalues/",\ + "packageDependencies": [\ + ["lodash.mapvalues", "npm:4.6.0"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["lodash.merge", [\ ["npm:4.6.2", {\ "packageLocation": "./.yarn/cache/lodash.merge-npm-4.6.2-77cb4416bf-ad580b4bdb.zip/node_modules/lodash.merge/",\ @@ -6613,6 +6646,16 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["sort-any", [\ + ["npm:2.0.0", {\ + "packageLocation": "./.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip/node_modules/sort-any/",\ + "packageDependencies": [\ + ["sort-any", "npm:2.0.0"],\ + ["lodash", "npm:4.17.21"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["source-map", [\ ["npm:0.6.1", {\ "packageLocation": "./.yarn/cache/source-map-npm-0.6.1-1a3621db16-59ce8640cf.zip/node_modules/source-map/",\ diff --git a/.prettierignore b/.prettierignore index e3d7ce45..a9436136 100644 --- a/.prettierignore +++ b/.prettierignore @@ -51,3 +51,4 @@ yarn.lock .idea .gitkeep .yarn/ +transcend-yml-schema-*.json diff --git a/.yarn/cache/@transcend-io-privacy-types-npm-4.94.0-c5ce6b7558-8cc1a8dd54.zip b/.yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip similarity index 89% rename from .yarn/cache/@transcend-io-privacy-types-npm-4.94.0-c5ce6b7558-8cc1a8dd54.zip rename to .yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip index 0a9da4e9..b67220c3 100644 Binary files a/.yarn/cache/@transcend-io-privacy-types-npm-4.94.0-c5ce6b7558-8cc1a8dd54.zip and b/.yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip differ diff --git a/.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip b/.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip new file mode 100644 index 00000000..72ad4c96 Binary files /dev/null and b/.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip differ diff --git a/.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip b/.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip new file mode 100644 index 00000000..48eef882 Binary files /dev/null and b/.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip differ diff --git a/.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip b/.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip new file mode 100644 index 00000000..063b2daa Binary files /dev/null and b/.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip differ diff --git a/.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip b/.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip new file mode 100644 index 00000000..0be1d132 Binary files /dev/null and b/.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip differ diff --git a/README.md b/README.md index 1b0a9cd6..f15b1b2d 100644 --- a/README.md +++ b/README.md @@ -118,30 +118,35 @@ - [Authentication](#authentication-26) - [Arguments](#arguments-26) - [Usage](#usage-27) - - [tr-upload-consent-preferences](#tr-upload-consent-preferences) + - [tr-upload-preferences](#tr-upload-preferences) - [Authentication](#authentication-27) + - [Authentication](#authentication-28) - [Arguments](#arguments-27) - [Usage](#usage-28) - - [tr-pull-consent-preferences](#tr-pull-consent-preferences) - - [Authentication](#authentication-28) + - [tr-upload-consent-preferences](#tr-upload-consent-preferences) + - [Authentication](#authentication-29) - [Arguments](#arguments-28) - [Usage](#usage-29) - - [tr-upload-data-flows-from-csv](#tr-upload-data-flows-from-csv) - - [Authentication](#authentication-29) + - [tr-pull-consent-preferences](#tr-pull-consent-preferences) + - [Authentication](#authentication-30) - [Arguments](#arguments-29) - [Usage](#usage-30) - - [tr-upload-cookies-from-csv](#tr-upload-cookies-from-csv) - - [Authentication](#authentication-30) + - [tr-upload-data-flows-from-csv](#tr-upload-data-flows-from-csv) + - [Authentication](#authentication-31) - [Arguments](#arguments-30) - [Usage](#usage-31) - - [tr-generate-api-keys](#tr-generate-api-keys) - - [Authentication](#authentication-31) + - [tr-upload-cookies-from-csv](#tr-upload-cookies-from-csv) + - [Authentication](#authentication-32) - [Arguments](#arguments-31) - [Usage](#usage-32) - - [tr-build-xdi-sync-endpoint](#tr-build-xdi-sync-endpoint) - - [Authentication](#authentication-32) + - [tr-generate-api-keys](#tr-generate-api-keys) + - [Authentication](#authentication-33) - [Arguments](#arguments-32) - [Usage](#usage-33) + - [tr-build-xdi-sync-endpoint](#tr-build-xdi-sync-endpoint) + - [Authentication](#authentication-34) + - [Arguments](#arguments-33) + - [Usage](#usage-34) - [Prompt Manager](#prompt-manager) - [Proxy usage](#proxy-usage) @@ -2296,6 +2301,71 @@ Specify the backend URL, needed for US hosted backend infrastructure: yarn tr-pull-datapoints --auth=$TRANSCEND_API_KEY --file=./datapoints.csv --transcendUrl=https://api.us.transcend.io ``` +### tr-upload-preferences + +This command allows for updating of preference management data to your Transcend Preference Store. + +This command uses [inquirer](https://github.com/SBoudrias/Inquirer.js/) to prompt the user to +map the shape of the CSV to the shape of the Transcend API. There is no requirement for the +shape of the incoming CSV, as the script will handle the mapping process. + +The script will also produce a JSON cache file, that allows for the mappings to be preserved between runs. +This can be useful if you have the same CSV shape that needs to be imported multiple times. +Once the mapping process is done once, it does not need to be done again. + +Additionally, the JSON cache file will store the result of any preferences that fail to be uploaded so that the +script can be run multiple times if an issue happens. + +#### Authentication + +In order to use this cli, you will first need to generate an API key on the Transcend Admin Dashboard (https://app.transcend.io/infrastructure/api-keys). + +The API key needs the following scopes: + +- Modify User Stored Preferences +- View Managed Consent Database Admin API +- View Preference Store Settings + +#### Authentication + +In order to use this cli, you will first need to follow [this guide](https://docs.transcend.io/docs/consent/reference/managed-consent-database#authenticate-a-user's-consent) in order + +#### Arguments + +| Argument | Description | Type | Default | Required | +| -------------------- | ------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------- | -------- | +| auth | The Transcend API key with the scopes necessary for the command. | string | N/A | true | +| partition | The partition key to download consent preferences to | string | N/A | true | +| sombraAuth | The sombra internal key, use for additional authentication when self-hosting sombra. | string | N/A | false | +| transcendUrl | URL of the Transcend backend. Use https://consent.us.transcend.io for US hosting. | string - URL | https://consent.transcend.io | false | +| file | Path to the CSV file to load preferences from | string - file-path | ./preferences.csv | false | +| dryRun | Whether to do a dry run only - will write results ti receiptFilepath without updating Transcend | boolean | false | false | +| skipWorkflowTriggers | Whether to skip workflow triggers when uploading to preference store | boolean | false | false | +| skipConflictUpdates | Whether to skip uploading of any records where the preference store and file have a hard conflict | boolean | false | false | +| isSilent | Whether to skip sending emails in workflows | boolean | true | false | +| attributes | Attributes to add to any DSR request if created | string | Tags:transcend-cli,Source:transcend-cli | false | +| receiptFilepath | Store resulting, continuing where left off | string - file-path | ./preference-management-upload-receipts.json | false | + +#### Usage + +Upload consent preferences to partition key `4d1c5daa-90b7-4d18-aa40-f86a43d2c726` + +```sh +yarn tr-upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 +``` + +Upload consent preferences with additional options + +```sh +yarn tr-upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 --file=./preferences.csv --dryRun=true --skipWorkflowTriggers=true --skipConflictUpdates=true --isSilent=false --attributes="Tags:transcend-cli,Source:transcend-cli" --receiptFilepath=./preference-management-upload-receipts.json +``` + +Specifying the backend URL, needed for US hosted backend infrastructure. + +```sh +yarn tr-upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 --transcendUrl=https://consent.us.transcend.io +``` + ### tr-upload-consent-preferences This command allows for updating of consent preferences to the [Managed Consent Database](https://docs.transcend.io/docs/consent/reference/managed-consent-database). diff --git a/package.json b/package.json index 3d94b336..880a3a71 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "author": "Transcend Inc.", "name": "@transcend-io/cli", "description": "Small package containing useful typescript utilities.", - "version": "6.9.0", + "version": "6.10.0", "homepage": "https://github.com/transcend-io/cli", "repository": { "type": "git", @@ -44,7 +44,8 @@ "tr-update-consent-manager": "./build/cli-update-consent-manager-to-latest.js", "tr-upload-consent-preferences": "./build/cli-upload-consent-preferences.js", "tr-upload-cookies-from-csv": "./build/cli-upload-cookies-from-csv.js", - "tr-upload-data-flows-from-csv": "./build/cli-upload-data-flows-from-csv.js" + "tr-upload-data-flows-from-csv": "./build/cli-upload-data-flows-from-csv.js", + "tr-upload-preferences": "./build/cli-upload-preferences.js" }, "files": [ "build/**/*", @@ -66,7 +67,7 @@ "@transcend-io/handlebars-utils": "^1.1.0", "@transcend-io/internationalization": "^1.6.0", "@transcend-io/persisted-state": "^1.0.4", - "@transcend-io/privacy-types": "^4.94.0", + "@transcend-io/privacy-types": "^4.98.0", "@transcend-io/secret-value": "^1.2.0", "@transcend-io/type-utils": "^1.5.0", "bluebird": "^3.7.2", @@ -99,6 +100,7 @@ "@types/chai": "^4.3.4", "@types/cli-progress": "^3.11.0", "@types/colors": "^1.2.1", + "@types/deep-equal-in-any-order": "1.0.1", "@types/fuzzysearch": "^1.0.0", "@types/global-agent": "^2.1.1", "@types/inquirer": "^7.3.1", @@ -115,6 +117,7 @@ "@typescript-eslint/parser": "^5.58.0", "@yarnpkg/sdks": "^3.0.0-rc.42", "chai": "^4.3.7", + "deep-equal-in-any-order": "^1.0.28", "depcheck": "^1.4.3", "eslint": "^8.38.0", "eslint-config-airbnb-base": "^15.0.0", diff --git a/src/cli-request-upload.ts b/src/cli-request-upload.ts index 2d94a65f..557f73e2 100644 --- a/src/cli-request-upload.ts +++ b/src/cli-request-upload.ts @@ -73,7 +73,7 @@ async function main(): Promise { skipSendingReceipt: skipSendingReceipt === 'true', emailIsVerified: emailIsVerified === 'true', isTest: isTest === 'true', - dryRun: dryRun === 'true', + dryRun: dryRun !== 'false', }); } diff --git a/src/cli-upload-consent-preferences.ts b/src/cli-upload-consent-preferences.ts index 20ffdb01..5464f7b8 100644 --- a/src/cli-upload-consent-preferences.ts +++ b/src/cli-upload-consent-preferences.ts @@ -5,7 +5,7 @@ import colors from 'colors'; import { logger } from './logger'; import { DEFAULT_TRANSCEND_CONSENT_API } from './constants'; -import { uploadConsentPreferences } from './consent-manager/uploadConsentPreferences'; +import { uploadConsents } from './consent-manager/uploadConsents'; import { ConsentPreferenceUpload } from './consent-manager/types'; import { readCsv } from './requests'; @@ -75,7 +75,7 @@ async function main(): Promise { const preferences = readCsv(file, ConsentPreferenceUpload); // Upload cookies - await uploadConsentPreferences({ + await uploadConsents({ base64EncryptionKey, base64SigningKey, preferences, diff --git a/src/cli-upload-preferences.ts b/src/cli-upload-preferences.ts new file mode 100644 index 00000000..b63550ec --- /dev/null +++ b/src/cli-upload-preferences.ts @@ -0,0 +1,89 @@ +#!/usr/bin/env node + +import yargs from 'yargs-parser'; +import colors from 'colors'; + +import { logger } from './logger'; +import { DEFAULT_TRANSCEND_API } from './constants'; +import { uploadPreferenceManagementPreferencesInteractive } from './preference-management'; +import { splitCsvToList } from './requests'; + +/** + * Upload consent preferences to the managed consent database + * + * Requires following documentation found at: + * https://docs.transcend.io/docs/consent/reference/managed-consent-database + * + * Dev Usage: + * yarn ts-node ./src/cli-upload-consent-preferences-interactive --base64EncryptionKey=$TRANSCEND_CONSENT_ENCRYPTION_KEY \ + * --base64SigningKey=$TRANSCEND_CONSENT_SIGNING_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 + * + * Standard usage: + * yarn tr-upload-consent-preferences-interactive --base64EncryptionKey=$TRANSCEND_CONSENT_ENCRYPTION_KEY \ + * --base64SigningKey=$TRANSCEND_CONSENT_SIGNING_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 + */ +async function main(): Promise { + // Parse command line arguments + const { + /** File to load preferences from */ + file = './preferences.csv', + /** Transcend URL */ + transcendUrl = DEFAULT_TRANSCEND_API, + /** API key */ + auth, + /** Sombra API key */ + sombraAuth, + /** Partition key to load into */ + partition, + /** Whether to do a dry run */ + dryRun = 'false', + /** Whether to skip workflow triggers */ + skipWorkflowTriggers = 'false', + /** Whether to skip conflict updates */ + skipConflictUpdates = 'false', + /** Whether to skip sending emails */ + isSilent = 'true', + /** Attributes to add to any DSR request if created */ + attributes = 'Tags:transcend-cli,Source:transcend-cli', + /** Store resulting, continuing where left off */ + receiptFilepath = './preference-management-upload-receipts.json', + } = yargs(process.argv.slice(2)) as { [k in string]: string }; + + // Ensure auth is passed + if (!auth) { + logger.error( + colors.red( + 'A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY', + ), + ); + process.exit(1); + } + + // Ensure partition + if (!partition) { + logger.error( + colors.red( + 'A partition must be provided. ' + + 'You can specify using --partition=ee1a0845-694e-4820-9d51-50c7d0a23467', + ), + ); + process.exit(1); + } + + // Upload cookies + await uploadPreferenceManagementPreferencesInteractive({ + receiptFilepath, + auth, + sombraAuth, + file, + partition, + transcendUrl, + skipConflictUpdates: skipConflictUpdates !== 'false', + skipWorkflowTriggers: skipWorkflowTriggers !== 'false', + isSilent: isSilent !== 'false', + dryRun: dryRun !== 'false', + attributes: splitCsvToList(attributes), + }); +} + +main(); diff --git a/src/consent-manager/index.ts b/src/consent-manager/index.ts index 77b4bca0..a1f4e911 100644 --- a/src/consent-manager/index.ts +++ b/src/consent-manager/index.ts @@ -7,3 +7,4 @@ export * from './consentManagersToBusinessEntities'; export * from './domainToHost'; export * from './createConsentToken'; export * from './fetchConsentPreferences'; +export * from './uploadConsents'; diff --git a/src/consent-manager/uploadConsentPreferences.ts b/src/consent-manager/uploadConsents.ts similarity index 99% rename from src/consent-manager/uploadConsentPreferences.ts rename to src/consent-manager/uploadConsents.ts index 17d83d4b..72acbd2e 100644 --- a/src/consent-manager/uploadConsentPreferences.ts +++ b/src/consent-manager/uploadConsents.ts @@ -22,7 +22,7 @@ export const PurposeMap = t.record( * * @param options - Options */ -export async function uploadConsentPreferences({ +export async function uploadConsents({ base64EncryptionKey, base64SigningKey, preferences, diff --git a/src/graphql/createSombraGotInstance.ts b/src/graphql/createSombraGotInstance.ts index d02b5195..aca717ce 100644 --- a/src/graphql/createSombraGotInstance.ts +++ b/src/graphql/createSombraGotInstance.ts @@ -19,7 +19,6 @@ export async function createSombraGotInstance( ): Promise { // Create GraphQL client to connect to Transcend backend const client = buildTranscendGraphQLClient(transcendUrl, transcendApiKey); - // Grab metadata about organization's sombra from GraphQL endpoint const { organization } = await makeGraphQLRequest<{ /** Requests */ @@ -31,7 +30,6 @@ export async function createSombraGotInstance( }; }; }>(client, ORGANIZATION); - // Create got instance with default values return got.extend({ prefixUrl: organization.sombra.customerUrl, diff --git a/src/graphql/fetchAllPreferenceTopics.ts b/src/graphql/fetchAllPreferenceTopics.ts new file mode 100644 index 00000000..2de6d4bf --- /dev/null +++ b/src/graphql/fetchAllPreferenceTopics.ts @@ -0,0 +1,65 @@ +import { GraphQLClient } from 'graphql-request'; +import { PREFERENCE_TOPICS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; + +export interface PreferenceTopic { + /** ID of preference topic */ + id: string; + /** Slug of preference topic */ + slug: string; + /** Type of preference topic */ + type: PreferenceTopicType; + /** Option values */ + preferenceOptionValues: { + /** Slug of value */ + slug: string; + }[]; + /** Related purpose */ + purpose: { + /** Slug */ + trackingType: string; + }; +} + +const PAGE_SIZE = 20; + +/** + * Fetch all preference topics in the organization + * + * @param client - GraphQL client + * @returns All preference topics in the organization + */ +export async function fetchAllPreferenceTopics( + client: GraphQLClient, +): Promise { + const preferenceTopics: PreferenceTopic[] = []; + let offset = 0; + + // Whether to continue looping + let shouldContinue = false; + do { + const { + preferenceTopics: { nodes }, + // eslint-disable-next-line no-await-in-loop + } = await makeGraphQLRequest<{ + /** Preference topics */ + preferenceTopics: { + /** List */ + nodes: PreferenceTopic[]; + }; + }>(client, PREFERENCE_TOPICS, { + first: PAGE_SIZE, + offset, + }); + preferenceTopics.push(...nodes); + offset += PAGE_SIZE; + shouldContinue = nodes.length === PAGE_SIZE; + } while (shouldContinue); + + return preferenceTopics.sort((a, b) => + `${a.slug}:${a.purpose.trackingType}`.localeCompare( + `${b.slug}:${b.purpose.trackingType}`, + ), + ); +} diff --git a/src/graphql/fetchAllPurposes.ts b/src/graphql/fetchAllPurposes.ts new file mode 100644 index 00000000..73564d23 --- /dev/null +++ b/src/graphql/fetchAllPurposes.ts @@ -0,0 +1,64 @@ +import { GraphQLClient } from 'graphql-request'; +import { PURPOSES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; + +export interface Purpose { + /** ID of purpose */ + id: string; + /** Name of purpose */ + name: string; + /** Slug of purpose */ + trackingType: string; + /** Whether the purpose is active */ + isActive: boolean; + /** Whether the purpose is deleted */ + deletedAt?: string; +} + +const PAGE_SIZE = 20; + +/** + * Fetch all purposes in the organization + * + * @param client - GraphQL client + * @param input - Input + * @returns All purposes in the organization + */ +export async function fetchAllPurposes( + client: GraphQLClient, + { + includeDeleted = false, + }: { + /** Whether to include deleted purposes */ + includeDeleted?: boolean; + } = {}, +): Promise { + const purposes: Purpose[] = []; + let offset = 0; + + // Whether to continue looping + let shouldContinue = false; + do { + const { + purposes: { nodes }, + // eslint-disable-next-line no-await-in-loop + } = await makeGraphQLRequest<{ + /** Purposes */ + purposes: { + /** List */ + nodes: Purpose[]; + }; + }>(client, PURPOSES, { + first: PAGE_SIZE, + offset, + input: { + includeDeleted, + }, + }); + purposes.push(...nodes); + offset += PAGE_SIZE; + shouldContinue = nodes.length === PAGE_SIZE; + } while (shouldContinue); + + return purposes.sort((a, b) => a.trackingType.localeCompare(b.trackingType)); +} diff --git a/src/graphql/fetchConsentManagerId.ts b/src/graphql/fetchConsentManagerId.ts index efa3ca96..6420d54a 100644 --- a/src/graphql/fetchConsentManagerId.ts +++ b/src/graphql/fetchConsentManagerId.ts @@ -19,7 +19,6 @@ import { FETCH_CONSENT_MANAGER_ID, FETCH_CONSENT_MANAGER, EXPERIENCES, - PURPOSES, CONSENT_MANAGER_ANALYTICS_DATA, FETCH_CONSENT_MANAGER_THEME, } from './gqls'; @@ -118,27 +117,6 @@ export interface ConsentPurpose { trackingType: string; } -/** - * Fetch consent manager purposes - * - * @param client - GraphQL client - * @returns Consent manager purposes in the organization - */ -export async function fetchPurposes( - client: GraphQLClient, -): Promise { - const { - purposes: { purposes }, - } = await makeGraphQLRequest<{ - /** Consent manager query */ - purposes: { - /** Consent manager object */ - purposes: ConsentPurpose[]; - }; - }>(client, PURPOSES); - return purposes; -} - const PAGE_SIZE = 50; export interface ConsentExperience { diff --git a/src/graphql/gqls/consentManager.ts b/src/graphql/gqls/consentManager.ts index 3ac36fcd..de4cca45 100644 --- a/src/graphql/gqls/consentManager.ts +++ b/src/graphql/gqls/consentManager.ts @@ -1,18 +1,6 @@ /* eslint-disable max-lines */ import { gql } from 'graphql-request'; -export const PURPOSES = gql` - query TranscendCliPurposes { - purposes { - purposes { - id - name - trackingType - } - } - } -`; - // TODO: https://transcend.height.app/T-27909 - order by createdAt // TODO: https://transcend.height.app/T-27909 - enable optimizations // isExportCsv: true diff --git a/src/graphql/gqls/index.ts b/src/graphql/gqls/index.ts index b39f98b3..24b61da2 100644 --- a/src/graphql/gqls/index.ts +++ b/src/graphql/gqls/index.ts @@ -18,6 +18,8 @@ export * from './request'; export * from './message'; export * from './RequestEnricher'; export * from './assessment'; +export * from './purpose'; +export * from './preferenceTopic'; export * from './assessmentTemplate'; export * from './prompt'; export * from './RequestEnricher'; diff --git a/src/graphql/gqls/preferenceTopic.ts b/src/graphql/gqls/preferenceTopic.ts new file mode 100644 index 00000000..4a47b376 --- /dev/null +++ b/src/graphql/gqls/preferenceTopic.ts @@ -0,0 +1,30 @@ +import { gql } from 'graphql-request'; + +// TODO: https://transcend.height.app/T-27909 - enable optimizations +// isExportCsv: true +// useMaster: false +// orderBy: [ +// { field: createdAt, direction: ASC } +// { field: name, direction: ASC } +// ] +export const PREFERENCE_TOPICS = gql` + query TranscendCliPreferenceTopics( + $first: Int! + $offset: Int! + $filterBy: PreferenceTopicFilterInput + ) { + preferenceTopics(first: $first, offset: $offset, filterBy: $filterBy) { + nodes { + id + slug + type + preferenceOptionValues { + slug + } + purpose { + trackingType + } + } + } + } +`; diff --git a/src/graphql/gqls/purpose.ts b/src/graphql/gqls/purpose.ts new file mode 100644 index 00000000..2f4be7b8 --- /dev/null +++ b/src/graphql/gqls/purpose.ts @@ -0,0 +1,32 @@ +import { gql } from 'graphql-request'; + +// TODO: https://transcend.height.app/T-27909 - enable optimizations +// isExportCsv: true +// useMaster: false +// orderBy: [ +// { field: createdAt, direction: ASC } +// { field: name, direction: ASC } +// ] +export const PURPOSES = gql` + query TranscendCliPurposes( + $first: Int! + $offset: Int! + $filterBy: TrackingPurposeFiltersInput + $input: TrackingPurposeInput! + ) { + purposes( + first: $first + offset: $offset + filterBy: $filterBy + input: $input + ) { + nodes { + id + name + trackingType + isActive + deletedAt + } + } + } +`; diff --git a/src/graphql/index.ts b/src/graphql/index.ts index 74ca41a9..c41dfb1c 100644 --- a/src/graphql/index.ts +++ b/src/graphql/index.ts @@ -32,10 +32,12 @@ export * from './fetchAllRequests'; export * from './fetchAllRequestIdentifiers'; export * from './fetchAllRequestEnrichers'; export * from './fetchRequestDataSilo'; +export * from './fetchAllPreferenceTopics'; export * from './fetchAllAttributes'; export * from './syncAttribute'; export * from './fetchAllDataFlows'; export * from './syncActionItems'; +export * from './fetchAllPurposes'; export * from './syncActionItemCollections'; export * from './fetchAllCookies'; export * from './fetchAllActionItems'; diff --git a/src/graphql/syncConsentManager.ts b/src/graphql/syncConsentManager.ts index f921fb92..c6327645 100644 --- a/src/graphql/syncConsentManager.ts +++ b/src/graphql/syncConsentManager.ts @@ -19,7 +19,6 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; import { fetchConsentManagerId, fetchConsentManagerExperiences, - fetchPurposes, } from './fetchConsentManagerId'; import keyBy from 'lodash/keyBy'; import { map } from 'bluebird'; @@ -30,6 +29,7 @@ import { import { logger } from '../logger'; import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { fetchPartitions } from './syncPartitions'; +import { fetchAllPurposes } from './fetchAllPurposes'; const PURPOSES_LINK = 'https://app.transcend.io/consent-manager/regional-experiences/purposes'; @@ -49,7 +49,7 @@ export async function syncConsentManagerExperiences( const experienceLookup = keyBy(existingExperiences, 'name'); // Fetch existing purposes - const purposes = await fetchPurposes(client); + const purposes = await fetchAllPurposes(client); const purposeLookup = keyBy(purposes, 'trackingType'); // Bulk update or create experiences diff --git a/src/graphql/syncCookies.ts b/src/graphql/syncCookies.ts index c529c8ce..7bc590e2 100644 --- a/src/graphql/syncCookies.ts +++ b/src/graphql/syncCookies.ts @@ -24,7 +24,7 @@ export async function updateOrCreateCookies( const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchPurposes(client); + // const purposes = await fetchAllPurposes(client); // const purposeNameToId = keyBy(purposes, 'name'); await mapSeries(chunk(cookieInputs, MAX_PAGE_SIZE), async (page) => { diff --git a/src/graphql/syncDataFlows.ts b/src/graphql/syncDataFlows.ts index 3e0bcd15..6a8b1f3e 100644 --- a/src/graphql/syncDataFlows.ts +++ b/src/graphql/syncDataFlows.ts @@ -27,7 +27,7 @@ export async function updateDataFlows( const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchPurposes(client); + // const purposes = await fetchAllPurposes(client); // const purposeNameToId = keyBy(purposes, 'name'); await mapSeries(chunk(dataFlowInputs, MAX_PAGE_SIZE), async (page) => { @@ -74,7 +74,7 @@ export async function createDataFlows( ): Promise { const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchPurposes(client); + // const purposes = await fetchAllPurposes(client); // const purposeNameToId = keyBy(purposes, 'name'); await mapSeries(chunk(dataFlowInputs, MAX_PAGE_SIZE), async (page) => { await makeGraphQLRequest(client, CREATE_DATA_FLOWS, { diff --git a/src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts b/src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts new file mode 100644 index 00000000..17b5525c --- /dev/null +++ b/src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts @@ -0,0 +1,93 @@ +import { + PreferenceQueryResponseItem, + PreferenceStorePurposeResponse, + PreferenceTopicType, +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; + +/** + * Check if the pending set of updates are exactly the same as the current consent record. + * + * @param options - Options + * @returns Whether the pending updates already exist in the preference store + */ +export function checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, +}: { + /** The current consent record */ + currentConsentRecord: PreferenceQueryResponseItem; + /** The pending updates */ + pendingUpdates: { + [purposeName in string]: Omit; + }; + /** The preference topic configurations */ + preferenceTopics: PreferenceTopic[]; +}): boolean { + // Check each update + return Object.entries(pendingUpdates).every( + ([purposeName, { preferences = [], enabled }]) => { + // Ensure the purpose exists + const currentPurpose = currentConsentRecord.purposes.find( + (existingPurpose) => existingPurpose.purpose === purposeName, + ); + + // Ensure purpose.enabled is in sync + // Also false if the purpose does not exist + const enabledIsInSync = + !!currentPurpose && currentPurpose.enabled === enabled; + if (!enabledIsInSync) { + return false; + } + + // Compare the preferences are in sync + return preferences.every( + ({ topic, choice }) => + // ensure preferences exist on record + currentPurpose.preferences && + currentPurpose.preferences.find((existingPreference) => { + // find matching topic + if (existingPreference.topic !== topic) { + return false; + } + + // Determine type of preference topic + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === topic && x.purpose.trackingType === purposeName, + ); + if (!preferenceTopic) { + throw new Error(`Could not find preference topic for ${topic}`); + } + + // Handle comparison based on type + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: + return ( + existingPreference.choice.booleanValue === choice.booleanValue + ); + case PreferenceTopicType.Select: + return ( + existingPreference.choice.selectValue === choice.selectValue + ); + case PreferenceTopicType.MultiSelect: + // eslint-disable-next-line no-case-declarations + const sortedCurrentValues = ( + existingPreference.choice.selectValues || [] + ).sort(); + // eslint-disable-next-line no-case-declarations + const sortedNewValues = (choice.selectValues || []).sort(); + return ( + sortedCurrentValues.length === sortedNewValues.length && + sortedCurrentValues.every((x, i) => x === sortedNewValues[i]) + ); + default: + throw new Error( + `Unknown preference topic type: ${preferenceTopic.type}`, + ); + } + }), + ); + }, + ); +} diff --git a/src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts b/src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts new file mode 100644 index 00000000..a14b4845 --- /dev/null +++ b/src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts @@ -0,0 +1,94 @@ +import { + PreferenceQueryResponseItem, + PreferenceStorePurposeResponse, + PreferenceTopicType, +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; + +/** + * Check if the pending set of updates will result in a change of + * value to an existing purpose or preference in the preference store. + * + * @param options - Options + * @returns True if conflict, false if no conflict and just adding new data for first time + */ +export function checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, +}: { + /** The current consent record */ + currentConsentRecord: PreferenceQueryResponseItem; + /** The pending updates */ + pendingUpdates: { + [purposeName in string]: Omit; + }; + /** The preference topic configurations */ + preferenceTopics: PreferenceTopic[]; +}): boolean { + // Check if any update has conflict + return !!Object.entries(pendingUpdates).find( + ([purposeName, { preferences = [], enabled }]) => { + // Ensure the purpose exists + const currentPurpose = currentConsentRecord.purposes.find( + (existingPurpose) => existingPurpose.purpose === purposeName, + ); + + // If no purpose exists, then it is not a conflict + if (!currentPurpose) { + return false; + } + + // If purpose.enabled value is off, this is a conflict + if (currentPurpose.enabled !== enabled) { + return true; + } + + // Check if any preferences are out of sync + return !!preferences.find(({ topic, choice }) => { + // find matching topic + const currentPreference = (currentPurpose.preferences || []).find( + (existingPreference) => existingPreference.topic === topic, + ); + + // if no topic exists, no conflict + if (!currentPreference) { + return false; + } + + // Determine type of preference topic + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === topic && x.purpose.trackingType === purposeName, + ); + if (!preferenceTopic) { + throw new Error(`Could not find preference topic for ${topic}`); + } + + // Handle comparison based on type + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: + return ( + currentPreference.choice.booleanValue !== choice.booleanValue + ); + case PreferenceTopicType.Select: + return currentPreference.choice.selectValue !== choice.selectValue; + case PreferenceTopicType.MultiSelect: + // eslint-disable-next-line no-case-declarations + const sortedCurrentValues = ( + currentPreference.choice.selectValues || [] + ).sort(); + // eslint-disable-next-line no-case-declarations + const sortedNewValues = (choice.selectValues || []).sort(); + return ( + sortedCurrentValues.length !== sortedNewValues.length || + !sortedCurrentValues.every((x, i) => x === sortedNewValues[i]) + ); + default: + throw new Error( + `Unknown preference topic type: ${preferenceTopic.type}`, + ); + } + }); + }, + ); +} diff --git a/src/preference-management/codecs.ts b/src/preference-management/codecs.ts new file mode 100644 index 00000000..0b07c507 --- /dev/null +++ b/src/preference-management/codecs.ts @@ -0,0 +1,116 @@ +import { + PreferenceQueryResponseItem, + PreferenceUpdateItem, +} from '@transcend-io/privacy-types'; +import * as t from 'io-ts'; + +export const PurposeRowMapping = t.type({ + /** + * The slug or trackingType of the purpose to map to + * + * e.g. `Marketing` + */ + purpose: t.string, + /** + * If the column maps to a preference instead of a purpose + * this is the slug of the purpose. + * + * null value indicates that this column maps to the true/false + * value of the purpose + */ + preference: t.union([t.string, t.null]), + /** + * The mapping between each row value and purpose/preference value. + * + * e.g. for a boolean preference or purpose + * { + * 'true': true, + * 'false': false, + * '': true, + * } + * + * or for a single or multi select preference + * { + * '': true, + * 'value1': 'Value1', + * 'value2': 'Value2', + * } + */ + valueMapping: t.record(t.string, t.union([t.string, t.boolean, t.null])), +}); + +/** Override type */ +export type PurposeRowMapping = t.TypeOf; + +export const FileMetadataState = t.intersection([ + t.type({ + /** + * Definition of how to map each column in the CSV to + * the relevant purpose and preference definitions in transcend + */ + columnToPurposeName: t.record(t.string, PurposeRowMapping), + /** Last time the file was last parsed at */ + lastFetchedAt: t.string, + /** + * Mapping of userId to the rows in the file that need to be uploaded + * These uploads are overwriting non-existent preferences and are safe + */ + pendingSafeUpdates: t.record(t.string, t.record(t.string, t.string)), + /** + * Mapping of userId to the rows in the file that need to be uploaded + * these records have conflicts with existing consent preferences + */ + pendingConflictUpdates: t.record( + t.string, + t.type({ + record: PreferenceQueryResponseItem, + row: t.record(t.string, t.string), + }), + ), + /** + * Mapping of userId to the rows in the file that can be skipped because + * their preferences are already in the store + */ + skippedUpdates: t.record(t.string, t.record(t.string, t.string)), + }), + t.partial({ + /** Determine which column name in file maps to consent record identifier to upload on */ + identifierColumn: t.string, + /** Determine which column name in file maps to the timestamp */ + timestampColum: t.string, + }), +]); + +/** Override type */ +export type FileMetadataState = t.TypeOf; + +/** Persist this data between runs of the script */ +export const PreferenceState = t.type({ + /** + * Store a cache of previous files read in + */ + fileMetadata: t.record(t.string, FileMetadataState), + /** + * The set of successful uploads to Transcend + * Mapping from userId to the upload metadata + */ + failingUpdates: t.record( + t.string, + t.type({ + /** Time upload ran at */ + uploadedAt: t.string, + /** Attempts to upload that resulted in an error */ + error: t.string, + /** The update body */ + update: PreferenceUpdateItem, + }), + ), + /** + * The set of pending uploads to Transcend + * Mapping from userId to the upload metadata + */ + pendingUpdates: t.record(t.string, PreferenceUpdateItem), +}); + +/** Override type */ +export type PreferenceState = t.TypeOf; diff --git a/src/preference-management/getPreferenceUpdatesFromRow.ts b/src/preference-management/getPreferenceUpdatesFromRow.ts new file mode 100644 index 00000000..32d63f68 --- /dev/null +++ b/src/preference-management/getPreferenceUpdatesFromRow.ts @@ -0,0 +1,196 @@ +import { + PreferenceStorePurposeResponse, + PreferenceTopicType, +} from '@transcend-io/privacy-types'; +import { PurposeRowMapping } from './codecs'; +import { apply } from '@transcend-io/type-utils'; +import { PreferenceTopic } from '../graphql'; +import { splitCsvToList } from '../requests'; + +/** + * Parse an arbitrary object to the Transcend PUT /v1/preference update shape + * by using a mapping of column names to purpose/preference slugs. + * + * columnToPurposeName looks like: + * { + * 'my_purpose': 'Marketing', + * 'has_topic_1': 'Marketing->BooleanPreference1', + * 'has_topic_2': 'Marketing->BooleanPreference2' + * } + * + * row looks like: + * { + * 'my_purpose': 'true', + * 'has_topic_1': 'true', + * 'has_topic_2': 'false' + * } + * + * @param options - Options + * @returns The parsed row + */ +export function getPreferenceUpdatesFromRow({ + row, + columnToPurposeName, + purposeSlugs, + preferenceTopics, +}: { + /** Row to parse */ + row: Record; + /** Mapping from column name ot row */ + columnToPurposeName: Record; + /** The set of allowed purpose slugs */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; +}): { + [k in string]: Omit; +} { + // Create a result object to store the parsed preferences + const result: { + [k in string]: Partial; + } = {}; + + // Iterate over each column and map to the purpose or preference + Object.entries(columnToPurposeName).forEach( + ([columnName, { purpose, preference, valueMapping }]) => { + // Ensure the purpose is valid + if (!purposeSlugs.includes(purpose)) { + throw new Error( + `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join( + ', ', + )}`, + ); + } + + // CHeck if parsing a preference or just the top level purpose + if (preference) { + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === preference && x.purpose.trackingType === purpose, + ); + if (!preferenceTopic) { + const allowedTopics = preferenceTopics + .filter((x) => x.purpose.trackingType === purpose) + .map((x) => x.slug); + throw new Error( + `Invalid preference slug: ${preference} for purpose: ${purpose}. ` + + `Allowed preference slugs for purpose are: ${allowedTopics.join( + ',', + )}`, + ); + } + + // If parsing preferences, default to an empty array + if (!result[purpose]) { + result[purpose] = { + preferences: [], + }; + } + if (!result[purpose].preferences) { + result[purpose].preferences = []; + } + + // The value to parse + const rawValue = row[columnName]; + const rawMapping = valueMapping[rawValue]; + const trimmedMapping = + typeof rawMapping === 'string' ? rawMapping.trim() || null : null; + + // handle each type of preference + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: + if (typeof rawMapping !== 'boolean') { + throw new Error( + `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}`, + ); + } + result[purpose].preferences!.push({ + topic: preference, + choice: { + booleanValue: rawMapping, + }, + }); + break; + case PreferenceTopicType.Select: + if (typeof rawMapping !== 'string' && rawMapping !== null) { + throw new Error( + `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}`, + ); + } + + if ( + trimmedMapping && + !preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .includes(trimmedMapping) + ) { + throw new Error( + `Invalid value for select preference: ${preference}, expected one of: ` + + `${preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .join(', ')}, got: ${rawValue}`, + ); + } + + // Update preferences + result[purpose].preferences!.push({ + topic: preference, + choice: { + selectValue: trimmedMapping, + }, + }); + break; + case PreferenceTopicType.MultiSelect: + if (typeof rawValue !== 'string') { + throw new Error( + `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}`, + ); + } + // Update preferences + result[purpose].preferences!.push({ + topic: preference, + choice: { + selectValues: splitCsvToList(rawValue) + .map((val) => { + const result = valueMapping[val]; + if (typeof result !== 'string') { + throw new Error( + `Invalid value for multi select preference: ${preference}, ` + + `expected one of: ${preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .join(', ')}, got: ${val}`, + ); + } + return result; + }) + .sort((a, b) => a.localeCompare(b)), + }, + }); + break; + default: + throw new Error(`Unknown preference type: ${preferenceTopic.type}`); + } + } else if (!result[purpose]) { + // Handle updating top level purpose for the first time + result[purpose] = { + enabled: valueMapping[row[columnName]] === true, + }; + } else { + // Handle updating top level purpose but preserve preference updates + result[purpose].enabled = valueMapping[row[columnName]] === true; + } + }, + ); + + // Ensure that enabled is provided + return apply(result, (x, purposeName) => { + if (typeof x.enabled !== 'boolean') { + throw new Error( + `No mapping provided for purpose.enabled=true/false value: ${purposeName}`, + ); + } + return { + ...x, + enabled: x.enabled!, + }; + }); +} diff --git a/src/preference-management/getPreferencesForIdentifiers.ts b/src/preference-management/getPreferencesForIdentifiers.ts new file mode 100644 index 00000000..ceb6c471 --- /dev/null +++ b/src/preference-management/getPreferencesForIdentifiers.ts @@ -0,0 +1,125 @@ +import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; +import type { Got } from 'got'; +import colors from 'colors'; +import cliProgress from 'cli-progress'; +import chunk from 'lodash/chunk'; +import { decodeCodec } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { map } from 'bluebird'; +import { logger } from '../logger'; + +const PreferenceRecordsQueryResponse = t.intersection([ + t.type({ + nodes: t.array(PreferenceQueryResponseItem), + }), + t.partial({ + /** The base64 encoded(PreferenceStorePaginationKey) cursor for pagination */ + cursor: t.string, + }), +]); + +const MSGS = ['ETIMEDOUT', '504 Gateway Time-out', 'Task timed out after']; + +/** + * Grab the current consent preference values for a list of identifiers + * + * @param sombra - Backend to make API call to + * @param options - Options + * @returns Plaintext context information + */ +export async function getPreferencesForIdentifiers( + sombra: Got, + { + identifiers, + partitionKey, + skipLogging = false, + }: { + /** The list of identifiers to look up */ + identifiers: { + /** The value of the identifier */ + value: string; + }[]; + /** The partition key to look up */ + partitionKey: string; + /** Whether to skip logging */ + skipLogging?: boolean; + }, +): Promise { + const results: PreferenceQueryResponseItem[] = []; + const groupedIdentifiers = chunk(identifiers, 100); + + // create a new progress bar instance and use shades_classic theme + const t0 = new Date().getTime(); + const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, + ); + if (!skipLogging) { + progressBar.start(identifiers.length, 0); + } + + let total = 0; + await map( + groupedIdentifiers, + async (group) => { + // Make the request with retry logic + let attempts = 0; + const maxAttempts = 3; + while (attempts < maxAttempts) { + try { + // eslint-disable-next-line no-await-in-loop + const rawResult = await sombra + .post(`v1/preferences/${partitionKey}/query`, { + json: { + filter: { + identifiers: group, + }, + limit: group.length, + }, + }) + .json(); + + const result = decodeCodec(PreferenceRecordsQueryResponse, rawResult); + results.push(...result.nodes); + total += group.length; + progressBar.update(total); + break; // Exit loop if successful + } catch (err) { + attempts += 1; + const msg = err?.response?.body || err?.message || ''; + if ( + attempts >= maxAttempts || + !MSGS.some((errorMessage) => msg.includes(errorMessage)) + ) { + throw new Error( + `Received an error from server after ${attempts} attempts: ${msg}`, + ); + } + + logger.warn( + colors.yellow( + `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` + + `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`, + ), + ); + } + } + }, + { + concurrency: 40, + }, + ); + + progressBar.stop(); + const t1 = new Date().getTime(); + const totalTime = t1 - t0; + + if (!skipLogging) { + // Log completion time + logger.info( + colors.green(`Completed download in "${totalTime / 1000}" seconds.`), + ); + } + + return results; +} diff --git a/src/preference-management/index.ts b/src/preference-management/index.ts new file mode 100644 index 00000000..1f914460 --- /dev/null +++ b/src/preference-management/index.ts @@ -0,0 +1,11 @@ +export * from './uploadPreferenceManagementPreferencesInteractive'; +export * from './codecs'; +export * from './getPreferencesForIdentifiers'; +export * from './parsePreferenceManagementCsv'; +export * from './getPreferenceUpdatesFromRow'; +export * from './parsePreferenceManagementCsv'; +export * from './parsePreferenceIdentifiersFromCsv'; +export * from './parsePreferenceTimestampsFromCsv'; +export * from './parsePreferenceAndPurposeValuesFromCsv'; +export * from './checkIfPendingPreferenceUpdatesAreNoOp'; +export * from './checkIfPendingPreferenceUpdatesCauseConflict'; diff --git a/src/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts new file mode 100644 index 00000000..6de11dad --- /dev/null +++ b/src/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -0,0 +1,114 @@ +import uniq from 'lodash/uniq'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import difference from 'lodash/difference'; +import { FileMetadataState } from './codecs'; +import { logger } from '../logger'; +import { mapSeries } from 'bluebird'; +import { PreferenceTopic } from '../graphql'; + +/* eslint-disable no-param-reassign */ + +/** + * Parse out the purpose.enabled and preference values from a CSV file + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @param options - Options + * @returns The updated file metadata state + */ +export async function parsePreferenceAndPurposeValuesFromCsv( + preferences: Record[], + currentState: FileMetadataState, + { + purposeSlugs, + preferenceTopics, + }: { + /** The purpose slugs that are allowed to be updated */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; + }, +): Promise { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for identifier + const otherColumns = difference(columnNames, [ + ...(currentState.identifierColumn ? [currentState.identifierColumn] : []), + ...(currentState.timestampColum ? [currentState.timestampColum] : []), + ]); + if (otherColumns.length === 0) { + throw new Error('No other columns to process'); + } + + // The purpose and preferences to map to + const purposeNames = [ + ...purposeSlugs, + ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`), + ]; + + // Ensure all columns are accounted for + await mapSeries(otherColumns, async (col) => { + // Determine the unique values to map in this column + const uniqueValues = uniq(preferences.map((x) => x[col])); + + // Map the column to a purpose + let purposeMapping = currentState.columnToPurposeName[col]; + if (purposeMapping) { + logger.info( + colors.magenta( + `Column "${col}" is associated with purpose "${purposeMapping.purpose}"`, + ), + ); + } else { + const { purposeName } = await inquirer.prompt<{ + /** purpose name */ + purposeName: string; + }>([ + { + name: 'purposeName', + message: `Choose the purpose that column ${col} is associated with`, + type: 'list', + default: purposeNames[0], + choices: purposeNames, + }, + ]); + const [purposeSlug, preferenceSlug] = purposeName.split('->'); + purposeMapping = { + purpose: purposeSlug, + preference: preferenceSlug || null, + valueMapping: {}, + }; + } + + // map each value to the purpose value + await mapSeries(uniqueValues, async (value) => { + if (purposeMapping.valueMapping[value] !== undefined) { + logger.info( + colors.magenta( + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"`, + ), + ); + return; + } + const { purposeValue } = await inquirer.prompt<{ + /** purpose value */ + purposeValue: boolean; + }>([ + { + name: 'purposeValue', + message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, + type: 'confirm', + default: value !== 'false', + }, + ]); + purposeMapping.valueMapping[value] = purposeValue; + }); + + currentState.columnToPurposeName[col] = purposeMapping; + }); + + return currentState; +} +/* eslint-enable no-param-reassign */ diff --git a/src/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/preference-management/parsePreferenceIdentifiersFromCsv.ts new file mode 100644 index 00000000..f03c2e89 --- /dev/null +++ b/src/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -0,0 +1,140 @@ +import uniq from 'lodash/uniq'; +import groupBy from 'lodash/groupBy'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import difference from 'lodash/difference'; +import { FileMetadataState } from './codecs'; +import { logger } from '../logger'; +import { inquirerConfirmBoolean } from '../helpers'; + +/* eslint-disable no-param-reassign */ + +/** + * Parse identifiers from a CSV list of preferences + * + * Ensures that all rows have a valid identifier + * and that all identifiers are unique. + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @returns The updated file metadata state + */ +export async function parsePreferenceIdentifiersFromCsv( + preferences: Record[], + currentState: FileMetadataState, +): Promise<{ + /** The updated state */ + currentState: FileMetadataState; + /** The updated preferences */ + preferences: Record[]; +}> { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for identifier + const remainingColumnsForIdentifier = difference(columnNames, [ + ...(currentState.identifierColumn ? [currentState.identifierColumn] : []), + ...Object.keys(currentState.columnToPurposeName), + ]); + + // Determine the identifier column to work off of + if (!currentState.identifierColumn) { + const { identifierName } = await inquirer.prompt<{ + /** Identifier name */ + identifierName: string; + }>([ + { + name: 'identifierName', + message: + 'Choose the column that will be used as the identifier to upload consent preferences by', + type: 'list', + default: + remainingColumnsForIdentifier.find((col) => + col.toLowerCase().includes('email'), + ) || remainingColumnsForIdentifier[0], + choices: remainingColumnsForIdentifier, + }, + ]); + currentState.identifierColumn = identifierName; + } + logger.info( + colors.magenta( + `Using identifier column "${currentState.identifierColumn}"`, + ), + ); + + // Validate that the identifier column is present for all rows and unique + const identifierColumnsMissing = preferences + .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + if (identifierColumnsMissing.length > 0) { + const msg = `The identifier column "${ + currentState.identifierColumn + }" is missing a value for the following rows: ${identifierColumnsMissing.join( + ', ', + )}`; + logger.warn(colors.yellow(msg)); + + // Ask user if they would like to skip rows missing an identifier + const skip = await inquirerConfirmBoolean({ + message: 'Would you like to skip rows missing an identifier?', + }); + if (!skip) { + throw new Error(msg); + } + + // Filter out rows missing an identifier + const previous = preferences.length; + preferences = preferences.filter( + (pref) => pref[currentState.identifierColumn!], + ); + logger.info( + colors.yellow( + `Skipped ${previous - preferences.length} rows missing an identifier`, + ), + ); + } + logger.info( + colors.magenta( + `The identifier column "${currentState.identifierColumn}" is present for all rows`, + ), + ); + + // Validate that all identifiers are unique + const rowsByUserId = groupBy(preferences, currentState.identifierColumn); + const duplicateIdentifiers = Object.entries(rowsByUserId).filter( + ([, rows]) => rows.length > 1, + ); + if (duplicateIdentifiers.length > 0) { + const msg = `The identifier column "${ + currentState.identifierColumn + }" has duplicate values for the following rows: ${duplicateIdentifiers + .slice(0, 10) + .map(([userId, rows]) => `${userId} (${rows.length})`) + .join('\n')}`; + logger.warn(colors.yellow(msg)); + + // Ask user if they would like to take the most recent update + // for each duplicate identifier + const skip = await inquirerConfirmBoolean({ + message: 'Would you like to automatically take the latest update?', + }); + if (!skip) { + throw new Error(msg); + } + preferences = Object.entries(rowsByUserId) + .map(([, rows]) => { + const sorted = rows.sort( + (a, b) => + new Date(b[currentState.timestampColum!]).getTime() - + new Date(a[currentState.timestampColum!]).getTime(), + ); + return sorted[0]; + }) + .filter((x) => x); + } + + return { currentState, preferences }; +} +/* eslint-enable no-param-reassign */ diff --git a/src/preference-management/parsePreferenceManagementCsv.ts b/src/preference-management/parsePreferenceManagementCsv.ts new file mode 100644 index 00000000..3aae3e38 --- /dev/null +++ b/src/preference-management/parsePreferenceManagementCsv.ts @@ -0,0 +1,174 @@ +import { PersistedState } from '@transcend-io/persisted-state'; +import type { Got } from 'got'; +import keyBy from 'lodash/keyBy'; +import * as t from 'io-ts'; +import colors from 'colors'; +import { FileMetadataState, PreferenceState } from './codecs'; +import { logger } from '../logger'; +import { readCsv } from '../requests'; +import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; +import { PreferenceTopic } from '../graphql'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; +import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; +import { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; + +/** + * Parse a file into the cache + * + * + * @param options - Options + * @param cache - The cache to store the parsed file in + * @returns The cache with the parsed file + */ +export async function parsePreferenceManagementCsvWithCache( + { + file, + sombra, + purposeSlugs, + preferenceTopics, + partitionKey, + }: { + /** File to parse */ + file: string; + /** The purpose slugs that are allowed to be updated */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; + /** Sombra got instance */ + sombra: Got; + /** Partition key */ + partitionKey: string; + }, + cache: PersistedState, +): Promise { + // Start the timer + const t0 = new Date().getTime(); + + // Get the current metadata + const fileMetadata = cache.getValue('fileMetadata'); + + // Read in the file + logger.info(colors.magenta(`Reading in file: "${file}"`)); + let preferences = readCsv(file, t.record(t.string, t.string)); + + // start building the cache, can use previous cache as well + let currentState: FileMetadataState = { + columnToPurposeName: {}, + pendingSafeUpdates: {}, + pendingConflictUpdates: {}, + skippedUpdates: {}, + // Load in the last fetched time + ...((fileMetadata[file] || {}) as Partial), + lastFetchedAt: new Date().toISOString(), + }; + + // Validate that all timestamps are present in the file + currentState = await parsePreferenceTimestampsFromCsv( + preferences, + currentState, + ); + fileMetadata[file] = currentState; + await cache.setValue(fileMetadata, 'fileMetadata'); + + // Validate that all identifiers are present and unique + const result = await parsePreferenceIdentifiersFromCsv( + preferences, + currentState, + ); + currentState = result.currentState; + preferences = result.preferences; + fileMetadata[file] = currentState; + await cache.setValue(fileMetadata, 'fileMetadata'); + + // Ensure all other columns are mapped to purpose and preference + // slug values + currentState = await parsePreferenceAndPurposeValuesFromCsv( + preferences, + currentState, + { + preferenceTopics, + purposeSlugs, + }, + ); + fileMetadata[file] = currentState; + await cache.setValue(fileMetadata, 'fileMetadata'); + + // Grab existing preference store records + const identifiers = preferences.map( + (pref) => pref[currentState.identifierColumn!], + ); + const existingConsentRecords = await getPreferencesForIdentifiers(sombra, { + identifiers: identifiers.map((x) => ({ value: x })), + partitionKey, + }); + const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId'); + + // Clear out previous updates + currentState.pendingConflictUpdates = {}; + currentState.pendingSafeUpdates = {}; + currentState.skippedUpdates = {}; + + // Process each row + preferences.forEach((pref) => { + // Grab unique Id for the user + const userId = pref[currentState.identifierColumn!]; + + // determine updates for user + const pendingUpdates = getPreferenceUpdatesFromRow({ + row: pref, + columnToPurposeName: currentState.columnToPurposeName, + preferenceTopics, + purposeSlugs, + }); + + // Grab current state of the update + const currentConsentRecord = consentRecordByIdentifier[userId]; + + // Check if the update can be skipped + // this is the case if a record exists, and the purpose + // and preference values are all in sync + if ( + currentConsentRecord && + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, + }) + ) { + currentState.skippedUpdates[userId] = pref; + return; + } + + // Determine if there are any conflicts + if ( + currentConsentRecord && + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, + }) + ) { + currentState.pendingConflictUpdates[userId] = { + row: pref, + record: currentConsentRecord, + }; + return; + } + + // Add to pending updates + currentState.pendingSafeUpdates[userId] = pref; + }); + + // Read in the file + fileMetadata[file] = currentState; + await cache.setValue(fileMetadata, 'fileMetadata'); + const t1 = new Date().getTime(); + logger.info( + colors.green( + `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, + ), + ); +} diff --git a/src/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/preference-management/parsePreferenceTimestampsFromCsv.ts new file mode 100644 index 00000000..cbaa33b2 --- /dev/null +++ b/src/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -0,0 +1,88 @@ +import uniq from 'lodash/uniq'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import difference from 'lodash/difference'; +import { FileMetadataState } from './codecs'; +import { logger } from '../logger'; + +export const NONE_PREFERENCE_MAP = '[NONE]'; + +/* eslint-disable no-param-reassign */ + +/** + * Parse timestamps from a CSV list of preferences + * + * When timestamp is requested, this script + * ensures that all rows have a valid timestamp. + * + * Error is throw if timestamp is missing + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @returns The updated file metadata state + */ +export async function parsePreferenceTimestampsFromCsv( + preferences: Record[], + currentState: FileMetadataState, +): Promise { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for timestamp + const remainingColumnsForTimestamp = difference(columnNames, [ + ...(currentState.identifierColumn ? [currentState.identifierColumn] : []), + ...Object.keys(currentState.columnToPurposeName), + ]); + + // Determine the timestamp column to work off of + if (!currentState.timestampColum) { + const { timestampName } = await inquirer.prompt<{ + /** timestamp name */ + timestampName: string; + }>([ + { + name: 'timestampName', + message: + 'Choose the column that will be used as the timestamp of last preference update', + type: 'list', + default: + remainingColumnsForTimestamp.find((col) => + col.toLowerCase().includes('date'), + ) || + remainingColumnsForTimestamp.find((col) => + col.toLowerCase().includes('time'), + ) || + remainingColumnsForTimestamp[0], + choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], + }, + ]); + currentState.timestampColum = timestampName; + } + logger.info( + colors.magenta(`Using timestamp column "${currentState.timestampColum}"`), + ); + + // Validate that all rows have valid timestamp + if (currentState.timestampColum !== NONE_PREFERENCE_MAP) { + const timestampColumnsMissing = preferences + .map((pref, ind) => (pref[currentState.timestampColum!] ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + if (timestampColumnsMissing.length > 0) { + throw new Error( + `The timestamp column "${ + currentState.timestampColum + }" is missing a value for the following rows: ${timestampColumnsMissing.join( + '\n', + )}`, + ); + } + logger.info( + colors.magenta( + `The timestamp column "${currentState.timestampColum}" is present for all row`, + ), + ); + } + return currentState; +} +/* eslint-enable no-param-reassign */ diff --git a/src/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts b/src/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts new file mode 100644 index 00000000..cddba6e3 --- /dev/null +++ b/src/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts @@ -0,0 +1,433 @@ +/* eslint-disable max-lines */ +import { expect } from 'chai'; + +import { checkIfPendingPreferenceUpdatesAreNoOp } from '../index'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../../graphql'; + +const DEFAULT_VALUES = { + userId: 'test@transcend.io', + timestamp: '2024-11-30T00:00:15.327Z', + partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', + system: { + decryptionStatus: 'DECRYPTED' as const, + updatedAt: '2024-11-30T00:00:16.506Z', + }, + consentManagement: { + usp: null, + gpp: null, + tcf: null, + airgapVersion: null, + }, + metadata: [], +}; + +const PREFERENCE_TOPICS: PreferenceTopic[] = [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, +]; + +describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { + it('should return true for simple purpose comparison', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + preferences: [], + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); + + it('should return false for simple purpose comparison', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: true, + preferences: [], + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return true for simple purpose comparison with extra preference', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + ], + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); + + it('should return false for simple purpose comparison with extra preference in update', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return true for preferences being same', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); + + it('should return false for boolean preference changing', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: false, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return false for single select preference changing', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value2', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return false for multi select preference changing', () => { + expect( + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); +}); +/* eslint-enable max-lines */ diff --git a/src/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts b/src/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts new file mode 100644 index 00000000..bbc109c7 --- /dev/null +++ b/src/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts @@ -0,0 +1,456 @@ +/* eslint-disable max-lines */ +import { expect } from 'chai'; + +import { checkIfPendingPreferenceUpdatesCauseConflict } from '../index'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../../graphql'; + +const DEFAULT_VALUES = { + userId: 'test@transcend.io', + timestamp: '2024-11-30T00:00:15.327Z', + partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', + system: { + decryptionStatus: 'DECRYPTED' as const, + updatedAt: '2024-11-30T00:00:16.506Z', + }, + consentManagement: { + usp: null, + gpp: null, + tcf: null, + airgapVersion: null, + }, + metadata: [], +}; + +const PREFERENCE_TOPICS: PreferenceTopic[] = [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, +]; + +describe.only('checkIfPendingPreferenceUpdatesCauseConflict', () => { + it('should return false for simple purpose comparison', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + preferences: [], + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return false if purpose missing', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + preferences: [], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return true for simple purpose comparison', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: true, + preferences: [], + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); + + it('should return true for simple purpose comparison with extra preference', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + ], + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return false for simple purpose comparison with extra preference in update', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'SalesOutreach', + enabled: false, + }, + ], + }, + pendingUpdates: { + SalesOutreach: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return false for preferences being same', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(false); + }); + + it('should return true for boolean preference changing', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: false, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); + + it('should return true for single select preference changing', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value2', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); + + it('should return true for multi select preference changing', () => { + expect( + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord: { + ...DEFAULT_VALUES, + purposes: [ + { + purpose: 'Marketing', + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value2'], + }, + }, + ], + }, + ], + }, + pendingUpdates: { + Marketing: { + enabled: false, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }, + preferenceTopics: PREFERENCE_TOPICS, + }), + ).to.equal(true); + }); +}); +/* eslint-enable max-lines */ diff --git a/src/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/preference-management/tests/getPreferenceUpdatesFromRow.test.ts new file mode 100644 index 00000000..2b78deb0 --- /dev/null +++ b/src/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -0,0 +1,540 @@ +/* eslint-disable max-lines */ +import chai, { expect } from 'chai'; +import deepEqualInAnyOrder from 'deep-equal-in-any-order'; + +import { getPreferenceUpdatesFromRow } from '../index'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; + +chai.use(deepEqualInAnyOrder); + +describe('getPreferenceUpdatesFromRow', () => { + it('should parse boolean updates', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_1: { + purpose: 'Marketing', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_2: { + purpose: 'Marketing', + preference: 'BooleanPreference2', + valueMapping: { + true: true, + false: false, + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'BooleanPreference2', + choice: { + booleanValue: false, + }, + }, + ], + }, + }); + }); + + it('should parse a single select', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_3: 'Option 1', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_3: { + purpose: 'Marketing', + preference: 'SingleSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + ], + }, + }); + }); + + it('should parse a multi select example', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_4: 'Option 2,Option 1', + has_topic_5: 'Option 1', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_4: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + has_topic_5: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1'], + }, + }, + ], + }, + }); + }); + + it('should parse boolean, single select, multi select example', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', + has_topic_3: 'Option 1', + has_topic_4: 'Option 2,Option 1', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_1: { + purpose: 'Marketing', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_2: { + purpose: 'Marketing', + preference: 'BooleanPreference2', + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_3: { + purpose: 'Marketing', + preference: 'SingleSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + has_topic_4: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'BooleanPreference2', + choice: { + booleanValue: false, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }); + }); + + it('should error if missing purpose', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'Marketing', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.include('No mapping provided'); + } + }); + + it('should error if purpose name is not valid', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing', 'Advertising'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'InvalidPurpose', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.equal( + 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', + ); + } + }); + + it('should error if single select option is invalid', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'Marketing', + preference: 'SingleSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.equal( + 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', + ); + } + }); + + it('should error if multi select value is invalid', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.equal( + 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', + ); + } + }); +}); +/* eslint-enable max-lines */ diff --git a/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts new file mode 100644 index 00000000..2fb0040c --- /dev/null +++ b/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -0,0 +1,281 @@ +import { + buildTranscendGraphQLClient, + createSombraGotInstance, + fetchAllPurposes, +} from '../graphql'; +import colors from 'colors'; +import { map } from 'bluebird'; +import chunk from 'lodash/chunk'; +import { DEFAULT_TRANSCEND_CONSENT_API } from '../constants'; +import { logger } from '../logger'; +import cliProgress from 'cli-progress'; +import { parseAttributesFromString } from '../requests'; +import { PersistedState } from '@transcend-io/persisted-state'; +import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; +import { PreferenceState } from './codecs'; +import { fetchAllPreferenceTopics } from '../graphql/fetchAllPreferenceTopics'; +import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; + +/** + * Upload a set of consent preferences + * + * @param options - Options + */ +export async function uploadPreferenceManagementPreferencesInteractive({ + auth, + sombraAuth, + receiptFilepath, + file, + partition, + isSilent = true, + dryRun = false, + skipWorkflowTriggers = false, + skipConflictUpdates = false, + attributes = [], + transcendUrl = DEFAULT_TRANSCEND_CONSENT_API, +}: { + /** The Transcend API key */ + auth: string; + /** Sombra API key authentication */ + sombraAuth?: string; + /** Partition key */ + partition: string; + /** File where to store receipt and continue from where left off */ + receiptFilepath: string; + /** The file to process */ + file: string; + /** API URL for Transcend backend */ + transcendUrl?: string; + /** Whether to do a dry run */ + dryRun?: boolean; + /** Whether to upload as isSilent */ + isSilent?: boolean; + /** Attributes string pre-parse. In format Key:Value */ + attributes?: string[]; + /** Skip workflow triggers */ + skipWorkflowTriggers?: boolean; + /** + * When true, only update preferences that do not conflict with existing + * preferences. When false, update all preferences in CSV based on timestamp. + */ + skipConflictUpdates?: boolean; +}): Promise { + // Parse out the extra attributes to apply to all requests uploaded + const parsedAttributes = parseAttributesFromString(attributes); + + // Create a new state file to store the requests from this run + const preferenceState = new PersistedState(receiptFilepath, PreferenceState, { + fileMetadata: {}, + failingUpdates: {}, + pendingUpdates: {}, + }); + const failingRequests = preferenceState.getValue('failingUpdates'); + const pendingRequests = preferenceState.getValue('pendingUpdates'); + let fileMetadata = preferenceState.getValue('fileMetadata'); + + logger.info( + colors.magenta( + 'Restored cache, there are: \n' + + `${ + Object.values(failingRequests).length + } failing requests to be retried\n` + + `${ + Object.values(pendingRequests).length + } pending requests to be processed\n` + + `The following files are stored in cache and will be used:\n${Object.keys( + fileMetadata, + ) + .map((x) => x) + .join('\n')}\n` + + `The following file will be processed: ${file}\n`, + ), + ); + + // Create GraphQL client to connect to Transcend backend + const client = buildTranscendGraphQLClient(transcendUrl, auth); + + const [sombra, purposes, preferenceTopics] = await Promise.all([ + // Create sombra instance to communicate with + createSombraGotInstance(transcendUrl, auth, sombraAuth), + // get all purposes and topics + fetchAllPurposes(client), + fetchAllPreferenceTopics(client), + ]); + + // Process the file + await parsePreferenceManagementCsvWithCache( + { + file, + purposeSlugs: purposes.map((x) => x.trackingType), + preferenceTopics, + sombra, + partitionKey: partition, + }, + preferenceState, + ); + + // Construct the pending updates + const pendingUpdates: Record = {}; + fileMetadata = preferenceState.getValue('fileMetadata'); + const metadata = fileMetadata[file]; + + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.pendingSafeUpdates).length + } safe updates in ${file}`, + ), + ); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.pendingConflictUpdates).length + } conflict updates in ${file}`, + ), + ); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.skippedUpdates).length + } skipped updates in ${file}`, + ), + ); + + // Update either safe updates only or safe + conflict + Object.entries({ + ...metadata.pendingSafeUpdates, + ...(skipConflictUpdates + ? {} + : apply(metadata.pendingConflictUpdates, ({ row }) => row)), + }).forEach(([userId, update]) => { + // Determine timestamp + const timestamp = + metadata.timestampColum === NONE_PREFERENCE_MAP + ? new Date() + : new Date(update[metadata.timestampColum!]); + + // Determine updates + const updates = getPreferenceUpdatesFromRow({ + row: update, + columnToPurposeName: metadata.columnToPurposeName, + preferenceTopics, + purposeSlugs: purposes.map((x) => x.trackingType), + }); + pendingUpdates[userId] = { + userId, + partition, + timestamp: timestamp.toISOString(), + purposes: Object.entries(updates).map(([purpose, value]) => ({ + ...value, + purpose, + workflowSettings: { + attributes: parsedAttributes, + isSilent, + skipWorkflowTrigger: skipWorkflowTriggers, + }, + })), + }; + }); + await preferenceState.setValue(pendingUpdates, 'pendingUpdates'); + await preferenceState.setValue({}, 'failingUpdates'); + + // Exist early if dry run + if (dryRun) { + logger.info( + colors.green( + `Dry run complete, exiting. ${ + Object.values(pendingUpdates).length + } pending updates. Check file: ${receiptFilepath}`, + ), + ); + return; + } + + logger.info( + colors.magenta( + `Uploading ${ + Object.values(pendingUpdates).length + } preferences to partition: ${partition}`, + ), + ); + + // Time duration + const t0 = new Date().getTime(); + + // create a new progress bar instance and use shades_classic theme + const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, + ); + + // Build a GraphQL client + let total = 0; + const updatesToRun = Object.entries(pendingUpdates); + const chunkedUpdates = chunk(updatesToRun, skipWorkflowTriggers ? 100 : 10); + progressBar.start(updatesToRun.length, 0); + await map( + chunkedUpdates, + async (currentChunk) => { + // Make the request + try { + await sombra + .put('v1/preferences', { + json: { + records: currentChunk.map(([, update]) => update), + skipWorkflowTriggers, + }, + }) + .json(); + } catch (err) { + try { + const parsed = JSON.parse(err?.response?.body || '{}'); + if (parsed.error) { + logger.error(colors.red(`Error: ${parsed.error}`)); + } + } catch (e) { + // continue + } + logger.error( + colors.red( + `Failed to upload ${ + currentChunk.length + } user preferences to partition ${partition}: ${ + err?.response?.body || err?.message + }`, + ), + ); + const failingUpdates = preferenceState.getValue('failingUpdates'); + currentChunk.forEach(([userId, update]) => { + failingUpdates[userId] = { + uploadedAt: new Date().toISOString(), + update, + error: err?.response?.body || err?.message || 'Unknown error', + }; + }); + await preferenceState.setValue(failingUpdates, 'failingUpdates'); + } + + total += currentChunk.length; + progressBar.update(total); + }, + { + concurrency: 40, + }, + ); + + progressBar.stop(); + const t1 = new Date().getTime(); + const totalTime = t1 - t0; + logger.info( + colors.green( + `Successfully uploaded ${ + updatesToRun.length + } user preferences to partition ${partition} in "${ + totalTime / 1000 + }" seconds!`, + ), + ); +} diff --git a/src/requests/bulkRestartRequests.ts b/src/requests/bulkRestartRequests.ts index 6d614f9d..36fc69d7 100644 --- a/src/requests/bulkRestartRequests.ts +++ b/src/requests/bulkRestartRequests.ts @@ -201,7 +201,7 @@ export async function bulkRestartRequests({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - state.setValue(restartedRequests, 'restartedRequests'); + await state.setValue(restartedRequests, 'restartedRequests'); } catch (err) { const msg = `${err.message} - ${JSON.stringify( err.response?.body, @@ -219,7 +219,7 @@ export async function bulkRestartRequests({ attemptedAt: new Date().toISOString(), error: clientError || msg, }); - state.setValue(failingRequests, 'failingRequests'); + await state.setValue(failingRequests, 'failingRequests'); } total += 1; progressBar.update(total); diff --git a/src/requests/mapColumnsToAttributes.ts b/src/requests/mapColumnsToAttributes.ts index b4ff8f3e..15e65988 100644 --- a/src/requests/mapColumnsToAttributes.ts +++ b/src/requests/mapColumnsToAttributes.ts @@ -53,9 +53,11 @@ export async function mapColumnsToAttributes( }; }), ); - Object.entries(attributeNameMap).forEach(([k, v]) => { - state.setValue(v, 'attributeNames', k); - }); + await Promise.all( + Object.entries(attributeNameMap).map(([k, v]) => + state.setValue(v, 'attributeNames', k), + ), + ); return { ...state.getValue('attributeNames'), diff --git a/src/requests/mapColumnsToIdentifiers.ts b/src/requests/mapColumnsToIdentifiers.ts index 7a0c0c6d..6bec8709 100644 --- a/src/requests/mapColumnsToIdentifiers.ts +++ b/src/requests/mapColumnsToIdentifiers.ts @@ -59,9 +59,11 @@ export async function mapColumnsToIdentifiers( }; }), ); - Object.entries(identifierNameMap).forEach(([k, v]) => { - state.setValue(v, 'identifierNames', k); - }); + await Promise.all( + Object.entries(identifierNameMap).map(([k, v]) => + state.setValue(v, 'identifierNames', k), + ), + ); return { ...state.getValue('identifierNames'), diff --git a/src/requests/mapCsvColumnsToApi.ts b/src/requests/mapCsvColumnsToApi.ts index 0e479b75..567562cf 100644 --- a/src/requests/mapCsvColumnsToApi.ts +++ b/src/requests/mapCsvColumnsToApi.ts @@ -59,8 +59,10 @@ export async function mapCsvColumnsToApi( }), ); - getEntries(columnNameMap).forEach(([k, v]) => { - state.setValue(v, 'columnNames', k); - }); + await Promise.all( + getEntries(columnNameMap).map(([k, v]) => + state.setValue(v, 'columnNames', k), + ), + ); return columnNameMap; } diff --git a/src/requests/mapRequestEnumValues.ts b/src/requests/mapRequestEnumValues.ts index d119c94a..9e807604 100644 --- a/src/requests/mapRequestEnumValues.ts +++ b/src/requests/mapRequestEnumValues.ts @@ -56,7 +56,10 @@ export async function mapRequestEnumValues( Object.values(RequestAction), state.getValue('requestTypeToRequestAction'), ); - state.setValue(requestTypeToRequestAction, 'requestTypeToRequestAction'); + await state.setValue( + requestTypeToRequestAction, + 'requestTypeToRequestAction', + ); // Map data subject type logger.info(colors.magenta('Determining mapping of columns for subject')); @@ -66,7 +69,7 @@ export async function mapRequestEnumValues( internalSubjects.map(({ type }) => type), state.getValue('subjectTypeToSubjectName'), ); - state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName'); + await state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName'); // Map locale logger.info(colors.magenta('Determining mapping of columns for locale')); @@ -75,7 +78,7 @@ export async function mapRequestEnumValues( Object.values(LanguageKey), state.getValue('languageToLocale'), ); - state.setValue(languageToLocale, 'languageToLocale'); + await state.setValue(languageToLocale, 'languageToLocale'); logger.info( colors.magenta('Determining mapping of columns for request status'), ); @@ -95,7 +98,7 @@ export async function mapRequestEnumValues( [...Object.values(CompletedRequestStatus), NONE], state.getValue('statusToRequestStatus'), ); - state.setValue(statusToRequestStatus, 'statusToRequestStatus'); + await state.setValue(statusToRequestStatus, 'statusToRequestStatus'); // Map country logger.info(colors.magenta('Determining mapping of columns for country')); @@ -110,7 +113,7 @@ export async function mapRequestEnumValues( [...Object.values(IsoCountryCode), NONE], state.getValue('regionToCountry'), ); - state.setValue(regionToCountry, 'regionToCountry'); + await state.setValue(regionToCountry, 'regionToCountry'); // Map country sub division logger.info( @@ -127,5 +130,8 @@ export async function mapRequestEnumValues( [...Object.values(IsoCountrySubdivisionCode), NONE], state.getValue('regionToCountrySubDivision'), ); - state.setValue(regionToCountrySubDivision, 'regionToCountrySubDivision'); + await state.setValue( + regionToCountrySubDivision, + 'regionToCountrySubDivision', + ); } diff --git a/src/requests/uploadPrivacyRequestsFromCsv.ts b/src/requests/uploadPrivacyRequestsFromCsv.ts index 80fee0f2..84063de0 100644 --- a/src/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/requests/uploadPrivacyRequestsFromCsv.ts @@ -259,7 +259,7 @@ export async function uploadPrivacyRequestsFromCsv({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - requestState.setValue(successfulRequests, 'successfulRequests'); + await requestState.setValue(successfulRequests, 'successfulRequests'); } catch (err) { const msg = `${err.message} - ${JSON.stringify( err.response?.body, @@ -286,7 +286,7 @@ export async function uploadPrivacyRequestsFromCsv({ rowIndex: ind, attemptedAt: new Date().toISOString(), }); - requestState.setValue(duplicateRequests, 'duplicateRequests'); + await requestState.setValue(duplicateRequests, 'duplicateRequests'); } else { const failingRequests = requestState.getValue('failingRequests'); failingRequests.push({ @@ -295,7 +295,7 @@ export async function uploadPrivacyRequestsFromCsv({ error: clientError || msg, attemptedAt: new Date().toISOString(), }); - requestState.setValue(failingRequests, 'failingRequests'); + await requestState.setValue(failingRequests, 'failingRequests'); if (debug) { logger.error(colors.red(clientError || msg)); logger.error( diff --git a/transcend-yml-schema-v6.json b/transcend-yml-schema-v6.json index a0b12cb2..a8e42058 100644 --- a/transcend-yml-schema-v6.json +++ b/transcend-yml-schema-v6.json @@ -11,7 +11,11 @@ "allOf": [ { "type": "object", - "required": ["title", "type", "collections"], + "required": [ + "title", + "type", + "collections" + ], "properties": { "title": { "type": "string" @@ -74,7 +78,13 @@ "properties": { "priority": { "type": "string", - "enum": ["WONT_DO", "LOW", "MEDIUM", "HIGH", "CRITICAL"] + "enum": [ + "WONT_DO", + "LOW", + "MEDIUM", + "HIGH", + "CRITICAL" + ] }, "customerExperienceActionItemId": { "type": "string" @@ -107,7 +117,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -132,7 +145,10 @@ "allOf": [ { "type": "object", - "required": ["title", "productLine"], + "required": [ + "title", + "productLine" + ], "properties": { "title": { "type": "string" @@ -178,7 +194,9 @@ "type": "array", "items": { "type": "object", - "required": ["title"], + "required": [ + "title" + ], "properties": { "title": { "type": "string" @@ -192,7 +210,10 @@ "allOf": [ { "type": "object", - "required": ["name", "description"], + "required": [ + "name", + "description" + ], "properties": { "name": { "type": "string" @@ -327,7 +348,9 @@ "type": "array", "items": { "type": "object", - "required": ["title"], + "required": [ + "title" + ], "properties": { "title": { "type": "string" @@ -341,7 +364,10 @@ "allOf": [ { "type": "object", - "required": ["title", "output-identifiers"], + "required": [ + "title", + "output-identifiers" + ], "properties": { "title": { "type": "string" @@ -395,7 +421,10 @@ }, "transitionRequestStatus": { "type": "string", - "enum": ["CANCELED", "ON_HOLD"] + "enum": [ + "CANCELED", + "ON_HOLD" + ] }, "phoneNumbers": { "type": "array", @@ -5806,7 +5835,10 @@ "allOf": [ { "type": "object", - "required": ["name", "value"], + "required": [ + "name", + "value" + ], "properties": { "name": { "type": "string" @@ -5865,7 +5897,10 @@ "allOf": [ { "type": "object", - "required": ["name", "type"], + "required": [ + "name", + "type" + ], "properties": { "name": { "type": "string" @@ -5928,7 +5963,9 @@ "allOf": [ { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -5960,7 +5997,9 @@ "allOf": [ { "type": "object", - "required": ["title"], + "required": [ + "title" + ], "properties": { "title": { "type": "string" @@ -11379,7 +11418,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -11416,7 +11458,9 @@ "allOf": [ { "type": "object", - "required": ["title"], + "required": [ + "title" + ], "properties": { "title": { "type": "string" @@ -16856,7 +16900,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -16881,7 +16928,10 @@ "allOf": [ { "type": "object", - "required": ["name", "category"], + "required": [ + "name", + "category" + ], "properties": { "name": { "type": "string" @@ -16935,7 +16985,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -16960,7 +17013,10 @@ "allOf": [ { "type": "object", - "required": ["name", "purpose"], + "required": [ + "name", + "purpose" + ], "properties": { "name": { "type": "string" @@ -17007,7 +17063,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -17032,7 +17091,9 @@ "allOf": [ { "type": "object", - "required": ["type"], + "required": [ + "type" + ], "properties": { "type": { "type": "string" @@ -17089,7 +17150,9 @@ "allOf": [ { "type": "object", - "required": ["type"], + "required": [ + "type" + ], "properties": { "type": { "type": "string", @@ -17134,7 +17197,11 @@ }, "regionDetectionMethod": { "type": "string", - "enum": ["DISABLED", "AUTO", "FORM"] + "enum": [ + "DISABLED", + "AUTO", + "FORM" + ] }, "regionList": { "type": "array", @@ -27929,7 +27996,10 @@ "allOf": [ { "type": "object", - "required": ["name", "type"], + "required": [ + "name", + "type" + ], "properties": { "name": { "type": "string" @@ -28010,7 +28080,10 @@ "allOf": [ { "type": "object", - "required": ["title", "integrationName"], + "required": [ + "title", + "integrationName" + ], "properties": { "title": { "type": "string" @@ -28041,7 +28114,10 @@ "allOf": [ { "type": "object", - "required": ["name", "value"], + "required": [ + "name", + "value" + ], "properties": { "name": { "type": "string" @@ -28101,7 +28177,9 @@ "allOf": [ { "type": "object", - "required": ["key"], + "required": [ + "key" + ], "properties": { "key": { "type": "string" @@ -28221,7 +28299,9 @@ "allOf": [ { "type": "object", - "required": ["key"], + "required": [ + "key" + ], "properties": { "key": { "type": "string" @@ -28248,7 +28328,10 @@ "type": "array", "items": { "type": "object", - "required": ["purpose", "name"], + "required": [ + "purpose", + "name" + ], "properties": { "purpose": { "type": "string", @@ -28280,7 +28363,9 @@ "allOf": [ { "type": "object", - "required": ["category"], + "required": [ + "category" + ], "properties": { "category": { "type": "string", @@ -28334,7 +28419,9 @@ "allOf": [ { "type": "object", - "required": ["category"], + "required": [ + "category" + ], "properties": { "category": { "type": "string", @@ -28380,7 +28467,11 @@ }, "confidenceLabel": { "type": "string", - "enum": ["HIGH", "MEDIUM", "LOW"] + "enum": [ + "HIGH", + "MEDIUM", + "LOW" + ] }, "confidence": { "type": "number" @@ -28408,7 +28499,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -28455,7 +28549,10 @@ }, "send-type": { "type": "string", - "enum": ["PER_DSR", "CROSS_DSR"] + "enum": [ + "PER_DSR", + "CROSS_DSR" + ] }, "include-identifiers-attachment": { "type": "boolean" @@ -33870,7 +33967,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -33895,14 +33995,23 @@ "allOf": [ { "type": "object", - "required": ["value", "type"], + "required": [ + "value", + "type" + ], "properties": { "value": { "type": "string" }, "type": { "type": "string", - "enum": ["HOST", "PATH", "QUERY_PARAM", "REGEX", "CSP"] + "enum": [ + "HOST", + "PATH", + "QUERY_PARAM", + "REGEX", + "CSP" + ] } } }, @@ -33923,7 +34032,10 @@ }, "status": { "type": "string", - "enum": ["LIVE", "NEEDS_REVIEW"] + "enum": [ + "LIVE", + "NEEDS_REVIEW" + ] }, "owners": { "type": "array", @@ -33941,7 +34053,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -33966,7 +34081,9 @@ "allOf": [ { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -33993,7 +34110,10 @@ }, "status": { "type": "string", - "enum": ["LIVE", "NEEDS_REVIEW"] + "enum": [ + "LIVE", + "NEEDS_REVIEW" + ] }, "owners": { "type": "array", @@ -34011,7 +34131,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" @@ -34053,30 +34176,52 @@ }, "consentPrecedence": { "type": "string", - "enum": ["user", "signal"] + "enum": [ + "user", + "signal" + ] }, "unknownRequestPolicy": { "type": "string", - "enum": ["ALLOW", "REQUIRE_FULL_CONSENT", "BLOCK"] + "enum": [ + "ALLOW", + "REQUIRE_FULL_CONSENT", + "BLOCK" + ] }, "unknownCookiePolicy": { "type": "string", - "enum": ["ALLOW", "REQUIRE_FULL_CONSENT", "BLOCK"] + "enum": [ + "ALLOW", + "REQUIRE_FULL_CONSENT", + "BLOCK" + ] }, "syncEndpoint": { "type": "string" }, "telemetryPartitioning": { "type": "string", - "enum": ["origin", "path", "url"] + "enum": [ + "origin", + "path", + "url" + ] }, "signedIabAgreement": { "type": "string", - "enum": ["yes", "no", "unknown"] + "enum": [ + "yes", + "no", + "unknown" + ] }, "uspapi": { "type": "string", - "enum": ["on", "off"] + "enum": [ + "on", + "off" + ] }, "experiences": { "type": "array", @@ -34084,7 +34229,9 @@ "allOf": [ { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -39500,14 +39647,21 @@ }, "onConsentExpiry": { "type": "string", - "enum": ["Prompt", "ResetAll", "ResetOptIns"] + "enum": [ + "Prompt", + "ResetAll", + "ResetOptIns" + ] }, "consentExpiry": { "type": "number" }, "operator": { "type": "string", - "enum": ["IN", "NOT_IN"] + "enum": [ + "IN", + "NOT_IN" + ] }, "displayPriority": { "type": "number" @@ -39539,7 +39693,9 @@ "type": "array", "items": { "type": "object", - "required": ["trackingType"], + "required": [ + "trackingType" + ], "properties": { "trackingType": { "type": "string" @@ -39551,7 +39707,9 @@ "type": "array", "items": { "type": "object", - "required": ["trackingType"], + "required": [ + "trackingType" + ], "properties": { "trackingType": { "type": "string" @@ -40277,7 +40435,10 @@ "type": "array", "items": { "type": "object", - "required": ["title", "content"], + "required": [ + "title", + "content" + ], "properties": { "title": { "type": "string" @@ -40292,7 +40453,10 @@ "type": "array", "items": { "type": "object", - "required": ["title", "content"], + "required": [ + "title", + "content" + ], "properties": { "title": { "type": "string" @@ -40307,7 +40471,11 @@ "type": "array", "items": { "type": "object", - "required": ["title", "description", "prompts"], + "required": [ + "title", + "description", + "prompts" + ], "properties": { "title": { "type": "string" @@ -40356,14 +40524,21 @@ }, "large-language-model": { "type": "object", - "required": ["name", "client"], + "required": [ + "name", + "client" + ], "properties": { "name": { "type": "string" }, "client": { "type": "string", - "enum": ["openai", "claude", "llama"] + "enum": [ + "openai", + "claude", + "llama" + ] } } } @@ -40411,7 +40586,11 @@ "type": "array", "items": { "type": "object", - "required": ["name", "description", "parameters"], + "required": [ + "name", + "description", + "parameters" + ], "properties": { "name": { "type": "string" @@ -40431,7 +40610,12 @@ "allOf": [ { "type": "object", - "required": ["name", "fileId", "size", "purpose"], + "required": [ + "name", + "fileId", + "size", + "purpose" + ], "properties": { "name": { "type": "string" @@ -40769,7 +40953,10 @@ "allOf": [ { "type": "object", - "required": ["name", "url"], + "required": [ + "name", + "url" + ], "properties": { "name": { "type": "string" @@ -40786,7 +40973,10 @@ "type": "array", "items": { "type": "object", - "required": ["name", "url"], + "required": [ + "name", + "url" + ], "properties": { "name": { "type": "string" @@ -40805,7 +40995,10 @@ "allOf": [ { "type": "object", - "required": ["name", "url"], + "required": [ + "name", + "url" + ], "properties": { "name": { "type": "string" @@ -40822,7 +41015,10 @@ "type": "array", "items": { "type": "object", - "required": ["name", "url"], + "required": [ + "name", + "url" + ], "properties": { "name": { "type": "string" @@ -40849,7 +41045,9 @@ "allOf": [ { "type": "object", - "required": ["title"], + "required": [ + "title" + ], "properties": { "title": { "type": "string" @@ -40954,7 +41152,9 @@ "allOf": [ { "type": "object", - "required": ["id"], + "required": [ + "id" + ], "properties": { "id": { "type": "string" @@ -41187,7 +41387,9 @@ "allOf": [ { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -41211,7 +41413,9 @@ "allOf": [ { "type": "object", - "required": ["title"], + "required": [ + "title" + ], "properties": { "title": { "type": "string" @@ -41227,7 +41431,10 @@ "allOf": [ { "type": "object", - "required": ["title", "questions"], + "required": [ + "title", + "questions" + ], "properties": { "title": { "type": "string" @@ -41238,7 +41445,10 @@ "allOf": [ { "type": "object", - "required": ["title", "type"], + "required": [ + "title", + "type" + ], "properties": { "title": { "type": "string" @@ -41301,11 +41511,16 @@ "allOf": [ { "type": "object", - "required": ["action"], + "required": [ + "action" + ], "properties": { "action": { "type": "string", - "enum": ["SHOW", "SKIP"] + "enum": [ + "SHOW", + "SKIP" + ] } } }, @@ -41411,7 +41626,9 @@ "type": "array", "items": { "type": "object", - "required": ["value"], + "required": [ + "value" + ], "properties": { "value": { "type": "string" @@ -41570,11 +41787,18 @@ }, "status": { "type": "string", - "enum": ["DRAFT", "PUBLISHED"] + "enum": [ + "DRAFT", + "PUBLISHED" + ] }, "source": { "type": "string", - "enum": ["MANUAL", "DATA_INVENTORY", "IMPORT"] + "enum": [ + "MANUAL", + "DATA_INVENTORY", + "IMPORT" + ] }, "creator": { "type": "string" @@ -41599,7 +41823,11 @@ }, "retention-schedule": { "type": "object", - "required": ["type", "duration-days", "operand"], + "required": [ + "type", + "duration-days", + "operand" + ], "properties": { "type": { "type": "string", @@ -41613,7 +41841,11 @@ }, "operand": { "type": "string", - "enum": ["FULL_DELETE", "PARTIAL_DELETE", "NONE"] + "enum": [ + "FULL_DELETE", + "PARTIAL_DELETE", + "NONE" + ] } } }, @@ -41634,7 +41866,10 @@ "allOf": [ { "type": "object", - "required": ["title", "group"], + "required": [ + "title", + "group" + ], "properties": { "title": { "type": "string" @@ -41653,7 +41888,10 @@ "allOf": [ { "type": "object", - "required": ["title", "questions"], + "required": [ + "title", + "questions" + ], "properties": { "title": { "type": "string" @@ -41664,7 +41902,10 @@ "allOf": [ { "type": "object", - "required": ["title", "type"], + "required": [ + "title", + "type" + ], "properties": { "title": { "type": "string" @@ -41727,11 +41968,16 @@ "allOf": [ { "type": "object", - "required": ["action"], + "required": [ + "action" + ], "properties": { "action": { "type": "string", - "enum": ["SHOW", "SKIP"] + "enum": [ + "SHOW", + "SKIP" + ] } } }, @@ -41837,7 +42083,9 @@ "type": "array", "items": { "type": "object", - "required": ["value"], + "required": [ + "value" + ], "properties": { "value": { "type": "string" @@ -42061,7 +42309,10 @@ "type": "array", "items": { "type": "object", - "required": ["title", "type"], + "required": [ + "title", + "type" + ], "properties": { "title": { "type": "string" @@ -42102,7 +42353,10 @@ "type": "array", "items": { "type": "object", - "required": ["title", "type"], + "required": [ + "title", + "type" + ], "properties": { "title": { "type": "string" @@ -42141,7 +42395,11 @@ }, "retention-schedule": { "type": "object", - "required": ["type", "duration-days", "operand"], + "required": [ + "type", + "duration-days", + "operand" + ], "properties": { "type": { "type": "string", @@ -42155,7 +42413,11 @@ }, "operand": { "type": "string", - "enum": ["FULL_DELETE", "PARTIAL_DELETE", "NONE"] + "enum": [ + "FULL_DELETE", + "PARTIAL_DELETE", + "NONE" + ] } } }, @@ -42163,7 +42425,10 @@ "type": "array", "items": { "type": "object", - "required": ["key", "values"], + "required": [ + "key", + "values" + ], "properties": { "key": { "type": "string" diff --git a/yarn.lock b/yarn.lock index 02ff583d..14a6f72d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -515,13 +515,14 @@ __metadata: "@transcend-io/handlebars-utils": ^1.1.0 "@transcend-io/internationalization": ^1.6.0 "@transcend-io/persisted-state": ^1.0.4 - "@transcend-io/privacy-types": ^4.94.0 + "@transcend-io/privacy-types": ^4.98.0 "@transcend-io/secret-value": ^1.2.0 "@transcend-io/type-utils": ^1.5.0 "@types/bluebird": ^3.5.38 "@types/chai": ^4.3.4 "@types/cli-progress": ^3.11.0 "@types/colors": ^1.2.1 + "@types/deep-equal-in-any-order": 1.0.1 "@types/fuzzysearch": ^1.0.0 "@types/global-agent": ^2.1.1 "@types/inquirer": ^7.3.1 @@ -542,6 +543,7 @@ __metadata: cli-progress: ^3.11.2 colors: ^1.4.0 csv-parse: =4.9.1 + deep-equal-in-any-order: ^1.0.28 depcheck: ^1.4.3 eslint: ^8.38.0 eslint-config-airbnb-base: ^15.0.0 @@ -608,6 +610,7 @@ __metadata: tr-upload-consent-preferences: ./build/cli-upload-consent-preferences.js tr-upload-cookies-from-csv: ./build/cli-upload-cookies-from-csv.js tr-upload-data-flows-from-csv: ./build/cli-upload-data-flows-from-csv.js + tr-upload-preferences: ./build/cli-upload-preferences.js languageName: unknown linkType: soft @@ -643,14 +646,14 @@ __metadata: languageName: node linkType: hard -"@transcend-io/privacy-types@npm:^4.94.0": - version: 4.94.0 - resolution: "@transcend-io/privacy-types@npm:4.94.0" +"@transcend-io/privacy-types@npm:^4.98.0": + version: 4.98.0 + resolution: "@transcend-io/privacy-types@npm:4.98.0" dependencies: "@transcend-io/type-utils": ^1.0.5 fp-ts: ^2.16.1 io-ts: ^2.2.21 - checksum: 8cc1a8dd546f7660a7363937c50bf217026c7e492e5a3ef9c7a482c70bd71f4b51db85dcc264128e5100698dac99d99508f9403e6709e47ee1d4f0c3ae743a86 + checksum: bee61228fa7ce58f2fbfe1dcb2e31812e480ef16012aa2f73deccf3812fc330225a332d57730303d46764de98079a8fc905969894419a3dc9a33744e8ec012d1 languageName: node linkType: hard @@ -777,6 +780,13 @@ __metadata: languageName: node linkType: hard +"@types/deep-equal-in-any-order@npm:1.0.1": + version: 1.0.1 + resolution: "@types/deep-equal-in-any-order@npm:1.0.1" + checksum: 09a76203438bc2ed9e3ac3f22b6fd43f3561ac4ae5403cf62b4e69359f3016680fbd6ce31f80704e310c3616f4d134e3252ba70e963fc4ca4dd261719c6eb4fd + languageName: node + linkType: hard + "@types/emscripten@npm:^1.39.6": version: 1.39.6 resolution: "@types/emscripten@npm:1.39.6" @@ -2148,6 +2158,16 @@ __metadata: languageName: node linkType: hard +"deep-equal-in-any-order@npm:^1.0.28": + version: 1.1.20 + resolution: "deep-equal-in-any-order@npm:1.1.20" + dependencies: + lodash.mapvalues: ^4.6.0 + sort-any: ^2.0.0 + checksum: 3fd4a571269e86f8958e797e4994df9f8aaa7c8cc438204a15b9c170e55b53f4b674e4609d8eaccb0b444a55c1932fd74026ef072f4a5412fd879599000e499f + languageName: node + linkType: hard + "deep-is@npm:^0.1.3": version: 0.1.4 resolution: "deep-is@npm:0.1.4" @@ -4255,6 +4275,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"lodash.mapvalues@npm:^4.6.0": + version: 4.6.0 + resolution: "lodash.mapvalues@npm:4.6.0" + checksum: 0ff1b252fda318fc36e47c296984925e98fbb0fc5a2ecc4ef458f3c739a9476d47e40c95ac653e8314d132aa59c746d4276527b99d6e271940555c6e12d2babd + languageName: node + linkType: hard + "lodash.merge@npm:^4.6.2": version: 4.6.2 resolution: "lodash.merge@npm:4.6.2" @@ -5592,6 +5619,15 @@ resolve@^1.18.1: languageName: node linkType: hard +"sort-any@npm:^2.0.0": + version: 2.0.0 + resolution: "sort-any@npm:2.0.0" + dependencies: + lodash: ^4.17.21 + checksum: d2dc6cc4f56298ce50b13ce6aafd8bf16010a8f8d0ae75345e8424118ac2cc017a8ffea0eedd136dd67751c09e7b0edb09c959d052ed7ed02afcfbab63c68277 + languageName: node + linkType: hard + "source-map-js@npm:^1.0.2": version: 1.0.2 resolution: "source-map-js@npm:1.0.2"