diff --git a/.pnp.cjs b/.pnp.cjs index c6c2be26..b8422964 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -39,6 +39,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/chai", "npm:4.3.4"],\ ["@types/cli-progress", "npm:3.11.0"],\ ["@types/colors", "npm:1.2.1"],\ + ["@types/deep-equal-in-any-order", "npm:1.0.1"],\ ["@types/fuzzysearch", "npm:1.0.0"],\ ["@types/global-agent", "npm:2.1.1"],\ ["@types/inquirer", "npm:7.3.3"],\ @@ -59,6 +60,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["cli-progress", "npm:3.11.2"],\ ["colors", "npm:1.4.0"],\ ["csv-parse", "npm:4.9.1"],\ + ["deep-equal-in-any-order", "npm:1.1.20"],\ ["depcheck", "npm:1.4.3"],\ ["eslint", "npm:8.38.0"],\ ["eslint-config-airbnb-base", "virtual:05f71a130d973c89ca25c909f694e525ea840a597fa145c0b76e367faaae056c000c13c8dc936d4a8d6e8c6aa1f1290e6e296ad4666bf503fd491f764698380a#npm:15.0.0"],\ @@ -689,6 +691,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/chai", "npm:4.3.4"],\ ["@types/cli-progress", "npm:3.11.0"],\ ["@types/colors", "npm:1.2.1"],\ + ["@types/deep-equal-in-any-order", "npm:1.0.1"],\ ["@types/fuzzysearch", "npm:1.0.0"],\ ["@types/global-agent", "npm:2.1.1"],\ ["@types/inquirer", "npm:7.3.3"],\ @@ -709,6 +712,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["cli-progress", "npm:3.11.2"],\ ["colors", "npm:1.4.0"],\ ["csv-parse", "npm:4.9.1"],\ + ["deep-equal-in-any-order", "npm:1.1.20"],\ ["depcheck", "npm:1.4.3"],\ ["eslint", "npm:8.38.0"],\ ["eslint-config-airbnb-base", "virtual:05f71a130d973c89ca25c909f694e525ea840a597fa145c0b76e367faaae056c000c13c8dc936d4a8d6e8c6aa1f1290e6e296ad4666bf503fd491f764698380a#npm:15.0.0"],\ @@ -929,6 +933,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@types/deep-equal-in-any-order", [\ + ["npm:1.0.1", {\ + "packageLocation": "./.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip/node_modules/@types/deep-equal-in-any-order/",\ + "packageDependencies": [\ + ["@types/deep-equal-in-any-order", "npm:1.0.1"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@types/emscripten", [\ ["npm:1.39.6", {\ "packageLocation": "./.yarn/cache/@types-emscripten-npm-1.39.6-c9c4021365-437f2f9cdf.zip/node_modules/@types/emscripten/",\ @@ -2617,6 +2630,17 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["deep-equal-in-any-order", [\ + ["npm:1.1.20", {\ + "packageLocation": "./.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip/node_modules/deep-equal-in-any-order/",\ + "packageDependencies": [\ + ["deep-equal-in-any-order", "npm:1.1.20"],\ + ["lodash.mapvalues", "npm:4.6.0"],\ + ["sort-any", "npm:2.0.0"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["deep-is", [\ ["npm:0.1.4", {\ "packageLocation": "./.yarn/cache/deep-is-npm-0.1.4-88938b5a67-edb65dd0d7.zip/node_modules/deep-is/",\ @@ -5137,6 +5161,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["lodash.mapvalues", [\ + ["npm:4.6.0", {\ + "packageLocation": "./.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip/node_modules/lodash.mapvalues/",\ + "packageDependencies": [\ + ["lodash.mapvalues", "npm:4.6.0"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["lodash.merge", [\ ["npm:4.6.2", {\ "packageLocation": "./.yarn/cache/lodash.merge-npm-4.6.2-77cb4416bf-ad580b4bdb.zip/node_modules/lodash.merge/",\ @@ -6613,6 +6646,16 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["sort-any", [\ + ["npm:2.0.0", {\ + "packageLocation": "./.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip/node_modules/sort-any/",\ + "packageDependencies": [\ + ["sort-any", "npm:2.0.0"],\ + ["lodash", "npm:4.17.21"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["source-map", [\ ["npm:0.6.1", {\ "packageLocation": "./.yarn/cache/source-map-npm-0.6.1-1a3621db16-59ce8640cf.zip/node_modules/source-map/",\ diff --git a/.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip b/.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip new file mode 100644 index 00000000..72ad4c96 Binary files /dev/null and b/.yarn/cache/@types-deep-equal-in-any-order-npm-1.0.1-94ef7dc492-09a7620343.zip differ diff --git a/.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip b/.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip new file mode 100644 index 00000000..48eef882 Binary files /dev/null and b/.yarn/cache/deep-equal-in-any-order-npm-1.1.20-9c0bb76c30-3fd4a57126.zip differ diff --git a/.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip b/.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip new file mode 100644 index 00000000..063b2daa Binary files /dev/null and b/.yarn/cache/lodash.mapvalues-npm-4.6.0-4664380119-0ff1b252fd.zip differ diff --git a/.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip b/.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip new file mode 100644 index 00000000..0be1d132 Binary files /dev/null and b/.yarn/cache/sort-any-npm-2.0.0-e58427c6cb-d2dc6cc4f5.zip differ diff --git a/package.json b/package.json index 1572d464..880a3a71 100644 --- a/package.json +++ b/package.json @@ -100,6 +100,7 @@ "@types/chai": "^4.3.4", "@types/cli-progress": "^3.11.0", "@types/colors": "^1.2.1", + "@types/deep-equal-in-any-order": "1.0.1", "@types/fuzzysearch": "^1.0.0", "@types/global-agent": "^2.1.1", "@types/inquirer": "^7.3.1", @@ -116,6 +117,7 @@ "@typescript-eslint/parser": "^5.58.0", "@yarnpkg/sdks": "^3.0.0-rc.42", "chai": "^4.3.7", + "deep-equal-in-any-order": "^1.0.28", "depcheck": "^1.4.3", "eslint": "^8.38.0", "eslint-config-airbnb-base": "^15.0.0", diff --git a/src/cli-upload-preferences.ts b/src/cli-upload-preferences.ts index d04c6580..b63550ec 100644 --- a/src/cli-upload-preferences.ts +++ b/src/cli-upload-preferences.ts @@ -26,7 +26,7 @@ async function main(): Promise { // Parse command line arguments const { /** File to load preferences from */ - files = './preferences.csv', + file = './preferences.csv', /** Transcend URL */ transcendUrl = DEFAULT_TRANSCEND_API, /** API key */ @@ -39,6 +39,8 @@ async function main(): Promise { dryRun = 'false', /** Whether to skip workflow triggers */ skipWorkflowTriggers = 'false', + /** Whether to skip conflict updates */ + skipConflictUpdates = 'false', /** Whether to skip sending emails */ isSilent = 'true', /** Attributes to add to any DSR request if created */ @@ -73,9 +75,10 @@ async function main(): Promise { receiptFilepath, auth, sombraAuth, - files: splitCsvToList(files), + file, partition, transcendUrl, + skipConflictUpdates: skipConflictUpdates !== 'false', skipWorkflowTriggers: skipWorkflowTriggers !== 'false', isSilent: isSilent !== 'false', dryRun: dryRun !== 'false', diff --git a/src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts b/src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts new file mode 100644 index 00000000..17b5525c --- /dev/null +++ b/src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts @@ -0,0 +1,93 @@ +import { + PreferenceQueryResponseItem, + PreferenceStorePurposeResponse, + PreferenceTopicType, +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; + +/** + * Check if the pending set of updates are exactly the same as the current consent record. + * + * @param options - Options + * @returns Whether the pending updates already exist in the preference store + */ +export function checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, +}: { + /** The current consent record */ + currentConsentRecord: PreferenceQueryResponseItem; + /** The pending updates */ + pendingUpdates: { + [purposeName in string]: Omit; + }; + /** The preference topic configurations */ + preferenceTopics: PreferenceTopic[]; +}): boolean { + // Check each update + return Object.entries(pendingUpdates).every( + ([purposeName, { preferences = [], enabled }]) => { + // Ensure the purpose exists + const currentPurpose = currentConsentRecord.purposes.find( + (existingPurpose) => existingPurpose.purpose === purposeName, + ); + + // Ensure purpose.enabled is in sync + // Also false if the purpose does not exist + const enabledIsInSync = + !!currentPurpose && currentPurpose.enabled === enabled; + if (!enabledIsInSync) { + return false; + } + + // Compare the preferences are in sync + return preferences.every( + ({ topic, choice }) => + // ensure preferences exist on record + currentPurpose.preferences && + currentPurpose.preferences.find((existingPreference) => { + // find matching topic + if (existingPreference.topic !== topic) { + return false; + } + + // Determine type of preference topic + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === topic && x.purpose.trackingType === purposeName, + ); + if (!preferenceTopic) { + throw new Error(`Could not find preference topic for ${topic}`); + } + + // Handle comparison based on type + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: + return ( + existingPreference.choice.booleanValue === choice.booleanValue + ); + case PreferenceTopicType.Select: + return ( + existingPreference.choice.selectValue === choice.selectValue + ); + case PreferenceTopicType.MultiSelect: + // eslint-disable-next-line no-case-declarations + const sortedCurrentValues = ( + existingPreference.choice.selectValues || [] + ).sort(); + // eslint-disable-next-line no-case-declarations + const sortedNewValues = (choice.selectValues || []).sort(); + return ( + sortedCurrentValues.length === sortedNewValues.length && + sortedCurrentValues.every((x, i) => x === sortedNewValues[i]) + ); + default: + throw new Error( + `Unknown preference topic type: ${preferenceTopic.type}`, + ); + } + }), + ); + }, + ); +} diff --git a/src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts b/src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts new file mode 100644 index 00000000..a14b4845 --- /dev/null +++ b/src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts @@ -0,0 +1,94 @@ +import { + PreferenceQueryResponseItem, + PreferenceStorePurposeResponse, + PreferenceTopicType, +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; + +/** + * Check if the pending set of updates will result in a change of + * value to an existing purpose or preference in the preference store. + * + * @param options - Options + * @returns True if conflict, false if no conflict and just adding new data for first time + */ +export function checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, +}: { + /** The current consent record */ + currentConsentRecord: PreferenceQueryResponseItem; + /** The pending updates */ + pendingUpdates: { + [purposeName in string]: Omit; + }; + /** The preference topic configurations */ + preferenceTopics: PreferenceTopic[]; +}): boolean { + // Check if any update has conflict + return !!Object.entries(pendingUpdates).find( + ([purposeName, { preferences = [], enabled }]) => { + // Ensure the purpose exists + const currentPurpose = currentConsentRecord.purposes.find( + (existingPurpose) => existingPurpose.purpose === purposeName, + ); + + // If no purpose exists, then it is not a conflict + if (!currentPurpose) { + return false; + } + + // If purpose.enabled value is off, this is a conflict + if (currentPurpose.enabled !== enabled) { + return true; + } + + // Check if any preferences are out of sync + return !!preferences.find(({ topic, choice }) => { + // find matching topic + const currentPreference = (currentPurpose.preferences || []).find( + (existingPreference) => existingPreference.topic === topic, + ); + + // if no topic exists, no conflict + if (!currentPreference) { + return false; + } + + // Determine type of preference topic + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === topic && x.purpose.trackingType === purposeName, + ); + if (!preferenceTopic) { + throw new Error(`Could not find preference topic for ${topic}`); + } + + // Handle comparison based on type + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: + return ( + currentPreference.choice.booleanValue !== choice.booleanValue + ); + case PreferenceTopicType.Select: + return currentPreference.choice.selectValue !== choice.selectValue; + case PreferenceTopicType.MultiSelect: + // eslint-disable-next-line no-case-declarations + const sortedCurrentValues = ( + currentPreference.choice.selectValues || [] + ).sort(); + // eslint-disable-next-line no-case-declarations + const sortedNewValues = (choice.selectValues || []).sort(); + return ( + sortedCurrentValues.length !== sortedNewValues.length || + !sortedCurrentValues.every((x, i) => x === sortedNewValues[i]) + ); + default: + throw new Error( + `Unknown preference topic type: ${preferenceTopic.type}`, + ); + } + }); + }, + ); +} diff --git a/src/preference-management/codecs.ts b/src/preference-management/codecs.ts index 3223f43d..0b07c507 100644 --- a/src/preference-management/codecs.ts +++ b/src/preference-management/codecs.ts @@ -5,10 +5,38 @@ import { import * as t from 'io-ts'; export const PurposeRowMapping = t.type({ - /** Name of the purpose to map to */ + /** + * The slug or trackingType of the purpose to map to + * + * e.g. `Marketing` + */ purpose: t.string, - /** Mapping from value in row to value in transcend API */ - valueMapping: t.record(t.string, t.union([t.string, t.boolean])), + /** + * If the column maps to a preference instead of a purpose + * this is the slug of the purpose. + * + * null value indicates that this column maps to the true/false + * value of the purpose + */ + preference: t.union([t.string, t.null]), + /** + * The mapping between each row value and purpose/preference value. + * + * e.g. for a boolean preference or purpose + * { + * 'true': true, + * 'false': false, + * '': true, + * } + * + * or for a single or multi select preference + * { + * '': true, + * 'value1': 'Value1', + * 'value2': 'Value2', + * } + */ + valueMapping: t.record(t.string, t.union([t.string, t.boolean, t.null])), }); /** Override type */ @@ -16,9 +44,12 @@ export type PurposeRowMapping = t.TypeOf; export const FileMetadataState = t.intersection([ t.type({ - /** Mapping of column name to it's relevant purpose in Transcend */ + /** + * Definition of how to map each column in the CSV to + * the relevant purpose and preference definitions in transcend + */ columnToPurposeName: t.record(t.string, PurposeRowMapping), - /** Last time the file was fetched */ + /** Last time the file was last parsed at */ lastFetchedAt: t.string, /** * Mapping of userId to the rows in the file that need to be uploaded diff --git a/src/preference-management/getPreferenceUpdatesFromRow.ts b/src/preference-management/getPreferenceUpdatesFromRow.ts new file mode 100644 index 00000000..32d63f68 --- /dev/null +++ b/src/preference-management/getPreferenceUpdatesFromRow.ts @@ -0,0 +1,196 @@ +import { + PreferenceStorePurposeResponse, + PreferenceTopicType, +} from '@transcend-io/privacy-types'; +import { PurposeRowMapping } from './codecs'; +import { apply } from '@transcend-io/type-utils'; +import { PreferenceTopic } from '../graphql'; +import { splitCsvToList } from '../requests'; + +/** + * Parse an arbitrary object to the Transcend PUT /v1/preference update shape + * by using a mapping of column names to purpose/preference slugs. + * + * columnToPurposeName looks like: + * { + * 'my_purpose': 'Marketing', + * 'has_topic_1': 'Marketing->BooleanPreference1', + * 'has_topic_2': 'Marketing->BooleanPreference2' + * } + * + * row looks like: + * { + * 'my_purpose': 'true', + * 'has_topic_1': 'true', + * 'has_topic_2': 'false' + * } + * + * @param options - Options + * @returns The parsed row + */ +export function getPreferenceUpdatesFromRow({ + row, + columnToPurposeName, + purposeSlugs, + preferenceTopics, +}: { + /** Row to parse */ + row: Record; + /** Mapping from column name ot row */ + columnToPurposeName: Record; + /** The set of allowed purpose slugs */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; +}): { + [k in string]: Omit; +} { + // Create a result object to store the parsed preferences + const result: { + [k in string]: Partial; + } = {}; + + // Iterate over each column and map to the purpose or preference + Object.entries(columnToPurposeName).forEach( + ([columnName, { purpose, preference, valueMapping }]) => { + // Ensure the purpose is valid + if (!purposeSlugs.includes(purpose)) { + throw new Error( + `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join( + ', ', + )}`, + ); + } + + // CHeck if parsing a preference or just the top level purpose + if (preference) { + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === preference && x.purpose.trackingType === purpose, + ); + if (!preferenceTopic) { + const allowedTopics = preferenceTopics + .filter((x) => x.purpose.trackingType === purpose) + .map((x) => x.slug); + throw new Error( + `Invalid preference slug: ${preference} for purpose: ${purpose}. ` + + `Allowed preference slugs for purpose are: ${allowedTopics.join( + ',', + )}`, + ); + } + + // If parsing preferences, default to an empty array + if (!result[purpose]) { + result[purpose] = { + preferences: [], + }; + } + if (!result[purpose].preferences) { + result[purpose].preferences = []; + } + + // The value to parse + const rawValue = row[columnName]; + const rawMapping = valueMapping[rawValue]; + const trimmedMapping = + typeof rawMapping === 'string' ? rawMapping.trim() || null : null; + + // handle each type of preference + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: + if (typeof rawMapping !== 'boolean') { + throw new Error( + `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}`, + ); + } + result[purpose].preferences!.push({ + topic: preference, + choice: { + booleanValue: rawMapping, + }, + }); + break; + case PreferenceTopicType.Select: + if (typeof rawMapping !== 'string' && rawMapping !== null) { + throw new Error( + `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}`, + ); + } + + if ( + trimmedMapping && + !preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .includes(trimmedMapping) + ) { + throw new Error( + `Invalid value for select preference: ${preference}, expected one of: ` + + `${preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .join(', ')}, got: ${rawValue}`, + ); + } + + // Update preferences + result[purpose].preferences!.push({ + topic: preference, + choice: { + selectValue: trimmedMapping, + }, + }); + break; + case PreferenceTopicType.MultiSelect: + if (typeof rawValue !== 'string') { + throw new Error( + `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}`, + ); + } + // Update preferences + result[purpose].preferences!.push({ + topic: preference, + choice: { + selectValues: splitCsvToList(rawValue) + .map((val) => { + const result = valueMapping[val]; + if (typeof result !== 'string') { + throw new Error( + `Invalid value for multi select preference: ${preference}, ` + + `expected one of: ${preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .join(', ')}, got: ${val}`, + ); + } + return result; + }) + .sort((a, b) => a.localeCompare(b)), + }, + }); + break; + default: + throw new Error(`Unknown preference type: ${preferenceTopic.type}`); + } + } else if (!result[purpose]) { + // Handle updating top level purpose for the first time + result[purpose] = { + enabled: valueMapping[row[columnName]] === true, + }; + } else { + // Handle updating top level purpose but preserve preference updates + result[purpose].enabled = valueMapping[row[columnName]] === true; + } + }, + ); + + // Ensure that enabled is provided + return apply(result, (x, purposeName) => { + if (typeof x.enabled !== 'boolean') { + throw new Error( + `No mapping provided for purpose.enabled=true/false value: ${purposeName}`, + ); + } + return { + ...x, + enabled: x.enabled!, + }; + }); +} diff --git a/src/preference-management/index.ts b/src/preference-management/index.ts index d46382df..1f914460 100644 --- a/src/preference-management/index.ts +++ b/src/preference-management/index.ts @@ -1,4 +1,11 @@ export * from './uploadPreferenceManagementPreferencesInteractive'; export * from './codecs'; export * from './getPreferencesForIdentifiers'; -export * from './parsePreferenceManagementCsvWithCache'; +export * from './parsePreferenceManagementCsv'; +export * from './getPreferenceUpdatesFromRow'; +export * from './parsePreferenceManagementCsv'; +export * from './parsePreferenceIdentifiersFromCsv'; +export * from './parsePreferenceTimestampsFromCsv'; +export * from './parsePreferenceAndPurposeValuesFromCsv'; +export * from './checkIfPendingPreferenceUpdatesAreNoOp'; +export * from './checkIfPendingPreferenceUpdatesCauseConflict'; diff --git a/src/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts new file mode 100644 index 00000000..6de11dad --- /dev/null +++ b/src/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -0,0 +1,114 @@ +import uniq from 'lodash/uniq'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import difference from 'lodash/difference'; +import { FileMetadataState } from './codecs'; +import { logger } from '../logger'; +import { mapSeries } from 'bluebird'; +import { PreferenceTopic } from '../graphql'; + +/* eslint-disable no-param-reassign */ + +/** + * Parse out the purpose.enabled and preference values from a CSV file + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @param options - Options + * @returns The updated file metadata state + */ +export async function parsePreferenceAndPurposeValuesFromCsv( + preferences: Record[], + currentState: FileMetadataState, + { + purposeSlugs, + preferenceTopics, + }: { + /** The purpose slugs that are allowed to be updated */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; + }, +): Promise { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for identifier + const otherColumns = difference(columnNames, [ + ...(currentState.identifierColumn ? [currentState.identifierColumn] : []), + ...(currentState.timestampColum ? [currentState.timestampColum] : []), + ]); + if (otherColumns.length === 0) { + throw new Error('No other columns to process'); + } + + // The purpose and preferences to map to + const purposeNames = [ + ...purposeSlugs, + ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`), + ]; + + // Ensure all columns are accounted for + await mapSeries(otherColumns, async (col) => { + // Determine the unique values to map in this column + const uniqueValues = uniq(preferences.map((x) => x[col])); + + // Map the column to a purpose + let purposeMapping = currentState.columnToPurposeName[col]; + if (purposeMapping) { + logger.info( + colors.magenta( + `Column "${col}" is associated with purpose "${purposeMapping.purpose}"`, + ), + ); + } else { + const { purposeName } = await inquirer.prompt<{ + /** purpose name */ + purposeName: string; + }>([ + { + name: 'purposeName', + message: `Choose the purpose that column ${col} is associated with`, + type: 'list', + default: purposeNames[0], + choices: purposeNames, + }, + ]); + const [purposeSlug, preferenceSlug] = purposeName.split('->'); + purposeMapping = { + purpose: purposeSlug, + preference: preferenceSlug || null, + valueMapping: {}, + }; + } + + // map each value to the purpose value + await mapSeries(uniqueValues, async (value) => { + if (purposeMapping.valueMapping[value] !== undefined) { + logger.info( + colors.magenta( + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"`, + ), + ); + return; + } + const { purposeValue } = await inquirer.prompt<{ + /** purpose value */ + purposeValue: boolean; + }>([ + { + name: 'purposeValue', + message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, + type: 'confirm', + default: value !== 'false', + }, + ]); + purposeMapping.valueMapping[value] = purposeValue; + }); + + currentState.columnToPurposeName[col] = purposeMapping; + }); + + return currentState; +} +/* eslint-enable no-param-reassign */ diff --git a/src/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/preference-management/parsePreferenceIdentifiersFromCsv.ts new file mode 100644 index 00000000..f03c2e89 --- /dev/null +++ b/src/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -0,0 +1,140 @@ +import uniq from 'lodash/uniq'; +import groupBy from 'lodash/groupBy'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import difference from 'lodash/difference'; +import { FileMetadataState } from './codecs'; +import { logger } from '../logger'; +import { inquirerConfirmBoolean } from '../helpers'; + +/* eslint-disable no-param-reassign */ + +/** + * Parse identifiers from a CSV list of preferences + * + * Ensures that all rows have a valid identifier + * and that all identifiers are unique. + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @returns The updated file metadata state + */ +export async function parsePreferenceIdentifiersFromCsv( + preferences: Record[], + currentState: FileMetadataState, +): Promise<{ + /** The updated state */ + currentState: FileMetadataState; + /** The updated preferences */ + preferences: Record[]; +}> { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for identifier + const remainingColumnsForIdentifier = difference(columnNames, [ + ...(currentState.identifierColumn ? [currentState.identifierColumn] : []), + ...Object.keys(currentState.columnToPurposeName), + ]); + + // Determine the identifier column to work off of + if (!currentState.identifierColumn) { + const { identifierName } = await inquirer.prompt<{ + /** Identifier name */ + identifierName: string; + }>([ + { + name: 'identifierName', + message: + 'Choose the column that will be used as the identifier to upload consent preferences by', + type: 'list', + default: + remainingColumnsForIdentifier.find((col) => + col.toLowerCase().includes('email'), + ) || remainingColumnsForIdentifier[0], + choices: remainingColumnsForIdentifier, + }, + ]); + currentState.identifierColumn = identifierName; + } + logger.info( + colors.magenta( + `Using identifier column "${currentState.identifierColumn}"`, + ), + ); + + // Validate that the identifier column is present for all rows and unique + const identifierColumnsMissing = preferences + .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + if (identifierColumnsMissing.length > 0) { + const msg = `The identifier column "${ + currentState.identifierColumn + }" is missing a value for the following rows: ${identifierColumnsMissing.join( + ', ', + )}`; + logger.warn(colors.yellow(msg)); + + // Ask user if they would like to skip rows missing an identifier + const skip = await inquirerConfirmBoolean({ + message: 'Would you like to skip rows missing an identifier?', + }); + if (!skip) { + throw new Error(msg); + } + + // Filter out rows missing an identifier + const previous = preferences.length; + preferences = preferences.filter( + (pref) => pref[currentState.identifierColumn!], + ); + logger.info( + colors.yellow( + `Skipped ${previous - preferences.length} rows missing an identifier`, + ), + ); + } + logger.info( + colors.magenta( + `The identifier column "${currentState.identifierColumn}" is present for all rows`, + ), + ); + + // Validate that all identifiers are unique + const rowsByUserId = groupBy(preferences, currentState.identifierColumn); + const duplicateIdentifiers = Object.entries(rowsByUserId).filter( + ([, rows]) => rows.length > 1, + ); + if (duplicateIdentifiers.length > 0) { + const msg = `The identifier column "${ + currentState.identifierColumn + }" has duplicate values for the following rows: ${duplicateIdentifiers + .slice(0, 10) + .map(([userId, rows]) => `${userId} (${rows.length})`) + .join('\n')}`; + logger.warn(colors.yellow(msg)); + + // Ask user if they would like to take the most recent update + // for each duplicate identifier + const skip = await inquirerConfirmBoolean({ + message: 'Would you like to automatically take the latest update?', + }); + if (!skip) { + throw new Error(msg); + } + preferences = Object.entries(rowsByUserId) + .map(([, rows]) => { + const sorted = rows.sort( + (a, b) => + new Date(b[currentState.timestampColum!]).getTime() - + new Date(a[currentState.timestampColum!]).getTime(), + ); + return sorted[0]; + }) + .filter((x) => x); + } + + return { currentState, preferences }; +} +/* eslint-enable no-param-reassign */ diff --git a/src/preference-management/parsePreferenceManagementCsv.ts b/src/preference-management/parsePreferenceManagementCsv.ts new file mode 100644 index 00000000..4af95af5 --- /dev/null +++ b/src/preference-management/parsePreferenceManagementCsv.ts @@ -0,0 +1,175 @@ +import { PersistedState } from '@transcend-io/persisted-state'; +import type { Got } from 'got'; +import keyBy from 'lodash/keyBy'; +import * as t from 'io-ts'; +import colors from 'colors'; +import { FileMetadataState, PreferenceState } from './codecs'; +import { logger } from '../logger'; +import { readCsv } from '../requests'; +import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; +import { PreferenceTopic } from '../graphql'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; +import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; +import { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; + +/** + * Parse a file into the cache + * + * + * @param options - Options + * @param cache - The cache to store the parsed file in + * @returns The cache with the parsed file + */ +export async function parsePreferenceManagementCsvWithCache( + { + file, + sombra, + purposeSlugs, + preferenceTopics, + partitionKey, + }: { + /** File to parse */ + file: string; + /** The purpose slugs that are allowed to be updated */ + purposeSlugs: string[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; + /** Sombra got instance */ + sombra: Got; + /** Partition key */ + partitionKey: string; + }, + cache: PersistedState, +): Promise { + // Start the timer + const t0 = new Date().getTime(); + + // Get the current metadata + const fileMetadata = cache.getValue('fileMetadata'); + + // Read in the file + logger.info(colors.magenta(`Reading in file: "${file}"`)); + // FIXME + let preferences = readCsv(file, t.record(t.string, t.string)).slice(0, 20000); + + // start building the cache, can use previous cache as well + let currentState: FileMetadataState = { + columnToPurposeName: {}, + pendingSafeUpdates: {}, + pendingConflictUpdates: {}, + skippedUpdates: {}, + // Load in the last fetched time + ...((fileMetadata[file] || {}) as Partial), + lastFetchedAt: new Date().toISOString(), + }; + + // Validate that all timestamps are present in the file + currentState = await parsePreferenceTimestampsFromCsv( + preferences, + currentState, + ); + fileMetadata[file] = currentState; + cache.setValue(fileMetadata, 'fileMetadata'); + + // Validate that all identifiers are present and unique + const result = await parsePreferenceIdentifiersFromCsv( + preferences, + currentState, + ); + currentState = result.currentState; + preferences = result.preferences; + fileMetadata[file] = currentState; + cache.setValue(fileMetadata, 'fileMetadata'); + + // Ensure all other columns are mapped to purpose and preference + // slug values + currentState = await parsePreferenceAndPurposeValuesFromCsv( + preferences, + currentState, + { + preferenceTopics, + purposeSlugs, + }, + ); + fileMetadata[file] = currentState; + cache.setValue(fileMetadata, 'fileMetadata'); + + // Grab existing preference store records + const identifiers = preferences.map( + (pref) => pref[currentState.identifierColumn!], + ); + const existingConsentRecords = await getPreferencesForIdentifiers(sombra, { + identifiers: identifiers.map((x) => ({ value: x })), + partitionKey, + }); + const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId'); + + // Clear out previous updates + currentState.pendingConflictUpdates = {}; + currentState.pendingSafeUpdates = {}; + currentState.skippedUpdates = {}; + + // Process each row + preferences.forEach((pref) => { + // Grab unique Id for the user + const userId = pref[currentState.identifierColumn!]; + + // determine updates for user + const pendingUpdates = getPreferenceUpdatesFromRow({ + row: pref, + columnToPurposeName: currentState.columnToPurposeName, + preferenceTopics, + purposeSlugs, + }); + + // Grab current state of the update + const currentConsentRecord = consentRecordByIdentifier[userId]; + + // Check if the update can be skipped + // this is the case if a record exists, and the purpose + // and preference values are all in sync + if ( + currentConsentRecord && + checkIfPendingPreferenceUpdatesAreNoOp({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, + }) + ) { + currentState.skippedUpdates[userId] = pref; + return; + } + + // Determine if there are any conflicts + if ( + currentConsentRecord && + checkIfPendingPreferenceUpdatesCauseConflict({ + currentConsentRecord, + pendingUpdates, + preferenceTopics, + }) + ) { + currentState.pendingConflictUpdates[userId] = { + row: pref, + record: currentConsentRecord, + }; + return; + } + + // Add to pending updates + currentState.pendingSafeUpdates[userId] = pref; + }); + + // Read in the file + fileMetadata[file] = currentState; + cache.setValue(fileMetadata, 'fileMetadata'); + const t1 = new Date().getTime(); + logger.info( + colors.green( + `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, + ), + ); +} diff --git a/src/preference-management/parsePreferenceManagementCsvWithCache.ts b/src/preference-management/parsePreferenceManagementCsvWithCache.ts deleted file mode 100644 index 867dc4ed..00000000 --- a/src/preference-management/parsePreferenceManagementCsvWithCache.ts +++ /dev/null @@ -1,452 +0,0 @@ -/* eslint-disable max-lines */ -import { PersistedState } from '@transcend-io/persisted-state'; -import difference from 'lodash/difference'; -import type { Got } from 'got'; -import groupBy from 'lodash/groupBy'; -import uniq from 'lodash/uniq'; -import keyBy from 'lodash/keyBy'; -import inquirer from 'inquirer'; -import * as t from 'io-ts'; -import colors from 'colors'; -import { - FileMetadataState, - PreferenceState, - PurposeRowMapping, -} from './codecs'; -import { logger } from '../logger'; -import { readCsv } from '../requests'; -import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; -import { Purpose, PreferenceTopic } from '../graphql'; -import { mapSeries } from 'bluebird'; -import { inquirerConfirmBoolean } from '../helpers'; -import { PreferenceStorePurposeResponse } from '@transcend-io/privacy-types'; -import { apply } from '@transcend-io/type-utils'; - -export const NONE_PREFERENCE_MAP = '[NONE]'; - -/** - * Parse a row into its purposes and preferences - * - * @param options - Options - * @returns The parsed row - */ -export function getUpdatesFromPreferenceRow({ - row, - columnToPurposeName, -}: { - /** Row to parse */ - row: Record; - /** Column names to parse */ - columnToPurposeName: Record; -}): { - [k in string]: Omit; -} { - const result: { - [k in string]: Partial; - } = {}; - - Object.keys(columnToPurposeName).forEach((col) => { - const mappings = columnToPurposeName[col]; - const [purpose, preference] = mappings.purpose.split('->'); - - if (preference) { - if (!result[purpose]) { - result[purpose] = { - preferences: [], - }; - } - result[purpose].preferences!.push({ - topic: preference, - choice: { - // FIXME select and multi select - booleanValue: mappings.valueMapping[row[col]] === true, - }, - }); - } else if (!result[purpose]) { - result[purpose] = { - enabled: mappings.valueMapping[row[col]] === true, - }; - } else { - result[purpose].enabled = mappings.valueMapping[row[col]] === true; - } - }); - - return apply(result, (x) => ({ - ...x, - enabled: x.enabled!, - })); -} - -/** - * Parse a file into the cache - * - * - * @param options - Options - * @param cache - The cache to store the parsed file in - * @returns The cache with the parsed file - */ -export async function parsePreferenceManagementCsvWithCache( - { - file, - sombra, - purposes, - preferenceTopics, - partitionKey, - }: { - /** File to parse */ - file: string; - /** The purposes */ - purposes: Purpose[]; - /** The preference topics */ - preferenceTopics: PreferenceTopic[]; - /** Sombra got instance */ - sombra: Got; - /** Partition key */ - partitionKey: string; - }, - cache: PersistedState, -): Promise { - const t0 = new Date().getTime(); - - // Get the current metadata - const fileMetadata = cache.getValue('fileMetadata'); - - // Read in the file - logger.info(colors.magenta(`Reading in file: "${file}"`)); - let preferences = readCsv(file, t.record(t.string, t.string)); - - // start building the cache, can use previous cache as well - const currentState: FileMetadataState = { - columnToPurposeName: {}, - pendingSafeUpdates: {}, - pendingConflictUpdates: {}, - skippedUpdates: {}, - // Load in the last fetched time - ...((fileMetadata[file] || {}) as Partial), - lastFetchedAt: new Date().toISOString(), - }; - - // Determine columns to map - const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); - - // Determine the identifier column to work off of - const remainingColumnsForTimestamp = difference( - columnNames, - currentState.identifierColumn ? [currentState.identifierColumn] : [], - ); - if (!currentState.timestampColum) { - const { timestampName } = await inquirer.prompt<{ - /** timestamp name */ - timestampName: string; - }>([ - { - name: 'timestampName', - message: - 'Choose the column that will be used as the timestamp of last preference update', - type: 'list', - default: - remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes('date'), - ) || - remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes('time'), - ) || - remainingColumnsForTimestamp[0], - choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], - }, - ]); - currentState.timestampColum = timestampName; - fileMetadata[file] = currentState; - cache.setValue(fileMetadata, 'fileMetadata'); - } - logger.info( - colors.magenta( - `Using timestamp column "${currentState.timestampColum}" in file: "${file}"`, - ), - ); - - // Validate that all rows have valid timestamp - if (currentState.timestampColum !== NONE_PREFERENCE_MAP) { - const timestampColumnsMissing = preferences - .map((pref, ind) => (pref[currentState.timestampColum!] ? null : [ind])) - .filter((x): x is number[] => !!x) - .flat(); - if (timestampColumnsMissing.length > 0) { - throw new Error( - `The timestamp column "${ - currentState.timestampColum - }" is missing a value for the following rows: ${timestampColumnsMissing.join( - '\n', - )} in file "${file}"`, - ); - } - logger.info( - colors.magenta( - `The timestamp column "${currentState.timestampColum}" is present for all rows in file: "${file}"`, - ), - ); - } - - // Determine the identifier column to work off of - const remainingColumnsForIdentifier = difference( - columnNames, - currentState.identifierColumn ? [currentState.identifierColumn] : [], - ); - if (!currentState.identifierColumn) { - const { identifierName } = await inquirer.prompt<{ - /** Identifier name */ - identifierName: string; - }>([ - { - name: 'identifierName', - message: - 'Choose the column that will be used as the identifier to upload consent preferences by', - type: 'list', - default: - remainingColumnsForIdentifier.find((col) => - col.toLowerCase().includes('email'), - ) || remainingColumnsForIdentifier[0], - choices: remainingColumnsForIdentifier, - }, - ]); - currentState.identifierColumn = identifierName; - fileMetadata[file] = currentState; - cache.setValue(fileMetadata, 'fileMetadata'); - } - logger.info( - colors.magenta( - `Using identifier column "${currentState.identifierColumn}" in file: "${file}"`, - ), - ); - - // Validate that the identifier column is present for all rows and unique - const identifierColumnsMissing = preferences - .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind])) - .filter((x): x is number[] => !!x) - .flat(); - if (identifierColumnsMissing.length > 0) { - logger.warn( - colors.yellow( - `The identifier column "${ - currentState.identifierColumn - }" is missing a value for the following rows: ${identifierColumnsMissing.join( - ', ', - )} in file "${file}"`, - ), - ); - const skip = await inquirerConfirmBoolean({ - message: 'Would you like to skip rows missing an identifier?', - }); - if (!skip) { - throw new Error( - `The identifier column "${ - currentState.identifierColumn - }" is missing a value for the following rows: ${identifierColumnsMissing.join( - ', ', - )} in file "${file}"`, - ); - } - const previous = preferences.length; - preferences = preferences.filter( - (pref) => pref[currentState.identifierColumn!], - ); - logger.info( - colors.yellow( - `Skipped ${ - previous - preferences.length - } rows missing an identifier in file "${file}"`, - ), - ); - } - logger.info( - colors.magenta( - `The identifier column "${currentState.identifierColumn}" is present for all rows in file: "${file}"`, - ), - ); - - // Validate that all identifiers are unique - const rowsByUserId = groupBy(preferences, currentState.identifierColumn); - const duplicateIdentifiers = Object.entries(rowsByUserId).filter( - ([, rows]) => rows.length > 1, - ); - if (duplicateIdentifiers.length > 0) { - const msg = `The identifier column "${ - currentState.identifierColumn - }" has duplicate values for the following rows: ${duplicateIdentifiers - .slice(0, 10) - .map(([userId, rows]) => `${userId} (${rows.length})`) - .join('\n')} in file "${file}"`; - logger.warn(colors.yellow(msg)); - const skip = await inquirerConfirmBoolean({ - message: 'Would you like to automatically take the latest update?', - }); - if (!skip) { - throw new Error(msg); - } - preferences = Object.entries(rowsByUserId) - .map(([, rows]) => { - const sorted = rows.sort( - (a, b) => - new Date(b[currentState.timestampColum!]).getTime() - - new Date(a[currentState.timestampColum!]).getTime(), - ); - return sorted[0]; - }) - .filter((x) => x); - } - - // Ensure all rows are accounted for - const otherColumns = difference(columnNames, [ - currentState.identifierColumn, - currentState.timestampColum, - ]); - const purposeNames = [ - ...purposes.map((x) => x.trackingType), - ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`), - ]; - if (otherColumns.length === 0) { - throw new Error(`No other columns to process in file "${file}"`); - } - await mapSeries(otherColumns, async (col) => { - // Determine the unique values to map in this column - const uniqueValues = uniq(preferences.map((x) => x[col])); - - // Map the column to a purpose - let purposeMapping = currentState.columnToPurposeName[col]; - if (purposeMapping) { - logger.info( - colors.magenta( - `Column "${col}" is associated with purpose "${purposeMapping.purpose}" in file: "${file}"`, - ), - ); - } else { - const { purposeName } = await inquirer.prompt<{ - /** purpose name */ - purposeName: string; - }>([ - { - name: 'purposeName', - message: `Choose the purpose that column ${col} is associated with`, - type: 'list', - default: purposeNames[0], - choices: purposeNames, - }, - ]); - purposeMapping = { - purpose: purposeName, - valueMapping: {}, - }; - } - - // map each value to the purpose value - await mapSeries(uniqueValues, async (value) => { - if (purposeMapping.valueMapping[value] !== undefined) { - logger.info( - colors.magenta( - `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}" in file: "${file}"`, - ), - ); - return; - } - const { purposeValue } = await inquirer.prompt<{ - /** purpose value */ - purposeValue: boolean; - }>([ - { - name: 'purposeValue', - message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: 'confirm', - }, - ]); - purposeMapping.valueMapping[value] = purposeValue; - }); - - currentState.columnToPurposeName[col] = purposeMapping; - fileMetadata[file] = currentState; - cache.setValue(fileMetadata, 'fileMetadata'); - }); - - // Grab existing preference store records - const identifiers = preferences.map( - (pref) => pref[currentState.identifierColumn!], - ); - const existingConsentRecords = await getPreferencesForIdentifiers(sombra, { - identifiers: identifiers.map((x) => ({ value: x })), - partitionKey, - }); - const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId'); - - // Clear out previous updates - currentState.pendingConflictUpdates = {}; - currentState.pendingSafeUpdates = {}; - currentState.skippedUpdates = {}; - - // Process each row - preferences.forEach((pref) => { - // used to compare if the preference has already been processed - const userId = pref[currentState.identifierColumn!]; - const purposeMapping = otherColumns.reduce( - (acc, col) => - Object.assign(acc, { - [currentState.columnToPurposeName[col].purpose]: - currentState.columnToPurposeName[col].valueMapping[pref[col]], - }), - {} as Record, - ); - - // Grab current state of the update - const currentConsentRecord = consentRecordByIdentifier[userId]; - - // Check if the update can be skipped - if ( - currentConsentRecord && - Object.entries(purposeMapping).every( - ([key, value]) => - currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === key, - )?.enabled === value, - ) - ) { - currentState.skippedUpdates[userId] = pref; - return; - } - // console.log({ - // purposeMapping, - // purposes: currentConsentRecord?.purposes, - // currentConsentRecord, - // userId, - // }); - - // Determine if there are any conflicts - const hasConflicts = - currentConsentRecord && - Object.entries(purposeMapping).find(([key, value]) => { - const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === key, - ); - return currentPurpose && currentPurpose.enabled !== value; - }); - if (hasConflicts) { - currentState.pendingConflictUpdates[userId] = { - row: pref, - record: currentConsentRecord, - }; - return; - } - - // Add to pending updates - currentState.pendingSafeUpdates[userId] = pref; - }); - - // Read in the file - fileMetadata[file] = currentState; - cache.setValue(fileMetadata, 'fileMetadata'); - const t1 = new Date().getTime(); - logger.info( - colors.green( - `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, - ), - ); -} - -/* eslint-enable max-lines */ diff --git a/src/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/preference-management/parsePreferenceTimestampsFromCsv.ts new file mode 100644 index 00000000..cbaa33b2 --- /dev/null +++ b/src/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -0,0 +1,88 @@ +import uniq from 'lodash/uniq'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import difference from 'lodash/difference'; +import { FileMetadataState } from './codecs'; +import { logger } from '../logger'; + +export const NONE_PREFERENCE_MAP = '[NONE]'; + +/* eslint-disable no-param-reassign */ + +/** + * Parse timestamps from a CSV list of preferences + * + * When timestamp is requested, this script + * ensures that all rows have a valid timestamp. + * + * Error is throw if timestamp is missing + * + * @param preferences - List of preferences + * @param currentState - The current file metadata state for parsing this list + * @returns The updated file metadata state + */ +export async function parsePreferenceTimestampsFromCsv( + preferences: Record[], + currentState: FileMetadataState, +): Promise { + // Determine columns to map + const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + + // Determine the columns that could potentially be used for timestamp + const remainingColumnsForTimestamp = difference(columnNames, [ + ...(currentState.identifierColumn ? [currentState.identifierColumn] : []), + ...Object.keys(currentState.columnToPurposeName), + ]); + + // Determine the timestamp column to work off of + if (!currentState.timestampColum) { + const { timestampName } = await inquirer.prompt<{ + /** timestamp name */ + timestampName: string; + }>([ + { + name: 'timestampName', + message: + 'Choose the column that will be used as the timestamp of last preference update', + type: 'list', + default: + remainingColumnsForTimestamp.find((col) => + col.toLowerCase().includes('date'), + ) || + remainingColumnsForTimestamp.find((col) => + col.toLowerCase().includes('time'), + ) || + remainingColumnsForTimestamp[0], + choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], + }, + ]); + currentState.timestampColum = timestampName; + } + logger.info( + colors.magenta(`Using timestamp column "${currentState.timestampColum}"`), + ); + + // Validate that all rows have valid timestamp + if (currentState.timestampColum !== NONE_PREFERENCE_MAP) { + const timestampColumnsMissing = preferences + .map((pref, ind) => (pref[currentState.timestampColum!] ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + if (timestampColumnsMissing.length > 0) { + throw new Error( + `The timestamp column "${ + currentState.timestampColum + }" is missing a value for the following rows: ${timestampColumnsMissing.join( + '\n', + )}`, + ); + } + logger.info( + colors.magenta( + `The timestamp column "${currentState.timestampColum}" is present for all row`, + ), + ); + } + return currentState; +} +/* eslint-enable no-param-reassign */ diff --git a/src/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts b/src/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts new file mode 100644 index 00000000..d9c6d467 --- /dev/null +++ b/src/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts @@ -0,0 +1 @@ +// FIXME diff --git a/src/preference-management/tests/checkIfPurposeOrPreferencesAreChanging.test.ts b/src/preference-management/tests/checkIfPurposeOrPreferencesAreChanging.test.ts new file mode 100644 index 00000000..d9c6d467 --- /dev/null +++ b/src/preference-management/tests/checkIfPurposeOrPreferencesAreChanging.test.ts @@ -0,0 +1 @@ +// FIXME diff --git a/src/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/preference-management/tests/getPreferenceUpdatesFromRow.test.ts new file mode 100644 index 00000000..2b78deb0 --- /dev/null +++ b/src/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -0,0 +1,540 @@ +/* eslint-disable max-lines */ +import chai, { expect } from 'chai'; +import deepEqualInAnyOrder from 'deep-equal-in-any-order'; + +import { getPreferenceUpdatesFromRow } from '../index'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; + +chai.use(deepEqualInAnyOrder); + +describe('getPreferenceUpdatesFromRow', () => { + it('should parse boolean updates', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_1: { + purpose: 'Marketing', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_2: { + purpose: 'Marketing', + preference: 'BooleanPreference2', + valueMapping: { + true: true, + false: false, + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'BooleanPreference2', + choice: { + booleanValue: false, + }, + }, + ], + }, + }); + }); + + it('should parse a single select', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_3: 'Option 1', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_3: { + purpose: 'Marketing', + preference: 'SingleSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + ], + }, + }); + }); + + it('should parse a multi select example', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_4: 'Option 2,Option 1', + has_topic_5: 'Option 1', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_4: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + has_topic_5: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1'], + }, + }, + ], + }, + }); + }); + + it('should parse boolean, single select, multi select example', () => { + expect( + getPreferenceUpdatesFromRow({ + row: { + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', + has_topic_3: 'Option 1', + has_topic_4: 'Option 2,Option 1', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + my_purpose: { + purpose: 'Marketing', + preference: null, + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_1: { + purpose: 'Marketing', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_2: { + purpose: 'Marketing', + preference: 'BooleanPreference2', + valueMapping: { + true: true, + false: false, + }, + }, + has_topic_3: { + purpose: 'Marketing', + preference: 'SingleSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + has_topic_4: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }), + ).to.deep.equal({ + Marketing: { + enabled: true, + preferences: [ + { + topic: 'BooleanPreference1', + choice: { + booleanValue: true, + }, + }, + { + topic: 'BooleanPreference2', + choice: { + booleanValue: false, + }, + }, + { + topic: 'SingleSelectPreference', + choice: { + selectValue: 'Value1', + }, + }, + { + topic: 'MultiSelectPreference', + choice: { + selectValues: ['Value1', 'Value2'], + }, + }, + ], + }, + }); + }); + + it('should error if missing purpose', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'Marketing', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.include('No mapping provided'); + } + }); + + it('should error if purpose name is not valid', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing', 'Advertising'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + { + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', + type: PreferenceTopicType.Boolean, + preferenceOptionValues: [], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'InvalidPurpose', + preference: 'BooleanPreference1', + valueMapping: { + true: true, + false: false, + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.equal( + 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', + ); + } + }); + + it('should error if single select option is invalid', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + type: PreferenceTopicType.Select, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'Marketing', + preference: 'SingleSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.equal( + 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', + ); + } + }); + + it('should error if multi select value is invalid', () => { + try { + getPreferenceUpdatesFromRow({ + row: { + has_topic_1: 'true', + }, + purposeSlugs: ['Marketing'], + preferenceTopics: [ + { + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', + type: PreferenceTopicType.MultiSelect, + preferenceOptionValues: [ + { + slug: 'Value1', + }, + { + slug: 'Value2', + }, + ], + purpose: { + trackingType: 'Marketing', + }, + }, + ], + columnToPurposeName: { + has_topic_1: { + purpose: 'Marketing', + preference: 'MultiSelectPreference', + valueMapping: { + 'Option 1': 'Value1', + 'Option 2': 'Value2', + }, + }, + }, + }); + expect.fail('Should have thrown'); + } catch (err) { + expect(err.message).to.equal( + 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', + ); + } + }); +}); +/* eslint-enable max-lines */ diff --git a/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index d5251d65..5c8a3eac 100644 --- a/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -4,22 +4,20 @@ import { fetchAllPurposes, } from '../graphql'; import colors from 'colors'; -import { map, mapSeries } from 'bluebird'; +import { map } from 'bluebird'; import chunk from 'lodash/chunk'; import { DEFAULT_TRANSCEND_CONSENT_API } from '../constants'; import { logger } from '../logger'; import cliProgress from 'cli-progress'; import { parseAttributesFromString } from '../requests'; import { PersistedState } from '@transcend-io/persisted-state'; -import { - NONE_PREFERENCE_MAP, - getUpdatesFromPreferenceRow, - parsePreferenceManagementCsvWithCache, -} from './parsePreferenceManagementCsvWithCache'; +import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; import { PreferenceState } from './codecs'; import { fetchAllPreferenceTopics } from '../graphql/fetchAllPreferenceTopics'; import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; import { apply } from '@transcend-io/type-utils'; +import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; /** * Upload a set of consent preferences @@ -30,11 +28,12 @@ export async function uploadPreferenceManagementPreferencesInteractive({ auth, sombraAuth, receiptFilepath, - files, + file, partition, isSilent = true, dryRun = false, skipWorkflowTriggers = false, + skipConflictUpdates = false, attributes = [], transcendUrl = DEFAULT_TRANSCEND_CONSENT_API, }: { @@ -46,8 +45,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ partition: string; /** File where to store receipt and continue from where left off */ receiptFilepath: string; - /** The files to process */ - files: string[]; + /** The file to process */ + file: string; /** API URL for Transcend backend */ transcendUrl?: string; /** Whether to do a dry run */ @@ -58,6 +57,11 @@ export async function uploadPreferenceManagementPreferencesInteractive({ attributes?: string[]; /** Skip workflow triggers */ skipWorkflowTriggers?: boolean; + /** + * When true, only update preferences that do not conflict with existing + * preferences. When false, update all preferences in CSV based on timestamp. + */ + skipConflictUpdates?: boolean; }): Promise { // Parse out the extra attributes to apply to all requests uploaded const parsedAttributes = parseAttributesFromString(attributes); @@ -70,7 +74,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }); const failingRequests = preferenceState.getValue('failingUpdates'); const pendingRequests = preferenceState.getValue('pendingUpdates'); - const fileMetadata = preferenceState.getValue('fileMetadata'); + let fileMetadata = preferenceState.getValue('fileMetadata'); logger.info( colors.magenta( @@ -86,9 +90,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ ) .map((x) => x) .join('\n')}\n` + - `The following files will be read in and refreshed in the cache:\n${files.join( - '\n', - )}\n`, + `The following file will be processed: ${file}\n`, ), ); @@ -103,123 +105,91 @@ export async function uploadPreferenceManagementPreferencesInteractive({ fetchAllPreferenceTopics(client), ]); - // Process each file - await mapSeries(files, async (file) => { - await parsePreferenceManagementCsvWithCache( - { - file, - purposes, - preferenceTopics, - sombra, - partitionKey: partition, - }, - preferenceState, - ); - }); + // Process the file + await parsePreferenceManagementCsvWithCache( + { + file, + purposeSlugs: purposes.map((x) => x.trackingType), + preferenceTopics, + sombra, + partitionKey: partition, + }, + preferenceState, + ); // Construct the pending updates const pendingUpdates: Record = {}; - files.forEach((file) => { - const fileMetadata = preferenceState.getValue('fileMetadata'); - const metadata = fileMetadata[file]; + fileMetadata = preferenceState.getValue('fileMetadata'); + const metadata = fileMetadata[file]; - logger.info( - colors.magenta( - `Found ${ - Object.entries(metadata.pendingSafeUpdates).length - } safe updates in ${file}`, - ), - ); - logger.info( - colors.magenta( - `Found ${ - Object.entries(metadata.pendingConflictUpdates).length - } conflict updates in ${file}`, - ), - ); - logger.info( - colors.magenta( - `Found ${ - Object.entries(metadata.skippedUpdates).length - } skipped updates in ${file}`, - ), - ); - Object.entries({ - ...metadata.pendingSafeUpdates, - ...apply(metadata.pendingConflictUpdates, ({ row }) => row), - }).forEach(([userId, update]) => { - const currentUpdate = pendingUpdates[userId]; - const timestamp = - metadata.timestampColum === NONE_PREFERENCE_MAP - ? new Date() - : new Date(update[metadata.timestampColum!]); - const updates = getUpdatesFromPreferenceRow({ - row: update, - columnToPurposeName: metadata.columnToPurposeName, - }); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.pendingSafeUpdates).length + } safe updates in ${file}`, + ), + ); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.pendingConflictUpdates).length + } conflict updates in ${file}`, + ), + ); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.skippedUpdates).length + } skipped updates in ${file}`, + ), + ); - if (currentUpdate) { - const newPurposes = Object.entries(updates).map(([purpose, value]) => ({ - ...value, - purpose, - workflowSettings: { - attributes: parsedAttributes, - isSilent, - skipWorkflowTrigger: skipWorkflowTriggers, - }, - })); - (currentUpdate.purposes || []).forEach((purpose) => { - if (updates[purpose.purpose].enabled !== purpose.enabled) { - logger.warn( - colors.yellow( - `Conflict detected for user: ${userId} and purpose: ${purpose.purpose}`, - ), - ); - } - }); - pendingUpdates[userId] = { - userId, - partition, - // take the most recent timestamp - timestamp: - timestamp > new Date(currentUpdate.timestamp) - ? timestamp.toISOString() - : currentUpdate.timestamp, - purposes: [ - ...(currentUpdate.purposes || []), - ...newPurposes.filter( - (newPurpose) => - !(currentUpdate.purposes || []).find( - (currentPurpose) => - currentPurpose.purpose === newPurpose.purpose, - ), - ), - ], - }; - } else { - pendingUpdates[userId] = { - userId, - partition, - timestamp: timestamp.toISOString(), - purposes: Object.entries(updates).map(([purpose, value]) => ({ - ...value, - purpose, - workflowSettings: { - attributes: parsedAttributes, - isSilent, - skipWorkflowTrigger: skipWorkflowTriggers, - }, - })), - }; - } + // Update either safe updates only or safe + conflict + Object.entries({ + ...metadata.pendingSafeUpdates, + ...(skipConflictUpdates + ? {} + : apply(metadata.pendingConflictUpdates, ({ row }) => row)), + }).forEach(([userId, update]) => { + // Determine timestamp + const timestamp = + metadata.timestampColum === NONE_PREFERENCE_MAP + ? new Date() + : new Date(update[metadata.timestampColum!]); + + // Determine updates + const updates = getPreferenceUpdatesFromRow({ + row: update, + columnToPurposeName: metadata.columnToPurposeName, + preferenceTopics, + purposeSlugs: purposes.map((x) => x.trackingType), }); + pendingUpdates[userId] = { + userId, + partition, + timestamp: timestamp.toISOString(), + purposes: Object.entries(updates).map(([purpose, value]) => ({ + ...value, + purpose, + workflowSettings: { + attributes: parsedAttributes, + isSilent, + skipWorkflowTrigger: skipWorkflowTriggers, + }, + })), + }; }); preferenceState.setValue(pendingUpdates, 'pendingUpdates'); preferenceState.setValue({}, 'failingUpdates'); + // Exist early if dry run if (dryRun) { logger.info( - colors.green(`Dry run complete, exiting. Check file: ${receiptFilepath}`), + colors.green( + `Dry run complete, exiting. ${ + Object.values(pendingUpdates).length + } pending updates. Check file: ${receiptFilepath}`, + ), ); return; } diff --git a/yarn.lock b/yarn.lock index b32a4a48..14a6f72d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -522,6 +522,7 @@ __metadata: "@types/chai": ^4.3.4 "@types/cli-progress": ^3.11.0 "@types/colors": ^1.2.1 + "@types/deep-equal-in-any-order": 1.0.1 "@types/fuzzysearch": ^1.0.0 "@types/global-agent": ^2.1.1 "@types/inquirer": ^7.3.1 @@ -542,6 +543,7 @@ __metadata: cli-progress: ^3.11.2 colors: ^1.4.0 csv-parse: =4.9.1 + deep-equal-in-any-order: ^1.0.28 depcheck: ^1.4.3 eslint: ^8.38.0 eslint-config-airbnb-base: ^15.0.0 @@ -778,6 +780,13 @@ __metadata: languageName: node linkType: hard +"@types/deep-equal-in-any-order@npm:1.0.1": + version: 1.0.1 + resolution: "@types/deep-equal-in-any-order@npm:1.0.1" + checksum: 09a76203438bc2ed9e3ac3f22b6fd43f3561ac4ae5403cf62b4e69359f3016680fbd6ce31f80704e310c3616f4d134e3252ba70e963fc4ca4dd261719c6eb4fd + languageName: node + linkType: hard + "@types/emscripten@npm:^1.39.6": version: 1.39.6 resolution: "@types/emscripten@npm:1.39.6" @@ -2149,6 +2158,16 @@ __metadata: languageName: node linkType: hard +"deep-equal-in-any-order@npm:^1.0.28": + version: 1.1.20 + resolution: "deep-equal-in-any-order@npm:1.1.20" + dependencies: + lodash.mapvalues: ^4.6.0 + sort-any: ^2.0.0 + checksum: 3fd4a571269e86f8958e797e4994df9f8aaa7c8cc438204a15b9c170e55b53f4b674e4609d8eaccb0b444a55c1932fd74026ef072f4a5412fd879599000e499f + languageName: node + linkType: hard + "deep-is@npm:^0.1.3": version: 0.1.4 resolution: "deep-is@npm:0.1.4" @@ -4256,6 +4275,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"lodash.mapvalues@npm:^4.6.0": + version: 4.6.0 + resolution: "lodash.mapvalues@npm:4.6.0" + checksum: 0ff1b252fda318fc36e47c296984925e98fbb0fc5a2ecc4ef458f3c739a9476d47e40c95ac653e8314d132aa59c746d4276527b99d6e271940555c6e12d2babd + languageName: node + linkType: hard + "lodash.merge@npm:^4.6.2": version: 4.6.2 resolution: "lodash.merge@npm:4.6.2" @@ -5593,6 +5619,15 @@ resolve@^1.18.1: languageName: node linkType: hard +"sort-any@npm:^2.0.0": + version: 2.0.0 + resolution: "sort-any@npm:2.0.0" + dependencies: + lodash: ^4.17.21 + checksum: d2dc6cc4f56298ce50b13ce6aafd8bf16010a8f8d0ae75345e8424118ac2cc017a8ffea0eedd136dd67751c09e7b0edb09c959d052ed7ed02afcfbab63c68277 + languageName: node + linkType: hard + "source-map-js@npm:^1.0.2": version: 1.0.2 resolution: "source-map-js@npm:1.0.2"