diff --git a/.pnp.cjs b/.pnp.cjs index a5312f4c..c6c2be26 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -32,7 +32,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@transcend-io/handlebars-utils", "npm:1.1.0"],\ ["@transcend-io/internationalization", "npm:1.6.0"],\ ["@transcend-io/persisted-state", "npm:1.0.4"],\ - ["@transcend-io/privacy-types", "npm:4.96.0"],\ + ["@transcend-io/privacy-types", "npm:4.98.0"],\ ["@transcend-io/secret-value", "npm:1.2.0"],\ ["@transcend-io/type-utils", "npm:1.5.0"],\ ["@types/bluebird", "npm:3.5.38"],\ @@ -682,7 +682,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@transcend-io/handlebars-utils", "npm:1.1.0"],\ ["@transcend-io/internationalization", "npm:1.6.0"],\ ["@transcend-io/persisted-state", "npm:1.0.4"],\ - ["@transcend-io/privacy-types", "npm:4.96.0"],\ + ["@transcend-io/privacy-types", "npm:4.98.0"],\ ["@transcend-io/secret-value", "npm:1.2.0"],\ ["@transcend-io/type-utils", "npm:1.5.0"],\ ["@types/bluebird", "npm:3.5.38"],\ @@ -781,10 +781,10 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { }]\ ]],\ ["@transcend-io/privacy-types", [\ - ["npm:4.96.0", {\ - "packageLocation": "./.yarn/cache/@transcend-io-privacy-types-npm-4.96.0-b86f5e1f6c-0ed38f70c7.zip/node_modules/@transcend-io/privacy-types/",\ + ["npm:4.98.0", {\ + "packageLocation": "./.yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip/node_modules/@transcend-io/privacy-types/",\ "packageDependencies": [\ - ["@transcend-io/privacy-types", "npm:4.96.0"],\ + ["@transcend-io/privacy-types", "npm:4.98.0"],\ ["@transcend-io/type-utils", "npm:1.0.5"],\ ["fp-ts", "npm:2.16.1"],\ ["io-ts", "virtual:a57afaf9d13087a7202de8c93ac4854c9e2828bad7709250829ec4c7bc9dc95ecc2858c25612aa1774c986aedc232c76957076a1da3156fd2ab63ae5551b086f#npm:2.2.21"]\ diff --git a/.yarn/cache/@transcend-io-privacy-types-npm-4.96.0-b86f5e1f6c-0ed38f70c7.zip b/.yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip similarity index 94% rename from .yarn/cache/@transcend-io-privacy-types-npm-4.96.0-b86f5e1f6c-0ed38f70c7.zip rename to .yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip index 80162a9c..b67220c3 100644 Binary files a/.yarn/cache/@transcend-io-privacy-types-npm-4.96.0-b86f5e1f6c-0ed38f70c7.zip and b/.yarn/cache/@transcend-io-privacy-types-npm-4.98.0-49ec094def-bee61228fa.zip differ diff --git a/README.md b/README.md index 849eb7fe..46b63a7c 100644 --- a/README.md +++ b/README.md @@ -2326,7 +2326,7 @@ The API key needs the following scopes: - Modify User Stored Preferences - View Managed Consent Database Admin API -- View Global Attributes +- View Preference Store Settings #### Authentication diff --git a/package.json b/package.json index 3b905001..12ddaa29 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "@transcend-io/handlebars-utils": "^1.1.0", "@transcend-io/internationalization": "^1.6.0", "@transcend-io/persisted-state": "^1.0.4", - "@transcend-io/privacy-types": "^4.96.0", + "@transcend-io/privacy-types": "^4.98.0", "@transcend-io/secret-value": "^1.2.0", "@transcend-io/type-utils": "^1.5.0", "bluebird": "^3.7.2", diff --git a/src/graphql/fetchAllPreferenceTopics.ts b/src/graphql/fetchAllPreferenceTopics.ts new file mode 100644 index 00000000..2de6d4bf --- /dev/null +++ b/src/graphql/fetchAllPreferenceTopics.ts @@ -0,0 +1,65 @@ +import { GraphQLClient } from 'graphql-request'; +import { PREFERENCE_TOPICS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; + +export interface PreferenceTopic { + /** ID of preference topic */ + id: string; + /** Slug of preference topic */ + slug: string; + /** Type of preference topic */ + type: PreferenceTopicType; + /** Option values */ + preferenceOptionValues: { + /** Slug of value */ + slug: string; + }[]; + /** Related purpose */ + purpose: { + /** Slug */ + trackingType: string; + }; +} + +const PAGE_SIZE = 20; + +/** + * Fetch all preference topics in the organization + * + * @param client - GraphQL client + * @returns All preference topics in the organization + */ +export async function fetchAllPreferenceTopics( + client: GraphQLClient, +): Promise { + const preferenceTopics: PreferenceTopic[] = []; + let offset = 0; + + // Whether to continue looping + let shouldContinue = false; + do { + const { + preferenceTopics: { nodes }, + // eslint-disable-next-line no-await-in-loop + } = await makeGraphQLRequest<{ + /** Preference topics */ + preferenceTopics: { + /** List */ + nodes: PreferenceTopic[]; + }; + }>(client, PREFERENCE_TOPICS, { + first: PAGE_SIZE, + offset, + }); + preferenceTopics.push(...nodes); + offset += PAGE_SIZE; + shouldContinue = nodes.length === PAGE_SIZE; + } while (shouldContinue); + + return preferenceTopics.sort((a, b) => + `${a.slug}:${a.purpose.trackingType}`.localeCompare( + `${b.slug}:${b.purpose.trackingType}`, + ), + ); +} diff --git a/src/graphql/fetchAllPurposes.ts b/src/graphql/fetchAllPurposes.ts new file mode 100644 index 00000000..73564d23 --- /dev/null +++ b/src/graphql/fetchAllPurposes.ts @@ -0,0 +1,64 @@ +import { GraphQLClient } from 'graphql-request'; +import { PURPOSES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; + +export interface Purpose { + /** ID of purpose */ + id: string; + /** Name of purpose */ + name: string; + /** Slug of purpose */ + trackingType: string; + /** Whether the purpose is active */ + isActive: boolean; + /** Whether the purpose is deleted */ + deletedAt?: string; +} + +const PAGE_SIZE = 20; + +/** + * Fetch all purposes in the organization + * + * @param client - GraphQL client + * @param input - Input + * @returns All purposes in the organization + */ +export async function fetchAllPurposes( + client: GraphQLClient, + { + includeDeleted = false, + }: { + /** Whether to include deleted purposes */ + includeDeleted?: boolean; + } = {}, +): Promise { + const purposes: Purpose[] = []; + let offset = 0; + + // Whether to continue looping + let shouldContinue = false; + do { + const { + purposes: { nodes }, + // eslint-disable-next-line no-await-in-loop + } = await makeGraphQLRequest<{ + /** Purposes */ + purposes: { + /** List */ + nodes: Purpose[]; + }; + }>(client, PURPOSES, { + first: PAGE_SIZE, + offset, + input: { + includeDeleted, + }, + }); + purposes.push(...nodes); + offset += PAGE_SIZE; + shouldContinue = nodes.length === PAGE_SIZE; + } while (shouldContinue); + + return purposes.sort((a, b) => a.trackingType.localeCompare(b.trackingType)); +} diff --git a/src/graphql/fetchConsentManagerId.ts b/src/graphql/fetchConsentManagerId.ts index efa3ca96..6420d54a 100644 --- a/src/graphql/fetchConsentManagerId.ts +++ b/src/graphql/fetchConsentManagerId.ts @@ -19,7 +19,6 @@ import { FETCH_CONSENT_MANAGER_ID, FETCH_CONSENT_MANAGER, EXPERIENCES, - PURPOSES, CONSENT_MANAGER_ANALYTICS_DATA, FETCH_CONSENT_MANAGER_THEME, } from './gqls'; @@ -118,27 +117,6 @@ export interface ConsentPurpose { trackingType: string; } -/** - * Fetch consent manager purposes - * - * @param client - GraphQL client - * @returns Consent manager purposes in the organization - */ -export async function fetchPurposes( - client: GraphQLClient, -): Promise { - const { - purposes: { purposes }, - } = await makeGraphQLRequest<{ - /** Consent manager query */ - purposes: { - /** Consent manager object */ - purposes: ConsentPurpose[]; - }; - }>(client, PURPOSES); - return purposes; -} - const PAGE_SIZE = 50; export interface ConsentExperience { diff --git a/src/graphql/gqls/consentManager.ts b/src/graphql/gqls/consentManager.ts index 3ac36fcd..de4cca45 100644 --- a/src/graphql/gqls/consentManager.ts +++ b/src/graphql/gqls/consentManager.ts @@ -1,18 +1,6 @@ /* eslint-disable max-lines */ import { gql } from 'graphql-request'; -export const PURPOSES = gql` - query TranscendCliPurposes { - purposes { - purposes { - id - name - trackingType - } - } - } -`; - // TODO: https://transcend.height.app/T-27909 - order by createdAt // TODO: https://transcend.height.app/T-27909 - enable optimizations // isExportCsv: true diff --git a/src/graphql/gqls/index.ts b/src/graphql/gqls/index.ts index b39f98b3..24b61da2 100644 --- a/src/graphql/gqls/index.ts +++ b/src/graphql/gqls/index.ts @@ -18,6 +18,8 @@ export * from './request'; export * from './message'; export * from './RequestEnricher'; export * from './assessment'; +export * from './purpose'; +export * from './preferenceTopic'; export * from './assessmentTemplate'; export * from './prompt'; export * from './RequestEnricher'; diff --git a/src/graphql/gqls/preferenceTopic.ts b/src/graphql/gqls/preferenceTopic.ts new file mode 100644 index 00000000..4a47b376 --- /dev/null +++ b/src/graphql/gqls/preferenceTopic.ts @@ -0,0 +1,30 @@ +import { gql } from 'graphql-request'; + +// TODO: https://transcend.height.app/T-27909 - enable optimizations +// isExportCsv: true +// useMaster: false +// orderBy: [ +// { field: createdAt, direction: ASC } +// { field: name, direction: ASC } +// ] +export const PREFERENCE_TOPICS = gql` + query TranscendCliPreferenceTopics( + $first: Int! + $offset: Int! + $filterBy: PreferenceTopicFilterInput + ) { + preferenceTopics(first: $first, offset: $offset, filterBy: $filterBy) { + nodes { + id + slug + type + preferenceOptionValues { + slug + } + purpose { + trackingType + } + } + } + } +`; diff --git a/src/graphql/gqls/purpose.ts b/src/graphql/gqls/purpose.ts new file mode 100644 index 00000000..2f4be7b8 --- /dev/null +++ b/src/graphql/gqls/purpose.ts @@ -0,0 +1,32 @@ +import { gql } from 'graphql-request'; + +// TODO: https://transcend.height.app/T-27909 - enable optimizations +// isExportCsv: true +// useMaster: false +// orderBy: [ +// { field: createdAt, direction: ASC } +// { field: name, direction: ASC } +// ] +export const PURPOSES = gql` + query TranscendCliPurposes( + $first: Int! + $offset: Int! + $filterBy: TrackingPurposeFiltersInput + $input: TrackingPurposeInput! + ) { + purposes( + first: $first + offset: $offset + filterBy: $filterBy + input: $input + ) { + nodes { + id + name + trackingType + isActive + deletedAt + } + } + } +`; diff --git a/src/graphql/index.ts b/src/graphql/index.ts index 74ca41a9..c41dfb1c 100644 --- a/src/graphql/index.ts +++ b/src/graphql/index.ts @@ -32,10 +32,12 @@ export * from './fetchAllRequests'; export * from './fetchAllRequestIdentifiers'; export * from './fetchAllRequestEnrichers'; export * from './fetchRequestDataSilo'; +export * from './fetchAllPreferenceTopics'; export * from './fetchAllAttributes'; export * from './syncAttribute'; export * from './fetchAllDataFlows'; export * from './syncActionItems'; +export * from './fetchAllPurposes'; export * from './syncActionItemCollections'; export * from './fetchAllCookies'; export * from './fetchAllActionItems'; diff --git a/src/graphql/syncConsentManager.ts b/src/graphql/syncConsentManager.ts index f921fb92..c6327645 100644 --- a/src/graphql/syncConsentManager.ts +++ b/src/graphql/syncConsentManager.ts @@ -19,7 +19,6 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; import { fetchConsentManagerId, fetchConsentManagerExperiences, - fetchPurposes, } from './fetchConsentManagerId'; import keyBy from 'lodash/keyBy'; import { map } from 'bluebird'; @@ -30,6 +29,7 @@ import { import { logger } from '../logger'; import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { fetchPartitions } from './syncPartitions'; +import { fetchAllPurposes } from './fetchAllPurposes'; const PURPOSES_LINK = 'https://app.transcend.io/consent-manager/regional-experiences/purposes'; @@ -49,7 +49,7 @@ export async function syncConsentManagerExperiences( const experienceLookup = keyBy(existingExperiences, 'name'); // Fetch existing purposes - const purposes = await fetchPurposes(client); + const purposes = await fetchAllPurposes(client); const purposeLookup = keyBy(purposes, 'trackingType'); // Bulk update or create experiences diff --git a/src/graphql/syncCookies.ts b/src/graphql/syncCookies.ts index c529c8ce..7bc590e2 100644 --- a/src/graphql/syncCookies.ts +++ b/src/graphql/syncCookies.ts @@ -24,7 +24,7 @@ export async function updateOrCreateCookies( const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchPurposes(client); + // const purposes = await fetchAllPurposes(client); // const purposeNameToId = keyBy(purposes, 'name'); await mapSeries(chunk(cookieInputs, MAX_PAGE_SIZE), async (page) => { diff --git a/src/graphql/syncDataFlows.ts b/src/graphql/syncDataFlows.ts index 3e0bcd15..6a8b1f3e 100644 --- a/src/graphql/syncDataFlows.ts +++ b/src/graphql/syncDataFlows.ts @@ -27,7 +27,7 @@ export async function updateDataFlows( const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchPurposes(client); + // const purposes = await fetchAllPurposes(client); // const purposeNameToId = keyBy(purposes, 'name'); await mapSeries(chunk(dataFlowInputs, MAX_PAGE_SIZE), async (page) => { @@ -74,7 +74,7 @@ export async function createDataFlows( ): Promise { const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchPurposes(client); + // const purposes = await fetchAllPurposes(client); // const purposeNameToId = keyBy(purposes, 'name'); await mapSeries(chunk(dataFlowInputs, MAX_PAGE_SIZE), async (page) => { await makeGraphQLRequest(client, CREATE_DATA_FLOWS, { diff --git a/src/preference-management/codecs.ts b/src/preference-management/codecs.ts index e6a6868c..62895b48 100644 --- a/src/preference-management/codecs.ts +++ b/src/preference-management/codecs.ts @@ -1,13 +1,23 @@ import { PreferenceQueryResponseItem, - PreferenceStorePurposeUpdate, + PreferenceUpdateItem, } from '@transcend-io/privacy-types'; import * as t from 'io-ts'; +export const PurposeRowMapping = t.type({ + /** Name of the purpose to map to */ + purpose: t.string, + /** Mapping from value in row to value in transcend API */ + valueMapping: t.record(t.string, t.union([t.string, t.boolean])), +}); + +/** Override type */ +export type PurposeRowMapping = t.TypeOf; + export const FileMetadataState = t.intersection([ t.type({ /** Mapping of column name to it's relevant purpose in Transcend */ - columnToPurposeName: t.record(t.string, t.string), + columnToPurposeName: t.record(t.string, PurposeRowMapping), /** Last time the file was fetched */ lastFetchedAt: t.string, /** @@ -25,13 +35,6 @@ export const FileMetadataState = t.intersection([ * their preferences are already in the store */ skippedUpdates: t.record(t.string, t.record(t.string, t.string)), - /** - * Mapping of userId to the rows in the file that have been successfully uploaded - */ - successfulUpdates: t.record( - t.string, - t.array(t.record(t.string, t.string)), - ), }), t.partial({ /** Determine which column name in file maps to consent record identifier to upload on */ @@ -49,7 +52,10 @@ export const PreferenceState = t.type({ /** * Mapping from core userId to preference store record */ - preferenceStoreRecords: t.record(t.string, PreferenceQueryResponseItem), + preferenceStoreRecords: t.record( + t.string, + t.union([PreferenceQueryResponseItem, t.null]), + ), /** * Store a cache of previous files read in */ @@ -65,7 +71,7 @@ export const PreferenceState = t.type({ /** Time upload ran at */ uploadedAt: t.string, /** The update body */ - update: PreferenceStorePurposeUpdate, + update: PreferenceUpdateItem, }), ), ), @@ -75,22 +81,20 @@ export const PreferenceState = t.type({ */ failingUpdates: t.record( t.string, - t.array( - t.type({ - /** Time upload ran at */ - uploadedAt: t.string, - /** Attempts to upload that resulted in an error */ - error: t.string, - /** The update body */ - update: PreferenceStorePurposeUpdate, - }), - ), + t.type({ + /** Time upload ran at */ + uploadedAt: t.string, + /** Attempts to upload that resulted in an error */ + error: t.string, + /** The update body */ + update: PreferenceUpdateItem, + }), ), /** * The set of pending uploads to Transcend * Mapping from userId to the upload metadata */ - pendingUpdates: t.record(t.string, PreferenceStorePurposeUpdate), + pendingUpdates: t.record(t.string, PreferenceUpdateItem), }); /** Override type */ diff --git a/src/preference-management/getPreferencesFromEmailsWithCache.ts b/src/preference-management/getPreferencesFromIdentifiersWithCache.ts similarity index 56% rename from src/preference-management/getPreferencesFromEmailsWithCache.ts rename to src/preference-management/getPreferencesFromIdentifiersWithCache.ts index 7b7ae633..87453ea0 100644 --- a/src/preference-management/getPreferencesFromEmailsWithCache.ts +++ b/src/preference-management/getPreferencesFromIdentifiersWithCache.ts @@ -14,15 +14,15 @@ import { logger } from '../logger'; * @param cache - The cache to store the preferences in * @returns The preferences for the emails */ -export async function getPreferencesFromEmailsWithCache( +export async function getPreferencesFromIdentifiersWithCache( { - emails, + identifiers, ignoreCache = false, sombra, partitionKey, }: { - /** Emails to fetch */ - emails: string[]; + /** Identifiers to fetch */ + identifiers: string[]; /** Whether to use or ignore cache */ ignoreCache?: boolean; /** Sombra got instance */ @@ -31,56 +31,70 @@ export async function getPreferencesFromEmailsWithCache( partitionKey: string; }, cache: PersistedState, -): Promise { +): Promise<(PreferenceQueryResponseItem | null)[]> { // current cache value let preferenceStoreRecords = cache.getValue('preferenceStoreRecords'); // ignore cache if (ignoreCache) { logger.info( - colors.magenta(`Ignoring cache, pulling ${emails.length} emails`), + colors.magenta( + `Ignoring cache, pulling ${identifiers.length} identifiers`, + ), ); const response = await getPreferencesForIdentifiers(sombra, { - identifiers: emails.map((email) => ({ value: email })), + identifiers: identifiers.map((identifier) => ({ value: identifier })), partitionKey, }); preferenceStoreRecords = { ...preferenceStoreRecords, + ...Object.fromEntries( + identifiers.map((identifier) => [identifier, null]), + ), ...Object.fromEntries(response.map((record) => [record.userId, record])), }; cache.setValue(preferenceStoreRecords, 'preferenceStoreRecords'); - logger.info(colors.green(`Successfully pulled ${emails.length} emails`)); + logger.info( + colors.green(`Successfully pulled ${identifiers.length} identifiers`), + ); return response; } // group emails by whether they are in the cache - const { missing = [], existing = [] } = groupBy(emails, (email) => - preferenceStoreRecords[email] ? 'existing' : 'missing', + const { missing = [], existing = [] } = groupBy(identifiers, (email) => + preferenceStoreRecords[email] || preferenceStoreRecords[email] === null + ? 'existing' + : 'missing', ); logger.info( colors.magenta( - `Found ${existing.length} emails in cache, pulling ${missing.length} emails`, + `Found ${existing.length} identifiers in cache, pulling ${missing.length} identifiers`, ), ); - // fetch missing emails + // fetch missing identifiers if (missing.length > 0) { const response = await getPreferencesForIdentifiers(sombra, { - identifiers: missing.map((email) => ({ value: email })), + identifiers: missing.map((identifier) => ({ value: identifier })), partitionKey, }); const newPreferenceStoreRecords = { ...preferenceStoreRecords, + ...Object.fromEntries(missing.map((identifier) => [identifier, null])), ...Object.fromEntries(response.map((record) => [record.userId, record])), }; cache.setValue(newPreferenceStoreRecords, 'preferenceStoreRecords'); - logger.info(colors.green(`Successfully pulled ${missing.length} emails`)); - return existing.map((email) => newPreferenceStoreRecords[email]); + logger.info( + colors.green(`Successfully pulled ${missing.length} identifiers`), + ); + return identifiers.map( + (identifier) => newPreferenceStoreRecords[identifier], + ); } - logger.info(colors.green('No emails pulled, full cache hit')); + logger.info(colors.green('No identifiers pulled, full cache hit')); - // return existing emails - return existing.map((email) => preferenceStoreRecords[email]); + // return existing identifiers + return identifiers.map((identifier) => preferenceStoreRecords[identifier]); } diff --git a/src/preference-management/index.ts b/src/preference-management/index.ts index 95e5f093..2ad7a33d 100644 --- a/src/preference-management/index.ts +++ b/src/preference-management/index.ts @@ -1,5 +1,5 @@ export * from './uploadPreferenceManagementPreferencesInteractive'; export * from './codecs'; export * from './getPreferencesForIdentifiers'; -export * from './getPreferencesFromEmailsWithCache'; +export * from './getPreferencesFromIdentifiersWithCache'; export * from './parsePreferenceManagementCsvWithCache'; diff --git a/src/preference-management/parsePreferenceManagementCsvWithCache.ts b/src/preference-management/parsePreferenceManagementCsvWithCache.ts index af0c6b45..1e96f260 100644 --- a/src/preference-management/parsePreferenceManagementCsvWithCache.ts +++ b/src/preference-management/parsePreferenceManagementCsvWithCache.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { PersistedState } from '@transcend-io/persisted-state'; import difference from 'lodash/difference'; import type { Got } from 'got'; @@ -7,12 +8,55 @@ import keyBy from 'lodash/keyBy'; import inquirer from 'inquirer'; import * as t from 'io-ts'; import colors from 'colors'; -import { FileMetadataState, PreferenceState } from './codecs'; +import { + FileMetadataState, + PreferenceState, + PurposeRowMapping, +} from './codecs'; import { logger } from '../logger'; import { readCsv } from '../requests'; -import { getPreferencesFromEmailsWithCache } from './getPreferencesFromEmailsWithCache'; +import { getPreferencesFromIdentifiersWithCache } from './getPreferencesFromIdentifiersWithCache'; +import { Purpose, PreferenceTopic } from '../graphql'; +import { mapSeries } from 'bluebird'; -const NONE = '[NONE]'; +export const NONE_PREFERENCE_MAP = '[NONE]'; + +/** + * Parse a row into its purposes and preferences + * + * @param options - Options + * @returns The parsed row + */ +export function getUpdatesFromPreferenceRow({ + row, + columnToPurposeName, +}: { + /** Row to parse */ + row: Record; + /** Column names to parse */ + columnToPurposeName: Record; +}): { + [k in string]: { + /** Purpose enabled */ + enabled: boolean; + }; +} { + return Object.keys(columnToPurposeName).reduce( + (acc, col) => + Object.assign(acc, { + [columnToPurposeName[col].purpose]: { + enabled: columnToPurposeName[col].valueMapping[row[col]] === true, + }, + }), + {} as Record< + string, + { + /** Enabled */ + enabled: boolean; + } + >, + ); +} /** * Parse a file into the cache @@ -27,10 +71,17 @@ export async function parsePreferenceManagementCsvWithCache( file, ignoreCache, sombra, + purposes, + // FIXME + // preferenceTopics, partitionKey, }: { /** File to parse */ file: string; + /** The purposes */ + purposes: Purpose[]; + /** The preference topics */ + preferenceTopics: PreferenceTopic[]; /** Whether to use or ignore cache */ ignoreCache?: boolean; /** Sombra got instance */ @@ -55,7 +106,6 @@ export async function parsePreferenceManagementCsvWithCache( pendingSafeUpdates: {}, pendingConflictUpdates: {}, skippedUpdates: {}, - successfulUpdates: {}, // Load in the last fetched time ...((fileMetadata[file] || {}) as Partial), lastFetchedAt: new Date().toISOString(), @@ -144,7 +194,7 @@ export async function parsePreferenceManagementCsvWithCache( remainingColumns.find((col) => col.toLowerCase().includes('date')) || remainingColumns.find((col) => col.toLowerCase().includes('time')) || remainingColumns[0], - choices: [...remainingColumns, NONE], + choices: [...remainingColumns, NONE_PREFERENCE_MAP], }, ]); currentState.timestampColum = timestampName; @@ -157,23 +207,103 @@ export async function parsePreferenceManagementCsvWithCache( ), ); + // Validate that all rows have valid timestamp + if (currentState.timestampColum !== NONE_PREFERENCE_MAP) { + const timestampColumnsMissing = preferences + .map((pref, ind) => (pref[currentState.timestampColum!] ? null : [ind])) + .filter((x): x is number[] => !!x) + .flat(); + if (timestampColumnsMissing.length > 0) { + throw new Error( + `The timestamp column "${ + currentState.timestampColum + }" is missing a value for the following rows: ${timestampColumnsMissing.join( + ', ', + )} in file "${file}"`, + ); + } + logger.info( + colors.magenta( + `The timestamp column "${currentState.timestampColum}" is present for all rows in file: "${file}"`, + ), + ); + } + // Ensure all rows are accounted for const otherColumns = difference(columnNames, [ currentState.identifierColumn, currentState.timestampColum, ]); - // FIXME - if (otherColumns.length > 0) { - logger.error(colors.red(`Other columns: ${otherColumns.join(', ')}`)); + const purposeNames = purposes.map((x) => x.trackingType); + if (otherColumns.length === 0) { + throw new Error(`No other columns to process in file "${file}"`); } + await mapSeries(otherColumns, async (col) => { + // Determine the unique values to map in this column + const uniqueValues = uniq(preferences.map((x) => x[col])); + + // Map the column to a purpose + let purposeMapping = currentState.columnToPurposeName[col]; + if (purposeMapping) { + logger.info( + colors.magenta( + `Column "${col}" is associated with purpose "${purposeMapping.purpose}" in file: "${file}"`, + ), + ); + } else { + const { purposeName } = await inquirer.prompt<{ + /** purpose name */ + purposeName: string; + }>([ + { + name: 'purposeName', + message: `Choose the purpose that column ${col} is associated with`, + type: 'list', + default: purposeNames[0], + choices: purposeNames, + }, + ]); + purposeMapping = { + purpose: purposeName, + valueMapping: {}, + }; + } + + // map each value to the purpose value + await mapSeries(uniqueValues, async (value) => { + if (purposeMapping.valueMapping[value] !== undefined) { + logger.info( + colors.magenta( + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}" in file: "${file}"`, + ), + ); + return; + } + const { purposeValue } = await inquirer.prompt<{ + /** purpose value */ + purposeValue: boolean; + }>([ + { + name: 'purposeValue', + message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, + type: 'confirm', + }, + ]); + purposeMapping.valueMapping[value] = purposeValue; + }); + + currentState.columnToPurposeName[col] = purposeMapping; + fileMetadata[file] = currentState; + cache.setValue(fileMetadata, 'fileMetadata'); + }); // Grab existing preference store records - const emails = preferences.map( + const identifiers = preferences.map( (pref) => pref[currentState.identifierColumn!], ); - const existingConsentRecords = await getPreferencesFromEmailsWithCache( + const existingConsentRecords = await getPreferencesFromIdentifiersWithCache( { - emails, + identifiers, ignoreCache, sombra, partitionKey, @@ -182,36 +312,57 @@ export async function parsePreferenceManagementCsvWithCache( ); const consentRecordByEmail = keyBy(existingConsentRecords, 'userId'); + // Clear out previous updates + currentState.pendingConflictUpdates = {}; + currentState.pendingSafeUpdates = {}; + currentState.skippedUpdates = {}; + // Process each row preferences.forEach((pref) => { // used to compare if the preference has already been processed - const stringifiedPref = JSON.stringify(pref); + const userId = pref[currentState.identifierColumn!]; + const purposeMapping = otherColumns.reduce( + (acc, col) => + Object.assign(acc, { + [currentState.columnToPurposeName[col].purpose]: + currentState.columnToPurposeName[col].valueMapping[pref[col]], + }), + {} as Record, + ); // Grab current state of the update - const previousSuccesses = - currentState.successfulUpdates[pref[currentState.identifierColumn!]] || - []; - const currentConsentRecord = - consentRecordByEmail[pref[currentState.identifierColumn!]]; - const pendingConflictUpdate = - currentState.pendingConflictUpdates[pref[currentState.identifierColumn!]]; - const pendingSafeUpdate = - currentState.pendingSafeUpdates[pref[currentState.identifierColumn!]]; - const skippedUpdate = - currentState.skippedUpdates[pref[currentState.identifierColumn!]]; - - // Check if change was already processed - // no need to do anything here as there is already an audit record for this event - if (previousSuccesses.find((x) => JSON.stringify(x) === stringifiedPref)) { + const currentConsentRecord = consentRecordByEmail[userId]; + + // Check if the update can be skipped + if ( + currentConsentRecord && + Object.entries(purposeMapping).every( + ([key, value]) => + currentConsentRecord.purposes.find( + (existingPurpose) => existingPurpose.purpose === key, + )?.enabled === value, + ) + ) { + currentState.skippedUpdates[userId] = pref; return; } - console.log({ - pendingConflictUpdate, - pendingSafeUpdate, - skippedUpdate, - currentConsentRecord, - }); + // Determine if there are any conflicts + const hasConflicts = + currentConsentRecord && + Object.entries(purposeMapping).find(([key, value]) => { + const currentPurpose = currentConsentRecord.purposes.find( + (existingPurpose) => existingPurpose.purpose === key, + ); + return currentPurpose && currentPurpose.enabled !== value; + }); + if (hasConflicts) { + currentState.pendingConflictUpdates[userId] = pref; + return; + } + + // Add to pending updates + currentState.pendingSafeUpdates[userId] = pref; }); // Read in the file @@ -224,3 +375,23 @@ export async function parsePreferenceManagementCsvWithCache( ), ); } + +// // Ensure usp strings are valid +// const invalidUspStrings = preferences.filter( +// (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp), +// ); +// if (invalidUspStrings.length > 0) { +// throw new Error( +// `Received invalid usp strings: ${JSON.stringify( +// invalidUspStrings, +// null, +// 2, +// )}`, +// ); +// } +// // parse usp string +// const [, saleStatus] = consent.usp +// ? USP_STRING_REGEX.exec(consent.usp) || [] +// : []; + +/* eslint-enable max-lines */ diff --git a/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index a03cd6a8..676347f7 100644 --- a/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -1,22 +1,29 @@ -import { createSombraGotInstance } from '../graphql'; +import { + buildTranscendGraphQLClient, + createSombraGotInstance, + fetchAllPurposes, +} from '../graphql'; import colors from 'colors'; +import chunk from 'lodash/chunk'; import { DEFAULT_TRANSCEND_CONSENT_API } from '../constants'; import { mapSeries } from 'bluebird'; import { logger } from '../logger'; -// import cliProgress from 'cli-progress'; -// import { decodeCodec } from '@transcend-io/type-utils'; -// import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types'; -// import { USP_STRING_REGEX } from '../consent-manager'; +import cliProgress from 'cli-progress'; import { parseAttributesFromString } from '../requests'; import { PersistedState } from '@transcend-io/persisted-state'; -import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsvWithCache'; +import { + NONE_PREFERENCE_MAP, + getUpdatesFromPreferenceRow, + parsePreferenceManagementCsvWithCache, +} from './parsePreferenceManagementCsvWithCache'; import { PreferenceState } from './codecs'; +import { fetchAllPreferenceTopics } from '../graphql/fetchAllPreferenceTopics'; +import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; +import { getPreferencesFromIdentifiersWithCache } from './getPreferencesFromIdentifiersWithCache'; /** * Upload a set of consent preferences * - * FIXME pick up from left off with dryRun? - * * @param options - Options */ export async function uploadPreferenceManagementPreferencesInteractive({ @@ -93,14 +100,24 @@ export async function uploadPreferenceManagementPreferencesInteractive({ ), ); - // Create sombra instance to communicate with - const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth); + // Create GraphQL client to connect to Transcend backend + const client = buildTranscendGraphQLClient(transcendUrl, auth); + + const [sombra, purposes, preferenceTopics] = await Promise.all([ + // Create sombra instance to communicate with + createSombraGotInstance(transcendUrl, auth, sombraAuth), + // get all purposes and topics + fetchAllPurposes(client), + fetchAllPreferenceTopics(client), + ]); // Process each file await mapSeries(files, async (file) => { await parsePreferenceManagementCsvWithCache( { file, + purposes, + preferenceTopics, ignoreCache: refreshPreferenceStoreCache, sombra, partitionKey: partition, @@ -109,138 +126,193 @@ export async function uploadPreferenceManagementPreferencesInteractive({ ); }); - // // Ensure usp strings are valid - // const invalidUspStrings = preferences.filter( - // (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp), - // ); - // if (invalidUspStrings.length > 0) { - // throw new Error( - // `Received invalid usp strings: ${JSON.stringify( - // invalidUspStrings, - // null, - // 2, - // )}`, - // ); - // } - - // if (invalidPurposeMaps.length > 0) { - // throw new Error( - // `Received invalid purpose maps: ${JSON.stringify( - // invalidPurposeMaps, - // null, - // 2, - // )}`, - // ); - // } - - // // Ensure usp or preferences are provided - // const invalidInputs = preferences.filter( - // (pref) => !pref.usp && !pref.purposes, - // ); - // if (invalidInputs.length > 0) { - // throw new Error( - // `Received invalid inputs, expected either purposes or usp to be defined: ${JSON.stringify( - // invalidInputs, - // null, - // 2, - // )}`, - // ); - // } - - // logger.info( - // colors.magenta( - // `Uploading ${preferences.length} user preferences to partition ${partition}`, - // ), - // ); - - // // Time duration - // const t0 = new Date().getTime(); - // // create a new progress bar instance and use shades_classic theme - // const progressBar = new cliProgress.SingleBar( - // {}, - // cliProgress.Presets.shades_classic, - // ); - - // // Build a GraphQL client - // let total = 0; - // progressBar.start(preferences.length, 0); - // await map( - // preferences, - // async ({ - // userId, - // confirmed = 'true', - // updated, - // prompted, - // purposes, - // ...consent - // }) => { - // const token = createConsentToken( - // userId, - // base64EncryptionKey, - // base64SigningKey, - // ); - - // // parse usp string - // const [, saleStatus] = consent.usp - // ? USP_STRING_REGEX.exec(consent.usp) || [] - // : []; - - // const input = { - // token, - // partition, - // consent: { - // confirmed: confirmed === 'true', - // purposes: purposes - // ? decodeCodec(PurposeMap, purposes) - // : consent.usp - // ? { SaleOfInfo: saleStatus === 'Y' } - // : {}, - // ...(updated ? { updated: updated === 'true' } : {}), - // ...(prompted ? { prompted: prompted === 'true' } : {}), - // ...consent, - // }, - // } as ConsentPreferencesBody; - - // // Make the request - // try { - // await transcendConsentApi - // .post('sync', { - // json: input, - // }) - // .json(); - // } catch (err) { - // try { - // const parsed = JSON.parse(err?.response?.body || '{}'); - // if (parsed.error) { - // logger.error(colors.red(`Error: ${parsed.error}`)); - // } - // } catch (e) { - // // continue - // } - // throw new Error( - // `Received an error from server: ${ - // err?.response?.body || err?.message - // }`, - // ); - // } - - // total += 1; - // progressBar.update(total); - // }, - // { concurrency }, - // ); - - // progressBar.stop(); - // const t1 = new Date().getTime(); - // const totalTime = t1 - t0; - - // logger.info( - // colors.green( - // `Successfully uploaded ${ - // preferences.length - // } user preferences to partition ${partition} in "${ - // totalTime / 1000 - // }" seconds!`, - // ), - // ); + // Construct the pending updates + const pendingUpdates: Record = {}; + files.forEach((file) => { + const fileMetadata = preferenceState.getValue('fileMetadata'); + const metadata = fileMetadata[file]; + + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.pendingSafeUpdates).length + } safe updates in ${file}`, + ), + ); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.pendingConflictUpdates).length + } conflict updates in ${file}`, + ), + ); + logger.info( + colors.magenta( + `Found ${ + Object.entries(metadata.skippedUpdates).length + } skipped updates in ${file}`, + ), + ); + Object.entries({ + ...metadata.pendingSafeUpdates, + ...metadata.pendingConflictUpdates, + }).forEach(([userId, update]) => { + const currentUpdate = pendingUpdates[userId]; + const timestamp = + metadata.timestampColum === NONE_PREFERENCE_MAP + ? new Date() + : new Date(update[metadata.timestampColum!]); + const updates = getUpdatesFromPreferenceRow({ + row: update, + columnToPurposeName: metadata.columnToPurposeName, + }); + + if (currentUpdate) { + const newPurposes = Object.entries(updates).map(([purpose, value]) => ({ + ...value, + purpose, + })); + (currentUpdate.purposes || []).forEach((purpose) => { + if (updates[purpose.purpose].enabled !== purpose.enabled) { + logger.warn( + colors.yellow( + `Conflict detected for user: ${userId} and purpose: ${purpose.purpose}`, + ), + ); + } + }); + pendingUpdates[userId] = { + userId, + partition, + // take the most recent timestamp + timestamp: + timestamp > new Date(currentUpdate.timestamp) + ? timestamp.toISOString() + : currentUpdate.timestamp, + purposes: [ + ...(currentUpdate.purposes || []), + ...newPurposes.filter( + (newPurpose) => + !(currentUpdate.purposes || []).find( + (currentPurpose) => + currentPurpose.purpose === newPurpose.purpose, + ), + ), + ], + }; + } else { + pendingUpdates[userId] = { + userId, + partition, + timestamp: timestamp.toISOString(), + purposes: Object.entries(updates).map(([purpose, value]) => ({ + ...value, + purpose, + })), + }; + } + }); + }); + preferenceState.setValue(pendingUpdates, 'pendingUpdates'); + preferenceState.setValue({}, 'failingUpdates'); + + if (dryRun) { + logger.info( + colors.green(`Dry run complete, exiting. Check file: ${receiptFilepath}`), + ); + return; + } + + logger.info( + colors.magenta(`Uploading preferences to partition: ${partition}`), + ); + + // Time duration + const t0 = new Date().getTime(); + + // create a new progress bar instance and use shades_classic theme + const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, + ); + + // Build a GraphQL client + let total = 0; + const updatesToRun = Object.entries(pendingUpdates); + const chunkedUpdates = chunk(updatesToRun, 100); + progressBar.start(updatesToRun.length, 0); + await mapSeries(chunkedUpdates, async (chunkedUpdates) => { + const failingUpdates = preferenceState.getValue('failingUpdates'); + const successfulUpdates = preferenceState.getValue('successfulUpdates'); + const pendingUpdates = preferenceState.getValue('pendingUpdates'); + + // Make the request + try { + await sombra + .put('/v1/preferences', { + json: { + records: chunkedUpdates, + }, + }) + .json(); + chunkedUpdates.forEach(([userId, update]) => { + successfulUpdates[userId].push({ + uploadedAt: new Date().toISOString(), + update, + }); + delete pendingUpdates[userId]; + }); + preferenceState.setValue(pendingUpdates, 'pendingUpdates'); + preferenceState.setValue(successfulUpdates, 'successfulUpdates'); + } catch (err) { + try { + const parsed = JSON.parse(err?.response?.body || '{}'); + if (parsed.error) { + logger.error(colors.red(`Error: ${parsed.error}`)); + } + } catch (e) { + // continue + } + + chunkedUpdates.forEach(([userId, update]) => { + failingUpdates[userId] = { + uploadedAt: new Date().toISOString(), + update, + error: err?.response?.body || err?.message || 'Unknown error', + }; + delete pendingUpdates[userId]; + }); + preferenceState.setValue(failingUpdates, 'failingUpdates'); + preferenceState.setValue(pendingUpdates, 'pendingUpdates'); + } + + total += chunkedUpdates.length; + progressBar.update(total); + }); + + progressBar.stop(); + const t1 = new Date().getTime(); + const totalTime = t1 - t0; + logger.info( + colors.green( + `Successfully uploaded ${ + updatesToRun.length + } user preferences to partition ${partition} in "${ + totalTime / 1000 + }" seconds!`, + ), + ); + + logger.info(colors.magenta('Refreshing cache...')); + + const updateIdentifiers = Object.keys(pendingUpdates); + await getPreferencesFromIdentifiersWithCache( + { + identifiers: updateIdentifiers, + ignoreCache: true, + sombra, + partitionKey: partition, + }, + preferenceState, + ); } diff --git a/yarn.lock b/yarn.lock index d8b0230d..b32a4a48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -515,7 +515,7 @@ __metadata: "@transcend-io/handlebars-utils": ^1.1.0 "@transcend-io/internationalization": ^1.6.0 "@transcend-io/persisted-state": ^1.0.4 - "@transcend-io/privacy-types": ^4.96.0 + "@transcend-io/privacy-types": ^4.98.0 "@transcend-io/secret-value": ^1.2.0 "@transcend-io/type-utils": ^1.5.0 "@types/bluebird": ^3.5.38 @@ -644,14 +644,14 @@ __metadata: languageName: node linkType: hard -"@transcend-io/privacy-types@npm:^4.96.0": - version: 4.96.0 - resolution: "@transcend-io/privacy-types@npm:4.96.0" +"@transcend-io/privacy-types@npm:^4.98.0": + version: 4.98.0 + resolution: "@transcend-io/privacy-types@npm:4.98.0" dependencies: "@transcend-io/type-utils": ^1.0.5 fp-ts: ^2.16.1 io-ts: ^2.2.21 - checksum: 0ed38f70c7e45c0cf3f084c03284e76f82366add723c74ac76d17e06e39e0ab402a5cc3df5c71e4f6b3e7d2ce49a8b9b4034f9b61f86f21d43f965537b2d8067 + checksum: bee61228fa7ce58f2fbfe1dcb2e31812e480ef16012aa2f73deccf3812fc330225a332d57730303d46764de98079a8fc905969894419a3dc9a33744e8ec012d1 languageName: node linkType: hard