Wokring script

package.json

@@ -100,6 +100,7 @@
     "@types/chai": "^4.3.4",
     "@types/cli-progress": "^3.11.0",
     "@types/colors": "^1.2.1",
+    "@types/deep-equal-in-any-order": "1.0.1",
     "@types/fuzzysearch": "^1.0.0",
     "@types/global-agent": "^2.1.1",
     "@types/inquirer": "^7.3.1",
@@ -116,6 +117,7 @@
     "@typescript-eslint/parser": "^5.58.0",
     "@yarnpkg/sdks": "^3.0.0-rc.42",
     "chai": "^4.3.7",
+    "deep-equal-in-any-order": "^1.0.28",
     "depcheck": "^1.4.3",
     "eslint": "^8.38.0",
     "eslint-config-airbnb-base": "^15.0.0",

src/cli-upload-preferences.ts

@@ -26,7 +26,7 @@ async function main(): Promise<void> {
   // Parse command line arguments
   const {
     /** File to load preferences from */
-    files = './preferences.csv',
+    file = './preferences.csv',
     /** Transcend URL */
     transcendUrl = DEFAULT_TRANSCEND_API,
     /** API key */
@@ -39,6 +39,8 @@ async function main(): Promise<void> {
     dryRun = 'false',
     /** Whether to skip workflow triggers */
     skipWorkflowTriggers = 'false',
+    /** Whether to skip conflict updates */
+    skipConflictUpdates = 'false',
     /** Whether to skip sending emails */
     isSilent = 'true',
     /** Attributes to add to any DSR request if created */
@@ -73,9 +75,10 @@ async function main(): Promise<void> {
     receiptFilepath,
     auth,
     sombraAuth,
-    files: splitCsvToList(files),
+    file,
     partition,
     transcendUrl,
+    skipConflictUpdates: skipConflictUpdates !== 'false',
     skipWorkflowTriggers: skipWorkflowTriggers !== 'false',
     isSilent: isSilent !== 'false',
     dryRun: dryRun !== 'false',

src/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts

@@ -0,0 +1,93 @@
+import {
+  PreferenceQueryResponseItem,
+  PreferenceStorePurposeResponse,
+  PreferenceTopicType,
+} from '@transcend-io/privacy-types';
+import { PreferenceTopic } from '../graphql';
+
+/**
+ * Check if the pending set of updates are exactly the same as the current consent record.
+ *
+ * @param options - Options
+ * @returns Whether the pending updates already exist in the preference store
+ */
+export function checkIfPendingPreferenceUpdatesAreNoOp({
+  currentConsentRecord,
+  pendingUpdates,
+  preferenceTopics,
+}: {
+  /** The current consent record */
+  currentConsentRecord: PreferenceQueryResponseItem;
+  /** The pending updates */
+  pendingUpdates: {
+    [purposeName in string]: Omit<PreferenceStorePurposeResponse, 'purpose'>;
+  };
+  /** The preference topic configurations */
+  preferenceTopics: PreferenceTopic[];
+}): boolean {
+  // Check each update
+  return Object.entries(pendingUpdates).every(
+    ([purposeName, { preferences = [], enabled }]) => {
+      // Ensure the purpose exists
+      const currentPurpose = currentConsentRecord.purposes.find(
+        (existingPurpose) => existingPurpose.purpose === purposeName,
+      );
+
+      // Ensure purpose.enabled is in sync
+      // Also false if the purpose does not exist
+      const enabledIsInSync =
+        !!currentPurpose && currentPurpose.enabled === enabled;
+      if (!enabledIsInSync) {
+        return false;
+      }
+
+      // Compare the preferences are in sync
+      return preferences.every(
+        ({ topic, choice }) =>
+          // ensure preferences exist on record
+          currentPurpose.preferences &&
+          currentPurpose.preferences.find((existingPreference) => {
+            // find matching topic
+            if (existingPreference.topic !== topic) {
+              return false;
+            }
+
+            // Determine type of preference topic
+            const preferenceTopic = preferenceTopics.find(
+              (x) => x.slug === topic && x.purpose.trackingType === purposeName,
+            );
+            if (!preferenceTopic) {
+              throw new Error(`Could not find preference topic for ${topic}`);
+            }
+
+            // Handle comparison based on type
+            switch (preferenceTopic.type) {
+              case PreferenceTopicType.Boolean:
+                return (
+                  existingPreference.choice.booleanValue === choice.booleanValue
+                );
+              case PreferenceTopicType.Select:
+                return (
+                  existingPreference.choice.selectValue === choice.selectValue
+                );
+              case PreferenceTopicType.MultiSelect:
+                // eslint-disable-next-line no-case-declarations
+                const sortedCurrentValues = (
+                  existingPreference.choice.selectValues || []
+                ).sort();
+                // eslint-disable-next-line no-case-declarations
+                const sortedNewValues = (choice.selectValues || []).sort();
+                return (
+                  sortedCurrentValues.length === sortedNewValues.length &&
+                  sortedCurrentValues.every((x, i) => x === sortedNewValues[i])
+                );
+              default:
+                throw new Error(
+                  `Unknown preference topic type: ${preferenceTopic.type}`,
+                );
+            }
+          }),
+      );
+    },
+  );
+}

src/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts

@@ -0,0 +1,94 @@
+import {
+  PreferenceQueryResponseItem,
+  PreferenceStorePurposeResponse,
+  PreferenceTopicType,
+} from '@transcend-io/privacy-types';
+import { PreferenceTopic } from '../graphql';
+
+/**
+ * Check if the pending set of updates will result in a change of
+ * value to an existing purpose or preference in the preference store.
+ *
+ * @param options - Options
+ * @returns True if conflict, false if no conflict and just adding new data for first time
+ */
+export function checkIfPendingPreferenceUpdatesCauseConflict({
+  currentConsentRecord,
+  pendingUpdates,
+  preferenceTopics,
+}: {
+  /** The current consent record */
+  currentConsentRecord: PreferenceQueryResponseItem;
+  /** The pending updates */
+  pendingUpdates: {
+    [purposeName in string]: Omit<PreferenceStorePurposeResponse, 'purpose'>;
+  };
+  /** The preference topic configurations */
+  preferenceTopics: PreferenceTopic[];
+}): boolean {
+  // Check if any update has conflict
+  return !!Object.entries(pendingUpdates).find(
+    ([purposeName, { preferences = [], enabled }]) => {
+      // Ensure the purpose exists
+      const currentPurpose = currentConsentRecord.purposes.find(
+        (existingPurpose) => existingPurpose.purpose === purposeName,
+      );
+
+      // If no purpose exists, then it is not a conflict
+      if (!currentPurpose) {
+        return false;
+      }
+
+      // If purpose.enabled value is off, this is a conflict
+      if (currentPurpose.enabled !== enabled) {
+        return true;
+      }
+
+      // Check if any preferences are out of sync
+      return !!preferences.find(({ topic, choice }) => {
+        // find matching topic
+        const currentPreference = (currentPurpose.preferences || []).find(
+          (existingPreference) => existingPreference.topic === topic,
+        );
+
+        // if no topic exists, no conflict
+        if (!currentPreference) {
+          return false;
+        }
+
+        // Determine type of preference topic
+        const preferenceTopic = preferenceTopics.find(
+          (x) => x.slug === topic && x.purpose.trackingType === purposeName,
+        );
+        if (!preferenceTopic) {
+          throw new Error(`Could not find preference topic for ${topic}`);
+        }
+
+        // Handle comparison based on type
+        switch (preferenceTopic.type) {
+          case PreferenceTopicType.Boolean:
+            return (
+              currentPreference.choice.booleanValue !== choice.booleanValue
+            );
+          case PreferenceTopicType.Select:
+            return currentPreference.choice.selectValue !== choice.selectValue;
+          case PreferenceTopicType.MultiSelect:
+            // eslint-disable-next-line no-case-declarations
+            const sortedCurrentValues = (
+              currentPreference.choice.selectValues || []
+            ).sort();
+            // eslint-disable-next-line no-case-declarations
+            const sortedNewValues = (choice.selectValues || []).sort();
+            return (
+              sortedCurrentValues.length !== sortedNewValues.length ||
+              !sortedCurrentValues.every((x, i) => x === sortedNewValues[i])
+            );
+          default:
+            throw new Error(
+              `Unknown preference topic type: ${preferenceTopic.type}`,
+            );
+        }
+      });
+    },
+  );
+}

src/preference-management/codecs.ts

@@ -5,20 +5,51 @@ import {
 import * as t from 'io-ts';
 
 export const PurposeRowMapping = t.type({
-  /** Name of the purpose to map to */
+  /**
+   * The slug or trackingType of the purpose to map to
+   *
+   * e.g. `Marketing`
+   */
   purpose: t.string,
-  /** Mapping from value in row to value in transcend API */
-  valueMapping: t.record(t.string, t.union([t.string, t.boolean])),
+  /**
+   * If the column maps to a preference instead of a purpose
+   * this is the slug of the purpose.
+   *
+   * null value indicates that this column maps to the true/false
+   * value of the purpose
+   */
+  preference: t.union([t.string, t.null]),
+  /**
+   * The mapping between each row value and purpose/preference value.
+   *
+   * e.g. for a boolean preference or purpose
+   * {
+   *  'true': true,
+   *  'false': false,
+   *   '': true,
+   * }
+   *
+   * or for a single or multi select preference
+   * {
+   *   '': true,
+   *   'value1': 'Value1',
+   *   'value2': 'Value2',
+   * }
+   */
+  valueMapping: t.record(t.string, t.union([t.string, t.boolean, t.null])),
 });
 
 /** Override type */
 export type PurposeRowMapping = t.TypeOf<typeof PurposeRowMapping>;
 
 export const FileMetadataState = t.intersection([
   t.type({
-    /** Mapping of column name to it's relevant purpose in Transcend */
+    /**
+     * Definition of how to map each column in the CSV to
+     * the relevant purpose and preference definitions in transcend
+     */
     columnToPurposeName: t.record(t.string, PurposeRowMapping),
-    /** Last time the file was fetched */
+    /** Last time the file was last parsed at */
     lastFetchedAt: t.string,
     /**
      * Mapping of userId to the rows in the file that need to be uploaded