DBP API changes for WebUI - Release (#2233)

Cherry picked from #2146 Task/Issue URL: https://app.asana.com/0/1204167627774280/1206480273655878/f **Description**: Add url property to data broker model, and changes result for WebUI call to support new UI designs
duckduckgo · Feb 21, 2024 · 5e2a8c0 · 5e2a8c0
1 parent 60c5f36
commit 5e2a8c0
Show file tree

Hide file tree

Showing 56 changed files with 691 additions and 286 deletions.
diff --git a/...kerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDataManager.swift b/...kerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDataManager.swift
@@ -290,4 +290,26 @@ extension InMemoryDataCache: DBPUICommunicationDelegate {
 
         return mapper.maintenanceScanState(brokerProfileQueryData)
     }
+
+    func getDataBrokers() async -> [DBPUIDataBroker] {
+        brokerProfileQueryData
+        // 1. We get all brokers (in this list brokers are repeated)
+            .map { $0.dataBroker }
+        // 2. We map the brokers to the UI model
+            .flatMap { dataBroker -> [DBPUIDataBroker] in
+                var result: [DBPUIDataBroker] = []
+                result.append(DBPUIDataBroker(name: dataBroker.name, url: dataBroker.url))
+
+                for mirrorSite in dataBroker.mirrorSites {
+                    result.append(DBPUIDataBroker(name: mirrorSite.name, url: mirrorSite.url))
+                }
+                return result
+            }
+        // 3. We delete duplicates
+            .reduce(into: [DBPUIDataBroker]()) { (result, dataBroker) in
+                if !result.contains(where: { $0.url == dataBroker.url }) {
+                    result.append(dataBroker)
+                }
+            }
+    }
 }
diff --git a/.../Sources/DataBrokerProtection/DebugUI/DataBaseBrowser/DataBrokerDatabaseBrowserView.swift b/.../Sources/DataBrokerProtection/DebugUI/DataBaseBrowser/DataBrokerDatabaseBrowserView.swift
@@ -49,6 +49,7 @@ struct DatabaseView: View {
     @State private var isPopoverVisible = false
     @State private var selectedData: String = ""
     let data: [DataBrokerDatabaseBrowserData.Row]
+    let rowHeight: CGFloat = 40.0
 
     var body: some View {
         if data.count > 0 {
@@ -62,6 +63,11 @@ struct DatabaseView: View {
         }
     }
 
+    private func spacerHeight(_ geometry: GeometryProxy) -> CGFloat {
+        let result = geometry.size.height - CGFloat(data.count) * rowHeight
+        return max(0, result)
+    }
+
     private func dataView() -> some View {
         GeometryReader { geometry in
             ScrollView([.horizontal, .vertical]) {
@@ -86,7 +92,8 @@ struct DatabaseView: View {
                             ForEach(row.data.keys.sorted(), id: \.self) { key in
                                 VStack {
                                     Text("\(row.data[key]?.description ?? "")")
-                                        .frame(maxWidth: 200, maxHeight: 50)
+                                        .frame(maxWidth: 200)
+                                        .frame(height: rowHeight)
                                         .frame(minWidth: 60)
                                         .onTapGesture {
                                             selectedData = row.data[key]?.description ?? ""
@@ -100,7 +107,8 @@ struct DatabaseView: View {
                             }
                         }
                     }
-                    Spacer(minLength: geometry.size.height)
+                    Spacer()
+                        .frame(height: spacerHeight(geometry))
                 }
                 .frame(minWidth: geometry.size.width, minHeight: 0, alignment: .topLeading)
             }

diff --git a/...ges/DataBrokerProtection/Sources/DataBrokerProtection/Model/DBPUICommunicationModel.swift b/...ges/DataBrokerProtection/Sources/DataBrokerProtection/Model/DBPUICommunicationModel.swift
@@ -103,12 +103,24 @@ struct DBPUIAddressAtIndex: Codable {
 /// Message Object representing a data broker
 struct DBPUIDataBroker: Codable, Hashable {
     let name: String
+    let url: String
+    let date: Double?
+
+    init(name: String, url: String, date: Double? = nil) {
+        self.name = name
+        self.url = url
+        self.date = date
+    }
 
     func hash(into hasher: inout Hasher) {
         hasher.combine(name)
     }
 }
 
+struct DBPUIDataBrokerList: DBPUISendableMessage {
+    let dataBrokers: [DBPUIDataBroker]
+}
+
 /// Message Object representing a requested change to the user profile's brith year
 struct DBPUIBirthYear: Codable {
     let year: Int
@@ -123,6 +135,7 @@ struct DBPUIDataBrokerProfileMatch: Codable {
     let addresses: [DBPUIUserProfileAddress]
     let alternativeNames: [String]
     let relatives: [String]
+    let date: Double? // Used in some methods to set the removedDate or found date
 }
 
 /// Protocol to represent a message that can be passed from the host to the UI
@@ -139,6 +152,10 @@ struct DBPUIScanAndOptOutMaintenanceState: DBPUISendableMessage {
 struct DBPUIOptOutMatch: DBPUISendableMessage {
     let dataBroker: DBPUIDataBroker
     let matches: Int
+    let name: String
+    let alternativeNames: [String]
+    let addresses: [DBPUIUserProfileAddress]
+    let date: Double
 }
 
 /// Data representing the initial scan progress

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift
@@ -32,13 +32,46 @@ extension Int {
 
 struct MirrorSite: Codable, Sendable {
     let name: String
+    let url: String
     let addedAt: Date
     let removedAt: Date?
+
+    enum CodingKeys: CodingKey {
+        case name
+        case url
+        case addedAt
+        case removedAt
+    }
+
+    init(name: String, url: String, addedAt: Date, removedAt: Date? = nil) {
+        self.name = name
+        self.url = url
+        self.addedAt = addedAt
+        self.removedAt = removedAt
+    }
+
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        name = try container.decode(String.self, forKey: .name)
+
+        // The older versions of the JSON file did not have a URL property.
+        // When decoding those cases, we fallback to its name, since the name was the URL.
+        do {
+            url = try container.decode(String.self, forKey: .url)
+        } catch {
+            url = name
+        }
+
+        addedAt = try container.decode(Date.self, forKey: .addedAt)
+        removedAt = try? container.decode(Date.self, forKey: .removedAt)
+
+    }
 }
 
 struct DataBroker: Codable, Sendable {
     let id: Int64?
     let name: String
+    let url: String
     let steps: [Step]
     let version: String
     let schedulingConfig: DataBrokerScheduleConfig
@@ -51,6 +84,7 @@ struct DataBroker: Codable, Sendable {
 
     enum CodingKeys: CodingKey {
         case name
+        case url
         case steps
         case version
         case schedulingConfig
@@ -60,6 +94,7 @@ struct DataBroker: Codable, Sendable {
 
     init(id: Int64? = nil,
          name: String,
+         url: String,
          steps: [Step],
          version: String,
          schedulingConfig: DataBrokerScheduleConfig,
@@ -68,6 +103,13 @@ struct DataBroker: Codable, Sendable {
     ) {
         self.id = id
         self.name = name
+
+        if url.isEmpty {
+            self.url = name
+        } else {
+            self.url = url
+        }
+
         self.steps = steps
         self.version = version
         self.schedulingConfig = schedulingConfig
@@ -78,6 +120,15 @@ struct DataBroker: Codable, Sendable {
     init(from decoder: Decoder) throws {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         name = try container.decode(String.self, forKey: .name)
+
+        // The older versions of the JSON file did not have a URL property.
+        // When decoding those cases, we fallback to its name, since the name was the URL.
+        do {
+            url = try container.decode(String.self, forKey: .url)
+        } catch {
+            url = name
+        }
+
         version = try container.decode(String.self, forKey: .version)
         steps = try container.decode([Step].self, forKey: .steps)
         schedulingConfig = try container.decode(DataBrokerScheduleConfig.self, forKey: .schedulingConfig)

diff --git a/...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift
@@ -99,7 +99,7 @@ extension DataBrokerOperation {
         if action.needsEmail {
             do {
                 stageCalculator?.setStage(.emailGenerate)
-                extractedProfile?.email = try await emailService.getEmail(dataBrokerName: query.dataBroker.name)
+                extractedProfile?.email = try await emailService.getEmail(dataBrokerURL: query.dataBroker.url)
                 stageCalculator?.fireOptOutEmailGenerate()
             } catch {
                 await onError(error: DataBrokerProtectionError.emailError(error as? EmailError))

diff --git a/...rotection/Sources/DataBrokerProtection/Operations/DataBrokerProtectionBrokerUpdater.swift b/...rotection/Sources/DataBrokerProtection/Operations/DataBrokerProtectionBrokerUpdater.swift
@@ -152,7 +152,7 @@ public struct DataBrokerProtectionBrokerUpdater {
     // 2. If does exist, we check the number version, if the version number is new, we update it
     // 3. If it does not exist, we add it, and we create the scan operations related to it
     private func update(_ broker: DataBroker) throws {
-        guard let savedBroker = try vault.fetchBroker(with: broker.name) else {
+        guard let savedBroker = try vault.fetchBroker(with: broker.url) else {
             // The broker does not exist in the current storage. We need to add it.
             try add(broker)
             return

diff --git a/...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json b/...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json
@@ -1,21 +1,22 @@
 {
-  "name": "advancedbackgroundchecks.com",
-  "version": "0.1.0",
+  "name": "AdvancedBackgroundChecks",
+  "url": "advancedbackgroundchecks.com",
+  "version": "0.1.4",
   "parent": "peoplefinders.com",
-  "addedDatetime": 1678078800000,
+  "addedDatetime": 1678082400000,
   "steps": [
     {
       "stepType": "scan",
       "scanType": "templatedUrl",
       "actions": [
         {
           "actionType": "navigate",
-          "id": "ef8031e6-5e61-4183-b57e-7df156c7129a",
+          "id": "c73ba931-9e01-4d37-9e15-2fd7a14eefa3",
           "url": "https://www.advancedbackgroundchecks.com/names/${firstName}-${lastName}_${city}-${state}_age_${age}"
         },
         {
           "actionType": "extract",
-          "id": "f3ed744c-6cfc-4a99-b46e-6095587eadfc",
+          "id": "94003082-0a9d-4418-ac88-68595c7f4953",
           "selector": ".card-block",
           "profile": {
             "name": {

diff --git a/...DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json b/...DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json
@@ -1,6 +1,7 @@
 {
   "name": "backgroundcheck.run",
-  "version": "0.1.1",
+  "url": "backgroundcheck.run",
+  "version": "0.1.4",
   "parent": "verecor.com",
   "addedDatetime": 1677736800000,
   "steps": [
@@ -10,12 +11,12 @@
       "actions": [
         {
           "actionType": "navigate",
-          "id": "aa12b430-8e5d-4c64-bb77-2961f19a1bc8",
+          "id": "5f90e39f-cb94-4b8d-94ed-48ba0060dc08",
           "url": "https://backgroundcheck.run/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}"
         },
         {
           "actionType": "extract",
-          "id": "75fd2e16-d84a-4bbe-9cf1-79c6d1cc4dec",
+          "id": "3225fa15-4e00-4e6a-bfc7-a85dfb504c86",
           "selector": ".b-pfl-list",
           "profile": {
             "name": {

diff --git a/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json b/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json
@@ -1,16 +1,17 @@
 {
-  "name": "centeda.com",
-  "version": "0.1.1",
+  "name": "Centeda",
+  "url": "centeda.com",
+  "version": "0.1.4",
   "parent": "verecor.com",
-  "addedDatetime": 1677733200000,
+  "addedDatetime": 1677736800000,
   "steps": [
     {
       "stepType": "scan",
       "scanType": "templatedUrl",
       "actions": [
         {
           "actionType": "navigate",
-          "id": "25990359-3d58-45de-bdfd-d524b1946e57",
+          "id": "2f6639c0-201f-4d5e-8467-ae0ba457b409",
           "url": "https://centeda.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}",
           "ageRange": [
             "18-30",
@@ -24,7 +25,7 @@
         },
         {
           "actionType": "extract",
-          "id": "7108af78-dbbf-47ec-8bb9-e44be505993e",
+          "id": "e2e236b0-515b-43b3-9154-0432ed9b7566",
           "selector": ".search-item",
           "profile": {
             "name": {
@@ -63,4 +64,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}
diff --git a/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json b/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json
@@ -1,6 +1,7 @@
 {
-  "name": "clubset.com",
-  "version": "0.1.1",
+  "name": "Clubset",
+  "url": "clubset.com",
+  "version": "0.1.4",
   "parent": "verecor.com",
   "addedDatetime": 1702965600000,
   "steps": [
@@ -10,7 +11,7 @@
       "actions": [
         {
           "actionType": "navigate",
-          "id": "917f5d40-2011-4fe5-9ef6-136d6bfaea35",
+          "id": "5c559c67-c13c-4055-a318-6ba35d62a2cf",
           "url": "https://clubset.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state|upcase}&city=${city|capitalize}&fage=${age|ageRange}",
           "ageRange": [
             "18-30",
@@ -24,7 +25,7 @@
         },
         {
           "actionType": "extract",
-          "id": "06e37215-ef34-4971-bf86-e5a03dfe46e8",
+          "id": "866bdfc5-069e-4734-9ce0-a19976fa796b",
           "selector": ".card",
           "profile": {
             "name": {

diff --git a/...ages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clustrmaps.com.json b/...ages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clustrmaps.com.json
@@ -1,21 +1,22 @@
 {
-  "name": "clustrmaps.com",
-  "version": "0.1.1",
+  "name": "ClustrMaps",
+  "url": "clustrmaps.com",
+  "version": "0.1.4",
   "parent": "neighbor.report",
-  "addedDatetime": 1692590400000,
+  "addedDatetime": 1692594000000,
   "steps": [
     {
       "stepType": "scan",
       "scanType": "templatedUrl",
       "actions": [
         {
           "actionType": "navigate",
-          "id": "a39655de-5c23-477d-9887-1d34966a1069",
+          "id": "e6929e37-4764-450a-be2a-73479f11842a",
           "url": "https://clustrmaps.com/persons/${firstName}-${lastName}/${state|stateFull|capitalize}/${city|hyphenated}"
         },
         {
           "actionType": "extract",
-          "id": "4e3a628e-3634-4a2b-b632-4fbb8ce0b52b",
+          "id": "06f39df7-89c2-40da-b288-cdf3ed0e4bfd",
           "selector": ".//div[@itemprop='Person']",
           "profile": {
             "name": {
@@ -55,4 +56,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}