Add identifier model to ExtractedProfile (#2188)

Task/Issue URL: https://app.asana.com/0/1204167627774280/1205703170200803/f **Description**: Add identifier model to ExtractedProfile
duckduckgo · Feb 22, 2024 · 469dd5b · 469dd5b
1 parent 3dad84b
commit 469dd5b
Show file tree

Hide file tree

Showing 17 changed files with 70 additions and 36 deletions.
diff --git a/DuckDuckGo.xcodeproj/project.pbxproj b/DuckDuckGo.xcodeproj/project.pbxproj
@@ -13538,7 +13538,7 @@
 			repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit";
 			requirement = {
 				kind = exactVersion;
-				version = 109.0.1;
+				version = 109.0.2;
 			};
 		};
 		AA06B6B52672AF8100F541C5 /* XCRemoteSwiftPackageReference "Sparkle" */ = {

diff --git a/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -14,17 +14,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/BrowserServicesKit",
       "state" : {
-        "revision" : "da6a822844922401d80e26963b8b11dcd6ef221a",
-        "version" : "109.0.1"
+        "revision" : "da5f8ae73e7ad7fc47931f82f5ac6c4fafa6ac94",
+        "version" : "109.0.2"
       }
     },
     {
       "identity" : "content-scope-scripts",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "063b560e59a50e03d9b00b88a7fcb2ed2b562395",
-        "version" : "4.61.0"
+        "revision" : "36ddba2cbac52a41b9a9275af06d32fa8a56d2d7",
+        "version" : "4.64.0"
       }
     },
     {

diff --git a/LocalPackages/DataBrokerProtection/Package.swift b/LocalPackages/DataBrokerProtection/Package.swift
@@ -29,7 +29,7 @@ let package = Package(
             targets: ["DataBrokerProtection"])
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
         .package(path: "../PixelKit"),
         .package(path: "../SwiftUIExtensions"),
         .package(path: "../XPCHelper")

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift
@@ -77,6 +77,7 @@ struct ExtractedProfile: Codable, Sendable {
     var email: String?
     var removedDate: Date?
     let fullName: String?
+    let identifier: String?
 
     enum CodingKeys: CodingKey {
         case id
@@ -92,6 +93,7 @@ struct ExtractedProfile: Codable, Sendable {
         case email
         case removedDate
         case fullName
+        case identifier
     }
 
     init(id: Int64? = nil,
@@ -105,7 +107,8 @@ struct ExtractedProfile: Codable, Sendable {
          reportId: String? = nil,
          age: String? = nil,
          email: String? = nil,
-         removedDate: Date? = nil) {
+         removedDate: Date? = nil,
+         identifier: String? = nil) {
         self.id = id
         self.name = name
         self.alternativeNames = alternativeNames
@@ -119,6 +122,29 @@ struct ExtractedProfile: Codable, Sendable {
         self.email = email
         self.removedDate = removedDate
         self.fullName = name
+        self.identifier = identifier
+    }
+
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        id = try container.decodeIfPresent(Int64.self, forKey: .id)
+        name = try container.decodeIfPresent(String.self, forKey: .name)
+        alternativeNames = try container.decodeIfPresent([String].self, forKey: .alternativeNames)
+        addressFull = try container.decodeIfPresent(String.self, forKey: .addressFull)
+        addresses = try container.decodeIfPresent([AddressCityState].self, forKey: .addresses)
+        phoneNumbers = try container.decodeIfPresent([String].self, forKey: .phoneNumbers)
+        relatives = try container.decodeIfPresent([String].self, forKey: .relatives)
+        profileUrl = try container.decode(String.self, forKey: .profileUrl)
+        reportId = try container.decodeIfPresent(String.self, forKey: .reportId)
+        age = try container.decodeIfPresent(String.self, forKey: .age)
+        email = try container.decodeIfPresent(String.self, forKey: .email)
+        removedDate = try container.decodeIfPresent(Date.self, forKey: .removedDate)
+        fullName = try container.decodeIfPresent(String.self, forKey: .fullName)
+        if let identifier = try container.decodeIfPresent(String.self, forKey: .identifier) {
+            self.identifier = identifier
+        } else {
+            self.identifier = profileUrl
+        }
     }
 
     func merge(with profile: ProfileQuery) -> ExtractedProfile {
@@ -134,7 +160,8 @@ struct ExtractedProfile: Codable, Sendable {
             reportId: self.reportId,
             age: self.age ?? String(profile.age),
             email: self.email,
-            removedDate: self.removedDate
+            removedDate: self.removedDate,
+            identifier: self.identifier
         )
     }
 }

diff --git a/...tion/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/...tion/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift
@@ -135,10 +135,10 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
 
                     // We check if the profile exists in the database.
                     let extractedProfilesForBroker = database.fetchExtractedProfiles(for: brokerId)
-                    let doesProfileExistsInDatabase = extractedProfilesForBroker.contains { $0.profileUrl == extractedProfile.profileUrl }
+                    let doesProfileExistsInDatabase = extractedProfilesForBroker.contains { $0.identifier == extractedProfile.identifier }
 
                     // If the profile exists we do not create a new opt-out operation
-                    if doesProfileExistsInDatabase, let alreadyInDatabaseProfile = extractedProfilesForBroker.first(where: { $0.profileUrl == extractedProfile.profileUrl }), let id = alreadyInDatabaseProfile.id {
+                    if doesProfileExistsInDatabase, let alreadyInDatabaseProfile = extractedProfilesForBroker.first(where: { $0.identifier == extractedProfile.identifier }), let id = alreadyInDatabaseProfile.id {
                         // If it was removed in the past but was found again when scanning, it means it appearead again, so we reset the remove date.
                         if alreadyInDatabaseProfile.removedDate != nil {
                             database.updateRemovedDate(nil, on: id)
@@ -176,7 +176,7 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
             // Check for removed profiles
             let removedProfiles = brokerProfileQueryData.extractedProfiles.filter { savedProfile in
                 !extractedProfiles.contains { recentlyFoundProfile in
-                    recentlyFoundProfile.profileUrl == savedProfile.profileUrl
+                    recentlyFoundProfile.identifier == savedProfile.identifier
                 }
             }
 

diff --git a/...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json b/...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json
@@ -1,22 +1,22 @@
 {
   "name": "AdvancedBackgroundChecks",
   "url": "advancedbackgroundchecks.com",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "parent": "peoplefinders.com",
-  "addedDatetime": 1678082400000,
+  "addedDatetime": 1678060800000,
   "steps": [
     {
       "stepType": "scan",
       "scanType": "templatedUrl",
       "actions": [
         {
           "actionType": "navigate",
-          "id": "c73ba931-9e01-4d37-9e15-2fd7a14eefa3",
+          "id": "7967f064-e3c5-442d-8380-99cf752fb8df",
           "url": "https://www.advancedbackgroundchecks.com/names/${firstName}-${lastName}_${city}-${state}_age_${age}"
         },
         {
           "actionType": "extract",
-          "id": "94003082-0a9d-4418-ac88-68595c7f4953",
+          "id": "6f6bb616-a4cb-4231-9abb-522722208f95",
           "selector": ".card-block",
           "profile": {
             "name": {
@@ -25,7 +25,8 @@
             },
             "alternativeNamesList": {
               "selector": "(.//p[@class='card-text max-lines-1'])[1]",
-              "afterText": "AKA:"
+              "afterText": "AKA:",
+              "separator": ","
             },
             "age": {
               "selector": ".card-title",
@@ -40,7 +41,8 @@
             },
             "relativesList": {
               "selector": "(.//p[@class='card-text max-lines-1'])[2]",
-              "afterText": "Related to:"
+              "afterText": "Related to:",
+              "separator": ","
             },
             "profileUrl": {
               "selector": ".link-to-details"
@@ -60,4 +62,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}
diff --git a/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json b/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json
@@ -3,15 +3,15 @@
   "url": "centeda.com",
   "version": "0.1.4",
   "parent": "verecor.com",
-  "addedDatetime": 1677736800000,
+  "addedDatetime": 1677715200000,
   "steps": [
     {
       "stepType": "scan",
       "scanType": "templatedUrl",
       "actions": [
         {
           "actionType": "navigate",
-          "id": "2f6639c0-201f-4d5e-8467-ae0ba457b409",
+          "id": "af9c9f03-e778-4c29-85fc-e5cbbfec563c",
           "url": "https://centeda.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}",
           "ageRange": [
             "18-30",
@@ -25,8 +25,8 @@
         },
         {
           "actionType": "extract",
-          "id": "e2e236b0-515b-43b3-9154-0432ed9b7566",
-          "selector": ".search-item",
+          "id": "79fa2a1c-65b4-417a-a8ac-2ca6d729ffc1",
+          "selector": ".search-result > a",
           "profile": {
             "name": {
               "selector": ".title",
@@ -47,7 +47,7 @@
               "selector": ".//div[@class='col-sm-24 col-md-8 related-to']//li"
             },
             "profileUrl": {
-              "selector": ".get-report-btn"
+              "selector": "a"
             }
           }
         }
@@ -64,4 +64,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}
diff --git a/...BrokerProtection/Sources/DataBrokerProtection/Resources/JSON/freepeopledirectory.com.json b/...BrokerProtection/Sources/DataBrokerProtection/Resources/JSON/freepeopledirectory.com.json
@@ -11,17 +11,21 @@
       "actions": [
         {
           "actionType": "navigate",
-          "id": "4c607417-36bc-47d4-8562-9c2244db354d",
+          "id": "b8b912b0-201d-4cd1-8237-235c34fe0fea",
           "url": "https://www.freepeopledirectory.com/name/${firstName}-${lastName}/${state|upcase}/${city}"
         },
         {
           "actionType": "extract",
-          "id": "a1637310-ca7a-40b0-b2f5-db22b43b5d54",
+          "id": "50e30922-ef1d-4820-abbd-f536378472d4",
           "selector": ".whole-card",
           "profile": {
             "name": {
               "selector": ".card-title"
             },
+            "alternativeNamesList": {
+              "selector": ".//h3/span[contains(text(),'AKA:')]/following-sibling::span",
+              "afterText": "No other aliases."
+            },
             "addressCityState": {
               "selector": ".city"
             },
@@ -50,4 +54,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}
diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift
@@ -233,7 +233,8 @@ struct MapperToModel {
                      reportId: extractedProfile.reportId,
                      age: extractedProfile.age,
                      email: extractedProfile.email,
-                     removedDate: extractedProfileDB.removedDate)
+                     removedDate: extractedProfileDB.removedDate,
+                     identifier: extractedProfile.identifier)
     }
 
     func mapToModel(_ scanEvent: ScanHistoryEventDB) throws -> HistoryEvent {

diff --git a/LocalPackages/LoginItems/Package.swift b/LocalPackages/LoginItems/Package.swift
@@ -13,7 +13,7 @@ let package = Package(
         ),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
     ],
     targets: [
         .target(

diff --git a/LocalPackages/NetworkProtectionMac/Package.swift b/LocalPackages/NetworkProtectionMac/Package.swift
@@ -31,7 +31,7 @@ let package = Package(
         .library(name: "NetworkProtectionUI", targets: ["NetworkProtectionUI"])
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
         .package(path: "../XPCHelper"),
         .package(path: "../SwiftUIExtensions"),
         .package(path: "../LoginItems")

diff --git a/LocalPackages/PixelKit/Package.swift b/LocalPackages/PixelKit/Package.swift
@@ -20,7 +20,7 @@ let package = Package(
         )
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
     ],
     targets: [
         .target(

diff --git a/LocalPackages/SubscriptionUI/Package.swift b/LocalPackages/SubscriptionUI/Package.swift
@@ -12,7 +12,7 @@ let package = Package(
             targets: ["SubscriptionUI"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
         .package(path: "../SwiftUIExtensions")
     ],
     targets: [

diff --git a/LocalPackages/SwiftUIExtensions/Package.swift b/LocalPackages/SwiftUIExtensions/Package.swift
@@ -11,7 +11,7 @@ let package = Package(
         .library(name: "PreferencesViews", targets: ["PreferencesViews"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
     ],
     targets: [
         .target(

diff --git a/LocalPackages/SyncUI/Package.swift b/LocalPackages/SyncUI/Package.swift
@@ -14,7 +14,7 @@ let package = Package(
     ],
     dependencies: [
         .package(path: "../SwiftUIExtensions"),
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
     ],
     targets: [
         .target(

diff --git a/LocalPackages/SystemExtensionManager/Package.swift b/LocalPackages/SystemExtensionManager/Package.swift
@@ -16,7 +16,7 @@ let package = Package(
         ),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
     ],
     targets: [
         // Targets are the basic building blocks of a package, defining a module or a test suite.

diff --git a/LocalPackages/XPCHelper/Package.swift b/LocalPackages/XPCHelper/Package.swift
@@ -30,7 +30,7 @@ let package = Package(
         .library(name: "XPCHelper", targets: ["XPCHelper"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"),
     ],
     targets: [
         .target(