From 469dd5bc110d5955933660f657e1620d4d7ed4cb Mon Sep 17 00:00:00 2001 From: Fernando Bunn Date: Thu, 22 Feb 2024 18:21:01 +0000 Subject: [PATCH] Add identifier model to ExtractedProfile (#2188) Task/Issue URL: https://app.asana.com/0/1204167627774280/1205703170200803/f **Description**: Add identifier model to ExtractedProfile --- DuckDuckGo.xcodeproj/project.pbxproj | 2 +- .../xcshareddata/swiftpm/Package.resolved | 8 ++--- .../DataBrokerProtection/Package.swift | 2 +- .../Model/ExtractedProfile.swift | 31 +++++++++++++++++-- ...taBrokerProfileQueryOperationManager.swift | 6 ++-- .../JSON/advancedbackgroundchecks.com.json | 16 +++++----- .../Resources/JSON/centeda.com.json | 12 +++---- .../JSON/freepeopledirectory.com.json | 10 ++++-- .../Storage/Mappers.swift | 3 +- LocalPackages/LoginItems/Package.swift | 2 +- .../NetworkProtectionMac/Package.swift | 2 +- LocalPackages/PixelKit/Package.swift | 2 +- LocalPackages/SubscriptionUI/Package.swift | 2 +- LocalPackages/SwiftUIExtensions/Package.swift | 2 +- LocalPackages/SyncUI/Package.swift | 2 +- .../SystemExtensionManager/Package.swift | 2 +- LocalPackages/XPCHelper/Package.swift | 2 +- 17 files changed, 70 insertions(+), 36 deletions(-) diff --git a/DuckDuckGo.xcodeproj/project.pbxproj b/DuckDuckGo.xcodeproj/project.pbxproj index 0a586a3376..68632bf6e3 100644 --- a/DuckDuckGo.xcodeproj/project.pbxproj +++ b/DuckDuckGo.xcodeproj/project.pbxproj @@ -13538,7 +13538,7 @@ repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit"; requirement = { kind = exactVersion; - version = 109.0.1; + version = 109.0.2; }; }; AA06B6B52672AF8100F541C5 /* XCRemoteSwiftPackageReference "Sparkle" */ = { diff --git a/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index e0a1c42442..dbe440a602 100644 --- a/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -14,8 +14,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/duckduckgo/BrowserServicesKit", "state" : { - "revision" : "da6a822844922401d80e26963b8b11dcd6ef221a", - "version" : "109.0.1" + "revision" : "da5f8ae73e7ad7fc47931f82f5ac6c4fafa6ac94", + "version" : "109.0.2" } }, { @@ -23,8 +23,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/duckduckgo/content-scope-scripts", "state" : { - "revision" : "063b560e59a50e03d9b00b88a7fcb2ed2b562395", - "version" : "4.61.0" + "revision" : "36ddba2cbac52a41b9a9275af06d32fa8a56d2d7", + "version" : "4.64.0" } }, { diff --git a/LocalPackages/DataBrokerProtection/Package.swift b/LocalPackages/DataBrokerProtection/Package.swift index 64fa721012..e8853bdc23 100644 --- a/LocalPackages/DataBrokerProtection/Package.swift +++ b/LocalPackages/DataBrokerProtection/Package.swift @@ -29,7 +29,7 @@ let package = Package( targets: ["DataBrokerProtection"]) ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), .package(path: "../PixelKit"), .package(path: "../SwiftUIExtensions"), .package(path: "../XPCHelper") diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift index 29757bc891..1fd0366c8e 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ExtractedProfile.swift @@ -77,6 +77,7 @@ struct ExtractedProfile: Codable, Sendable { var email: String? var removedDate: Date? let fullName: String? + let identifier: String? enum CodingKeys: CodingKey { case id @@ -92,6 +93,7 @@ struct ExtractedProfile: Codable, Sendable { case email case removedDate case fullName + case identifier } init(id: Int64? = nil, @@ -105,7 +107,8 @@ struct ExtractedProfile: Codable, Sendable { reportId: String? = nil, age: String? = nil, email: String? = nil, - removedDate: Date? = nil) { + removedDate: Date? = nil, + identifier: String? = nil) { self.id = id self.name = name self.alternativeNames = alternativeNames @@ -119,6 +122,29 @@ struct ExtractedProfile: Codable, Sendable { self.email = email self.removedDate = removedDate self.fullName = name + self.identifier = identifier + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + id = try container.decodeIfPresent(Int64.self, forKey: .id) + name = try container.decodeIfPresent(String.self, forKey: .name) + alternativeNames = try container.decodeIfPresent([String].self, forKey: .alternativeNames) + addressFull = try container.decodeIfPresent(String.self, forKey: .addressFull) + addresses = try container.decodeIfPresent([AddressCityState].self, forKey: .addresses) + phoneNumbers = try container.decodeIfPresent([String].self, forKey: .phoneNumbers) + relatives = try container.decodeIfPresent([String].self, forKey: .relatives) + profileUrl = try container.decode(String.self, forKey: .profileUrl) + reportId = try container.decodeIfPresent(String.self, forKey: .reportId) + age = try container.decodeIfPresent(String.self, forKey: .age) + email = try container.decodeIfPresent(String.self, forKey: .email) + removedDate = try container.decodeIfPresent(Date.self, forKey: .removedDate) + fullName = try container.decodeIfPresent(String.self, forKey: .fullName) + if let identifier = try container.decodeIfPresent(String.self, forKey: .identifier) { + self.identifier = identifier + } else { + self.identifier = profileUrl + } } func merge(with profile: ProfileQuery) -> ExtractedProfile { @@ -134,7 +160,8 @@ struct ExtractedProfile: Codable, Sendable { reportId: self.reportId, age: self.age ?? String(profile.age), email: self.email, - removedDate: self.removedDate + removedDate: self.removedDate, + identifier: self.identifier ) } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift index 1b043a6d41..73f08604f6 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift @@ -135,10 +135,10 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager { // We check if the profile exists in the database. let extractedProfilesForBroker = database.fetchExtractedProfiles(for: brokerId) - let doesProfileExistsInDatabase = extractedProfilesForBroker.contains { $0.profileUrl == extractedProfile.profileUrl } + let doesProfileExistsInDatabase = extractedProfilesForBroker.contains { $0.identifier == extractedProfile.identifier } // If the profile exists we do not create a new opt-out operation - if doesProfileExistsInDatabase, let alreadyInDatabaseProfile = extractedProfilesForBroker.first(where: { $0.profileUrl == extractedProfile.profileUrl }), let id = alreadyInDatabaseProfile.id { + if doesProfileExistsInDatabase, let alreadyInDatabaseProfile = extractedProfilesForBroker.first(where: { $0.identifier == extractedProfile.identifier }), let id = alreadyInDatabaseProfile.id { // If it was removed in the past but was found again when scanning, it means it appearead again, so we reset the remove date. if alreadyInDatabaseProfile.removedDate != nil { database.updateRemovedDate(nil, on: id) @@ -176,7 +176,7 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager { // Check for removed profiles let removedProfiles = brokerProfileQueryData.extractedProfiles.filter { savedProfile in !extractedProfiles.contains { recentlyFoundProfile in - recentlyFoundProfile.profileUrl == savedProfile.profileUrl + recentlyFoundProfile.identifier == savedProfile.identifier } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json index ee35af6d17..d7daac5036 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json @@ -1,9 +1,9 @@ { "name": "AdvancedBackgroundChecks", "url": "advancedbackgroundchecks.com", - "version": "0.1.4", + "version": "0.1.5", "parent": "peoplefinders.com", - "addedDatetime": 1678082400000, + "addedDatetime": 1678060800000, "steps": [ { "stepType": "scan", @@ -11,12 +11,12 @@ "actions": [ { "actionType": "navigate", - "id": "c73ba931-9e01-4d37-9e15-2fd7a14eefa3", + "id": "7967f064-e3c5-442d-8380-99cf752fb8df", "url": "https://www.advancedbackgroundchecks.com/names/${firstName}-${lastName}_${city}-${state}_age_${age}" }, { "actionType": "extract", - "id": "94003082-0a9d-4418-ac88-68595c7f4953", + "id": "6f6bb616-a4cb-4231-9abb-522722208f95", "selector": ".card-block", "profile": { "name": { @@ -25,7 +25,8 @@ }, "alternativeNamesList": { "selector": "(.//p[@class='card-text max-lines-1'])[1]", - "afterText": "AKA:" + "afterText": "AKA:", + "separator": "," }, "age": { "selector": ".card-title", @@ -40,7 +41,8 @@ }, "relativesList": { "selector": "(.//p[@class='card-text max-lines-1'])[2]", - "afterText": "Related to:" + "afterText": "Related to:", + "separator": "," }, "profileUrl": { "selector": ".link-to-details" @@ -60,4 +62,4 @@ "confirmOptOutScan": 72, "maintenanceScan": 240 } -} \ No newline at end of file +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json index bb15f0093f..7050bab137 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json @@ -3,7 +3,7 @@ "url": "centeda.com", "version": "0.1.4", "parent": "verecor.com", - "addedDatetime": 1677736800000, + "addedDatetime": 1677715200000, "steps": [ { "stepType": "scan", @@ -11,7 +11,7 @@ "actions": [ { "actionType": "navigate", - "id": "2f6639c0-201f-4d5e-8467-ae0ba457b409", + "id": "af9c9f03-e778-4c29-85fc-e5cbbfec563c", "url": "https://centeda.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", "ageRange": [ "18-30", @@ -25,8 +25,8 @@ }, { "actionType": "extract", - "id": "e2e236b0-515b-43b3-9154-0432ed9b7566", - "selector": ".search-item", + "id": "79fa2a1c-65b4-417a-a8ac-2ca6d729ffc1", + "selector": ".search-result > a", "profile": { "name": { "selector": ".title", @@ -47,7 +47,7 @@ "selector": ".//div[@class='col-sm-24 col-md-8 related-to']//li" }, "profileUrl": { - "selector": ".get-report-btn" + "selector": "a" } } } @@ -64,4 +64,4 @@ "confirmOptOutScan": 72, "maintenanceScan": 240 } -} \ No newline at end of file +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/freepeopledirectory.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/freepeopledirectory.com.json index c448989448..9d114d48b3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/freepeopledirectory.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/freepeopledirectory.com.json @@ -11,17 +11,21 @@ "actions": [ { "actionType": "navigate", - "id": "4c607417-36bc-47d4-8562-9c2244db354d", + "id": "b8b912b0-201d-4cd1-8237-235c34fe0fea", "url": "https://www.freepeopledirectory.com/name/${firstName}-${lastName}/${state|upcase}/${city}" }, { "actionType": "extract", - "id": "a1637310-ca7a-40b0-b2f5-db22b43b5d54", + "id": "50e30922-ef1d-4820-abbd-f536378472d4", "selector": ".whole-card", "profile": { "name": { "selector": ".card-title" }, + "alternativeNamesList": { + "selector": ".//h3/span[contains(text(),'AKA:')]/following-sibling::span", + "afterText": "No other aliases." + }, "addressCityState": { "selector": ".city" }, @@ -50,4 +54,4 @@ "confirmOptOutScan": 72, "maintenanceScan": 240 } -} \ No newline at end of file +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift index 56b08dd5f2..81cf02f70d 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift @@ -233,7 +233,8 @@ struct MapperToModel { reportId: extractedProfile.reportId, age: extractedProfile.age, email: extractedProfile.email, - removedDate: extractedProfileDB.removedDate) + removedDate: extractedProfileDB.removedDate, + identifier: extractedProfile.identifier) } func mapToModel(_ scanEvent: ScanHistoryEventDB) throws -> HistoryEvent { diff --git a/LocalPackages/LoginItems/Package.swift b/LocalPackages/LoginItems/Package.swift index a631a199d9..b7b7bb72d9 100644 --- a/LocalPackages/LoginItems/Package.swift +++ b/LocalPackages/LoginItems/Package.swift @@ -13,7 +13,7 @@ let package = Package( ), ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), ], targets: [ .target( diff --git a/LocalPackages/NetworkProtectionMac/Package.swift b/LocalPackages/NetworkProtectionMac/Package.swift index e2dc908671..f1e2140732 100644 --- a/LocalPackages/NetworkProtectionMac/Package.swift +++ b/LocalPackages/NetworkProtectionMac/Package.swift @@ -31,7 +31,7 @@ let package = Package( .library(name: "NetworkProtectionUI", targets: ["NetworkProtectionUI"]) ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), .package(path: "../XPCHelper"), .package(path: "../SwiftUIExtensions"), .package(path: "../LoginItems") diff --git a/LocalPackages/PixelKit/Package.swift b/LocalPackages/PixelKit/Package.swift index 1222931baa..0cc6651ae2 100644 --- a/LocalPackages/PixelKit/Package.swift +++ b/LocalPackages/PixelKit/Package.swift @@ -20,7 +20,7 @@ let package = Package( ) ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), ], targets: [ .target( diff --git a/LocalPackages/SubscriptionUI/Package.swift b/LocalPackages/SubscriptionUI/Package.swift index e5192fc149..b7af9e0bbb 100644 --- a/LocalPackages/SubscriptionUI/Package.swift +++ b/LocalPackages/SubscriptionUI/Package.swift @@ -12,7 +12,7 @@ let package = Package( targets: ["SubscriptionUI"]), ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), .package(path: "../SwiftUIExtensions") ], targets: [ diff --git a/LocalPackages/SwiftUIExtensions/Package.swift b/LocalPackages/SwiftUIExtensions/Package.swift index 10d667a750..249d2a6cbb 100644 --- a/LocalPackages/SwiftUIExtensions/Package.swift +++ b/LocalPackages/SwiftUIExtensions/Package.swift @@ -11,7 +11,7 @@ let package = Package( .library(name: "PreferencesViews", targets: ["PreferencesViews"]), ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), ], targets: [ .target( diff --git a/LocalPackages/SyncUI/Package.swift b/LocalPackages/SyncUI/Package.swift index b927525124..8928521a9e 100644 --- a/LocalPackages/SyncUI/Package.swift +++ b/LocalPackages/SyncUI/Package.swift @@ -14,7 +14,7 @@ let package = Package( ], dependencies: [ .package(path: "../SwiftUIExtensions"), - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), ], targets: [ .target( diff --git a/LocalPackages/SystemExtensionManager/Package.swift b/LocalPackages/SystemExtensionManager/Package.swift index 43bafef378..0ae14903d5 100644 --- a/LocalPackages/SystemExtensionManager/Package.swift +++ b/LocalPackages/SystemExtensionManager/Package.swift @@ -16,7 +16,7 @@ let package = Package( ), ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. diff --git a/LocalPackages/XPCHelper/Package.swift b/LocalPackages/XPCHelper/Package.swift index e62141ec7a..323f247b91 100644 --- a/LocalPackages/XPCHelper/Package.swift +++ b/LocalPackages/XPCHelper/Package.swift @@ -30,7 +30,7 @@ let package = Package( .library(name: "XPCHelper", targets: ["XPCHelper"]), ], dependencies: [ - .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.1"), + .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "109.0.2"), ], targets: [ .target(