Add scan and optout support for child sites (#1681)

duckduckgo · Sep 28, 2023 · d020765 · d020765
1 parent 1713fd4
commit d020765
Show file tree

Hide file tree

Showing 10 changed files with 161 additions and 7 deletions.
diff --git a/...BrokerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDatabase.swift b/...BrokerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDatabase.swift
@@ -23,6 +23,8 @@ protocol DataBrokerProtectionRepository {
     func save(_ profile: DataBrokerProtectionProfile) async
     func fetchProfile() -> DataBrokerProtectionProfile?
 
+    func fetchChildBrokers(for parentBroker: String) -> [DataBroker]
+
     func saveOptOutOperation(optOut: OptOutOperationData, extractedProfile: ExtractedProfile) throws
 
     func brokerProfileQueryData(for brokerId: Int64, and profileQueryId: Int64) -> BrokerProfileQueryData?
@@ -135,6 +137,16 @@ final class DataBrokerProtectionDatabase: DataBrokerProtectionRepository {
         }
     }
 
+    func fetchChildBrokers(for parentBroker: String) -> [DataBroker] {
+        do {
+            let vault = try DataBrokerProtectionSecureVaultFactory.makeVault(errorReporter: nil)
+            return try vault.fetchChildBrokers(for: parentBroker)
+        } catch {
+            os_log("Database error: fetchChildBrokers, error: %{public}@", log: .error, error.localizedDescription)
+            return [DataBroker]()
+        }
+    }
+
     func save(_ extractedProfile: ExtractedProfile, brokerId: Int64, profileQueryId: Int64) throws -> Int64 {
         let vault = try self.vault ?? DataBrokerProtectionSecureVaultFactory.makeVault(errorReporter: nil)
 

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift
@@ -36,6 +36,7 @@ struct DataBroker: Codable, Sendable {
     let steps: [Step]
     let version: String
     let schedulingConfig: DataBrokerScheduleConfig
+    let parent: String?
 
     var isFakeBroker: Bool {
         name.contains("fake") // A future improvement will be to add a property in the JSON file.
@@ -46,14 +47,22 @@ struct DataBroker: Codable, Sendable {
         case steps
         case version
         case schedulingConfig
+        case parent
     }
 
-    init(id: Int64? = nil, name: String, steps: [Step], version: String, schedulingConfig: DataBrokerScheduleConfig) {
+    init(id: Int64? = nil,
+         name: String,
+         steps: [Step],
+         version: String,
+         schedulingConfig: DataBrokerScheduleConfig,
+         parent: String? = nil
+    ) {
         self.id = id
         self.name = name
         self.steps = steps
         self.version = version
         self.schedulingConfig = schedulingConfig
+        self.parent = parent
     }
 
     init(from decoder: Decoder) throws {
@@ -62,6 +71,7 @@ struct DataBroker: Codable, Sendable {
         version = try container.decode(String.self, forKey: .version)
         steps = try container.decode([Step].self, forKey: .steps)
         schedulingConfig = try container.decode(DataBrokerScheduleConfig.self, forKey: .schedulingConfig)
+        parent = try? container.decode(String.self, forKey: .parent)
         id = nil
     }
 
@@ -74,14 +84,20 @@ struct DataBroker: Codable, Sendable {
         return scanStep
     }
 
-    func optOutStep() throws -> Step? {
+    func optOutStep() -> Step? {
         guard let optOutStep = steps.first(where: { $0.type == .optOut }) else {
             return nil
         }
 
         return optOutStep
     }
 
+    func performsOptOutWithinParent() -> Bool {
+        guard let optOutStep = optOutStep(), let optOutType = optOutStep.optOutType else { return false }
+
+        return optOutType == .parentSiteOptOut
+    }
+
     static func initFromResource(_ url: URL) -> DataBroker {
         // swiftlint:disable:next force_try
         let data = try! Data(contentsOf: url)

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Step.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Step.swift
@@ -23,22 +23,30 @@ enum StepType: String, Codable, Sendable {
     case optOut
 }
 
+enum OptOutType: String, Codable, Sendable {
+    case formOptOut
+    case parentSiteOptOut
+}
+
 struct Step: Codable, Sendable {
     let type: StepType
+    let optOutType: OptOutType?
     let actions: [Action]
 
     enum CodingKeys: String, CodingKey {
-        case actions, stepType
+        case actions, stepType, optOutType
     }
 
-    init(type: StepType, actions: [Action]) {
+    init(type: StepType, actions: [Action], optOutType: OptOutType? = nil) {
         self.type = type
         self.actions = actions
+        self.optOutType = optOutType
     }
 
     init(from decoder: Decoder) throws {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         type = try container.decode(StepType.self, forKey: .stepType)
+        optOutType = try? container.decode(OptOutType.self, forKey: .optOutType)
 
         let actionsList = try container.decode([[String: Any]].self, forKey: .actions)
         actions = try Step.parse(actionsList)
@@ -47,6 +55,8 @@ struct Step: Codable, Sendable {
     func encode(to encoder: Encoder) throws {
         var container = encoder.container(keyedBy: CodingKeys.self)
         try container.encode(type, forKey: .stepType)
+        try container.encode(optOutType, forKey: .optOutType)
+
         var actionsContainer = container.nestedUnkeyedContainer(forKey: .actions)
         for action in actions {
             if let navigateAction = action as? NavigateAction {

diff --git a/...tion/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/...tion/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift
@@ -136,18 +136,25 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
 
                         os_log("Extracted profile already exists in database: %@", log: .dataBrokerProtection, id.description)
                     } else {
+                        // If it's a new found profile, we'd like to opt-out ASAP
+                        // If this broker has a parent opt out, we set the preferred date to nil, as we will only perform the operation within the parent.
+                        let broker = brokerProfileQueryData.dataBroker
+                        let preferredRunOperation: Date? = broker.performsOptOutWithinParent() ? nil : Date()
+
                         // If profile does not exist we insert the new profile and we create the opt-out operation
                         //
                         // This is done inside a transaction on the database side. We insert the extracted profile and then
                         // we insert the opt-out operation, we do not want to do things separately in case creating an opt-out fails
                         // causing the extracted profile to be orphan.
                         let optOutOperationData = OptOutOperationData(brokerId: brokerId,
                                                                       profileQueryId: profileQueryId,
-                                                                      preferredRunDate: Date(), // If it's a new found profile, we'd like to opt-out ASAP
+                                                                      preferredRunDate: preferredRunOperation,
                                                                       historyEvents: [HistoryEvent](),
                                                                       extractedProfile: extractedProfile)
-                        os_log("Creating new opt-out operation data for: %@", log: .dataBrokerProtection, String(describing: extractedProfile.name))
+
                         try database.saveOptOutOperation(optOut: optOutOperationData, extractedProfile: extractedProfile)
+
+                        os_log("Creating new opt-out operation data for: %@", log: .dataBrokerProtection, String(describing: extractedProfile.name))
                     }
                 }
             } else {
@@ -264,6 +271,10 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
                                     showWebView: showWebView,
                                     shouldRunNextStep: shouldRunNextStep)
 
+            let updater = OperationPreferredDateUpdaterUseCase(database: database)
+            updater.updateChildrenBrokerForParentBroker(brokerProfileQueryData.dataBroker,
+                                                        profileQueryId: profileQueryId)
+
             database.addAttempt(extractedProfileId: extractedProfileId,
                                 attemptUUID: stageDurationCalculator.attemptId,
                                 dataBroker: stageDurationCalculator.dataBroker,

diff --git a/...kerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateUpdater.swift b/...kerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateUpdater.swift
@@ -26,6 +26,8 @@ protocol OperationPreferredDateUpdater {
                                   profileQueryId: Int64,
                                   extractedProfileId: Int64?,
                                   schedulingConfig: DataBrokerScheduleConfig) throws
+
+    func updateChildrenBrokerForParentBroker(_ parentBroker: DataBroker, profileQueryId: Int64)
 }
 
 struct OperationPreferredDateUpdaterUseCase: OperationPreferredDateUpdater {
@@ -74,6 +76,21 @@ struct OperationPreferredDateUpdaterUseCase: OperationPreferredDateUpdater {
         }
     }
 
+    /// 1, This method fetches scan operations with the profileQueryId and with child sites of parentBrokerId
+    /// 2. Then for each one it updates the preferredRunDate of the scan to its confirm scan
+    func updateChildrenBrokerForParentBroker(_ parentBroker: DataBroker, profileQueryId: Int64) {
+        let childBrokers = database.fetchChildBrokers(for: parentBroker.name)
+
+        childBrokers.forEach { childBroker in
+            if let childBrokerId = childBroker.id {
+                let confirmOptOutScanDate = Date().addingTimeInterval(childBroker.schedulingConfig.confirmOptOutScan.hoursToSeconds)
+                database.updatePreferredRunDate(confirmOptOutScanDate,
+                                                brokerId: childBrokerId,
+                                                profileQueryId: profileQueryId)
+            }
+        }
+    }
+
     private func updatePreferredRunDate( _ date: Date?,
                                          brokerId: Int64,
                                          profileQueryId: Int64,

diff --git a/...okerProtection/Sources/DataBrokerProtection/Storage/DataBrokerProtectionSecureVault.swift b/...okerProtection/Sources/DataBrokerProtection/Storage/DataBrokerProtectionSecureVault.swift
@@ -43,6 +43,7 @@ protocol DataBrokerProtectionSecureVault: SecureVault {
     func fetchBroker(with id: Int64) throws -> DataBroker?
     func fetchBroker(with name: String) throws -> DataBroker?
     func fetchAllBrokers() throws -> [DataBroker]
+    func fetchChildBrokers(for parentBroker: String) throws -> [DataBroker]
 
     func save(profileQuery: ProfileQuery, profileId: Int64) throws -> Int64
     func fetchProfileQuery(with id: Int64) throws -> ProfileQuery?
@@ -136,6 +137,13 @@ final class DefaultDataBrokerProtectionSecureVault<T: DataBrokerProtectionDataba
         return try self.providers.database.fetchAllBrokers().map(mapper.mapToModel(_:))
     }
 
+    func fetchChildBrokers(for parentBroker: String) throws -> [DataBroker] {
+        let mapper = MapperToModel(mechanism: l2Decrypt(data:))
+        let brokers = try self.providers.database.fetchAllBrokers().map(mapper.mapToModel(_:))
+
+        return brokers.filter { $0.parent == parentBroker }
+    }
+
     func save(profileQuery: ProfileQuery, profileId: Int64) throws -> Int64 {
         let mapper = MapperToDB(mechanism: l2Encrypt(data:))
         return try self.providers.database.save(mapper.mapToDB(profileQuery, relatedTo: profileId))

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Storage/Mappers.swift
@@ -170,7 +170,8 @@ struct MapperToModel {
             name: decodedBroker.name,
             steps: decodedBroker.steps,
             version: decodedBroker.version,
-            schedulingConfig: decodedBroker.schedulingConfig
+            schedulingConfig: decodedBroker.schedulingConfig,
+            parent: decodedBroker.parent
         )
     }
 

diff --git a/...tection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/...tection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift
@@ -561,6 +561,9 @@ final class MockDatabase: DataBrokerProtectionRepository {
     var lastPreferredRunDateOnOptOut: Date?
     var extractedProfileRemovedDate: Date?
     var extractedProfilesFromBroker = [ExtractedProfile]()
+    var childBrokers = [DataBroker]()
+    var lastParentBrokerWhereChildSitesWhereFetched: String?
+    var lastProfileQueryIdOnScanUpdatePreferredRunDate: Int64?
 
     lazy var callsList: [Bool] = [
         wasSaveProfileCalled,
@@ -612,6 +615,7 @@ final class MockDatabase: DataBrokerProtectionRepository {
 
     func updatePreferredRunDate(_ date: Date?, brokerId: Int64, profileQueryId: Int64) {
         lastPreferredRunDateOnScan = date
+        lastProfileQueryIdOnScanUpdatePreferredRunDate = profileQueryId
         wasUpdatedPreferredRunDateForScanCalled = true
     }
 
@@ -659,6 +663,11 @@ final class MockDatabase: DataBrokerProtectionRepository {
     func addAttempt(extractedProfileId: Int64, attemptUUID: UUID, dataBroker: String, lastStageDate: Date, startTime: Date) {
     }
 
+    func fetchChildBrokers(for parentBroker: String) -> [DataBroker] {
+        lastParentBrokerWhereChildSitesWhereFetched = parentBroker
+        return childBrokers
+    }
+
     func clear() {
         wasSaveProfileCalled = false
         wasFetchProfileCalled = false
@@ -678,6 +687,9 @@ final class MockDatabase: DataBrokerProtectionRepository {
         lastPreferredRunDateOnOptOut = nil
         extractedProfileRemovedDate = nil
         extractedProfilesFromBroker.removeAll()
+        childBrokers.removeAll()
+        lastParentBrokerWhereChildSitesWhereFetched = nil
+        lastProfileQueryIdOnScanUpdatePreferredRunDate = nil
     }
 }
 

diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift
@@ -546,6 +546,10 @@ final class DataBrokerProtectionSecureVaultMock: DataBrokerProtectionSecureVault
     func hasMatches() throws -> Bool {
         false
     }
+
+    func fetchChildBrokers(for parentBroker: String) throws -> [DataBroker] {
+        return [DataBroker]()
+    }
 }
 
 public class MockDataBrokerProtectionPixelsHandler: EventMapping<DataBrokerProtectionPixels> {

diff --git a/...BrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateUpdaterTests.swift b/...BrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateUpdaterTests.swift
@@ -0,0 +1,63 @@
+//
+//  OperationPreferredDateUpdaterTests.swift
+//
+//  Copyright © 2023 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import XCTest
+@testable import DataBrokerProtection
+
+final class OperationPreferredDateUpdaterTests: XCTestCase {
+
+    private let databaseMock = MockDatabase()
+
+    override func tearDown() {
+        databaseMock.clear()
+    }
+
+    func testWhenParentBrokerHasChildSites_thenThoseSitesScanPreferredRunDateIsUpdatedWithConfirm() {
+        let sut = OperationPreferredDateUpdaterUseCase(database: databaseMock)
+        let confirmOptOutScanHours = 48
+        let profileQueryId: Int64 = 11
+        let expectedDate = Date().addingTimeInterval(confirmOptOutScanHours.hoursToSeconds)
+        let childBroker = DataBroker(
+            id: 1,
+            name: "Child broker",
+            steps: [Step](),
+            version: "1.0",
+            schedulingConfig: DataBrokerScheduleConfig(
+                retryError: 1,
+                confirmOptOutScan: confirmOptOutScanHours,
+                maintenanceScan: 1
+            )
+        )
+        databaseMock.childBrokers = [childBroker]
+
+        sut.updateChildrenBrokerForParentBroker(.mock, profileQueryId: profileQueryId)
+
+        XCTAssertTrue(databaseMock.wasUpdatedPreferredRunDateForScanCalled)
+        XCTAssertEqual(databaseMock.lastParentBrokerWhereChildSitesWhereFetched, "Test broker")
+        XCTAssertEqual(databaseMock.lastProfileQueryIdOnScanUpdatePreferredRunDate, profileQueryId)
+        XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedDate, date2: databaseMock.lastPreferredRunDateOnScan))
+    }
+
+    func testWhenParentBrokerHasNoChildsites_thenNoCallsToTheDatabaseAreDone() {
+        let sut = OperationPreferredDateUpdaterUseCase(database: databaseMock)
+
+        sut.updateChildrenBrokerForParentBroker(.mock, profileQueryId: 1)
+
+        XCTAssertFalse(databaseMock.wasDatabaseCalled)
+    }
+}