Merge branch 'main' into fcappelli/subscription_oauth_api_v2

# Conflicts:
#	Sources/MaliciousSiteProtection/API/APIClient.swift
#	Sources/Subscription/Flows/Models/SubscriptionOptions.swift
#	Sources/Subscription/Managers/StorePurchaseManager/StorePurchaseManager.swift
#	Sources/SubscriptionTestingUtilities/SubscriptionFeatureMappingCacheMock.swift
#	Tests/SubscriptionTests/Flows/Models/SubscriptionOptionsTests.swift

Package.resolved

@@ -14,17 +14,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "c4bb146afdf0c7a93fb9a7d95b1cb255708a470d",
-        "version" : "6.41.0"
+        "revision" : "93ea6c3e771bc0b743b38cefbff548c10add9898",
+        "version" : "6.42.0"
       }
     },
     {
       "identity" : "duckduckgo-autofill",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/duckduckgo-autofill.git",
       "state" : {
-        "revision" : "c992041d16ec10d790e6204dce9abf9966d1363c",
-        "version" : "15.1.0"
+        "revision" : "88982a3802ac504e2f1a118a73bfdf2d8f4a7735",
+        "version" : "16.0.0"
       }
     },
     {
@@ -59,8 +59,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/privacy-dashboard",
       "state" : {
-        "revision" : "49db79829dcb166b3524afdbc1c680890452ce1c",
-        "version" : "7.2.1"
+        "revision" : "022c845b06ace6a4aa712a4fa3e79da32193d5c6",
+        "version" : "7.4.0"
       }
     },
     {

Package.swift

@@ -50,13 +50,13 @@ let package = Package(
         .library(name: "PrivacyStats", targets: ["PrivacyStats"])
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "15.1.0"),
+        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "16.0.0"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.4.2"),
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.3.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "3.0.0"),
-        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.41.0"),
-        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.2.1"),
+        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.42.0"),
+        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.4.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
         .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1"),

Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift

@@ -491,6 +491,7 @@ extension AutofillUserScript {
         case userInitiated
         case autoprompt
         case formSubmission
+        case partialSave
         case passwordGeneration
         case emailProtection
     }

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerDebugEvents.swift

@@ -46,10 +46,9 @@ public enum ContentBlockerDebugEvents {
 
     case contentBlockingCompilationFailed(listName: String, component: Component)
 
-    case contentBlockingCompilationTime
-
     case contentBlockingLookupRulesSucceeded
     case contentBlockingFetchLRCSucceeded
     case contentBlockingLRCMissing
     case contentBlockingNoMatchInLRC
+    case contentBlockingCompilationTaskPerformance(iterationCount: Int, timeBucketAggregation: Double)
 }

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift

@@ -79,7 +79,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
             self.identifier = identifier
         }
 
-        internal init(compilationResult: (compiledRulesList: WKContentRuleList, model: ContentBlockerRulesSourceModel)) {
+        internal init(compilationResult: CompilationResult) {
             let surrogateTDS = ContentBlockerRulesManager.extractSurrogates(from: compilationResult.model.tds)
             let encodedData = try? JSONEncoder().encode(surrogateTDS)
             let encodedTrackerData = String(data: encodedData!, encoding: .utf8)!
@@ -130,7 +130,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
     public var sourceManagers = [String: ContentBlockerRulesSourceManager]()
 
     private var currentTasks = [CompilationTask]()
-    private var compilationStartTime: TimeInterval?
 
     private let workQueue = DispatchQueue(label: "ContentBlockerManagerQueue", qos: .userInitiated)
 
@@ -229,7 +228,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         }
 
         state = .recompiling(currentTokens: [token])
-        compilationStartTime = compilationStartTime ?? CACurrentMediaTime()
         lock.unlock()
         return true
     }
@@ -242,26 +240,18 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         Logger.contentBlocking.debug("Lookup compiled rules")
         prepareSourceManagers()
         let initialCompilationTask = LookupRulesTask(sourceManagers: Array(sourceManagers.values))
-        let mutex = DispatchSemaphore(value: 0)
 
-        Task {
-            do {
-                try await initialCompilationTask.lookupCachedRulesLists()
-            } catch {
-                Logger.contentBlocking.debug("❌ Lookup failed: \(error.localizedDescription, privacy: .public)")
-            }
-            mutex.signal()
-        }
-        // We want to confine Compilation work to WorkQueue, so we wait to come back from async Task
-        mutex.wait()
+        do {
+            let result = try initialCompilationTask.lookupCachedRulesLists()
 
-        if let result = initialCompilationTask.result {
             let rules = result.map(Rules.init(compilationResult:))
-            Logger.contentBlocking.debug("🟩 Found \(rules.count, privacy: .public) rules")
+            Logger.contentBlocking.debug("🟩 Lookup Found \(rules.count, privacy: .public) rules")
             applyRules(rules)
             return true
+        } catch {
+            Logger.contentBlocking.debug("❌ Lookup failed: \(error.localizedDescription, privacy: .public)")
+            return false
         }
-        return false
     }
 
     /*
@@ -273,18 +263,10 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
 
         let initialCompilationTask = LastCompiledRulesLookupTask(sourceRules: rulesSource.contentBlockerRulesLists,
                                                                  lastCompiledRules: lastCompiledRules)
-        let mutex = DispatchSemaphore(value: 0)
-        Task {
-            try? await initialCompilationTask.fetchCachedRulesLists()
-            mutex.signal()
-        }
-        // We want to confine Compilation work to WorkQueue, so we wait to come back from async Task
-        mutex.wait()
-
-        let rulesFound = initialCompilationTask.getFetchedRules()
+        let rules = initialCompilationTask.fetchCachedRulesLists()
 
-        if let rulesFound {
-            applyRules(rulesFound)
+        if let rules {
+            applyRules(rules)
         } else {
             lock.lock()
             state = .idle
@@ -294,7 +276,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         // No matter if rules were found or not, we need to schedule recompilation, after all
         scheduleCompilation()
 
-        return rulesFound != nil
+        return rules != nil
     }
 
     private func prepareSourceManagers() {
@@ -315,7 +297,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
     }
 
     private func startCompilationProcess() {
-        Logger.contentBlocking.debug("Starting compilataion process")
         prepareSourceManagers()
 
         // Prepare compilation tasks based on the sources
@@ -375,10 +356,11 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
                 Logger.contentBlocking.debug("Failed to complete task \(task.rulesList.name, privacy: .public)")
                 return nil
             }
+
             let rules = Rules(compilationResult: result)
 
             let diff: ContentBlockerRulesIdentifier.Difference
-            if let id = _currentRules.first(where: {$0.name == task.rulesList.name })?.identifier {
+            if let id = _currentRules.first(where: { $0.name == task.rulesList.name })?.identifier {
                 diff = id.compare(with: result.model.rulesIdentifier)
             } else {
                 diff = result.model.rulesIdentifier.compare(with: ContentBlockerRulesIdentifier(name: task.rulesList.name,
@@ -388,6 +370,15 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
                                                                                                 unprotectedSitesHash: nil))
             }
 
+            if task.rulesList.name == DefaultContentBlockerRulesListsSource.Constants.trackerDataSetRulesListName &&
+                result.resultType == .rulesCompilation {
+                if let perfInfo = result.performanceInfo {
+                    self.errorReporting?.fire(.contentBlockingCompilationTaskPerformance(iterationCount: perfInfo.iterationCount,
+                                                                                         timeBucketAggregation: perfInfo.compilationTime),
+                                              parameters: ["compilationTime": String(perfInfo.compilationTime)])
+                }
+            }
+
             changes[task.rulesList.name] = diff
             return rules
         }
@@ -404,7 +395,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         _currentRules = rules
 
         let completionTokens: [CompletionToken]
-        let compilationTime = compilationStartTime.map { start in CACurrentMediaTime() - start }
         switch state {
         case .recompilingAndScheduled(let currentTokens, let pendingTokens):
             // New work has been scheduled - prepare for execution.
@@ -414,12 +404,10 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
 
             completionTokens = currentTokens
             state = .recompiling(currentTokens: pendingTokens)
-            compilationStartTime = CACurrentMediaTime()
 
         case .recompiling(let currentTokens):
             completionTokens = currentTokens
             state = .idle
-            compilationStartTime = nil
 
         case .idle:
             assertionFailure("Unexpected state")
@@ -432,10 +420,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         updatesSubject.send(UpdateEvent(rules: rules, changes: changes, completionTokens: completionTokens))
 
         DispatchQueue.main.async {
-            if let compilationTime = compilationTime {
-                self.errorReporting?.fire(.contentBlockingCompilationTime, parameters: ["compilationTime": String(compilationTime)])
-            }
-
             self.cleanup(currentIdentifiers: currentIdentifiers)
         }
     }

Sources/BrowserServicesKit/ContentBlocking/ContentBlockingRulesCompilationTask.swift

@@ -24,17 +24,42 @@ import os.log
 
 extension ContentBlockerRulesManager {
 
+    internal struct CompilationResult {
+        let compiledRulesList: WKContentRuleList
+        let model: ContentBlockerRulesSourceModel
+        let resultType: ResultType
+        let performanceInfo: PerformanceInfo?
+
+        struct PerformanceInfo {
+            let compilationTime: TimeInterval
+            let iterationCount: Int
+
+            // default iterationCount = 1 for successful compilation on one try
+                init(compilationTime: TimeInterval, iterationCount: Int = 1) {
+                    self.compilationTime = compilationTime
+                    self.iterationCount = iterationCount
+                }
+        }
+
+        enum ResultType {
+            case cacheLookup
+            case rulesCompilation
+        }
+    }
+
     /**
      Encapsulates compilation steps for a single Task
      */
     internal class CompilationTask {
         typealias Completion = (_ task: CompilationTask, _ success: Bool) -> Void
+
         let workQueue: DispatchQueue
         let rulesList: ContentBlockerRulesList
         let sourceManager: ContentBlockerRulesSourceManager
         var isCompleted: Bool { result != nil || compilationImpossible }
         private(set) var compilationImpossible = false
-        private(set) var result: (compiledRulesList: WKContentRuleList, model: ContentBlockerRulesSourceModel)?
+        private(set) var result: CompilationResult?
+        private(set) var compilationStartTime: TimeInterval?
 
         init(workQueue: DispatchQueue,
              rulesList: ContentBlockerRulesList,
@@ -53,6 +78,8 @@ extension ContentBlockerRulesManager {
                     return
                 }
 
+                self.compilationStartTime = CACurrentMediaTime()
+
                 guard !ignoreCache else {
                     Logger.contentBlocking.log("❗️ ignoring cache")
                     self.workQueue.async {
@@ -65,10 +92,14 @@ extension ContentBlockerRulesManager {
                 DispatchQueue.main.async {
                     let identifier = model.rulesIdentifier.stringValue
                     Logger.contentBlocking.debug("Lookup CBR with \(identifier, privacy: .public)")
+
                     WKContentRuleListStore.default()?.lookUpContentRuleList(forIdentifier: identifier) { ruleList, _ in
                         if let ruleList = ruleList {
                             Logger.contentBlocking.log("🟢 CBR loaded from cache: \(self.rulesList.name, privacy: .public)")
-                            self.compilationSucceeded(with: ruleList, model: model, completionHandler: completionHandler)
+                            self.compilationSucceeded(with: ruleList,
+                                                      model: model,
+                                                      resultType: .cacheLookup,
+                                                      completionHandler: completionHandler)
                         } else {
                             self.workQueue.async {
                                 self.compile(model: model, completionHandler: completionHandler)
@@ -81,9 +112,12 @@ extension ContentBlockerRulesManager {
 
         private func compilationSucceeded(with compiledRulesList: WKContentRuleList,
                                           model: ContentBlockerRulesSourceModel,
+                                          resultType: CompilationResult.ResultType,
                                           completionHandler: @escaping Completion) {
             workQueue.async {
-                self.result = (compiledRulesList, model)
+                self.result = self.getCompilationResult(ruleList: compiledRulesList,
+                                                        model: model,
+                                                        resultType: resultType)
                 completionHandler(self, true)
             }
         }
@@ -131,7 +165,10 @@ extension ContentBlockerRulesManager {
 
                     if let ruleList = ruleList {
                         Logger.contentBlocking.log("🟢 CBR compilation for \(self.rulesList.name, privacy: .public) succeeded")
-                        self.compilationSucceeded(with: ruleList, model: model, completionHandler: completionHandler)
+                        self.compilationSucceeded(with: ruleList,
+                                                  model: model,
+                                                  resultType: .rulesCompilation,
+                                                  completionHandler: completionHandler)
                     } else if let error = error {
                         self.compilationFailed(for: model, with: error, completionHandler: completionHandler)
                     } else {
@@ -140,6 +177,41 @@ extension ContentBlockerRulesManager {
                 }
             }
         }
+
+        func getCompilationResult(ruleList: WKContentRuleList,
+                                  model: ContentBlockerRulesSourceModel,
+                                  resultType: CompilationResult.ResultType) -> CompilationResult {
+            let compilationTime = self.compilationStartTime.map { CACurrentMediaTime() - $0 }
+
+            let perfInfo = compilationTime.map {
+                CompilationResult.PerformanceInfo(compilationTime: $0,
+                                                  iterationCount: getCompilationIterationCount())
+            }
+
+            return CompilationResult(compiledRulesList: ruleList,
+                                            model: model,
+                                            resultType: resultType,
+                                     performanceInfo: perfInfo)
+
+        }
+
+        func getCompilationIterationCount() -> Int {
+            guard let brokenSources = sourceManager.brokenSources else {
+                // if none of the sources are broken, we do 1 successful iteration and do not do any retries
+                return 1
+            }
+
+            let identifiers = [
+                brokenSources.allowListIdentifier,
+                brokenSources.tempListIdentifier,
+                brokenSources.unprotectedSitesIdentifier,
+                brokenSources.tdsIdentifier
+            ]
+
+            // Add one to account for the first compilation aside from any retries
+            return (identifiers.compactMap { $0 }.count) + 1
+        }
+
     }
 
 }