Merge branch 'main' into sam/remote-messaging-target-percentile

* main: Enable gzip compression for Sync PATCH payloads (#803) autofill: send current language in runtime config (#808) Password manager survey update (#811) on iOS allow bookmarks in top hits (#812) Rename tests to avoid conflicts (#813) Bat.js fix (fix for installing content blocking rules multiple times) (#779) Pixels automatic naming prefixing fixed (#810)
duckduckgo · May 10, 2024 · 0a93216 · 0a93216
2 parents ff34df3 + 72be4e7
commit 0a93216
Show file tree

Hide file tree

Showing 29 changed files with 1,033 additions and 159 deletions.
diff --git a/Package.resolved b/Package.resolved
@@ -23,8 +23,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/duckduckgo-autofill.git",
       "state" : {
-        "revision" : "6053999d6af384a716ab0ce7205dbab5d70ed1b3",
-        "version" : "11.0.1"
+        "revision" : "10aeff1ec7f533d1705233a9b14f9393a699b1c0",
+        "version" : "11.0.2"
       }
     },
     {
@@ -36,6 +36,15 @@
         "version" : "2.3.0"
       }
     },
+    {
+      "identity" : "gzipswift",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/1024jp/GzipSwift.git",
+      "state" : {
+        "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05",
+        "version" : "6.0.1"
+      }
+    },
     {
       "identity" : "privacy-dashboard",
       "kind" : "remoteSourceControl",

diff --git a/Package.swift b/Package.swift
@@ -38,7 +38,7 @@ let package = Package(
         .library(name: "PixelKitTestingUtilities", targets: ["PixelKitTestingUtilities"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "11.0.1"),
+        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "11.0.2"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.3.0"),
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "2.0.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"),
@@ -48,6 +48,7 @@ let package = Package(
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/wireguard-apple", exact: "1.1.3"),
+        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1")
     ],
     targets: [
         .target(
@@ -153,6 +154,7 @@ let package = Package(
                 "BrowserServicesKit",
                 "Common",
                 .product(name: "DDGSyncCrypto", package: "sync_crypto"),
+                .product(name: "Gzip", package: "GzipSwift"),
                 "Networking",
             ],
             resources: [

diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesIdentifier.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesIdentifier.swift
@@ -30,7 +30,7 @@ public class ContentBlockerRulesIdentifier: Equatable, Codable {
         return name + tdsEtag + tempListId + allowListId + unprotectedSitesHash
     }
 
-    public struct Difference: OptionSet {
+    public struct Difference: OptionSet, CustomDebugStringConvertible {
         public let rawValue: Int
 
         public init(rawValue: Int) {
@@ -43,6 +43,29 @@ public class ContentBlockerRulesIdentifier: Equatable, Codable {
         public static let unprotectedSites = Difference(rawValue: 1 << 3)
 
         public static let all: Difference = [.tdsEtag, .tempListId, .allowListId, .unprotectedSites]
+
+        public var debugDescription: String {
+            if self == .all {
+                return "all"
+            }
+            var result = "["
+            for i in 0...Int(log2(Double(max(self.rawValue, Self.all.rawValue)))) where self.contains(Self(rawValue: 1 << i)) {
+                if result.count > 1 {
+                    result += ", "
+                }
+                result += {
+                    switch Self(rawValue: 1 << i) {
+                    case .tdsEtag: ".tdsEtag"
+                    case .tempListId: ".tempListId"
+                    case .allowListId: ".allowListId"
+                    case .unprotectedSites: ".unprotectedSites"
+                    default: "1<<\(i)"
+                    }
+                }()
+            }
+            result += "]"
+            return result
+        }
     }
 
     private class func normalize(identifier: String?) -> String {

diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift
@@ -96,7 +96,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
     private let cache: ContentBlockerRulesCaching?
     public let exceptionsSource: ContentBlockerRulesExceptionsSource
 
-    public struct UpdateEvent {
+    public struct UpdateEvent: CustomDebugStringConvertible {
         public let rules: [ContentBlockerRulesManager.Rules]
         public let changes: [String: ContentBlockerRulesIdentifier.Difference]
         public let completionTokens: [ContentBlockerRulesManager.CompletionToken]
@@ -108,6 +108,14 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
             self.changes = changes
             self.completionTokens = completionTokens
         }
+
+        public var debugDescription: String {
+            """
+              rules: \(rules.map { "\($0.name):\($0.identifier) – \($0.rulesList) (\($0.etag))" }.joined(separator: ", "))
+              changes: \(changes)
+              completionTokens: \(completionTokens)
+            """
+        }
     }
     private let updatesSubject = PassthroughSubject<UpdateEvent, Never>()
     public var updatesPublisher: AnyPublisher<UpdateEvent, Never> {
@@ -193,6 +201,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
     @discardableResult
     public func scheduleCompilation() -> CompletionToken {
         let token = UUID().uuidString
+        os_log("Scheduling compilation with %{public}s", log: log, type: .default, token)
         workQueue.async {
             let shouldStartCompilation = self.updateCompilationState(token: token)
             if shouldStartCompilation {
@@ -228,19 +237,25 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
      Returns true if rules were found, false otherwise.
      */
     private func lookupCompiledRules() -> Bool {
+        os_log("Lookup compiled rules", log: log, type: .debug)
         prepareSourceManagers()
         let initialCompilationTask = LookupRulesTask(sourceManagers: Array(sourceManagers.values))
         let mutex = DispatchSemaphore(value: 0)
 
-        Task {
-            try? await initialCompilationTask.lookupCachedRulesLists()
+        Task { [log] in
+            do {
+                try await initialCompilationTask.lookupCachedRulesLists()
+            } catch {
+                os_log("❌ Lookup failed: %{public}s", log: log, type: .debug, error.localizedDescription)
+            }
             mutex.signal()
         }
         // We want to confine Compilation work to WorkQueue, so we wait to come back from async Task
         mutex.wait()
 
         if let result = initialCompilationTask.result {
             let rules = result.map(Rules.init(compilationResult:))
+            os_log("🟩 Found %{public}d rules", log: log, type: .debug, rules.count)
             applyRules(rules)
             return true
         }
@@ -252,6 +267,8 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
      Returns true if rules were found, false otherwise.
      */
     private func fetchLastCompiledRules(with lastCompiledRules: [LastCompiledRules]) {
+        os_log("Fetch last compiled rules: %{public}d", log: log, type: .debug, lastCompiledRules.count)
+
         let initialCompilationTask = LastCompiledRulesLookupTask(sourceRules: rulesSource.contentBlockerRulesLists,
                                                                  lastCompiledRules: lastCompiledRules)
         let mutex = DispatchSemaphore(value: 0)
@@ -294,6 +311,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
     }
 
     private func startCompilationProcess() {
+        os_log("Starting compilataion process", log: log, type: .debug)
         prepareSourceManagers()
 
         // Prepare compilation tasks based on the sources

diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockingRulesCompilationTask.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockingRulesCompilationTask.swift
@@ -53,12 +53,14 @@ extension ContentBlockerRulesManager {
         func start(ignoreCache: Bool = false, completionHandler: @escaping Completion) {
             self.workQueue.async {
                 guard let model = self.sourceManager.makeModel() else {
+                    os_log("❌ compilation impossible", log: self.log, type: .default)
                     self.compilationImpossible = true
                     completionHandler(self, false)
                     return
                 }
 
                 guard !ignoreCache else {
+                    os_log("❗️ ignoring cache", log: self.log, type: .default)
                     self.workQueue.async {
                         self.compile(model: model, completionHandler: completionHandler)
                     }
@@ -68,8 +70,10 @@ extension ContentBlockerRulesManager {
                 // Delegate querying to main thread - crashes were observed in background.
                 DispatchQueue.main.async {
                     let identifier = model.rulesIdentifier.stringValue
+                    os_log("Lookup CBR with %{public}s", log: self.log, type: .default, identifier)
                     WKContentRuleListStore.default()?.lookUpContentRuleList(forIdentifier: identifier) { ruleList, _ in
                         if let ruleList = ruleList {
+                            os_log("🟢 CBR loaded from cache: %{public}s", log: self.log, type: .default, self.rulesList.name)
                             self.compilationSucceeded(with: ruleList, model: model, completionHandler: completionHandler)
                         } else {
                             self.workQueue.async {
@@ -94,7 +98,7 @@ extension ContentBlockerRulesManager {
                                        with error: Error,
                                        completionHandler: @escaping Completion) {
             workQueue.async {
-                os_log("Failed to compile %{public}s rules %{public}s",
+                os_log("❌ Failed to compile %{public}s rules %{public}s",
                        log: self.log,
                        type: .error,
                        self.rulesList.name,
@@ -125,7 +129,7 @@ extension ContentBlockerRulesManager {
             do {
                 data = try JSONEncoder().encode(rules)
             } catch {
-                os_log("Failed to encode content blocking rules %{public}s", log: log, type: .error, rulesList.name)
+                os_log("❌ Failed to encode content blocking rules %{public}s", log: log, type: .error, rulesList.name)
                 compilationFailed(for: model, with: error, completionHandler: completionHandler)
                 return
             }
@@ -136,6 +140,7 @@ extension ContentBlockerRulesManager {
                                                                         encodedContentRuleList: ruleList) { ruleList, error in
 
                     if let ruleList = ruleList {
+                        os_log("🟢 CBR compilation for %{public}s succeeded", log: self.log, type: .default, self.rulesList.name)
                         self.compilationSucceeded(with: ruleList, model: model, completionHandler: completionHandler)
                     } else if let error = error {
                         self.compilationFailed(for: model, with: error, completionHandler: completionHandler)

diff --git a/Sources/BrowserServicesKit/ContentScopeScript/ContentScopeUserScript.swift b/Sources/BrowserServicesKit/ContentScopeScript/ContentScopeUserScript.swift
@@ -27,16 +27,29 @@ public final class ContentScopeProperties: Encodable {
     public let globalPrivacyControlValue: Bool
     public let debug: Bool = false
     public let sessionKey: String
+    public let languageCode: String
     public let platform = ContentScopePlatform()
     public let features: [String: ContentScopeFeature]
 
     public init(gpcEnabled: Bool, sessionKey: String, featureToggles: ContentScopeFeatureToggles) {
         self.globalPrivacyControlValue = gpcEnabled
         self.sessionKey = sessionKey
+        languageCode = Locale.current.languageCode ?? "en"
         features = [
             "autofill": ContentScopeFeature(featureToggles: featureToggles)
         ]
     }
+
+    enum CodingKeys: String, CodingKey {
+        // Rename 'languageCode' to 'language' to conform to autofill.js's interface.
+        case languageCode = "language"
+
+        case globalPrivacyControlValue
+        case debug
+        case sessionKey
+        case platform
+        case features
+    }
 }
 
 public struct ContentScopeFeature: Encodable {