BSK support for FB CTL update

Package.resolved

@@ -77,8 +77,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/TrackerRadarKit",
       "state" : {
-        "revision" : "6c84fd19139414fc0edbf9673ade06e532a564f0",
-        "version" : "2.0.0"
+        "revision" : "2a79006fccde6483f47f9d508b11f153f24da30d",
+        "version" : "2.1.0"
       }
     },
     {

Package.swift

@@ -40,7 +40,7 @@ let package = Package(
     dependencies: [
         .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "11.0.1"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.3.0"),
-        .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "2.0.0"),
+        .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "2.1.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "2.1.0"),
         .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "3.5.0"),

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift

@@ -317,8 +317,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         }
     }
 
-    static func extractSurrogates(from tds: TrackerData) -> TrackerData {
-
+    public static func extractSurrogates(from tds: TrackerData) -> TrackerData {
         let trackers = tds.trackers.filter { pair in
             return pair.value.rules?.first(where: { rule in
                 rule.surrogate != nil
@@ -345,7 +344,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
     }
 
     private func compilationCompleted() {
-
         var changes = [String: ContentBlockerRulesIdentifier.Difference]()
 
         lock.lock()

Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift

@@ -25,6 +25,7 @@ import Common
 public protocol SurrogatesUserScriptDelegate: NSObjectProtocol {
 
     func surrogatesUserScriptShouldProcessTrackers(_ script: SurrogatesUserScript) -> Bool
+    func surrogatesUserScriptShouldProcessCTLTrackers(_ script: SurrogatesUserScript) -> Bool
     func surrogatesUserScript(_ script: SurrogatesUserScript,
                               detectedTracker tracker: DetectedRequest,
                               withSurrogate host: String)
@@ -83,7 +84,7 @@ public class DefaultSurrogatesUserScriptConfig: SurrogatesUserScriptConfig {
     }
 }
 
-open class SurrogatesUserScript: NSObject, UserScript {
+open class SurrogatesUserScript: NSObject, UserScript, WKScriptMessageHandlerWithReply {
     struct TrackerDetectedKey {
         static let protectionId = "protectionId"
         static let blocked = "blocked"
@@ -111,25 +112,47 @@ open class SurrogatesUserScript: NSObject, UserScript {
 
     public var requiresRunInPageContentWorld: Bool = true
 
-    public var messageNames: [String] = [ "trackerDetectedMessage" ]
+    public var messageNames: [String] = [
+        "trackerDetectedMessage",
+        "isCTLEnabled"
+    ]
 
     public weak var delegate: SurrogatesUserScriptDelegate?
 
-    public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) {
+    public func userContentController(_ userContentController: WKUserContentController,
+                                      didReceive message: WKScriptMessage,
+                                      replyHandler: @escaping (Any?, String?) -> Void) {
+
         guard let delegate = delegate else { return }
-        guard delegate.surrogatesUserScriptShouldProcessTrackers(self) else { return }
 
-        guard let dict = message.body as? [String: Any] else { return }
-        guard let blocked = dict[TrackerDetectedKey.blocked] as? Bool else { return }
-        guard let urlString = dict[TrackerDetectedKey.url] as? String else { return }
-        guard let pageUrlStr = dict[TrackerDetectedKey.pageUrl] as? String else { return }
+        if message.name == "isCTLEnabled" {
+            let ctlEnabled = delegate.surrogatesUserScriptShouldProcessCTLTrackers(self)
+            replyHandler(ctlEnabled, nil)
+            return
+        } else if message.name == "trackerDetectedMessage"	{
+            guard delegate.surrogatesUserScriptShouldProcessTrackers(self) else { return }
+
+            guard let dict = message.body as? [String: Any] else { return }
+            guard let blocked = dict[TrackerDetectedKey.blocked] as? Bool else { return }
+            guard let urlString = dict[TrackerDetectedKey.url] as? String else { return }
+            guard let pageUrlStr = dict[TrackerDetectedKey.pageUrl] as? String else { return }
 
-        let tracker = trackerFromUrl(urlString.trimmingWhitespace(), pageUrlString: pageUrlStr, blocked)
+            let tracker = trackerFromUrl(urlString.trimmingWhitespace(), pageUrlString: pageUrlStr, blocked)
 
-        if let isSurrogate = dict[TrackerDetectedKey.isSurrogate] as? Bool, isSurrogate, let host = URL(string: urlString)?.host {
-            delegate.surrogatesUserScript(self, detectedTracker: tracker, withSurrogate: host)
+            if let isSurrogate = dict[TrackerDetectedKey.isSurrogate] as? Bool, isSurrogate, let host = URL(string: urlString)?.host {
+                delegate.surrogatesUserScript(self, detectedTracker: tracker, withSurrogate: host)
+            }
+            replyHandler(nil, nil)
+            return
         }
+
+        replyHandler(nil, "Unknown message")
     }
+
+    public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) {
+        assertionFailure("Should never be here!")
+    }
+
     private func trackerFromUrl(_ urlString: String, pageUrlString: String, _ blocked: Bool) -> DetectedRequest {
         let currentTrackerData = configuration.trackerData
         let knownTracker = currentTrackerData?.findTracker(forUrl: urlString)

Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift

@@ -221,7 +221,7 @@ fileprivate extension KnownTracker.Rule {
 private extension KnownTracker.ActionType {
 
     func toTrackerResolverRuleAction() -> TrackerResolver.RuleAction {
-        self == .block ? .blockRequest : .allowRequest
+        self == .block || self == .blockCtlFB ? .blockRequest : .allowRequest
     }
 
 }

Sources/BrowserServicesKit/ContentBlocking/UserScripts/surrogates.js

@@ -18,6 +18,9 @@
 //
 
 (function () {
+
+    let ctlSurrogates = ['fb-sdk.js']
+
     const duckduckgoDebugMessaging = (function () {
         let log = () => {}
         let signpostEvent = () => {}
@@ -42,6 +45,15 @@
         }
     }())
 
+    async function isCTLEnabled () {
+        try {
+            const response = await webkit.messageHandlers.isCTLEnabled.postMessage('') // message body required but ignored
+            return response
+        } catch (error) {
+            // webkit might not be defined
+        }
+    }
+
     function surrogateInjected (data) {
         try {
             webkit.messageHandlers.trackerDetectedMessage.postMessage(data)
@@ -472,9 +484,8 @@
     }
 
     // public
-    function shouldBlock (trackerUrl, type, element) {
+    async function shouldBlock (trackerUrl, type, element) {
         const startTime = performance.now()
-
         if (!blockingEnabled) {
             return false
         }
@@ -498,63 +509,71 @@
 
         const isSurrogate = !!(result.matchedRule && result.matchedRule.surrogate)
 
-        // Tracker blocking is dealt with by content rules
-        // Only handle surrogates here
-        if (blocked && isSurrogate && !isTrackerAllowlisted(topLevelUrl, trackerUrl)) {
-            // Remove error handlers on the original element
-            if (element && element.onerror) {
-                element.onerror = () => {}
-            }
-            try {
-                loadSurrogate(result.matchedRule.surrogate)
-                // Trigger a load event on the original element
-                if (element && element.onload) {
-                    element.onload(new Event('load'))
+        // set flag for CTL enable check if request is blocked and matches a CTL surrogate
+        const isCTLSurrogate = blocked && isSurrogate && ctlSurrogates.includes(result.matchedRule.surrogate)
+
+        // if a CTL surrogate, check if CTL is enabled first otherwise continue immediately
+        const promise = isCTLSurrogate ? isCTLEnabled() : Promise.resolve(true)
+
+        return promise.then ((surrogateEnabled) => {
+            // Tracker blocking is dealt with by content rules
+            // Only handle surrogates here
+            if (blocked && isSurrogate && !isTrackerAllowlisted(topLevelUrl, trackerUrl) && surrogateEnabled) {
+                // Remove error handlers on the original element
+                if (element && element.onerror) {
+                    element.onerror = () => {}
                 }
-            } catch (e) {
-                duckduckgoDebugMessaging.log(`error loading surrogate: ${e.toString()}`)
-            }
-            const pageUrl = window.location.href
-            surrogateInjected({
-                url: trackerUrl,
-                blocked: blocked,
-                reason: result.reason,
-                isSurrogate: isSurrogate,
-                pageUrl: pageUrl
-            })
+                try {
+                    loadSurrogate(result.matchedRule.surrogate)
+                    // Trigger a load event on the original element
+                    if (element && element.onload) {
+                        element.onload(new Event('load'))
+                    }
+                } catch (e) {
+                    duckduckgoDebugMessaging.log(`error loading surrogate: ${e.toString()}`)
+                }
+                const pageUrl = window.location.href
+                surrogateInjected({
+                    url: trackerUrl,
+                    blocked: blocked,
+                    reason: result.reason,
+                    isSurrogate: isSurrogate,
+                    pageUrl: pageUrl
+                })
 
-            duckduckgoDebugMessaging.signpostEvent({
-                event: 'Surrogate Injected',
-                url: trackerUrl,
-                time: performance.now() - startTime
-            })
+                duckduckgoDebugMessaging.signpostEvent({
+                    event: 'Surrogate Injected',
+                    url: trackerUrl,
+                    time: performance.now() - startTime
+                })
 
-            return true
-        }
+                return true
+            }
 
-        return false
+            return false
+        })
     }
 
-    const observer = new MutationObserver((records) => {
+    const observer = new MutationObserver(async (records) => {
         for (const record of records) {
-            record.addedNodes.forEach((node) => {
+            record.addedNodes.forEach(async (node) => {
                 if (node instanceof HTMLScriptElement) {
-                    if (shouldBlock(node.src, 'script', node)) {
+                    if (await shouldBlock(node.src, 'script', node)) {
                         duckduckgoDebugMessaging.log('blocking load')
                     }
                 }
             })
             if (record.target instanceof HTMLScriptElement) {
-                if (shouldBlock(record.target.src, 'script', record.target)) {
+                if (await shouldBlock(record.target.src, 'script', record.target)) {
                     duckduckgoDebugMessaging.log('blocking load')
                 }
             }
         }
     })
     const rootElement = document.body || document.documentElement
     observer.observe(rootElement, {
-        childList: true, 
-        subtree: true, 
+        childList: true,
+        subtree: true,
         attributeFilter: ['src']
     });
 

Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift

@@ -28,7 +28,7 @@ public enum PrivacyFeature: String {
     case gpc
     case httpsUpgrade = "https"
     case autoconsent
-    case clickToPlay
+    case clickToLoad
     case autofill
     case ampLinks
     case trackingParameters

Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesReferenceTests.swift

@@ -214,7 +214,7 @@ final class SurrogatesReferenceTests: XCTestCase {
             userScript.delegate = self.userScriptDelegateMock
 
             for messageName in userScript.messageNames {
-                configuration.userContentController.add(userScript, name: messageName)
+                configuration.userContentController.addScriptMessageHandler(userScript, contentWorld: .page, name: messageName)
             }
 
             configuration.userContentController.addUserScript(WKUserScript(source: userScript.source,

Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift

@@ -167,7 +167,7 @@ class SurrogatesUserScriptsTests: XCTestCase {
             userScript.delegate = self.userScriptDelegateMock
 
             for messageName in userScript.messageNames {
-                configuration.userContentController.add(userScript, name: messageName)
+                configuration.userContentController.addScriptMessageHandler(userScript, contentWorld: .page, name: messageName)
             }
 
             configuration.userContentController.addUserScript(WKUserScript(source: userScript.source,

Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift

@@ -73,6 +73,7 @@ final class MockRulesUserScriptDelegate: NSObject, ContentBlockerRulesUserScript
 final class MockSurrogatesUserScriptDelegate: NSObject, SurrogatesUserScriptDelegate {
 
     var shouldProcessTrackers = true
+    var shouldProcessCTLTrackers = true
 
     var onSurrogateDetected: ((DetectedRequest, String) -> Void)?
     var detectedSurrogates = Set<DetectedRequest>()
@@ -85,6 +86,10 @@ final class MockSurrogatesUserScriptDelegate: NSObject, SurrogatesUserScriptDele
         return shouldProcessTrackers
     }
 
+    func surrogatesUserScriptShouldProcessCTLTrackers(_ script: SurrogatesUserScript) -> Bool {
+        return shouldProcessCTLTrackers
+    }
+
     func surrogatesUserScript(_ script: SurrogatesUserScript,
                               detectedTracker tracker: DetectedRequest,
                               withSurrogate host: String) {