diff --git a/Package.resolved b/Package.resolved index b184b9292..2c4bd414d 100644 --- a/Package.resolved +++ b/Package.resolved @@ -86,8 +86,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/duckduckgo/TrackerRadarKit", "state" : { - "revision" : "c01e6a59d000356b58ec77053e0a99d538be56a5", - "version" : "2.1.1" + "revision" : "1403e17eeeb8493b92fb9d11eb8c846bb9776581", + "version" : "2.1.2" } }, { diff --git a/Package.swift b/Package.swift index dc2d51bb6..e85123951 100644 --- a/Package.swift +++ b/Package.swift @@ -40,7 +40,7 @@ let package = Package( dependencies: [ .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "11.0.2"), .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.3.0"), - .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "2.1.1"), + .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "2.1.2"), .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"), .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "2.1.0"), .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "5.15.0"), diff --git a/Sources/BrowserServicesKit/ContentBlocking/ClickToLoadRulesSplitter.swift b/Sources/BrowserServicesKit/ContentBlocking/ClickToLoadRulesSplitter.swift new file mode 100644 index 000000000..98ce846fc --- /dev/null +++ b/Sources/BrowserServicesKit/ContentBlocking/ClickToLoadRulesSplitter.swift @@ -0,0 +1,149 @@ +// +// ClickToLoadRulesSplitter.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import TrackerRadarKit + +public struct ClickToLoadRulesSplitter { + + public enum Constants { + + public static let clickToLoadRuleListPrefix = "CTL_" + public static let tdsRuleListPrefix = "TDS_" + + } + + private let rulesList: ContentBlockerRulesList + + public init(rulesList: ContentBlockerRulesList) { + self.rulesList = rulesList + } + + public func split() -> (withoutBlockCTL: ContentBlockerRulesList, withBlockCTL: ContentBlockerRulesList)? { + var withoutBlockCTL: (tds: TrackerData, etag: String)? + var withBlockCTL: (tds: TrackerData, etag: String)? + + if let trackerData = rulesList.trackerData { + let splitTDS = split(trackerData: trackerData) + guard !splitTDS.withBlockCTL.tds.trackers.isEmpty else { return nil } + + withoutBlockCTL = splitTDS.withoutBlockCTL + withBlockCTL = splitTDS.withBlockCTL + } + + return ( + ContentBlockerRulesList(name: rulesList.name, + trackerData: withoutBlockCTL, + fallbackTrackerData: split(trackerData: rulesList.fallbackTrackerData).withoutBlockCTL), + ContentBlockerRulesList(name: DefaultContentBlockerRulesListsSource.Constants.clickToLoadRulesListName, + trackerData: withBlockCTL, + fallbackTrackerData: split(trackerData: rulesList.fallbackTrackerData).withBlockCTL) + ) + } + + private func split(trackerData: TrackerDataManager.DataSet) -> (withoutBlockCTL: TrackerDataManager.DataSet, withBlockCTL: TrackerDataManager.DataSet) { + let (mainTrackers, ctlTrackers) = processCTLActions(trackerData.tds.trackers) + + let trackerDataWithoutBlockCTL = makeTrackerData(using: mainTrackers, originalTDS: trackerData.tds) + let trackerDataWithBlockCTL = makeTrackerData(using: ctlTrackers, originalTDS: trackerData.tds) + + return ( + (tds: trackerDataWithoutBlockCTL, etag: Constants.tdsRuleListPrefix + trackerData.etag), + (tds: trackerDataWithBlockCTL, etag: Constants.clickToLoadRuleListPrefix + trackerData.etag) + ) + } + + private func makeTrackerData(using trackers: [String: KnownTracker], originalTDS: TrackerData) -> TrackerData { + let entities = originalTDS.extractEntities(for: trackers) + let domains = extractDomains(from: entities) + return TrackerData(trackers: trackers, + entities: entities, + domains: domains, + cnames: originalTDS.cnames) + } + + private func processCTLActions(_ trackers: [String: KnownTracker]) -> (mainTrackers: [String: KnownTracker], ctlTrackers: [String: KnownTracker]) { + var mainTDSTrackers: [String: KnownTracker] = [:] + var ctlTrackers: [String: KnownTracker] = [:] + + for (key, tracker) in trackers { + guard tracker.containsCTLActions, let rules = tracker.rules else { + mainTDSTrackers[key] = tracker + continue + } + + // if we found some CTL rules, split out into its own list + var mainRules: [KnownTracker.Rule] = [] + var ctlRules: [KnownTracker.Rule] = [] + + for rule in rules.reversed() { + if let action = rule.action, action == .blockCTLFB { + ctlRules.insert(rule, at: 0) + } else { + ctlRules.insert(rule, at: 0) + mainRules.insert(rule, at: 0) + } + } + + let mainTracker = KnownTracker(domain: tracker.domain, + defaultAction: tracker.defaultAction, + owner: tracker.owner, + prevalence: tracker.prevalence, + subdomains: tracker.subdomains, + categories: tracker.categories, + rules: mainRules) + let ctlTracker = KnownTracker(domain: tracker.domain, + defaultAction: tracker.defaultAction, + owner: tracker.owner, + prevalence: tracker.prevalence, + subdomains: tracker.subdomains, + categories: tracker.categories, + rules: ctlRules) + mainTDSTrackers[key] = mainTracker + ctlTrackers[key] = ctlTracker + } + + return (mainTDSTrackers, ctlTrackers) + } + + private func extractDomains(from entities: [String: Entity]) -> [String: String] { + var domains = [String: String]() + for (key, entity) in entities { + for domain in entity.domains ?? [] { + domains[domain] = key + } + } + return domains + } + +} + +private extension TrackerData { + + func extractEntities(for trackers: [String: KnownTracker]) -> [String: Entity] { + let trackerOwners = Set(trackers.values.compactMap { $0.owner?.name }) + let entities = entities.filter { trackerOwners.contains($0.key) } + return entities + } + +} + +private extension KnownTracker { + + var containsCTLActions: Bool { rules?.first { $0.action == .blockCTLFB } != nil } + +} diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift index 6588035c7..19fdfc621 100644 --- a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift @@ -335,8 +335,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource { } } - static func extractSurrogates(from tds: TrackerData) -> TrackerData { - + public static func extractSurrogates(from tds: TrackerData) -> TrackerData { let trackers = tds.trackers.filter { pair in return pair.value.rules?.first(where: { rule in rule.surrogate != nil @@ -363,7 +362,6 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource { } private func compilationCompleted() { - var changes = [String: ContentBlockerRulesIdentifier.Difference]() lock.lock() diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift index a981f7ee4..7d815bde8 100644 --- a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift +++ b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift @@ -69,6 +69,7 @@ open class DefaultContentBlockerRulesListsSource: ContentBlockerRulesListsSource public struct Constants { public static let trackerDataSetRulesListName = "TrackerDataSet" + public static let clickToLoadRulesListName = "ClickToLoad" } private let trackerDataManager: TrackerDataManager diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift index ba2b69cfb..ba10bf76f 100644 --- a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/SurrogatesUserScript.swift @@ -25,6 +25,7 @@ import Common public protocol SurrogatesUserScriptDelegate: NSObjectProtocol { func surrogatesUserScriptShouldProcessTrackers(_ script: SurrogatesUserScript) -> Bool + func surrogatesUserScriptShouldProcessCTLTrackers(_ script: SurrogatesUserScript) -> Bool func surrogatesUserScript(_ script: SurrogatesUserScript, detectedTracker tracker: DetectedRequest, withSurrogate host: String) @@ -83,7 +84,7 @@ public class DefaultSurrogatesUserScriptConfig: SurrogatesUserScriptConfig { } } -open class SurrogatesUserScript: NSObject, UserScript { +open class SurrogatesUserScript: NSObject, UserScript, WKScriptMessageHandlerWithReply { struct TrackerDetectedKey { static let protectionId = "protectionId" static let blocked = "blocked" @@ -111,25 +112,47 @@ open class SurrogatesUserScript: NSObject, UserScript { public var requiresRunInPageContentWorld: Bool = true - public var messageNames: [String] = [ "trackerDetectedMessage" ] + public var messageNames: [String] = [ + "trackerDetectedMessage", + "isCTLEnabled" + ] public weak var delegate: SurrogatesUserScriptDelegate? - public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) { + public func userContentController(_ userContentController: WKUserContentController, + didReceive message: WKScriptMessage, + replyHandler: @escaping (Any?, String?) -> Void) { + guard let delegate = delegate else { return } - guard delegate.surrogatesUserScriptShouldProcessTrackers(self) else { return } - guard let dict = message.body as? [String: Any] else { return } - guard let blocked = dict[TrackerDetectedKey.blocked] as? Bool else { return } - guard let urlString = dict[TrackerDetectedKey.url] as? String else { return } - guard let pageUrlStr = dict[TrackerDetectedKey.pageUrl] as? String else { return } + if message.name == "isCTLEnabled" { + let ctlEnabled = delegate.surrogatesUserScriptShouldProcessCTLTrackers(self) + replyHandler(ctlEnabled, nil) + return + } else if message.name == "trackerDetectedMessage" { + guard delegate.surrogatesUserScriptShouldProcessTrackers(self) else { return } + + guard let dict = message.body as? [String: Any] else { return } + guard let blocked = dict[TrackerDetectedKey.blocked] as? Bool else { return } + guard let urlString = dict[TrackerDetectedKey.url] as? String else { return } + guard let pageUrlStr = dict[TrackerDetectedKey.pageUrl] as? String else { return } - let tracker = trackerFromUrl(urlString.trimmingWhitespace(), pageUrlString: pageUrlStr, blocked) + let tracker = trackerFromUrl(urlString.trimmingWhitespace(), pageUrlString: pageUrlStr, blocked) - if let isSurrogate = dict[TrackerDetectedKey.isSurrogate] as? Bool, isSurrogate, let host = URL(string: urlString)?.host { - delegate.surrogatesUserScript(self, detectedTracker: tracker, withSurrogate: host) + if let isSurrogate = dict[TrackerDetectedKey.isSurrogate] as? Bool, isSurrogate, let host = URL(string: urlString)?.host { + delegate.surrogatesUserScript(self, detectedTracker: tracker, withSurrogate: host) + } + replyHandler(nil, nil) + return } + + replyHandler(nil, "Unknown message") } + + public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) { + assertionFailure("Should never be here!") + } + private func trackerFromUrl(_ urlString: String, pageUrlString: String, _ blocked: Bool) -> DetectedRequest { let currentTrackerData = configuration.trackerData let knownTracker = currentTrackerData?.findTracker(forUrl: urlString) diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift index 7e1d98d51..8e1b2baae 100644 --- a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/TrackerResolver.swift @@ -206,7 +206,7 @@ fileprivate extension KnownTracker.Rule { func action(type: String, host: String) -> TrackerResolver.RuleAction? { // If there is a rule its default action is always block var resultAction: KnownTracker.ActionType? = action ?? .block - if resultAction == .block { + if resultAction == .block || resultAction == .blockCTLFB { if let options = options, !TrackerResolver.isMatching(options, host: host, resourceType: type) { resultAction = nil } else if let exceptions = exceptions, TrackerResolver.isMatching(exceptions, host: host, resourceType: type) { @@ -221,7 +221,7 @@ fileprivate extension KnownTracker.Rule { private extension KnownTracker.ActionType { func toTrackerResolverRuleAction() -> TrackerResolver.RuleAction { - self == .block ? .blockRequest : .allowRequest + self == .block || self == .blockCTLFB ? .blockRequest : .allowRequest } } diff --git a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/surrogates.js b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/surrogates.js index b15153721..35ec6f289 100644 --- a/Sources/BrowserServicesKit/ContentBlocking/UserScripts/surrogates.js +++ b/Sources/BrowserServicesKit/ContentBlocking/UserScripts/surrogates.js @@ -18,6 +18,9 @@ // (function () { + + let ctlSurrogates = ['fb-sdk.js'] + const duckduckgoDebugMessaging = (function () { let log = () => {} let signpostEvent = () => {} @@ -42,6 +45,15 @@ } }()) + async function isCTLEnabled () { + try { + const response = await webkit.messageHandlers.isCTLEnabled.postMessage('') // message body required but ignored + return response + } catch (error) { + // webkit might not be defined + } + } + function surrogateInjected (data) { try { webkit.messageHandlers.trackerDetectedMessage.postMessage(data) @@ -472,9 +484,8 @@ } // public - function shouldBlock (trackerUrl, type, element) { + async function shouldBlock (trackerUrl, type, element) { const startTime = performance.now() - if (!blockingEnabled) { return false } @@ -498,54 +509,62 @@ const isSurrogate = !!(result.matchedRule && result.matchedRule.surrogate) - // Tracker blocking is dealt with by content rules - // Only handle surrogates here - if (blocked && isSurrogate && !isTrackerAllowlisted(topLevelUrl, trackerUrl)) { - // Remove error handlers on the original element - if (element && element.onerror) { - element.onerror = () => {} - } - try { - loadSurrogate(result.matchedRule.surrogate) - // Trigger a load event on the original element - if (element && element.onload) { - element.onload(new Event('load')) + // set flag for CTL enable check if request is blocked and matches a CTL surrogate + const isCTLSurrogate = blocked && isSurrogate && ctlSurrogates.includes(result.matchedRule.surrogate) + + // if a CTL surrogate, check if CTL is enabled first otherwise continue immediately + const promise = isCTLSurrogate ? isCTLEnabled() : Promise.resolve(true) + + return promise.then ((surrogateEnabled) => { + // Tracker blocking is dealt with by content rules + // Only handle surrogates here + if (blocked && isSurrogate && !isTrackerAllowlisted(topLevelUrl, trackerUrl) && surrogateEnabled) { + // Remove error handlers on the original element + if (element && element.onerror) { + element.onerror = () => {} } - } catch (e) { - duckduckgoDebugMessaging.log(`error loading surrogate: ${e.toString()}`) - } - const pageUrl = window.location.href - surrogateInjected({ - url: trackerUrl, - blocked: blocked, - reason: result.reason, - isSurrogate: isSurrogate, - pageUrl: pageUrl - }) + try { + loadSurrogate(result.matchedRule.surrogate) + // Trigger a load event on the original element + if (element && element.onload) { + element.onload(new Event('load')) + } + } catch (e) { + duckduckgoDebugMessaging.log(`error loading surrogate: ${e.toString()}`) + } + const pageUrl = window.location.href + surrogateInjected({ + url: trackerUrl, + blocked: blocked, + reason: result.reason, + isSurrogate: isSurrogate, + pageUrl: pageUrl + }) - duckduckgoDebugMessaging.signpostEvent({ - event: 'Surrogate Injected', - url: trackerUrl, - time: performance.now() - startTime - }) + duckduckgoDebugMessaging.signpostEvent({ + event: 'Surrogate Injected', + url: trackerUrl, + time: performance.now() - startTime + }) - return true - } + return true + } - return false + return false + }) } - const observer = new MutationObserver((records) => { + const observer = new MutationObserver(async (records) => { for (const record of records) { - record.addedNodes.forEach((node) => { + record.addedNodes.forEach(async (node) => { if (node instanceof HTMLScriptElement) { - if (shouldBlock(node.src, 'script', node)) { + if (await shouldBlock(node.src, 'script', node)) { duckduckgoDebugMessaging.log('blocking load') } } }) if (record.target instanceof HTMLScriptElement) { - if (shouldBlock(record.target.src, 'script', record.target)) { + if (await shouldBlock(record.target.src, 'script', record.target)) { duckduckgoDebugMessaging.log('blocking load') } } @@ -553,8 +572,8 @@ }) const rootElement = document.body || document.documentElement observer.observe(rootElement, { - childList: true, - subtree: true, + childList: true, + subtree: true, attributeFilter: ['src'] }); diff --git a/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift b/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift index dfa67aad4..1f3e6e290 100644 --- a/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift +++ b/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift @@ -28,7 +28,7 @@ public enum PrivacyFeature: String { case gpc case httpsUpgrade = "https" case autoconsent - case clickToPlay + case clickToLoad case autofill case ampLinks case trackingParameters diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/ClickToLoadBlockingTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/ClickToLoadBlockingTests.swift new file mode 100644 index 000000000..b8d5df569 --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/ClickToLoadBlockingTests.swift @@ -0,0 +1,373 @@ +// +// ClickToLoadBlockingTests.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import os.log +import WebKit +import BrowserServicesKit +import TrackerRadarKit +import Common + +struct CTLTests: Decodable { + + struct Test: Decodable { + + let description: String + let site: String + let request: String + let ctlProtectionsEnabled: Bool + let isRequestLoaded: Bool + + } + + static let exampleRules = """ +{ + "trackers": { + "facebook.net": { + "domain": "facebook.net", + "owner": { + "name": "Facebook, Inc.", + "displayName": "Facebook", + "privacyPolicy": "https://www.facebook.com/privacy/explanation", + "url": "https://facebook.com" + }, + "prevalence": 0.268, + "fingerprinting": 2, + "cookies": 0.208, + "categories": [], + "default": "ignore", + "rules": [ + { + "rule": "facebook\\\\.net/.*/all\\\\.js", + "surrogate": "fb-sdk.js", + "action": "block-ctl-fb", + "fingerprinting": 1, + "cookies": 0.0000408 + }, + { + "rule": "facebook\\\\.net/.*/fbevents\\\\.js", + "fingerprinting": 1, + "cookies": 0.108 + }, + { + "rule": "facebook\\\\.net/[a-z_A-Z]+/sdk\\\\.js", + "surrogate": "fb-sdk.js", + "action": "block-ctl-fb", + "fingerprinting": 1, + "cookies": 0.000334 + }, + { + "rule": "facebook\\\\.net/signals/config/", + "fingerprinting": 1, + "cookies": 0.000101 + }, + { + "rule": "facebook\\\\.net\\\\/signals\\\\/plugins\\\\/openbridge3\\\\.js", + "fingerprinting": 1, + "cookies": 0 + }, + { + "rule": "facebook\\\\.net/.*/sdk/.*customerchat\\\\.js", + "fingerprinting": 1, + "cookies": 0.00000681 + }, + { + "rule": "facebook\\\\.net\\\\/en_US\\\\/messenger\\\\.Extensions\\\\.js", + "fingerprinting": 1, + "cookies": 0 + }, + { + "rule": "facebook\\\\.net\\\\/en_US\\\\/sdk\\\\/xfbml\\\\.save\\\\.js", + "fingerprinting": 1, + "cookies": 0 + }, + { + "rule": "facebook\\\\.net/", + "action": "block-ctl-fb" + } + ] + } + }, + "entities": { + "Facebook, Inc.": { + "domains": [ + "facebook.net" + ], + "displayName": "Facebook", + "prevalence": 0.1 + } + }, + "domains": { + "facebook.net": "Facebook, Inc." + }, + "cnames": {} +} +""" + + static let domainTests: [CTLTests.Test] = [ + CTLTests.Test(description: "non-CTL tracker request, CTL enabled", + site: "https://example.com", + request: "https://www.facebook.net/signals/config/config.js", + ctlProtectionsEnabled: true, + isRequestLoaded: false), + CTLTests.Test(description: "non-CTL tracker request, CTL disabled", + site: "https://www.example.com", + request: "https://www.facebook.net/signals/config/config.js", + ctlProtectionsEnabled: false, + isRequestLoaded: false), + CTLTests.Test(description: "CTL catch-all tracker, CTL enabled", + site: "https://www.example.com", + request: "https://www.facebook.net/some.js", + ctlProtectionsEnabled: true, + isRequestLoaded: false), + CTLTests.Test(description: "CTL catch-all tracker, CTL disabled", + site: "https://www.example.com", + request: "https://www.facebook.net/some.js", + ctlProtectionsEnabled: false, + isRequestLoaded: true), + CTLTests.Test(description: "CTL SDK request, CTL enabled", + site: "https://www.example.com", + request: "https://www.facebook.net/EN/fb-sdk.js", + ctlProtectionsEnabled: true, + isRequestLoaded: false), + CTLTests.Test(description: "CTL SDK request, CTL disabled", + site: "https://www.example.com", + request: "https://www.facebook.net/EN/fb-sdk.js", + ctlProtectionsEnabled: false, + isRequestLoaded: true) + ] +} + +class ClickToLoadBlockingTests: XCTestCase { + + let schemeHandler = TestSchemeHandler() + let userScriptDelegateMock = MockRulesUserScriptDelegate() + let navigationDelegateMock = MockNavigationDelegate() + let tld = TLD() + + var webView: WKWebView! + var tds: TrackerData! + var tests = CTLTests.domainTests + var mockWebsite: MockWebsite! + + var compiledCTLRules: WKContentRuleList! + var compiledNonCTLRules: WKContentRuleList! + + func setupWebView(trackerData: TrackerData, + ctlTrackerData: TrackerData, + userScriptDelegate: ContentBlockerRulesUserScriptDelegate, + schemeHandler: TestSchemeHandler, + completion: @escaping (WKWebView) -> Void) { + + WebKitTestHelper.prepareContentBlockingRules(trackerData: trackerData, + exceptions: [], + tempUnprotected: [], + trackerExceptions: [], + identifier: "nonCTLRules") { nonCTLRules in + + guard let nonCTLRules = nonCTLRules else { + XCTFail("Rules were not compiled properly") + return + } + + self.compiledNonCTLRules = nonCTLRules + WebKitTestHelper.prepareContentBlockingRules(trackerData: ctlTrackerData, + exceptions: [], + tempUnprotected: [], + trackerExceptions: [], + identifier: "ctlRules") { ctlRules in + + guard let ctlRules = ctlRules else { + XCTFail("Rules were not compiled properly") + return + } + + self.compiledCTLRules = ctlRules + + let configuration = WKWebViewConfiguration() + configuration.setURLSchemeHandler(schemeHandler, forURLScheme: schemeHandler.scheme) + + let webView = WKWebView(frame: .init(origin: .zero, size: .init(width: 500, height: 1000)), + configuration: configuration) + webView.navigationDelegate = self.navigationDelegateMock + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + + let config = TestSchemeContentBlockerUserScriptConfig(privacyConfiguration: privacyConfig, + trackerData: trackerData, + ctlTrackerData: ctlTrackerData, + tld: self.tld) + + let userScript = ContentBlockerRulesUserScript(configuration: config) + userScript.delegate = userScriptDelegate + + for messageName in userScript.messageNames { + configuration.userContentController.add(userScript, name: messageName) + } + + configuration.userContentController.addUserScript(WKUserScript(source: userScript.source, + injectionTime: .atDocumentStart, + forMainFrameOnly: false)) + configuration.userContentController.add(nonCTLRules) + + completion(webView) + } + } + } + + func filterFBTracker(from tds: TrackerData) -> TrackerData { + + guard let fbTracker = tds.trackers["facebook.net"] else { + XCTFail("Missing FB tracker") + return TrackerData(trackers: [:], entities: [:], domains: [:], cnames: [:]) + } + + return TrackerData(trackers: ["facebook.net": fbTracker], + entities: tds.entities, + domains: tds.domains, + cnames: [:]) + } + + func testDomainAllowlist() throws { + + let fullTDS = CTLTests.exampleRules.data(using: .utf8)! + let fullTrackerData = (try? JSONDecoder().decode(TrackerData.self, from: fullTDS))! + self.tds = fullTrackerData + + let dataSet = TrackerDataManager.DataSet(tds: fullTrackerData, etag: UUID().uuidString) + let ruleList = ContentBlockerRulesList(name: "TrackerDataSet", + trackerData: nil, + fallbackTrackerData: dataSet) + let ctlSplitter = ClickToLoadRulesSplitter(rulesList: ruleList) + + guard let splitRules = ctlSplitter.split() else { + XCTFail("Could not split rules") + return + } + + tests = CTLTests.domainTests + + let testsExecuted = expectation(description: "tests executed") + testsExecuted.expectedFulfillmentCount = tests.count + + setupWebView(trackerData: splitRules.withoutBlockCTL.fallbackTrackerData.tds, + ctlTrackerData: splitRules.withBlockCTL.fallbackTrackerData.tds, + userScriptDelegate: userScriptDelegateMock, + schemeHandler: schemeHandler) { webView in + self.webView = webView + + self.popTestAndExecute(onTestExecuted: testsExecuted) + } + + waitForExpectations(timeout: 30, handler: nil) + } + + // swiftlint:disable function_body_length + private func popTestAndExecute(onTestExecuted: XCTestExpectation) { + + guard let test = tests.popLast() else { + return + } + + os_log("TEST: %s", test.description) + + var siteURL = URL(string: test.site.testSchemeNormalized)! + if siteURL.absoluteString.hasSuffix(".com") { + siteURL = siteURL.appendingPathComponent("index.html") + } + let requestURL = URL(string: test.request.testSchemeNormalized)! + + let resource = MockWebsite.EmbeddedResource(type: .script, + url: requestURL) + + mockWebsite = MockWebsite(resources: [resource]) + + schemeHandler.reset() + schemeHandler.requestHandlers[siteURL] = { _ in + return self.mockWebsite.htmlRepresentation.data(using: .utf8)! + } + + userScriptDelegateMock.reset() + + if test.ctlProtectionsEnabled { + // CTL protections enabled - adding rule list + webView.configuration.userContentController.add(self.compiledCTLRules) + userScriptDelegateMock.shouldProcessCTLTrackers = true + } else { + // CTL protections disabled - removing rule list + webView.configuration.userContentController.remove(self.compiledCTLRules) + userScriptDelegateMock.shouldProcessCTLTrackers = false + } + + os_log("Loading %s ...", siteURL.absoluteString) + let request = URLRequest(url: siteURL) + + WKWebsiteDataStore.default().removeData(ofTypes: [WKWebsiteDataTypeDiskCache, + WKWebsiteDataTypeMemoryCache, + WKWebsiteDataTypeOfflineWebApplicationCache], + modifiedSince: Date(timeIntervalSince1970: 0), + completionHandler: { + self.webView.load(request) + }) + + navigationDelegateMock.onDidFinishNavigation = { + os_log("Website loaded") + if !test.isRequestLoaded { + // Only website request + XCTAssertEqual(self.schemeHandler.handledRequests.count, 1) + // Only resource request + XCTAssertEqual(self.userScriptDelegateMock.detectedTrackers.count, 1) + + if let tracker = self.userScriptDelegateMock.detectedTrackers.first { + XCTAssert(tracker.isBlocked) + } else { + XCTFail("Expected to detect tracker for test \(test.description)") + } + } else { + // Website request & resource request + XCTAssertEqual(self.schemeHandler.handledRequests.count, 2) + + if let pageEntity = self.tds.findEntity(forHost: siteURL.host!), + let trackerOwner = self.tds.findTracker(forUrl: requestURL.absoluteString)?.owner, + pageEntity.displayName == trackerOwner.name { + + // Nothing to detect - tracker and website have the same entity + } else { + XCTAssertEqual(self.userScriptDelegateMock.detectedTrackers.count, 1) + + if let tracker = self.userScriptDelegateMock.detectedTrackers.first { + XCTAssertFalse(tracker.isBlocked) + } else { + XCTFail("Expected to detect tracker for test \(test.description)") + } + } + } + + onTestExecuted.fulfill() + DispatchQueue.main.async { + self.popTestAndExecute(onTestExecuted: onTestExecuted) + } + } + } + // swiftlint:enable function_body_length + +} diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/ClickToLoadRulesSplitterTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/ClickToLoadRulesSplitterTests.swift new file mode 100644 index 000000000..24926338c --- /dev/null +++ b/Tests/BrowserServicesKitTests/ContentBlocker/ClickToLoadRulesSplitterTests.swift @@ -0,0 +1,286 @@ +// +// ClickToLoadRulesSplitterTests.swift +// +// Copyright © 2022 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +@testable import BrowserServicesKit +@testable import TrackerRadarKit + +final class ClickToLoadRulesSplitterTests: XCTestCase { + + private let ctlTdsName = DefaultContentBlockerRulesListsSource.Constants.clickToLoadRulesListName + private let mainTdsName = DefaultContentBlockerRulesListsSource.Constants.trackerDataSetRulesListName + + func testShoulNotdSplitTrackerDataWithoutCTLActions() { + // given + let etag = UUID().uuidString + + let dataSet = buildTrackerDataSet(rawTDS: exampleNonCTLRules, etag: etag) + XCTAssertNotNil(dataSet) + + let splitRules = splitTrackerDataSet(dataSet: dataSet!) + + // then + XCTAssertNil(splitRules) + + } + + func testShouldFallbackToCTLEmbeddedIfThereAreNoTrackers() { + // given + let etag = UUID().uuidString + let dataSet = buildTrackerDataSet(rawTDS: exampleCTLRules, etag: etag) + let rulesList = ContentBlockerRulesList(name: "TrackerDataSet", trackerData: nil, fallbackTrackerData: dataSet!) + let splitter = ClickToLoadRulesSplitter(rulesList: rulesList) + + // when + let result = splitter.split() + + // then + XCTAssertNotNil(result) + + XCTAssertNotNil(result?.withBlockCTL) + XCTAssertNil(result?.withBlockCTL.trackerData) + XCTAssertNotNil(result?.withBlockCTL.fallbackTrackerData) + + XCTAssertNotNil(result?.withoutBlockCTL) + XCTAssertNil(result?.withoutBlockCTL.trackerData) + XCTAssertNotNil(result?.withoutBlockCTL.fallbackTrackerData) + + } + + func testShouldSplitTrackerDataWithCTLActions() { + // given + let etag = UUID().uuidString + + let dataSet = buildTrackerDataSet(rawTDS: exampleCTLRules, etag: etag) + XCTAssertNotNil(dataSet) + + guard let splitRules = splitTrackerDataSet(dataSet: dataSet!) else { + XCTFail("Could not split rules") + return + } + + // then + XCTAssertNotNil(splitRules) + let rulesWithBlockCTL = splitRules.withBlockCTL + let rulesWithoutBlockCTL = splitRules.withoutBlockCTL + + // withBlockCTL list + XCTAssertEqual(rulesWithBlockCTL.name, ctlTdsName) + XCTAssertEqual(rulesWithBlockCTL.trackerData!.etag, "CTL_" + etag) + XCTAssertEqual(rulesWithBlockCTL.fallbackTrackerData.etag, "CTL_" + etag) + XCTAssertEqual(rulesWithBlockCTL.trackerData!.tds.trackers.count, 1) + XCTAssertEqual(rulesWithBlockCTL.trackerData!.tds.trackers.first?.key, "facebook.net") + + // withoutBlockCTL list + XCTAssertEqual(rulesWithoutBlockCTL.name, mainTdsName) + XCTAssertEqual(rulesWithoutBlockCTL.trackerData!.etag, "TDS_" + etag) + XCTAssertEqual(rulesWithoutBlockCTL.fallbackTrackerData.etag, "TDS_" + etag) + XCTAssertEqual(rulesWithoutBlockCTL.trackerData!.tds.trackers.count, 1) + XCTAssertEqual(rulesWithoutBlockCTL.trackerData!.tds.trackers.first?.key, "facebook.net") + + let (fbMainRules, mainCTLRuleCount) = getFBTrackerRules(ruleSet: rulesWithoutBlockCTL) + let (fbCTLRules, ctlCTLRuleCount) = getFBTrackerRules(ruleSet: rulesWithBlockCTL) + + let fbMainRuleCount = fbMainRules!.count + let fbCTLRuleCount = fbCTLRules!.count + + // ensure both rulesets contains facebook.net rules + XCTAssert(fbMainRuleCount == 6) + XCTAssert(fbCTLRuleCount == 9) + + // ensure FB CTL rules include CTL custom actions, and main rules FB do not + XCTAssert(mainCTLRuleCount == 0) + XCTAssert(ctlCTLRuleCount == 3) + + // ensure FB CTL rules are the sum of the main rules + CTL custom action rules + XCTAssert(fbMainRuleCount + ctlCTLRuleCount == fbCTLRuleCount) + + } + + private func makeEntity(withName name: String, domains: [String]) -> Entity { + Entity(displayName: name, domains: domains, prevalence: 5.0) + } + + private func makeKnownTracker(withName name: String, ownerName: String) -> KnownTracker { + KnownTracker(domain: name, + defaultAction: .block, + owner: .init(name: ownerName, displayName: ownerName), + prevalence: 5.0, + subdomains: nil, + categories: nil, + rules: nil) + } + + private func getFBTrackerRules(ruleSet: ContentBlockerRulesList) -> (rules: [KnownTracker.Rule]?, countCTLActions: Int) { + let tracker = ruleSet.trackerData?.tds.trackers["facebook.net"] + return (tracker?.rules, tracker?.countCTLActions ?? 0) + } + + private func buildTrackerDataSet(rawTDS: String, etag: String) -> TrackerDataManager.DataSet? { + let fullTDS = rawTDS.data(using: .utf8)! + let fullTrackerData = (try? JSONDecoder().decode(TrackerData.self, from: fullTDS))! + return TrackerDataManager.DataSet(tds: fullTrackerData, etag) + } + + private func splitTrackerDataSet(dataSet: TrackerDataManager.DataSet) -> (withoutBlockCTL: ContentBlockerRulesList, withBlockCTL: ContentBlockerRulesList)? { + let rulesList = ContentBlockerRulesList(name: "TrackerDataSet", + trackerData: dataSet, + fallbackTrackerData: dataSet) + let ctlSplitter = ClickToLoadRulesSplitter(rulesList: rulesList) + + return ctlSplitter.split() + } + +} + +private extension KnownTracker { + + var countCTLActions: Int { rules?.filter { $0.action == .blockCTLFB }.count ?? 0 } + +} + +let exampleCTLRules = """ +{ +"trackers": { + "facebook.net": { + "domain": "facebook.net", + "owner": { + "name": "Facebook, Inc.", + "displayName": "Facebook", + "privacyPolicy": "https://www.facebook.com/privacy/explanation", + "url": "https://facebook.com" + }, + "prevalence": 0.268, + "fingerprinting": 2, + "cookies": 0.208, + "categories": [], + "default": "ignore", + "rules": [ + { + "rule": "facebook\\\\.net/.*/all\\\\.js", + "surrogate": "fb-sdk.js", + "action": "block-ctl-fb", + "fingerprinting": 1, + "cookies": 0.0000408 + }, + { + "rule": "facebook\\\\.net/.*/fbevents\\\\.js", + "fingerprinting": 1, + "cookies": 0.108 + }, + { + "rule": "facebook\\\\.net/[a-z_A-Z]+/sdk\\\\.js", + "surrogate": "fb-sdk.js", + "action": "block-ctl-fb", + "fingerprinting": 1, + "cookies": 0.000334 + }, + { + "rule": "facebook\\\\.net/signals/config/", + "fingerprinting": 1, + "cookies": 0.000101 + }, + { + "rule": "facebook\\\\.net\\\\/signals\\\\/plugins\\\\/openbridge3\\\\.js", + "fingerprinting": 1, + "cookies": 0 + }, + { + "rule": "facebook\\\\.net/.*/sdk/.*customerchat\\\\.js", + "fingerprinting": 1, + "cookies": 0.00000681 + }, + { + "rule": "facebook\\\\.net\\\\/en_US\\\\/messenger\\\\.Extensions\\\\.js", + "fingerprinting": 1, + "cookies": 0 + }, + { + "rule": "facebook\\\\.net\\\\/en_US\\\\/sdk\\\\/xfbml\\\\.save\\\\.js", + "fingerprinting": 1, + "cookies": 0 + }, + { + "rule": "facebook\\\\.net/", + "action": "block-ctl-fb" + } + ] + }, +}, +"entities": { +"Facebook, Inc.": { + "domains": [ + "facebook.net" + ], + "displayName": "Facebook", + "prevalence": 0.1 +} +}, +"domains": { +"facebook.net": "Facebook, Inc." +}, +"cnames": {} +} +""" + +let exampleNonCTLRules = """ +{ +"trackers": { +"tracker.com": { + "domain": "tracker.com", + "default": "block", + "owner": { + "name": "Fake Tracking Inc", + "displayName": "FT Inc", + "privacyPolicy": "https://tracker.com/privacy", + "url": "http://tracker.com" + }, + "source": [ + "DDG" + ], + "prevalence": 0.002, + "fingerprinting": 0, + "cookies": 0.002, + "performance": { + "time": 1, + "size": 1, + "cpu": 1, + "cache": 3 + }, + "categories": [ + "Ad Motivated Tracking", + "Advertising", + "Analytics", + "Third-Party Analytics Marketing" + ] +} +}, +"entities": { +"Fake Tracking Inc": { + "domains": [ + "tracker.com" + ], + "displayName": "Fake Tracking Inc", + "prevalence": 0.1 +} +}, +"domains": { +"tracker.com": "Fake Tracking Inc" +} +} +""" diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesReferenceTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesReferenceTests.swift index e9dae8ab7..b1d77303b 100644 --- a/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesReferenceTests.swift +++ b/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesReferenceTests.swift @@ -214,7 +214,7 @@ final class SurrogatesReferenceTests: XCTestCase { userScript.delegate = self.userScriptDelegateMock for messageName in userScript.messageNames { - configuration.userContentController.add(userScript, name: messageName) + configuration.userContentController.addScriptMessageHandler(userScript, contentWorld: .page, name: messageName) } configuration.userContentController.addUserScript(WKUserScript(source: userScript.source, diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift index 3dc3f0968..5a33b0eb8 100644 --- a/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift +++ b/Tests/BrowserServicesKitTests/ContentBlocker/SurrogatesUserScriptTests.swift @@ -60,6 +60,41 @@ class SurrogatesUserScriptsTests: XCTestCase { "Analytics", "Third-Party Analytics Marketing" ] + }, + "ctl-tracker.com": { + "domain": "ctl-tracker.com", + "default": "ignore", + "rules": [ + { + "rule": "ctl-tracker\\\\.com\\\\/scripts\\\\/ctl\\\\.js", + "surrogate": "fb-sdk.js", + "action": "block-ctl-fb" + } + ], + "owner": { + "name": "Another Tracker Inc", + "displayName": "AT Inc", + "privacyPolicy": "https://ctl-tracker.com/privacy", + "url": "http://ctl-tracker.com" + }, + "source": [ + "DDG" + ], + "prevalence": 0.002, + "fingerprinting": 0, + "cookies": 0.002, + "performance": { + "time": 1, + "size": 1, + "cpu": 1, + "cache": 3 + }, + "categories": [ + "Ad Motivated Tracking", + "Advertising", + "Analytics", + "Third-Party Analytics Marketing" + ] } }, "entities": { @@ -69,11 +104,20 @@ class SurrogatesUserScriptsTests: XCTestCase { ], "displayName": "Fake Tracking Inc", "prevalence": 0.1 + }, + "Another Tracker Inc": { + "domains": [ + "ctl-tracker.com" + ], + "displayName": "AT Inc", + "prevalence": 0.1 } }, "domains": { - "tracker.com": "Fake Tracking Inc" - } + "tracker.com": "Fake Tracking Inc", + "ctl-tracker.com": "Another Tracker Inc" + }, + "cnames": {} } """ @@ -100,6 +144,20 @@ class SurrogatesUserScriptsTests: XCTestCase { window.surrT = surrogatesScriptTest })(); + ctl-tracker.com/fb-sdk.js application/javascript + (() => { + 'use strict'; + var ctlSurrogatesScriptTest = function() { + function ping() { + return "success" + } + return { + ping: ping + } + }() + window.ctlSurrT = ctlSurrogatesScriptTest + })(); + othertracker.net/track.js application/javascript (() => { 'use strict'; @@ -117,6 +175,7 @@ class SurrogatesUserScriptsTests: XCTestCase { let nonTrackerURL = URL(string: "test://nontracker.com/1.png")! let trackerURL = URL(string: "test://tracker.com/1.png")! let surrogateScriptURL = URL(string: "test://tracker.com/scripts/script.js")! + let surrogateScriptCTLURL = URL(string: "test://ctl-tracker.com/scripts/ctl.js")! let nonSurrogateScriptURL = URL(string: "test://tracker.com/other/script.js")! var website: MockWebsite! @@ -127,6 +186,7 @@ class SurrogatesUserScriptsTests: XCTestCase { website = MockWebsite(resources: [.init(type: .image, url: nonTrackerURL), .init(type: .image, url: trackerURL), .init(type: .script, url: surrogateScriptURL), + .init(type: .script, url: surrogateScriptCTLURL), .init(type: .script, url: nonSurrogateScriptURL)]) } @@ -167,7 +227,7 @@ class SurrogatesUserScriptsTests: XCTestCase { userScript.delegate = self.userScriptDelegateMock for messageName in userScript.messageNames { - configuration.userContentController.add(userScript, name: messageName) + configuration.userContentController.addScriptMessageHandler(userScript, contentWorld: .page, name: messageName) } configuration.userContentController.addUserScript(WKUserScript(source: userScript.source, @@ -215,16 +275,14 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true + let websiteURL = URL(string: "test://example.com")! let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.first?.url, self.surrogateScriptURL.absoluteString) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { result, err in XCTAssertNil(err) @@ -232,6 +290,84 @@ class SurrogatesUserScriptsTests: XCTestCase { XCTAssertEqual(result, "success") surrogateValidated.fulfill() } + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 2) + XCTAssertTrue(self.userScriptDelegateMock.detectedSurrogates.contains(where: { $0.url == self.surrogateScriptURL.absoluteString })) + websiteLoaded.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenThereIsCTLSurrogateRuleThenSurrogateIsInjected() { + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true + + let websiteURL = URL(string: "test://example.com")! + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + + self.webView?.evaluateJavaScript("window.ctlSurrT.ping()", completionHandler: { result, err in + XCTAssertNil(err) + if let result = result as? String { + XCTAssertEqual(result, "success") + surrogateValidated.fulfill() + } + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 2) + XCTAssertTrue(self.userScriptDelegateMock.detectedSurrogates.contains(where: { $0.url == self.surrogateScriptURL.absoluteString })) + websiteLoaded.fulfill() + }) + + let expectedRequests: Set = [websiteURL, self.nonTrackerURL] + XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) + } + + performTestFor(privacyConfig: privacyConfig, websiteURL: websiteURL) + + self.wait(for: [websiteLoaded, surrogateValidated], timeout: 15) + } + + func testWhenThereIsCTLIsDisabledThenCTLSurrogateIsNotInjected() { + + let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], + tempUnprotected: [], + trackerAllowlist: [:], + contentBlockingEnabled: true, + exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = false + + let websiteURL = URL(string: "test://example.com")! + + let websiteLoaded = self.expectation(description: "Website Loaded") + let surrogateValidated = self.expectation(description: "Validated surrogate injection") + + navigationDelegateMock.onDidFinishNavigation = { + + self.webView?.evaluateJavaScript("typeof(window.ctlSurrT) == \"undefined\"", completionHandler: { result, err in + XCTAssertNil(err) + if let result = result as? Bool { + XCTAssertEqual(result, true) + surrogateValidated.fulfill() + } + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) + XCTAssertTrue(self.userScriptDelegateMock.detectedSurrogates.contains(where: { $0.url == self.surrogateScriptURL.absoluteString })) + websiteLoaded.fulfill() }) let expectedRequests: Set = [websiteURL, self.nonTrackerURL] @@ -252,21 +388,22 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) - let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL, self.surrogateScriptCTLURL] XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) } @@ -282,6 +419,7 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteURL = URL(string: "test://sub.example.com")! @@ -289,10 +427,6 @@ class SurrogatesUserScriptsTests: XCTestCase { let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.first?.url, self.surrogateScriptURL.absoluteString) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { result, err in XCTAssertNil(err) @@ -300,6 +434,11 @@ class SurrogatesUserScriptsTests: XCTestCase { XCTAssertEqual(result, "success") surrogateValidated.fulfill() } + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 2) + XCTAssertTrue(self.userScriptDelegateMock.detectedSurrogates.contains(where: { $0.url == self.surrogateScriptURL.absoluteString })) + XCTAssertTrue(self.userScriptDelegateMock.detectedSurrogates.contains(where: { $0.url == self.surrogateScriptCTLURL.absoluteString })) + websiteLoaded.fulfill() }) let expectedRequests: Set = [websiteURL, self.nonTrackerURL] @@ -320,21 +459,22 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) - let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL, self.surrogateScriptCTLURL] XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) } @@ -352,21 +492,22 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) - let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL, self.surrogateScriptCTLURL] XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) } @@ -379,28 +520,29 @@ class SurrogatesUserScriptsTests: XCTestCase { let websiteURL = URL(string: "test://example.com/index.html")! - let allowlist = ["tracker.com": [PrivacyConfigurationData.TrackerAllowlist.Entry(rule: "tracker.com/", domains: ["example.com"])]] + let allowlist = ["tracker.com": [PrivacyConfigurationData.TrackerAllowlist.Entry(rule: "tracker.com/", domains: ["example.com"])], "ctl-tracker.com": [PrivacyConfigurationData.TrackerAllowlist.Entry(rule: "ctl-tracker.com/", domains: ["example.com"])]] let privacyConfig = WebKitTestHelper.preparePrivacyConfig(locallyUnprotected: [], tempUnprotected: [], trackerAllowlist: allowlist, contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) - let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL, self.surrogateScriptCTLURL] XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) } @@ -420,18 +562,19 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: allowlist, contentBlockingEnabled: true, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 1) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 2) + websiteLoaded.fulfill() }) let expectedRequests: Set = [websiteURL, self.nonTrackerURL] @@ -452,21 +595,22 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: ["example.com"]) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) - let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL, self.surrogateScriptCTLURL] XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) } @@ -484,21 +628,22 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: true, exceptions: ["example.com"]) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) - let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL] + let expectedRequests: Set = [websiteURL, self.nonTrackerURL, self.trackerURL, self.nonSurrogateScriptURL, self.surrogateScriptURL, self.surrogateScriptCTLURL] XCTAssertEqual(Set(self.schemeHandler.handledRequests), expectedRequests) } @@ -516,18 +661,19 @@ class SurrogatesUserScriptsTests: XCTestCase { trackerAllowlist: [:], contentBlockingEnabled: false, exceptions: []) + userScriptDelegateMock.shouldProcessCTLTrackers = true let websiteLoaded = self.expectation(description: "Website Loaded") let surrogateValidated = self.expectation(description: "Validated surrogate injection") navigationDelegateMock.onDidFinishNavigation = { - websiteLoaded.fulfill() - - XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) self.webView?.evaluateJavaScript("window.surrT.ping()", completionHandler: { _, err in XCTAssertNotNil(err) surrogateValidated.fulfill() + + XCTAssertEqual(self.userScriptDelegateMock.detectedSurrogates.count, 0) + websiteLoaded.fulfill() }) // Note: do not check the requests - they will be blocked as test setup adds content blocking rules diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift index b0bb57823..df1ff550b 100644 --- a/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift +++ b/Tests/BrowserServicesKitTests/ContentBlocker/TrackerResolverTests.swift @@ -161,6 +161,18 @@ class TrackerResolverTests: XCTestCase { options: nil, exceptions: KnownTracker.Rule.Matching(domains: ["attributed.com"], types: nil)), + KnownTracker.Rule(rule: "tracker\\.com/ctl-block/.*", + surrogate: nil, + action: .blockCTLFB, + options: nil, + exceptions: KnownTracker.Rule.Matching(domains: ["other.com"], + types: nil)), + KnownTracker.Rule(rule: "tracker\\.com/ctl-surrogate/.*", + surrogate: "fb-sdk.js", + action: .blockCTLFB, + options: nil, + exceptions: KnownTracker.Rule.Matching(domains: ["other.com"], + types: nil)), KnownTracker.Rule(rule: "tracker\\.com/ignore/.*", surrogate: nil, action: .ignore, @@ -267,6 +279,46 @@ class TrackerResolverTests: XCTestCase { XCTAssertEqual(blockTrackerRuleAttributedException?.entityName, entity.displayName) XCTAssertEqual(blockTrackerRuleAttributedException?.category, tracker.category) XCTAssertEqual(blockTrackerRuleAttributedException?.prevalence, tracker.prevalence) + + let blockTrackerRuleCTLBlock = resolver.trackerFromUrl("https://tracker.com/ctl-block/s.js", + pageUrlString: "https://example.com", + resourceType: "image", + potentiallyBlocked: true) + + XCTAssertNotNil(blockTrackerRuleCTLBlock) + XCTAssert(blockTrackerRuleCTLBlock?.isBlocked ?? false) + XCTAssertEqual(blockTrackerRuleCTLBlock?.state, .blocked) + XCTAssertEqual(blockTrackerRuleCTLBlock?.ownerName, tracker.owner?.name) + XCTAssertEqual(blockTrackerRuleCTLBlock?.entityName, entity.displayName) + XCTAssertEqual(blockTrackerRuleCTLBlock?.category, tracker.category) + XCTAssertEqual(blockTrackerRuleCTLBlock?.prevalence, tracker.prevalence) + + let blockTrackerRuleCTLSurrogate = resolver.trackerFromUrl("https://tracker.com/ctl-surrogate/s.js", + pageUrlString: "https://example.com", + resourceType: "image", + potentiallyBlocked: true) + + XCTAssertNotNil(blockTrackerRuleCTLSurrogate) + XCTAssert(blockTrackerRuleCTLSurrogate?.isBlocked ?? false) + XCTAssertEqual(blockTrackerRuleCTLSurrogate?.state, .blocked) + XCTAssertEqual(blockTrackerRuleCTLSurrogate?.ownerName, tracker.owner?.name) + XCTAssertEqual(blockTrackerRuleCTLSurrogate?.entityName, entity.displayName) + XCTAssertEqual(blockTrackerRuleCTLSurrogate?.category, tracker.category) + XCTAssertEqual(blockTrackerRuleCTLSurrogate?.prevalence, tracker.prevalence) + + let ignoreTrackerRuleCTLException = resolver.trackerFromUrl("https://tracker.com/ctl-block/s.js", + pageUrlString: "https://other.com", + resourceType: "image", + potentiallyBlocked: true) + + XCTAssertNotNil(ignoreTrackerRuleCTLException) + XCTAssertFalse(ignoreTrackerRuleCTLException?.isBlocked ?? true) + XCTAssertEqual(ignoreTrackerRuleCTLException?.state, BlockingState.allowed(reason: .ruleException)) + XCTAssertEqual(ignoreTrackerRuleCTLException?.ownerName, tracker.owner?.name) + XCTAssertEqual(ignoreTrackerRuleCTLException?.entityName, entity.displayName) + XCTAssertEqual(ignoreTrackerRuleCTLException?.category, tracker.category) + XCTAssertEqual(ignoreTrackerRuleCTLException?.prevalence, tracker.prevalence) + } func testWhenTrackerWithIgnoreActionHasRulesThenTheseAreRespected() { diff --git a/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift b/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift index 0091adc92..bf47acf94 100644 --- a/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift +++ b/Tests/BrowserServicesKitTests/ContentBlocker/WebViewTestHelper.swift @@ -40,6 +40,7 @@ final class MockNavigationDelegate: NSObject, WKNavigationDelegate { final class MockRulesUserScriptDelegate: NSObject, ContentBlockerRulesUserScriptDelegate { var shouldProcessTrackers = true + var shouldProcessCTLTrackers = true var onTrackerDetected: ((DetectedRequest) -> Void)? var detectedTrackers = Set() var onThirdPartyRequestDetected: ((DetectedRequest) -> Void)? @@ -54,7 +55,7 @@ final class MockRulesUserScriptDelegate: NSObject, ContentBlockerRulesUserScript } func contentBlockerRulesUserScriptShouldProcessCTLTrackers(_ script: ContentBlockerRulesUserScript) -> Bool { - return true + return shouldProcessCTLTrackers } func contentBlockerRulesUserScript(_ script: ContentBlockerRulesUserScript, @@ -73,6 +74,7 @@ final class MockRulesUserScriptDelegate: NSObject, ContentBlockerRulesUserScript final class MockSurrogatesUserScriptDelegate: NSObject, SurrogatesUserScriptDelegate { var shouldProcessTrackers = true + var shouldProcessCTLTrackers = false var onSurrogateDetected: ((DetectedRequest, String) -> Void)? var detectedSurrogates = Set() @@ -85,6 +87,10 @@ final class MockSurrogatesUserScriptDelegate: NSObject, SurrogatesUserScriptDele return shouldProcessTrackers } + func surrogatesUserScriptShouldProcessCTLTrackers(_ script: SurrogatesUserScript) -> Bool { + shouldProcessCTLTrackers + } + func surrogatesUserScript(_ script: SurrogatesUserScript, detectedTracker tracker: DetectedRequest, withSurrogate host: String) { @@ -169,13 +175,17 @@ final class WebKitTestHelper { trackerAllowlist: [String: [PrivacyConfigurationData.TrackerAllowlist.Entry]], contentBlockingEnabled: Bool, exceptions: [String], - httpsUpgradesEnabled: Bool = false) -> PrivacyConfiguration { + httpsUpgradesEnabled: Bool = false, + clickToLoadEnabled: Bool = true) -> PrivacyConfiguration { let contentBlockingExceptions = exceptions.map { PrivacyConfigurationData.ExceptionEntry(domain: $0, reason: nil) } let contentBlockingStatus = contentBlockingEnabled ? "enabled" : "disabled" let httpsStatus = httpsUpgradesEnabled ? "enabled" : "disabled" + let clickToLoadStatus = clickToLoadEnabled ? "enabled" : "disabled" let features = [PrivacyFeature.contentBlocking.rawValue: PrivacyConfigurationData.PrivacyFeature(state: contentBlockingStatus, exceptions: contentBlockingExceptions), - PrivacyFeature.httpsUpgrade.rawValue: PrivacyConfigurationData.PrivacyFeature(state: httpsStatus, exceptions: [])] + PrivacyFeature.httpsUpgrade.rawValue: PrivacyConfigurationData.PrivacyFeature(state: httpsStatus, exceptions: []), + PrivacyFeature.clickToLoad.rawValue: PrivacyConfigurationData.PrivacyFeature(state: clickToLoadStatus, + exceptions: contentBlockingExceptions)] let unprotectedTemporary = tempUnprotected.map { PrivacyConfigurationData.ExceptionEntry(domain: $0, reason: nil) } let privacyData = PrivacyConfigurationData(features: features, unprotectedTemporary: unprotectedTemporary, @@ -195,6 +205,7 @@ final class WebKitTestHelper { exceptions: [String], tempUnprotected: [String], trackerExceptions: [TrackerException], + identifier: String = "test", completion: @escaping (WKContentRuleList?) -> Void) { let rules = ContentBlockerRulesBuilder(trackerData: trackerData).buildRules(withExceptions: exceptions, @@ -207,7 +218,7 @@ final class WebKitTestHelper { // Replace https scheme regexp with test ruleList = ruleList.replacingOccurrences(of: "https", with: "test", options: [], range: nil) - WKContentRuleListStore.default().compileContentRuleList(forIdentifier: "test", encodedContentRuleList: ruleList) { list, _ in + WKContentRuleListStore.default().compileContentRuleList(forIdentifier: identifier, encodedContentRuleList: ruleList) { list, _ in DispatchQueue.main.async { completion(list)