Merge branch 'main' into fcappelli/subscription_refactoring_5

Package.resolved

@@ -14,17 +14,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "4689746e42b24c40c18896d697ea02b854e90d35",
-        "version" : "5.21.0"
+        "revision" : "7ac68ae3bc052fa59adbc1ba8fd5cb5849a6bc99",
+        "version" : "5.25.0"
       }
     },
     {
       "identity" : "duckduckgo-autofill",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/duckduckgo-autofill.git",
       "state" : {
-        "revision" : "10aeff1ec7f533d1705233a9b14f9393a699b1c0",
-        "version" : "11.0.2"
+        "revision" : "2b81745565db09eee8c1cd44d38eefa1011a9f0a",
+        "version" : "12.0.1"
       }
     },
     {

Package.swift

@@ -39,12 +39,12 @@ let package = Package(
         .library(name: "PixelKitTestingUtilities", targets: ["PixelKitTestingUtilities"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "11.0.2"),
+        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "12.0.1"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.3.0"),
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "2.1.2"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "2.1.0"),
-        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "5.21.0"),
+        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "5.25.0"),
         .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "4.1.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),

Sources/BrowserServicesKit/Autofill/AutofillPixelReporter.swift

@@ -29,6 +29,7 @@ public enum AutofillPixelEvent {
     case autofillToggledOff
     case autofillLoginsStacked
     case autofillCreditCardsStacked
+    case autofillIdentitiesStacked
 
     enum Parameter {
         static let countBucket = "count_bucket"
@@ -148,6 +149,10 @@ public final class AutofillPixelReporter {
             if let cardsCount = try? vault()?.creditCardsCount() {
                 eventMapping.fire(.autofillCreditCardsStacked, parameters: [AutofillPixelEvent.Parameter.countBucket: creditCardsBucketNameFrom(count: cardsCount)])
             }
+
+            if let identitiesCount = try? vault()?.identitiesCount() {
+                eventMapping.fire(.autofillIdentitiesStacked, parameters: [AutofillPixelEvent.Parameter.countBucket: identitiesBucketNameFrom(count: identitiesCount)])
+            }
         }
 
         switch type {
@@ -247,6 +252,18 @@ public final class AutofillPixelReporter {
         }
     }
 
+    private func identitiesBucketNameFrom(count: Int) -> String {
+        if count == 0 {
+            return BucketName.none.rawValue
+        } else if count < 5 {
+            return BucketName.some.rawValue
+        } else if count < 12 {
+            return BucketName.many.rawValue
+        } else {
+            return BucketName.lots.rawValue
+        }
+    }
+
 }
 
 public extension NSNotification.Name {

Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift

@@ -47,6 +47,7 @@ public enum PrivacyFeature: String {
     case privacyPro
     case sslCertificates
     case brokenSiteReportExperiment
+    case toggleReports
 }
 
 /// An abstraction to be implemented by any "subfeature" of a given `PrivacyConfiguration` feature.
@@ -91,14 +92,6 @@ public enum SyncSubfeature: String, PrivacySubfeature {
     case level3AllowCreateAccount
 }
 
-public enum PrivacyDashboardSubfeature: String, PrivacySubfeature {
-
-    public var parent: PrivacyFeature { .privacyDashboard }
-
-    case toggleReports
-
-}
-
 public enum AutoconsentSubfeature: String, PrivacySubfeature {
     public var parent: PrivacyFeature {
         .autoconsent
@@ -126,4 +119,5 @@ public enum sslCertificatesSubfeature: String, PrivacySubfeature {
 public enum DuckPlayerSubfeature: String, PrivacySubfeature {
     public var parent: PrivacyFeature { .duckPlayer }
     case pip
+    case autoplay
 }

Sources/BrowserServicesKit/SecureVault/AutofillDatabaseProvider.swift

@@ -51,6 +51,7 @@ public protocol AutofillDatabaseProvider: SecureStorageDatabaseProvider {
     func deleteNoteForNoteId(_ noteId: Int64) throws
 
     func identities() throws -> [SecureVaultModels.Identity]
+    func identitiesCount() throws -> Int
     func identityForIdentityId(_ identityId: Int64) throws -> SecureVaultModels.Identity?
     @discardableResult
     func storeIdentity(_ identity: SecureVaultModels.Identity) throws -> Int64
@@ -503,6 +504,13 @@ public final class DefaultAutofillDatabaseProvider: GRDBSecureStorageDatabasePro
         }
     }
 
+    public func identitiesCount() throws -> Int {
+        let count = try db.read {
+            try SecureVaultModels.Identity.fetchCount($0)
+        }
+        return count
+    }
+
     public func identityForIdentityId(_ identityId: Int64) throws -> SecureVaultModels.Identity? {
         try db.read {
             return try SecureVaultModels.Identity.fetchOne($0, sql: """

Sources/BrowserServicesKit/SecureVault/AutofillSecureVault.swift

@@ -80,6 +80,7 @@ public protocol AutofillSecureVault: SecureVault {
     func deleteNoteFor(noteId: Int64) throws
 
     func identities() throws -> [SecureVaultModels.Identity]
+    func identitiesCount() throws -> Int
     func identityFor(id: Int64) throws -> SecureVaultModels.Identity?
     func existingIdentityForAutofill(matching proposedIdentity: SecureVaultModels.Identity) throws -> SecureVaultModels.Identity?
     @discardableResult
@@ -476,6 +477,12 @@ public class DefaultAutofillSecureVault<T: AutofillDatabaseProvider>: AutofillSe
         }
     }
 
+    public func identitiesCount() throws -> Int {
+        return try executeThrowingDatabaseOperation {
+            return try self.providers.database.identitiesCount()
+        }
+    }
+
     public func identityFor(id: Int64) throws -> SecureVaultModels.Identity? {
         return try executeThrowingDatabaseOperation {
             return try self.providers.database.identityForIdentityId(id)

Sources/Common/Extensions/StringExtension.swift

@@ -110,6 +110,16 @@ public extension String {
         return false
     }
 
+    var isValidIpv4Host: Bool {
+        guard let toIPv4Host, !toIPv4Host.isEmpty else { return false }
+        return true
+    }
+
+    var toIPv4Host: String? {
+        guard let ipv4 = IPv4Address(self) else { return nil }
+        return [UInt8](ipv4.rawValue).map { String($0) }.joined(separator: ".")
+    }
+
     // MARK: Regex
 
     func matches(_ regex: NSRegularExpression) -> Bool {

Sources/Common/Extensions/WKWebViewExtension.swift

@@ -0,0 +1,48 @@
+//
+//  WKWebViewExtension.swift
+//
+//  Copyright © 2023 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import WebKit
+
+public extension WKWebView {
+
+    /// Calling this method is equivalent to calling `evaluateJavaScript:inFrame:inContentWorld:completionHandler:` with:
+    /// - A `frame` value of `nil` to represent the main frame
+    /// - A `contentWorld` value of `WKContentWorld.pageWorld`
+    @MainActor func evaluateJavaScript<T>(_ script: String) async throws -> T? {
+        try await withUnsafeThrowingContinuation { c in
+            evaluateJavaScript(script) { result, error in
+                if let error {
+                    c.resume(with: .failure(error))
+                } else {
+                    c.resume(with: .success(result as? T))
+                }
+            }
+        }
+    }
+
+    // This is meant to cause the `Ambiguous use` error, because async `evaluateJavaScript(_) -> Any`
+    // call will crash when its result is `nil`.
+    // Use typed `try await evaluateJavaScript(script) as Void?` (or other type you need),
+    // or even better `try await evaluateJavaScript(script, in: nil, in: .page|.defaultClient) -> Any?` (available in macOS 12/iOS 15)
+    @available(*, deprecated, message: "Use `try await evaluateJavaScript(script) as Void?` instead.")
+    @MainActor func evaluateJavaScript(_ script: String) async throws {
+        assertionFailure("Use `try await evaluateJavaScript(script) as Void?` instead of `try await evaluateJavaScript(script)` as it will crash in runtime")
+        try await evaluateJavaScript(script) as Void?
+    }
+
+}

Sources/Navigation/DistributedNavigationDelegate.swift

@@ -881,21 +881,21 @@ extension DistributedNavigationDelegate: WKNavigationDelegate {
             }
 
         } else {
-            // don‘t mark extra Session State Pop navigations as `current` when there‘s a `current` Anchor navigation stored in `startedNavigation`
-            let isCurrent = if let startedNavigation {
-                !(startedNavigation.navigationAction.navigationType.isSameDocumentNavigation && startedNavigation.isCurrent)
-            } else {
-                true
-            }
+            let shouldBecomeCurrent = {
+                guard let startedNavigation else { return true } // no current navigation, make the same-doc navigation current
+                guard startedNavigation.navigationAction.navigationType.isSameDocumentNavigation else { return false } // don‘t intrude into current non-same-doc navigation
+                // don‘t mark extra Session State Pop navigations as `current` when there‘s a `current` same-doc Anchor navigation stored in `startedNavigation`
+                return !startedNavigation.isCurrent
+            }()
 
-            navigation = Navigation(identity: NavigationIdentity(wkNavigation), responders: responders, state: .expected(nil), isCurrent: isCurrent)
+            navigation = Navigation(identity: NavigationIdentity(wkNavigation), responders: responders, state: .expected(nil), isCurrent: shouldBecomeCurrent)
             let request = wkNavigation?.request ?? URLRequest(url: webView.url ?? .empty)
             let navigationAction = NavigationAction(request: request, navigationType: .sameDocumentNavigation(navigationType), currentHistoryItemIdentity: currentHistoryItemIdentity, redirectHistory: nil, isUserInitiated: wkNavigation?.isUserInitiated ?? false, sourceFrame: .mainFrame(for: webView), targetFrame: .mainFrame(for: webView), shouldDownload: false, mainFrameNavigation: navigation)
             navigation.navigationActionReceived(navigationAction)
-            os_log("new same-doc navigation(.%d): %s (%s): %s, isCurrent: %d", log: log, type: .debug, wkNavigationType, wkNavigation.debugDescription, navigation.debugDescription, navigationAction.debugDescription, isCurrent ? 1 : 0)
+            os_log("new same-doc navigation(.%d): %s (%s): %s, isCurrent: %d", log: log, type: .debug, wkNavigationType, wkNavigation.debugDescription, navigation.debugDescription, navigationAction.debugDescription, shouldBecomeCurrent ? 1 : 0)
 
             // store `current` navigations in `startedNavigation` to get `currentNavigation` published
-            if isCurrent {
+            if shouldBecomeCurrent {
                 self.startedNavigation = navigation
             }
             // mark Navigation as finished as we‘re in __did__SameDocumentNavigation

Sources/Navigation/NavigationResponse.swift

@@ -50,7 +50,7 @@ public struct NavigationResponse {
 public extension NavigationResponse {
 
     var url: URL {
-        response.url!
+        response.url ?? .empty
     }
 
     var httpResponse: HTTPURLResponse? {

Sources/NetworkProtection/ExtensionMessage/ExtensionRequest.swift

@@ -25,6 +25,7 @@ public enum VPNCommand: Codable {
     case sendTestNotification
     case uninstallVPN
     case disableConnectOnDemandAndShutDown
+    case quitAgent
 }
 
 public enum ExtensionRequest: Codable {

Sources/NetworkProtection/NetworkProtectionDeviceManager.swift

@@ -43,12 +43,30 @@ public enum NetworkProtectionServerSelectionMethod: CustomDebugStringConvertible
     case failureRecovery(serverName: String)
 }
 
+public enum NetworkProtectionDNSSettings: Codable, Equatable, CustomStringConvertible {
+    case `default`
+    case custom([String])
+
+    public var usesCustomDNS: Bool {
+        guard case .custom(let servers) = self, !servers.isEmpty else { return false }
+        return true
+    }
+
+    public var description: String {
+        switch self {
+        case .default: return "DuckDuckGo"
+        case .custom(let servers): return servers.joined(separator: ", ")
+        }
+    }
+}
+
 public protocol NetworkProtectionDeviceManagement {
     typealias GenerateTunnelConfigurationResult = (tunnelConfiguration: TunnelConfiguration, server: NetworkProtectionServer)
 
     func generateTunnelConfiguration(selectionMethod: NetworkProtectionServerSelectionMethod,
                                      includedRoutes: [IPAddressRange],
                                      excludedRoutes: [IPAddressRange],
+                                     dnsSettings: NetworkProtectionDNSSettings,
                                      isKillSwitchEnabled: Bool,
                                      regenerateKey: Bool) async throws -> GenerateTunnelConfigurationResult
 
@@ -118,6 +136,7 @@ public actor NetworkProtectionDeviceManager: NetworkProtectionDeviceManagement {
     public func generateTunnelConfiguration(selectionMethod: NetworkProtectionServerSelectionMethod,
                                             includedRoutes: [IPAddressRange],
                                             excludedRoutes: [IPAddressRange],
+                                            dnsSettings: NetworkProtectionDNSSettings,
                                             isKillSwitchEnabled: Bool,
                                             regenerateKey: Bool) async throws -> GenerateTunnelConfigurationResult {
         var keyPair: KeyPair
@@ -156,6 +175,7 @@ public actor NetworkProtectionDeviceManager: NetworkProtectionDeviceManagement {
                                                         server: selectedServer,
                                                         includedRoutes: includedRoutes,
                                                         excludedRoutes: excludedRoutes,
+                                                        dnsSettings: dnsSettings,
                                                         isKillSwitchEnabled: isKillSwitchEnabled)
             return (configuration, selectedServer)
         } catch let error as NetworkProtectionError {
@@ -246,6 +266,7 @@ public actor NetworkProtectionDeviceManager: NetworkProtectionDeviceManagement {
                              server: NetworkProtectionServer,
                              includedRoutes: [IPAddressRange],
                              excludedRoutes: [IPAddressRange],
+                             dnsSettings: NetworkProtectionDNSSettings,
                              isKillSwitchEnabled: Bool) throws -> TunnelConfiguration {
 
         guard let allowedIPs = server.allowedIPs else {
@@ -266,11 +287,21 @@ public actor NetworkProtectionDeviceManager: NetworkProtectionDeviceManagement {
             throw NetworkProtectionError.couldNotGetInterfaceAddressRange
         }
 
+        let dns: [DNSServer]
+        switch dnsSettings {
+        case .default:
+            dns = [DNSServer(address: server.serverInfo.internalIP)]
+        case .custom(let servers):
+            dns = servers
+                .compactMap { IPv4Address($0) }
+                .map { DNSServer(address: $0) }
+        }
+
         let interface = interfaceConfiguration(privateKey: interfacePrivateKey,
                                                addressRange: interfaceAddressRange,
                                                includedRoutes: includedRoutes,
                                                excludedRoutes: excludedRoutes,
-                                               dns: [DNSServer(address: server.serverInfo.internalIP)],
+                                               dns: dns,
                                                isKillSwitchEnabled: isKillSwitchEnabled)
 
         return TunnelConfiguration(name: "DuckDuckGo VPN", interface: interface, peers: [peerConfiguration])

Sources/NetworkProtection/NetworkProtectionOptionKey.swift

@@ -23,6 +23,7 @@ public enum NetworkProtectionOptionKey {
     public static let selectedEnvironment = "selectedEnvironment"
     public static let selectedServer = "selectedServer"
     public static let selectedLocation = "selectedLocation"
+    public static let dnsSettings = "dnsSettings"
     public static let authToken = "authToken"
     public static let isOnDemand = "is-on-demand"
     public static let activationAttemptId = "activationAttemptId"