Malware protection 3: Refactor Data storing

Package.resolved

@@ -63,6 +63,24 @@
         "version" : "3.0.0"
       }
     },
+    {
+      "identity" : "swift-clocks",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-clocks.git",
+      "state" : {
+        "revision" : "b9b24b69e2adda099a1fa381cda1eeec272d5b53",
+        "version" : "1.0.5"
+      }
+    },
+    {
+      "identity" : "swift-concurrency-extras",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-concurrency-extras",
+      "state" : {
+        "revision" : "163409ef7dae9d960b87f34b51587b6609a76c1f",
+        "version" : "1.3.0"
+      }
+    },
     {
       "identity" : "swifter",
       "kind" : "remoteSourceControl",
@@ -89,6 +107,15 @@
         "revision" : "5de0a610a7927b638a5fd463a53032c9934a2c3b",
         "version" : "3.0.0"
       }
+    },
+    {
+      "identity" : "xctest-dynamic-overlay",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/xctest-dynamic-overlay",
+      "state" : {
+        "revision" : "a3f634d1a409c7979cabc0a71b3f26ffa9fc8af1",
+        "version" : "1.4.3"
+      }
     }
   ],
   "version" : 2

Package.swift

@@ -57,7 +57,8 @@ let package = Package(
         .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.2.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
-        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1")
+        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1"),
+        .package(url: "https://github.com/pointfreeco/swift-clocks.git", exact: "1.0.5"),
     ],
     targets: [
         .target(
@@ -644,7 +645,8 @@ let package = Package(
         .testTarget(
             name: "DuckPlayerTests",
             dependencies: [
-                "DuckPlayer"
+                "DuckPlayer",
+                "BrowserServicesKitTestsUtils",
             ]
         ),
 
@@ -653,6 +655,7 @@ let package = Package(
             dependencies: [
                 "TestUtils",
                 "MaliciousSiteProtection",
+                .product(name: "Clocks", package: "swift-clocks"),
             ],
             resources: [
                 .copy("Resources/phishingHashPrefixes.json"),

Sources/Common/Concurrency/TaskExtension.swift

@@ -18,51 +18,72 @@
 
 import Foundation
 
+public struct Sleeper {
+
+    public static let `default` = Sleeper(sleep: {
+        try await Task<Never, Never>.sleep(interval: $0)
+    })
+
+    private let sleep: (TimeInterval) async throws -> Void
+
+    public init(sleep: @escaping (TimeInterval) async throws -> Void) {
+        self.sleep = sleep
+    }
+
+    @available(macOS 13.0, iOS 16.0, *)
+    public init(clock: any Clock<Duration>) {
+        self.sleep = { interval in
+            try await clock.sleep(for: .nanoseconds(UInt64(interval * Double(NSEC_PER_SEC))))
+        }
+    }
+
+    public func sleep(for interval: TimeInterval) async throws {
+        try await sleep(interval)
+    }
+
+}
+
+public func performPeriodicJob(withDelay delay: TimeInterval? = nil,
+                               interval: TimeInterval,
+                               sleeper: Sleeper = .default,
+                               operation: @escaping @Sendable () async throws -> Void,
+                               cancellationHandler: (@Sendable () async -> Void)? = nil) async throws -> Never {
+
+    do {
+        if let delay {
+            try await sleeper.sleep(for: delay)
+        }
+
+        repeat {
+            try await operation()
+
+            try await sleeper.sleep(for: interval)
+        } while true
+    } catch let error as CancellationError {
+        await cancellationHandler?()
+        throw error
+    }
+}
+
 public extension Task where Success == Never, Failure == Error {
 
     static func periodic(delay: TimeInterval? = nil,
                          interval: TimeInterval,
+                         sleeper: Sleeper = .default,
                          operation: @escaping @Sendable () async -> Void,
                          cancellationHandler: (@Sendable () async -> Void)? = nil) -> Task {
 
-        Task {
-            do {
-                if let delay {
-                    try await Task<Never, Never>.sleep(interval: delay)
-                }
-
-                repeat {
-                    await operation()
-
-                    try await Task<Never, Never>.sleep(interval: interval)
-                } while true
-            } catch {
-                await cancellationHandler?()
-                throw error
-            }
-        }
+        return periodic(delay: delay, interval: interval, sleeper: sleeper, operation: { await operation() } as @Sendable () async throws -> Void, cancellationHandler: cancellationHandler)
     }
 
     static func periodic(delay: TimeInterval? = nil,
                          interval: TimeInterval,
+                         sleeper: Sleeper = .default,
                          operation: @escaping @Sendable () async throws -> Void,
                          cancellationHandler: (@Sendable () async -> Void)? = nil) -> Task {
 
         Task {
-            do {
-                if let delay {
-                    try await Task<Never, Never>.sleep(interval: delay)
-                }
-
-                repeat {
-                    try await operation()
-
-                    try await Task<Never, Never>.sleep(interval: interval)
-                } while true
-            } catch {
-                await cancellationHandler?()
-                throw error
-            }
+            try await performPeriodicJob(withDelay: delay, interval: interval, sleeper: sleeper, operation: operation, cancellationHandler: cancellationHandler)
         }
     }
 }

Sources/Common/Extensions/HashExtension.swift

@@ -42,8 +42,13 @@ extension Data {
 extension String {
 
     public var sha1: String {
-        let dataBytes = data(using: .utf8)!
-        return dataBytes.sha1
+        let result = utf8data.sha1
+        return result
+    }
+
+    public var sha256: String {
+        let result = utf8data.sha256
+        return result
     }
 
 }

Sources/Common/Extensions/StringExtension.swift

@@ -394,9 +394,9 @@ public extension String {
 
     // MARK: Regex
 
-    func matches(_ regex: NSRegularExpression) -> Bool {
-        let matches = regex.matches(in: self, options: .anchored, range: self.fullRange)
-        return matches.count == 1
+    func matches(_ regex: RegEx) -> Bool {
+        let firstMatch = firstMatch(of: regex, options: .anchored)
+        return firstMatch != nil
     }
 
     func matches(pattern: String, options: NSRegularExpression.Options = [.caseInsensitive]) -> Bool {
@@ -406,7 +406,7 @@ public extension String {
         return matches(regex)
     }
 
-    func replacing(_ regex: NSRegularExpression, with replacement: String) -> String {
+    func replacing(_ regex: RegEx, with replacement: String) -> String {
         regex.stringByReplacingMatches(in: self, range: self.fullRange, withTemplate: replacement)
     }
 

Sources/MaliciousSiteProtection/API/APIClient.swift

@@ -20,23 +20,21 @@ import Common
 import Foundation
 import Networking
 
-public protocol APIClientProtocol {
-    func load<Request: APIRequestProtocol>(_ requestConfig: Request) async throws -> Request.ResponseType
-}
-
-public extension APIClientProtocol where Self == APIClient {
-    static var production: APIClientProtocol { APIClient(environment: .production) }
-    static var staging: APIClientProtocol { APIClient(environment: .staging) }
+extension APIClient {
+    // used internally for testing
+    protocol Mockable {
+        func load<Request: APIClient.Request>(_ requestConfig: Request) async throws -> Request.Response
+    }
 }
+extension APIClient: APIClient.Mockable {}
 
 public protocol APIClientEnvironment {
-    func headers(for request: APIClient.Request) -> APIRequestV2.HeadersV2
-    func url(for request: APIClient.Request) -> URL
-    func timeout(for request: APIClient.Request) -> TimeInterval
+    func headers(for requestType: APIRequestType) -> APIRequestV2.HeadersV2
+    func url(for requestType: APIRequestType) -> URL
 }
 
-public extension APIClient {
-    enum DefaultEnvironment: APIClientEnvironment {
+public extension MaliciousSiteDetector {
+    enum APIEnvironment: APIClientEnvironment {
 
         case production
         case staging
@@ -49,7 +47,7 @@ public extension APIClient {
         }
 
         var defaultHeaders: APIRequestV2.HeadersV2 {
-            .init(userAgent: APIRequest.Headers.userAgent)
+            .init(userAgent: Networking.APIRequest.Headers.userAgent)
         }
 
         enum APIPath {
@@ -64,8 +62,8 @@ public extension APIClient {
             static let hashPrefix = "hashPrefix"
         }
 
-        public func url(for request: APIClient.Request) -> URL {
-            switch request {
+        public func url(for requestType: APIRequestType) -> URL {
+            switch requestType {
             case .hashPrefixSet(let configuration):
                 endpoint.appendingPathComponent(APIPath.hashPrefix).appendingParameters([
                     QueryParameter.category: configuration.threatKind.rawValue,
@@ -81,64 +79,50 @@ public extension APIClient {
             }
         }
 
-        public func headers(for request: APIClient.Request) -> APIRequestV2.HeadersV2 {
+        public func headers(for requestType: APIRequestType) -> APIRequestV2.HeadersV2 {
             defaultHeaders
         }
-
-        public func timeout(for request: APIClient.Request) -> TimeInterval {
-            switch request {
-            case .hashPrefixSet, .filterSet: 60
-            //  This could block navigation so we should favour navigation loading if the backend is degraded.
-            // On Android we're looking at a maximum 1 second timeout for this request.
-            case .matches: 1
-            }
-        }
     }
 
 }
 
-public struct APIClient: APIClientProtocol {
+struct APIClient {
 
     let environment: APIClientEnvironment
     private let service: APIService
 
-    public init(environment: Self.DefaultEnvironment = .production, service: APIService = DefaultAPIService(urlSession: .shared)) {
-        self.init(environment: environment as APIClientEnvironment, service: service)
-    }
-
-    public init(environment: APIClientEnvironment, service: APIService) {
+    init(environment: APIClientEnvironment, service: APIService = DefaultAPIService(urlSession: .shared)) {
         self.environment = environment
         self.service = service
     }
 
-    public func load<Request: APIRequestProtocol>(_ requestConfig: Request) async throws -> Request.ResponseType {
+    func load<R: Request>(_ requestConfig: R) async throws -> R.Response {
         let requestType = requestConfig.requestType
         let headers = environment.headers(for: requestType)
         let url = environment.url(for: requestType)
-        let timeout = environment.timeout(for: requestType)
 
-        let apiRequest = APIRequestV2(url: url, method: .get, headers: headers, timeoutInterval: timeout)
+        let apiRequest = APIRequestV2(url: url, method: .get, headers: headers)
         let response = try await service.fetch(request: apiRequest)
-        let result: Request.ResponseType = try response.decodeBody()
+        let result: R.Response = try response.decodeBody()
 
         return result
     }
 
 }
 
 // MARK: - Convenience
-extension APIClientProtocol {
-    public func filtersChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.FiltersChangeSet {
+extension APIClient.Mockable {
+    func filtersChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.FiltersChangeSet {
         let result = try await load(.filterSet(threatKind: threatKind, revision: revision))
         return result
     }
 
-    public func hashPrefixesChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.HashPrefixesChangeSet {
+    func hashPrefixesChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.HashPrefixesChangeSet {
         let result = try await load(.hashPrefixes(threatKind: threatKind, revision: revision))
         return result
     }
 
-    public func matches(forHashPrefix hashPrefix: String) async throws -> APIClient.Response.Matches {
+    func matches(forHashPrefix hashPrefix: String) async throws -> APIClient.Response.Matches {
         let result = try await load(.matches(hashPrefix: hashPrefix))
         return result
     }