Malware protection 3: Refactor Data storing (#1093)

<!--
Note: This checklist is a reminder of our shared engineering
expectations.
-->

Please review the release process for BrowserServicesKit
[here](https://app.asana.com/0/1200194497630846/1200837094583426).

**Required**:

Task/Issue URL:
https://app.asana.com/0/1202406491309510/1208033567421351/f
Tech design: https://app.asana.com/0/481882893211075/1208736595187321/f
iOS PR: 
macOS PR: duckduckgo/macos-browser#3598
What kind of version bump will this require?: Major

**Description**:
- Refactored Malicious Site `DataManager` to `actor` for thread safe
data storage with generic accessors and related file storage/update
manager
- Added dedicated FilterDictionary/HashPrefixSet structures storing
revision inside the structs and for faster data access

<!--
Tagging instructions
If this PR isn't ready to be merged for whatever reason it should be
marked with the `DO NOT MERGE` label (particularly if it's a draft)
If it's pending Product Review/PFR, please add the `Pending Product
Review` label.

If at any point it isn't actively being worked on/ready for
review/otherwise moving forward (besides the above PR/PFR exception)
strongly consider closing it (or not opening it in the first place). If
you decide not to close it, make sure it's labelled to make it clear the
PRs state and comment with more information.
-->

**Steps to test this PR**:
1. Activate Feature Flag override for malicious site protections in
Debug -> Feature Flag overrides
2. Visit https://privacy-test-pages.site/security/badware/phishing.html,
validate phishing detection works without changes
3. Validate tests pass and the update manager works as before

1.

<!--
Before submitting a PR, please ensure you have tested the combinations
you expect the reviewer to test, then delete configurations you *know*
do not need explicit testing.

Using a simulator where a physical device is unavailable is acceptable.
-->

**OS Testing**:

* [ ] iOS 14
* [ ] iOS 15
* [ ] iOS 16
* [ ] macOS 10.15
* [ ] macOS 11
* [ ] macOS 12

---
###### Internal references:
[Software Engineering
Expectations](https://app.asana.com/0/59792373528535/199064865822552)
[Technical Design
Template](https://app.asana.com/0/59792373528535/184709971311943)

Package.resolved

@@ -63,6 +63,24 @@
         "version" : "3.0.0"
       }
     },
+    {
+      "identity" : "swift-clocks",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-clocks.git",
+      "state" : {
+        "revision" : "b9b24b69e2adda099a1fa381cda1eeec272d5b53",
+        "version" : "1.0.5"
+      }
+    },
+    {
+      "identity" : "swift-concurrency-extras",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-concurrency-extras",
+      "state" : {
+        "revision" : "163409ef7dae9d960b87f34b51587b6609a76c1f",
+        "version" : "1.3.0"
+      }
+    },
     {
       "identity" : "swifter",
       "kind" : "remoteSourceControl",
@@ -89,6 +107,15 @@
         "revision" : "5de0a610a7927b638a5fd463a53032c9934a2c3b",
         "version" : "3.0.0"
       }
+    },
+    {
+      "identity" : "xctest-dynamic-overlay",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/xctest-dynamic-overlay",
+      "state" : {
+        "revision" : "a3f634d1a409c7979cabc0a71b3f26ffa9fc8af1",
+        "version" : "1.4.3"
+      }
     }
   ],
   "version" : 2

Package.swift

@@ -57,7 +57,8 @@ let package = Package(
         .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.2.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
-        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1")
+        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1"),
+        .package(url: "https://github.com/pointfreeco/swift-clocks.git", exact: "1.0.5"),
     ],
     targets: [
         .target(
@@ -644,7 +645,8 @@ let package = Package(
         .testTarget(
             name: "DuckPlayerTests",
             dependencies: [
-                "DuckPlayer"
+                "DuckPlayer",
+                "BrowserServicesKitTestsUtils",
             ]
         ),
 
@@ -653,6 +655,7 @@ let package = Package(
             dependencies: [
                 "TestUtils",
                 "MaliciousSiteProtection",
+                .product(name: "Clocks", package: "swift-clocks"),
             ],
             resources: [
                 .copy("Resources/phishingHashPrefixes.json"),

Sources/Common/Concurrency/TaskExtension.swift

@@ -18,51 +18,72 @@
 
 import Foundation
 
+public struct Sleeper {
+
+    public static let `default` = Sleeper(sleep: {
+        try await Task<Never, Never>.sleep(interval: $0)
+    })
+
+    private let sleep: (TimeInterval) async throws -> Void
+
+    public init(sleep: @escaping (TimeInterval) async throws -> Void) {
+        self.sleep = sleep
+    }
+
+    @available(macOS 13.0, iOS 16.0, *)
+    public init(clock: any Clock<Duration>) {
+        self.sleep = { interval in
+            try await clock.sleep(for: .nanoseconds(UInt64(interval * Double(NSEC_PER_SEC))))
+        }
+    }
+
+    public func sleep(for interval: TimeInterval) async throws {
+        try await sleep(interval)
+    }
+
+}
+
+public func performPeriodicJob(withDelay delay: TimeInterval? = nil,
+                               interval: TimeInterval,
+                               sleeper: Sleeper = .default,
+                               operation: @escaping @Sendable () async throws -> Void,
+                               cancellationHandler: (@Sendable () async -> Void)? = nil) async throws -> Never {
+
+    do {
+        if let delay {
+            try await sleeper.sleep(for: delay)
+        }
+
+        repeat {
+            try await operation()
+
+            try await sleeper.sleep(for: interval)
+        } while true
+    } catch let error as CancellationError {
+        await cancellationHandler?()
+        throw error
+    }
+}
+
 public extension Task where Success == Never, Failure == Error {
 
     static func periodic(delay: TimeInterval? = nil,
                          interval: TimeInterval,
+                         sleeper: Sleeper = .default,
                          operation: @escaping @Sendable () async -> Void,
                          cancellationHandler: (@Sendable () async -> Void)? = nil) -> Task {
 
-        Task {
-            do {
-                if let delay {
-                    try await Task<Never, Never>.sleep(interval: delay)
-                }
-
-                repeat {
-                    await operation()
-
-                    try await Task<Never, Never>.sleep(interval: interval)
-                } while true
-            } catch {
-                await cancellationHandler?()
-                throw error
-            }
-        }
+        return periodic(delay: delay, interval: interval, sleeper: sleeper, operation: { await operation() } as @Sendable () async throws -> Void, cancellationHandler: cancellationHandler)
     }
 
     static func periodic(delay: TimeInterval? = nil,
                          interval: TimeInterval,
+                         sleeper: Sleeper = .default,
                          operation: @escaping @Sendable () async throws -> Void,
                          cancellationHandler: (@Sendable () async -> Void)? = nil) -> Task {
 
         Task {
-            do {
-                if let delay {
-                    try await Task<Never, Never>.sleep(interval: delay)
-                }
-
-                repeat {
-                    try await operation()
-
-                    try await Task<Never, Never>.sleep(interval: interval)
-                } while true
-            } catch {
-                await cancellationHandler?()
-                throw error
-            }
+            try await performPeriodicJob(withDelay: delay, interval: interval, sleeper: sleeper, operation: operation, cancellationHandler: cancellationHandler)
         }
     }
 }

Sources/Common/Extensions/HashExtension.swift

@@ -42,8 +42,13 @@ extension Data {
 extension String {
 
     public var sha1: String {
-        let dataBytes = data(using: .utf8)!
-        return dataBytes.sha1
+        let result = utf8data.sha1
+        return result
+    }
+
+    public var sha256: String {
+        let result = utf8data.sha256
+        return result
     }
 
 }

Sources/Common/Extensions/StringExtension.swift

@@ -394,9 +394,9 @@ public extension String {
 
     // MARK: Regex
 
-    func matches(_ regex: NSRegularExpression) -> Bool {
-        let matches = regex.matches(in: self, options: .anchored, range: self.fullRange)
-        return matches.count == 1
+    func matches(_ regex: RegEx) -> Bool {
+        let firstMatch = firstMatch(of: regex, options: .anchored)
+        return firstMatch != nil
     }
 
     func matches(pattern: String, options: NSRegularExpression.Options = [.caseInsensitive]) -> Bool {
@@ -406,7 +406,7 @@ public extension String {
         return matches(regex)
     }
 
-    func replacing(_ regex: NSRegularExpression, with replacement: String) -> String {
+    func replacing(_ regex: RegEx, with replacement: String) -> String {
         regex.stringByReplacingMatches(in: self, range: self.fullRange, withTemplate: replacement)
     }
 

Sources/MaliciousSiteProtection/API/APIClient.swift

@@ -20,23 +20,21 @@ import Common
 import Foundation
 import Networking
 
-public protocol APIClientProtocol {
-    func load<Request: APIRequestProtocol>(_ requestConfig: Request) async throws -> Request.ResponseType
-}
-
-public extension APIClientProtocol where Self == APIClient {
-    static var production: APIClientProtocol { APIClient(environment: .production) }
-    static var staging: APIClientProtocol { APIClient(environment: .staging) }
+extension APIClient {
+    // used internally for testing
+    protocol Mockable {
+        func load<Request: APIClient.Request>(_ requestConfig: Request) async throws -> Request.Response
+    }
 }
+extension APIClient: APIClient.Mockable {}
 
 public protocol APIClientEnvironment {
-    func headers(for request: APIClient.Request) -> APIRequestV2.HeadersV2
-    func url(for request: APIClient.Request) -> URL
-    func timeout(for request: APIClient.Request) -> TimeInterval
+    func headers(for requestType: APIRequestType) -> APIRequestV2.HeadersV2
+    func url(for requestType: APIRequestType) -> URL
 }
 
-public extension APIClient {
-    enum DefaultEnvironment: APIClientEnvironment {
+public extension MaliciousSiteDetector {
+    enum APIEnvironment: APIClientEnvironment {
 
         case production
         case staging
@@ -49,7 +47,7 @@ public extension APIClient {
         }
 
         var defaultHeaders: APIRequestV2.HeadersV2 {
-            .init(userAgent: APIRequest.Headers.userAgent)
+            .init(userAgent: Networking.APIRequest.Headers.userAgent)
         }
 
         enum APIPath {
@@ -64,8 +62,8 @@ public extension APIClient {
             static let hashPrefix = "hashPrefix"
         }
 
-        public func url(for request: APIClient.Request) -> URL {
-            switch request {
+        public func url(for requestType: APIRequestType) -> URL {
+            switch requestType {
             case .hashPrefixSet(let configuration):
                 endpoint.appendingPathComponent(APIPath.hashPrefix).appendingParameters([
                     QueryParameter.category: configuration.threatKind.rawValue,
@@ -81,64 +79,50 @@ public extension APIClient {
             }
         }
 
-        public func headers(for request: APIClient.Request) -> APIRequestV2.HeadersV2 {
+        public func headers(for requestType: APIRequestType) -> APIRequestV2.HeadersV2 {
             defaultHeaders
         }
-
-        public func timeout(for request: APIClient.Request) -> TimeInterval {
-            switch request {
-            case .hashPrefixSet, .filterSet: 60
-            //  This could block navigation so we should favour navigation loading if the backend is degraded.
-            // On Android we're looking at a maximum 1 second timeout for this request.
-            case .matches: 1
-            }
-        }
     }
 
 }
 
-public struct APIClient: APIClientProtocol {
+struct APIClient {
 
     let environment: APIClientEnvironment
     private let service: APIService
 
-    public init(environment: Self.DefaultEnvironment = .production, service: APIService = DefaultAPIService(urlSession: .shared)) {
-        self.init(environment: environment as APIClientEnvironment, service: service)
-    }
-
-    public init(environment: APIClientEnvironment, service: APIService) {
+    init(environment: APIClientEnvironment, service: APIService = DefaultAPIService(urlSession: .shared)) {
         self.environment = environment
         self.service = service
     }
 
-    public func load<Request: APIRequestProtocol>(_ requestConfig: Request) async throws -> Request.ResponseType {
+    func load<R: Request>(_ requestConfig: R) async throws -> R.Response {
         let requestType = requestConfig.requestType
         let headers = environment.headers(for: requestType)
         let url = environment.url(for: requestType)
-        let timeout = environment.timeout(for: requestType)
 
-        let apiRequest = APIRequestV2(url: url, method: .get, headers: headers, timeoutInterval: timeout)
+        let apiRequest = APIRequestV2(url: url, method: .get, headers: headers)
         let response = try await service.fetch(request: apiRequest)
-        let result: Request.ResponseType = try response.decodeBody()
+        let result: R.Response = try response.decodeBody()
 
         return result
     }
 
 }
 
 // MARK: - Convenience
-extension APIClientProtocol {
-    public func filtersChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.FiltersChangeSet {
+extension APIClient.Mockable {
+    func filtersChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.FiltersChangeSet {
         let result = try await load(.filterSet(threatKind: threatKind, revision: revision))
         return result
     }
 
-    public func hashPrefixesChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.HashPrefixesChangeSet {
+    func hashPrefixesChangeSet(for threatKind: ThreatKind, revision: Int) async throws -> APIClient.Response.HashPrefixesChangeSet {
         let result = try await load(.hashPrefixes(threatKind: threatKind, revision: revision))
         return result
     }
 
-    public func matches(forHashPrefix hashPrefix: String) async throws -> APIClient.Response.Matches {
+    func matches(forHashPrefix hashPrefix: String) async throws -> APIClient.Response.Matches {
         let result = try await load(.matches(hashPrefix: hashPrefix))
         return result
     }