Enable gzip compression for Sync PATCH payloads

Package.resolved

@@ -36,6 +36,15 @@
         "version" : "2.3.0"
       }
     },
+    {
+      "identity" : "gzipswift",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/1024jp/GzipSwift.git",
+      "state" : {
+        "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05",
+        "version" : "6.0.1"
+      }
+    },
     {
       "identity" : "privacy-dashboard",
       "kind" : "remoteSourceControl",

Package.swift

@@ -48,6 +48,7 @@ let package = Package(
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/wireguard-apple", exact: "1.1.3"),
+        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1")
     ],
     targets: [
         .target(
@@ -153,6 +154,7 @@ let package = Package(
                 "BrowserServicesKit",
                 "Common",
                 .product(name: "DDGSyncCrypto", package: "sync_crypto"),
+                .product(name: "Gzip", package: "GzipSwift"),
                 "Networking",
             ],
             resources: [

Sources/DDGSync/internal/ProductionDependencies.swift

@@ -27,6 +27,7 @@ struct ProductionDependencies: SyncDependencies {
     let endpoints: Endpoints
     let account: AccountManaging
     let api: RemoteAPIRequestCreating
+    let payloadCompressor: SyncPayloadCompressing
     var keyValueStore: KeyValueStoring
     let secureStore: SecureStoring
     let crypter: CryptingInternal
@@ -72,6 +73,7 @@ struct ProductionDependencies: SyncDependencies {
         self.getLog = log
 
         api = RemoteAPIRequestCreator(log: log())
+        payloadCompressor = SyncPayloadCompressor()
 
         crypter = Crypter(secureStore: secureStore)
         account = AccountManager(endpoints: endpoints, api: api, crypter: crypter)

Sources/DDGSync/internal/SyncDependencies.swift

@@ -31,6 +31,7 @@ protocol SyncDependencies: SyncDependenciesDebuggingSupport {
     var endpoints: Endpoints { get }
     var account: AccountManaging { get }
     var api: RemoteAPIRequestCreating { get }
+    var payloadCompressor: SyncPayloadCompressing { get }
     var keyValueStore: KeyValueStoring { get }
     var secureStore: SecureStoring { get }
     var crypter: CryptingInternal { get }

Sources/DDGSync/internal/SyncOperation.swift

@@ -19,6 +19,7 @@
 import Foundation
 import Combine
 import Common
+import Gzip
 
 final class SyncOperation: Operation {
 
@@ -143,7 +144,7 @@ final class SyncOperation: Operation {
                         try checkCancellation()
                         let syncRequest = try await self.makeSyncRequest(for: dataProvider, fetchOnly: fetchOnly)
                         let clientTimestamp = Date()
-                        let httpRequest = try self.makeHTTPRequest(with: syncRequest, timestamp: clientTimestamp)
+                        let httpRequest = try self.makeHTTPRequest(for: dataProvider, with: syncRequest, timestamp: clientTimestamp)
 
                         try checkCancellation()
                         let httpResult: HTTPResult = try await httpRequest.execute()
@@ -211,10 +212,15 @@ final class SyncOperation: Operation {
         return SyncRequest(feature: dataProvider.feature, previousSyncTimestamp: dataProvider.lastSyncTimestamp, sent: localChanges)
     }
 
-    private func makeHTTPRequest(with syncRequest: SyncRequest, timestamp: Date) throws -> HTTPRequesting {
+    private func makeHTTPRequest(for dataProvider: DataProviding, with syncRequest: SyncRequest, timestamp: Date) throws -> HTTPRequesting {
         let hasLocalChanges = !syncRequest.sent.isEmpty
         if hasLocalChanges {
-            return try requestMaker.makePatchRequest(with: syncRequest, clientTimestamp: timestamp)
+            do {
+                return try requestMaker.makePatchRequest(with: syncRequest, clientTimestamp: timestamp, isCompressed: true)
+            } catch let error as GzipError {
+                dataProvider.handleSyncError(error)
+                return try requestMaker.makePatchRequest(with: syncRequest, clientTimestamp: timestamp, isCompressed: false)
+            }
         }
         return try requestMaker.makeGetRequest(with: syncRequest)
     }

Sources/DDGSync/internal/SyncPayloadCompressor.swift

@@ -0,0 +1,30 @@
+//
+//  SyncPayloadCompressor.swift
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+import Gzip
+
+protocol SyncPayloadCompressing {
+    func compress(_ payload: Data) throws -> Data
+}
+
+struct SyncPayloadCompressor: SyncPayloadCompressing {
+    func compress(_ payload: Data) throws -> Data {
+        try payload.gzipped()
+    }
+}

Sources/DDGSync/internal/SyncQueue.swift

@@ -69,6 +69,7 @@ final class SyncQueue {
             crypter: dependencies.crypter,
             api: dependencies.api,
             endpoints: dependencies.endpoints,
+            payloadCompressor: dependencies.payloadCompressor,
             log: dependencies.log
         )
     }
@@ -79,13 +80,14 @@ final class SyncQueue {
         crypter: Crypting,
         api: RemoteAPIRequestCreating,
         endpoints: Endpoints,
+        payloadCompressor: SyncPayloadCompressing,
         log: @escaping @autoclosure () -> OSLog = .disabled
     ) {
         self.dataProviders = dataProviders
         self.storage = storage
         self.crypter = crypter
         self.getLog = log
-        requestMaker = SyncRequestMaker(storage: storage, api: api, endpoints: endpoints)
+        requestMaker = SyncRequestMaker(storage: storage, api: api, endpoints: endpoints, payloadCompressor: payloadCompressor)
         syncDidFinishPublisher = syncDidFinishSubject.eraseToAnyPublisher()
         syncHTTPRequestErrorPublisher = syncHTTPRequestErrorSubject.eraseToAnyPublisher()
         isSyncInProgressPublisher = Publishers

Sources/DDGSync/internal/SyncRequestMaker.swift

@@ -17,16 +17,18 @@
 //
 
 import Foundation
+import Gzip
 
 protocol SyncRequestMaking {
     func makeGetRequest(with result: SyncRequest) throws -> HTTPRequesting
-    func makePatchRequest(with result: SyncRequest, clientTimestamp: Date) throws -> HTTPRequesting
+    func makePatchRequest(with result: SyncRequest, clientTimestamp: Date, isCompressed: Bool) throws -> HTTPRequesting
 }
 
 struct SyncRequestMaker: SyncRequestMaking {
     let storage: SecureStoring
     let api: RemoteAPIRequestCreating
     let endpoints: Endpoints
+    let payloadCompressor: SyncPayloadCompressing
     let dateFormatter = ISO8601DateFormatter()
 
     func makeGetRequest(with result: SyncRequest) throws -> HTTPRequesting {
@@ -41,7 +43,7 @@ struct SyncRequestMaker: SyncRequestMaking {
         return api.createAuthenticatedGetRequest(url: url, authToken: try getToken(), parameters: parameters)
     }
 
-    func makePatchRequest(with result: SyncRequest, clientTimestamp: Date) throws -> HTTPRequesting {
+    func makePatchRequest(with result: SyncRequest, clientTimestamp: Date, isCompressed: Bool) throws -> HTTPRequesting {
         var json = [String: Any]()
         let modelPayload: [String: Any?] = [
             "updates": result.sent.map(\.payload),
@@ -55,7 +57,18 @@ struct SyncRequestMaker: SyncRequestMaking {
         }
 
         let body = try JSONSerialization.data(withJSONObject: json, options: [])
-        return api.createAuthenticatedJSONRequest(url: endpoints.syncPatch, method: .PATCH, authToken: try getToken(), json: body)
+
+        guard isCompressed else {
+            return api.createAuthenticatedJSONRequest(url: endpoints.syncPatch, method: .PATCH, authToken: try getToken(), json: body)
+        }
+
+        let compressedBody = try payloadCompressor.compress(body)
+        return api.createAuthenticatedJSONRequest(
+            url: endpoints.syncPatch,
+            method: .PATCH,
+            authToken: try getToken(),
+            json: compressedBody,
+            headers: ["Content-Encoding": "gzip"])
     }
 
     private func getToken() throws -> String {

Tests/DDGSyncTests/Mocks/Mocks.swift

@@ -20,6 +20,7 @@ import BrowserServicesKit
 import Combine
 import Common
 import Foundation
+import Gzip
 import Persistence
 import TestUtils
 
@@ -199,6 +200,7 @@ struct MockSyncDependencies: SyncDependencies, SyncDependenciesDebuggingSupport
     var endpoints: Endpoints = Endpoints(baseURL: URL(string: "https://dev.null")!)
     var account: AccountManaging = AccountManagingMock()
     var api: RemoteAPIRequestCreating = RemoteAPIRequestCreatingMock()
+    var payloadCompressor: SyncPayloadCompressing = SyncPayloadCompressorMock()
     var secureStore: SecureStoring = SecureStorageStub()
     var crypter: CryptingInternal = CryptingMock()
     var scheduler: SchedulingInternal = SchedulerMock()
@@ -276,6 +278,45 @@ class RemoteAPIRequestCreatingMock: RemoteAPIRequestCreating {
     }
 }
 
+class InspectableSyncRequestMaker: SyncRequestMaking {
+
+    struct MakePatchRequestCallArgs {
+        let result: SyncRequest
+        let clientTimestamp: Date
+        let isCompressed: Bool
+    }
+
+    func makeGetRequest(with result: SyncRequest) throws -> HTTPRequesting {
+        try requestMaker.makeGetRequest(with: result)
+    }
+
+    func makePatchRequest(with result: SyncRequest, clientTimestamp: Date, isCompressed: Bool) throws -> HTTPRequesting {
+        makePatchRequestCallCount += 1
+        makePatchRequestCallArgs.append(.init(result: result, clientTimestamp: clientTimestamp, isCompressed: isCompressed))
+        return try requestMaker.makePatchRequest(with: result, clientTimestamp: clientTimestamp, isCompressed: isCompressed)
+    }
+
+    let requestMaker: SyncRequestMaker
+
+    init(requestMaker: SyncRequestMaker) {
+        self.requestMaker = requestMaker
+    }
+
+    var makePatchRequestCallCount = 0
+    var makePatchRequestCallArgs: [MakePatchRequestCallArgs] = []
+}
+
+class SyncPayloadCompressorMock: SyncPayloadCompressing {
+    var error: Error?
+
+    func compress(_ payload: Data) throws -> Data {
+        if let error {
+            throw error
+        }
+        return try payload.gzipped()
+    }
+}
+
 struct CryptingMock: CryptingInternal {
     var _encryptAndBase64Encode: (String) throws -> String = { "encrypted_\($0)" }
     var _base64DecodeAndDecrypt: (String) throws -> String = { $0.dropping(prefix: "encrypted_") }

Tests/DDGSyncTests/SyncOperationTests.swift

@@ -18,15 +18,17 @@
 
 import XCTest
 
+@testable import Gzip
 @testable import DDGSync
 
 class SyncOperationTests: XCTestCase {
     var apiMock: RemoteAPIRequestCreatingMock!
     var request: HTTPRequestingMock!
     var endpoints: Endpoints!
+    var payloadCompressor: SyncPayloadCompressorMock!
     var storage: SecureStorageStub!
     var crypter: CryptingMock!
-    var requestMaker: SyncRequestMaking!
+    var requestMaker: InspectableSyncRequestMaker!
 
     override func setUpWithError() throws {
         try super.setUpWithError()
@@ -35,6 +37,7 @@ class SyncOperationTests: XCTestCase {
         request = HTTPRequestingMock()
         apiMock.request = request
         endpoints = Endpoints(baseURL: URL(string: "https://example.com")!)
+        payloadCompressor = SyncPayloadCompressorMock()
         storage = SecureStorageStub()
         crypter = CryptingMock()
         try storage.persistAccount(
@@ -50,7 +53,9 @@ class SyncOperationTests: XCTestCase {
             )
         )
 
-        requestMaker = SyncRequestMaker(storage: storage, api: apiMock, endpoints: endpoints)
+        requestMaker = InspectableSyncRequestMaker(
+            requestMaker: .init(storage: storage, api: apiMock, endpoints: endpoints, payloadCompressor: payloadCompressor)
+        )
     }
 
     func testWhenThereAreNoChangesThenGetRequestIsFired() async throws {
@@ -167,16 +172,16 @@ class SyncOperationTests: XCTestCase {
 
         for body in bodies.compactMap({$0}) {
             do {
-                let payload = try JSONDecoder.snakeCaseKeys.decode(BookmarksPayload.self, from: body)
+                let payload = try JSONDecoder.snakeCaseKeys.decode(BookmarksPayload.self, from: body.gunzipped())
                 XCTAssertEqual(payload, bookmarks)
                 payloadCount -= 1
             } catch {
                 do {
-                    let payload = try JSONDecoder.snakeCaseKeys.decode(SettingsPayload.self, from: body)
+                    let payload = try JSONDecoder.snakeCaseKeys.decode(SettingsPayload.self, from: body.gunzipped())
                     XCTAssertEqual(payload, settings)
                     payloadCount -= 1
                 } catch {
-                    let payload = try JSONDecoder.snakeCaseKeys.decode(AutofillPayload.self, from: body)
+                    let payload = try JSONDecoder.snakeCaseKeys.decode(AutofillPayload.self, from: body.gunzipped())
                     XCTAssertEqual(payload, autofill)
                     payloadCount -= 1
                 }
@@ -245,6 +250,62 @@ class SyncOperationTests: XCTestCase {
 
         XCTAssertTrue(try sentModels.isJSONRepresentationEquivalent(to: objectsToSync))
     }
+
+    func testWhenPatchPayloadCompressionSucceedsThenPayloadIsSentCompressed() async throws {
+        let objectsToSync = [
+            Syncable(jsonObject: ["id": "1", "name": "bookmark1", "url": "https://example.com"]),
+        ]
+        let dataProvider = DataProvidingMock(feature: .init(name: "bookmarks"))
+        var sentModels: [Syncable] = []
+        var errors: [any Error] = []
+        dataProvider.updateSyncTimestamps(server: "1234", local: nil)
+        dataProvider._fetchChangedObjects = { _ in objectsToSync }
+        dataProvider._handleSyncError = { errors.append($0) }
+        dataProvider.handleSyncResponse = { sent, _, _, _, _ in
+            sentModels = sent
+        }
+
+        let syncOperation = SyncOperation(dataProviders: [dataProvider], storage: storage, crypter: crypter, requestMaker: requestMaker)
+        request.result = .init(data: nil, response: HTTPURLResponse(url: URL(string: "https://example.com")!, statusCode: 304, httpVersion: nil, headerFields: nil)!)
+
+        try await syncOperation.sync(fetchOnly: false)
+
+        XCTAssertTrue(try sentModels.isJSONRepresentationEquivalent(to: objectsToSync))
+        XCTAssertTrue(errors.isEmpty)
+        XCTAssertEqual(requestMaker.makePatchRequestCallCount, 1)
+        XCTAssertEqual(requestMaker.makePatchRequestCallArgs[0].isCompressed, true)
+    }
+
+    func testWhenPatchPayloadCompressionFailsThenPayloadIsSentUncompressed() async throws {
+        let errorCode = 100200300
+        payloadCompressor.error = GzipError(code: Int32(errorCode), msg: nil)
+
+        let objectsToSync = [
+            Syncable(jsonObject: ["id": "1", "name": "bookmark1", "url": "https://example.com"]),
+        ]
+        let dataProvider = DataProvidingMock(feature: .init(name: "bookmarks"))
+        var sentModels: [Syncable] = []
+        var errors: [any Error] = []
+        dataProvider.updateSyncTimestamps(server: "1234", local: nil)
+        dataProvider._fetchChangedObjects = { _ in objectsToSync }
+        dataProvider._handleSyncError = { errors.append($0) }
+        dataProvider.handleSyncResponse = { sent, _, _, _, _ in
+            sentModels = sent
+        }
+
+        let syncOperation = SyncOperation(dataProviders: [dataProvider], storage: storage, crypter: crypter, requestMaker: requestMaker)
+        request.result = .init(data: nil, response: HTTPURLResponse(url: URL(string: "https://example.com")!, statusCode: 304, httpVersion: nil, headerFields: nil)!)
+
+        try await syncOperation.sync(fetchOnly: false)
+
+        XCTAssertTrue(try sentModels.isJSONRepresentationEquivalent(to: objectsToSync))
+        XCTAssertEqual(errors.count, 1)
+        let error = try XCTUnwrap(errors.first as? GzipError)
+        XCTAssertEqual(error.kind, .unknown(code: errorCode))
+        XCTAssertEqual(requestMaker.makePatchRequestCallCount, 2)
+        XCTAssertEqual(requestMaker.makePatchRequestCallArgs[0].isCompressed, true)
+        XCTAssertEqual(requestMaker.makePatchRequestCallArgs[1].isCompressed, false)
+    }
 }
 
 private extension Array where Element == Syncable {