fix(storage): Fixing watchOS crash when dealing with big files (#3389)

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/Bytes.swift

@@ -16,22 +16,22 @@ enum Bytes {
     case kilobytes(Int)
     case bytes(Int)
 
-    var bytes: Int {
+    var bytes: UInt64 {
         switch self {
         case .terabytes(let tb):
-            return Int(pow(1_024.0, 4.0)) * tb
+            return UInt64(pow(1_024.0, 4.0)) * UInt64(tb)
         case .gigabytes(let gb):
-            return Int(pow(1_024.0, 3.0)) * gb
+            return UInt64(pow(1_024.0, 3.0)) * UInt64(gb)
         case .megabytes(let mb):
-            return Int(pow(1_024.0, 2.0)) * mb
+            return UInt64(pow(1_024.0, 2.0)) * UInt64(mb)
         case .kilobytes(let kb):
-            return 1_024 * kb
+            return 1_024 * UInt64(kb)
         case .bytes(let b):
-            return b
+            return UInt64(b)
         }
     }
 
-    var bits: Int {
+    var bits: UInt64 {
         return bytes * 8
     }
 }

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileHandle+UInt64.swift

@@ -0,0 +1,43 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import Foundation
+
+extension FileHandle {
+    /// Reads data synchronously up to the specified number of bytes.
+    /// - Parameter bytes: The number of bytes to read from the file handle.
+    /// - Parameter bytesReadLimit: The maximum number of bytes that can be read at a time. Defaults to `Int.max`.
+    /// - Returns: The data available through the receiver up to a maximum of length bytes, or the maximum size that can be represented by a Data object.
+    /// - Throws: An error if attempts to determine the file-handle type fail or if attempts to read from the file or channel fail.
+    func read(bytes: UInt64, bytesReadLimit: Int = Int.max) throws -> Data {
+        // Read as much as it's possible considering the `bytesReadLimit` maximum
+        let bytesRead = bytes <= bytesReadLimit ? Int(bytes) : bytesReadLimit
+        guard var data = try readData(upToCount: bytesRead) else {
+            // There is no more data to read from the file
+            return Data()
+        }
+
+        // If there's remaining bytes to read, do it and append to the current data
+        let remainingBytes = bytes - UInt64(bytesRead)
+        if remainingBytes > 0 {
+            try data.append(read(
+                bytes: remainingBytes,
+                bytesReadLimit: bytesReadLimit
+            ))
+        }
+
+        return data
+    }
+
+    private func readData(upToCount length: Int) throws -> Data? {
+        if #available(iOS 13.4, macOS 10.15.4, tvOS 13.4, *) {
+            return try read(upToCount: length)
+        } else {
+            return readData(ofLength: length)
+        }
+    }
+}

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileSystem.swift

@@ -185,7 +185,7 @@ class FileSystem {
     ///   - offset: position to start reading
     ///   - length: length of the part
     ///   - completionHandler: completion handler
-    func createPartialFile(fileURL: URL, offset: Int, length: Int, completionHandler: @escaping (Result<URL, Error>) -> Void) {
+    func createPartialFile(fileURL: URL, offset: UInt64, length: UInt64, completionHandler: @escaping (Result<URL, Error>) -> Void) {
         // 4.5 MB (1 MB per part)
         // 1024 1024 1024 1024 512
 
@@ -198,8 +198,8 @@ class FileSystem {
                 defer {
                     try? fileHandle.close()
                 }
-                try fileHandle.seek(toOffset: UInt64(offset))
-                let data = fileHandle.readData(ofLength: length)
+                try fileHandle.seek(toOffset: offset)
+                let data = try fileHandle.read(bytes: length)
                 let fileURL = try self.createTemporaryFile(data: data)
                 completionHandler(.success(fileURL))
             } catch {

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StoragePersistableTransferTask.swift

@@ -108,8 +108,8 @@ struct StoragePersistableMultipartUpload: Codable {
 struct StoragePersistableSubTask: Codable {
     let uploadId: UploadID
     let partNumber: PartNumber
-    let bytes: Int
-    let bytesTransferred: Int
+    let bytes: UInt64
+    let bytesTransferred: UInt64
     let taskIdentifier: TaskIdentifier? // once an UploadPart starts uploading it will have a taskIdentifier
     let eTag: String?
 

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageServiceSessionDelegate.swift

@@ -165,7 +165,7 @@ extension StorageServiceSessionDelegate: URLSessionTaskDelegate {
                 return
             }
 
-            multipartUploadSession.handle(uploadPartEvent: .progressUpdated(partNumber: partNumber, bytesTransferred: Int(bytesSent), taskIdentifier: task.taskIdentifier))
+            multipartUploadSession.handle(uploadPartEvent: .progressUpdated(partNumber: partNumber, bytesTransferred: UInt64(bytesSent), taskIdentifier: task.taskIdentifier))
         case .upload(let onEvent):
             let progress = Progress(totalUnitCount: totalBytesExpectedToSend)
             progress.completedUnitCount = totalBytesSent

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPart.swift

@@ -28,11 +28,11 @@ let minimumPartCount = 1
 let maximumPartCount = 10_000
 
 enum StorageUploadPart {
-    case pending(bytes: Int)
-    case queued(bytes: Int)
-    case inProgress(bytes: Int, bytesTransferred: Int, taskIdentifier: TaskIdentifier)
-    case failed(bytes: Int, bytesTransferred: Int, error: Error)
-    case completed(bytes: Int, eTag: String)
+    case pending(bytes: UInt64)
+    case queued(bytes: UInt64)
+    case inProgress(bytes: UInt64, bytesTransferred: UInt64, taskIdentifier: TaskIdentifier)
+    case failed(bytes: UInt64, bytesTransferred: UInt64, error: Error)
+    case completed(bytes: UInt64, eTag: String)
 
     var isPending: Bool {
         if case .pending = self {
@@ -90,8 +90,8 @@ enum StorageUploadPart {
         return result
     }
 
-    var bytes: Int {
-        let result: Int
+    var bytes: UInt64 {
+        let result: UInt64
         switch self {
         case .pending(let bytes), .queued(let bytes):
             result = bytes
@@ -106,8 +106,8 @@ enum StorageUploadPart {
         return result
     }
 
-    var bytesTransferred: Int {
-        let result: Int
+    var bytesTransferred: UInt64 {
+        let result: UInt64
         switch self {
         case .pending, .queued, .failed:
             result = 0
@@ -157,7 +157,7 @@ struct StorageUploadPartSize {
         case exceedsSupportedFileSize
         case exceedsMaximumObjectSize
     }
-    let size: Int
+    let size: UInt64
 
     static let `default`: StorageUploadPartSize = StorageUploadPartSize()
 
@@ -167,7 +167,7 @@ struct StorageUploadPartSize {
 
     /// Creates custom part size in bytes. Throws if file part is invalid.
     /// - Parameter size: part size
-    init(size: Int) throws {
+    init(size: UInt64) throws {
         if size < minimumPartSize {
             throw Failure.belowMinimumPartSize
         } else if size > maximumPartSize {
@@ -215,8 +215,8 @@ struct StorageUploadPartSize {
         }
     }
 
-    func offset(for partNumber: PartNumber) -> Int {
-        let result = (partNumber - 1) * size
+    func offset(for partNumber: PartNumber) -> UInt64 {
+        let result = UInt64(partNumber - 1) * size
         return result
     }
 
@@ -242,7 +242,7 @@ extension Array where Element == StorageUploadPart {
             throw Failure.partCountOverUpperLimit
         }
 
-        let remainingBytes = Int(fileSize % UInt64(size))
+        let remainingBytes = fileSize % size
         logger.debug("count = \(count), remainingBytes = \(remainingBytes), size = \(size), totalBytes = \(fileSize)")
 
         self.init(repeating: .pending(bytes: size), count: count)
@@ -298,13 +298,13 @@ extension Sequence where Element == StorageUploadPart {
         filter { $0.completed }
     }
 
-    var totalBytes: Int {
+    var totalBytes: UInt64 {
         reduce(into: 0) { result, part in
             result += part.bytes
         }
     }
 
-    var bytesTransferred: Int {
+    var bytesTransferred: UInt64 {
         reduce(into: 0) { result, part in
             result += part.bytesTransferred
         }

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPartEvent.swift

@@ -10,7 +10,7 @@ import Foundation
 enum StorageUploadPartEvent {
     case queued(partNumber: PartNumber)
     case started(partNumber: PartNumber, taskIdentifier: TaskIdentifier)
-    case progressUpdated(partNumber: PartNumber, bytesTransferred: Int, taskIdentifier: TaskIdentifier)
+    case progressUpdated(partNumber: PartNumber, bytesTransferred: UInt64, taskIdentifier: TaskIdentifier)
     case completed(partNumber: PartNumber, eTag: String, taskIdentifier: TaskIdentifier)
     case failed(partNumber: PartNumber, error: Error)
 

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileHandleTests.swift

@@ -0,0 +1,47 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import XCTest
+@testable import AWSS3StoragePlugin
+
+class FileHandleTests: XCTestCase {
+
+    /// Given: A FileHandle and a file
+    /// When: `read(bytes:bytesReadLimit)` is invoked with `bytesReadLimit` being lower than `bytes`
+    /// Then: Only `bytesReadLimit` bytes will be read at a time, but all `bytes` will be read and returned
+    func testRead_withBytesHigherThanLimit_shouldSucceedByReadingMultipleTimes() throws {
+        let sourceString = "012345678910" // 11 bytes
+        let sourceData = sourceString.data(using: .utf8)!
+        let sourceFile = try createFile(from: sourceData)
+        XCTAssertEqual(try StorageRequestUtils.getSize(sourceFile), UInt64(sourceString.count))
+
+        let fileSystem = FileSystem()
+        let bytesReadLimit = 2
+
+        let fileHandle = try FileHandle(forReadingFrom: sourceFile)
+        let firstPartData = try fileHandle.read(bytes: 5, bytesReadLimit: bytesReadLimit)
+        let firstPartString = String(decoding: firstPartData, as: UTF8.self)
+        XCTAssertEqual(firstPartString, "01234") // i.e. the first 5 bytes
+
+        let secondPartData = try fileHandle.read(bytes: 5, bytesReadLimit: bytesReadLimit)
+        let secondPartString = String(decoding: secondPartData, as: UTF8.self)
+        XCTAssertEqual(secondPartString, "56789") // i.e. the second 5 bytes
+
+        let thirdPartData = try fileHandle.read(bytes: 5, bytesReadLimit: bytesReadLimit)
+        let thirdPartString = String(decoding: thirdPartData, as: UTF8.self)
+        XCTAssertEqual(thirdPartString, "10") // i.e. the remaining bytes
+
+        try FileManager.default.removeItem(at: sourceFile)
+    }
+
+    private func createFile(from data: Data) throws -> URL {
+        let fileUrl = try FileManager.default.url(for: .cachesDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
+            .appendingPathComponent("\(UUID().uuidString).tmp")
+        try data.write(to: fileUrl, options: .atomic)
+        return fileUrl
+    }
+}

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileSystemTests.swift

@@ -18,7 +18,7 @@ extension FileSystem {
     /// - Parameter bytes: bytes
     /// - Returns: random data
     func randomData(bytes: Bytes) -> Data {
-        let count = bytes.bytes
+        let count = Int(bytes.bytes)
         var bytes = [Int8](repeating: 0, count: count)
         // Fill bytes with secure random data
         let status = SecRandomCopyBytes(
@@ -174,7 +174,7 @@ class FileSystemTests: XCTestCase {
         defer {
             fs.removeFileIfExists(fileURL: fileURL)
         }
-        var offset = 0
+        var offset: UInt64 = 0
         var step: ((Int) -> Void)?
         let queue = DispatchQueue(label: "done-count-queue")
 
@@ -190,7 +190,7 @@ class FileSystemTests: XCTestCase {
             let part = parts[index]
 
             print("Creating partial file [\(index)]")
-            fs.createPartialFile(fileURL: fileURL, offset: offset, length: part.count) { result in
+            fs.createPartialFile(fileURL: fileURL, offset: offset, length: UInt64(part.count)) { result in
                 do {
                     let partFileURL = try result.get()
                     let fileContents = try String(contentsOf: partFileURL)
@@ -210,7 +210,7 @@ class FileSystemTests: XCTestCase {
                     XCTFail("Failed to create partial file: \(error)")
                 }
             }
-            offset += part.count
+            offset += UInt64(part.count)
             step?(index + 1)
         }
 

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/MockMultipartUploadClient.swift

@@ -21,7 +21,7 @@ class MockMultipartUploadClient: StorageMultipartUploadClient {
 
     var didCreate: ((StorageMultipartUploadSession) -> Void)?
     var didStartPartUpload: ((StorageMultipartUploadSession, PartNumber) -> Void)?
-    var didTransferBytesForPartUpload: ((StorageMultipartUploadSession, PartNumber, Int) -> Void)?
+    var didTransferBytesForPartUpload: ((StorageMultipartUploadSession, PartNumber, UInt64) -> Void)?
     var shouldFailPartUpload: ((StorageMultipartUploadSession, PartNumber) -> Bool)?
     var didCompletePartUpload: ((StorageMultipartUploadSession, PartNumber, String, TaskIdentifier) -> Void)?
     var didFailPartUpload: ((StorageMultipartUploadSession, PartNumber, Error) -> Void)?

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageMultipartUploadSessionTests.swift

@@ -321,7 +321,7 @@ class StorageMultipartUploadSessionTests: XCTestCase {
     // MARK: - Private -
 
     private func createFile() throws -> URL {
-        let size = minimumPartSize
+        let size = Int(minimumPartSize)
         let parts: [String] = [
             Array(repeating: "a", count: size).joined(),
             Array(repeating: "b", count: size).joined(),

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageTransferDatabaseTests.swift

@@ -196,11 +196,11 @@ class StorageTransferDatabaseTests: XCTestCase {
                 subTask.uploadPart = .pending(bytes: Bytes.megabytes(5).bytes)
                 if index == 0 {
                     parts[index] = .inProgress(bytes: part.bytes,
-                                               bytesTransferred: Int(Double(part.bytes) * 0.75),
+                                               bytesTransferred: UInt64(Double(part.bytes) * 0.75),
                                                taskIdentifier: sessionTask.taskIdentifier)
                 } else if index == 1 {
                     parts[index] = .inProgress(bytes: part.bytes,
-                                               bytesTransferred: Int(Double(part.bytes) * 0.25),
+                                               bytesTransferred: UInt64(Double(part.bytes) * 0.25),
                                                taskIdentifier: sessionTask.taskIdentifier)
                 }
             } else {

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartSizeTests.swift

@@ -48,7 +48,7 @@ class StorageUploadPartSizeTests: XCTestCase {
 
     func testUploadPartSizeForLargeValidFile() throws {
         // use a file size which requires increasing from minimum part size
-        let fileSize = UInt64(minimumPartSize * maximumPartCount * 10)
+        let fileSize = minimumPartSize * UInt64(maximumPartCount) * 10
         let partSize = assertNoThrow(try StorageUploadPartSize(fileSize: fileSize))
         XCTAssertNotNil(partSize)
         if let partSize = partSize,
@@ -62,7 +62,7 @@ class StorageUploadPartSizeTests: XCTestCase {
 
     func testUploadPartSizeForSuperCrazyBigFile() throws {
         // use the maximum object size / max part count
-        let fileSize = UInt64(maximumObjectSize / maximumPartCount)
+        let fileSize = maximumObjectSize / UInt64(maximumPartCount)
         let partSize = assertNoThrow(try StorageUploadPartSize(fileSize: fileSize))
         XCTAssertNotNil(partSize)
         if let partSize = partSize,

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartTests.swift

@@ -15,9 +15,9 @@ class StorageUploadPartTests: XCTestCase {
     func testUploadPartCreation() throws {
         // Creates an array of upload parts with 21 parts
         // where the last part is 512 bytes.
-        let lastPartSize = 512
+        let lastPartSize: UInt64 = 512
         let partSize: StorageUploadPartSize = .default
-        let fileSize: UInt64 = UInt64(partSize.size * 20 + lastPartSize)
+        let fileSize = partSize.size * 20 + lastPartSize
         let parts = try StorageUploadParts(fileSize: fileSize, partSize: partSize)
         XCTAssertEqual(parts.count, 21)
         XCTAssertEqual(parts.pending.count, parts.count)
@@ -43,8 +43,8 @@ class StorageUploadPartTests: XCTestCase {
         XCTAssertEqual(parts.inProgress.count, parts.count)
         XCTAssertEqual(parts.failed.count, 0)
         XCTAssertEqual(parts.completed.count, 0)
-        XCTAssertEqual(parts.totalBytes, 100 * parts.count)
-        XCTAssertEqual(parts.bytesTransferred, 50 * parts.count)
+        XCTAssertEqual(parts.totalBytes, UInt64(100 * parts.count))
+        XCTAssertEqual(parts.bytesTransferred, UInt64(50 * parts.count))
         XCTAssertEqual(parts.percentTransferred, 0.5)
     }