feat!: Flexible checksum v2

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Resources/Package.Base.txt

@@ -103,7 +103,8 @@ private var runtimeTargets: [Target] {
                 .smithyEventStreamsAuthAPI,
                 .awsSDKCommon,
                 .awsSDKHTTPAuth,
-                .awsSDKIdentity
+                .awsSDKIdentity,
+                .awsSDKChecksums,
             ],
             path: "Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime",
             resources: [

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/AWSClientConfigDefaultsProvider.swift

@@ -18,6 +18,7 @@ import enum ClientRuntime.ClientLogMode
 import struct SmithyRetries.DefaultRetryStrategy
 import struct SmithyRetries.ExponentialBackoffStrategy
 import struct SmithyRetriesAPI.RetryStrategyOptions
+import enum AWSSDKChecksums.AWSChecksumCalculationMode
 
 typealias RuntimeConfigType = DefaultSDKRuntimeConfiguration<DefaultRetryStrategy, DefaultRetryErrorInfoProvider>
 
@@ -84,6 +85,40 @@ public class AWSClientConfigDefaultsProvider {
         return resolvedAppID
     }
 
+    public static func requestChecksumCalculation(
+        _ requestChecksumCalculation: AWSChecksumCalculationMode? = nil
+    ) throws -> AWSChecksumCalculationMode {
+        let fileBasedConfig = try CRTFileBasedConfiguration.make()
+        let resolvedRequestChecksumCalculation: AWSChecksumCalculationMode
+        if let requestChecksumCalculation {
+            resolvedRequestChecksumCalculation = requestChecksumCalculation
+        } else {
+            resolvedRequestChecksumCalculation = AWSChecksumsConfig.requestChecksumCalculation(
+                configValue: nil,
+                profileName: nil,
+                fileBasedConfig: fileBasedConfig
+            )
+        }
+        return resolvedRequestChecksumCalculation
+    }
+
+    public static func responseChecksumValidation(
+        _ responseChecksumValidation: AWSChecksumCalculationMode? = nil
+    ) throws -> AWSChecksumCalculationMode {
+        let fileBasedConfig = try CRTFileBasedConfiguration.make()
+        let resolvedResponseChecksumValidation: AWSChecksumCalculationMode
+        if let responseChecksumValidation {
+            resolvedResponseChecksumValidation = responseChecksumValidation
+        } else {
+            resolvedResponseChecksumValidation = AWSChecksumsConfig.responseChecksumValidation(
+                configValue: nil,
+                profileName: nil,
+                fileBasedConfig: fileBasedConfig
+            )
+        }
+        return resolvedResponseChecksumValidation
+    }
+
     public static func retryMode(_ retryMode: AWSRetryMode? = nil) throws -> AWSRetryMode {
         let fileBasedConfig = try CRTFileBasedConfiguration.make()
         let resolvedRetryMode: AWSRetryMode?

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Config/AWSChecksumsConfig.swift

@@ -0,0 +1,41 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import enum AWSSDKChecksums.AWSChecksumCalculationMode
+@_spi(FileBasedConfig) import AWSSDKCommon
+
+public enum AWSChecksumsConfig {
+    static func requestChecksumCalculation(
+        configValue: AWSChecksumCalculationMode?,
+        profileName: String?,
+        fileBasedConfig: FileBasedConfiguration
+    ) -> AWSChecksumCalculationMode {
+        return FieldResolver(
+            configValue: configValue,
+            envVarName: "AWS_REQUEST_CHECKSUM_CALCULATION",
+            configFieldName: "request_checksum_calculation",
+            fileBasedConfig: fileBasedConfig,
+            profileName: profileName,
+            converter: { AWSChecksumCalculationMode(caseInsensitiveRawValue: $0) }
+        ).value ?? .whenSupported
+    }
+
+    static func responseChecksumValidation(
+        configValue: AWSChecksumCalculationMode?,
+        profileName: String?,
+        fileBasedConfig: FileBasedConfiguration
+    ) -> AWSChecksumCalculationMode {
+        return FieldResolver(
+            configValue: configValue,
+            envVarName: "AWS_RESPONSE_CHECKSUM_VALIDATION",
+            configFieldName: "response_checksum_validation",
+            fileBasedConfig: fileBasedConfig,
+            profileName: profileName,
+            converter: { AWSChecksumCalculationMode(caseInsensitiveRawValue: $0) }
+        ).value ?? .whenSupported
+    }
+}

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Config/AWSDefaultClientConfiguration.swift

@@ -7,6 +7,7 @@
 
 import SmithyIdentity
 import SmithyIdentityAPI
+import enum AWSSDKChecksums.AWSChecksumCalculationMode
 
 public protocol AWSDefaultClientConfiguration {
     /// The AWS credential identity resolver to be used for AWS credentials.
@@ -45,4 +46,22 @@ public protocol AWSDefaultClientConfiguration {
     ///
     /// If set, this value gets used when resolving max attempts value from the standard progression of potential sources. If no value could be resolved, the SDK uses max attempts value of 3 by default.
     var maxAttempts: Int? { get set }
+
+    /// The AWS request checksum calculation mode to use.
+    ///
+    /// If `.whenRequired`, the client calculates checksum for the request payload only if the operation requires it.
+    /// If `.whenSupported`, the client calculates checksum for the request payload if the operation supports it.
+    ///
+    /// Default mode is `.whenSupported`.
+    ///
+    /// If no algorithm was chosen and no checksum was provided, CRC32 checksum algorithm is used by default.
+    var requestChecksumCalculation: AWSChecksumCalculationMode { get set }
+
+    /// The AWS response checksum calculation mode to use.
+    ///
+    /// If `.whenRequired`, the client validates checksum of the response only if the top-level input field for `requestValidationModeMember` is set to `.enabled` and SDK supports the checksum algorithm.
+    /// If `.whenSupported`, the client validates checksum of the response if the operation supports it and SDK supports at least one of the checksum algorithms returend by service.
+    ///
+    /// Default mode is `.whenSupported`.
+    var responseChecksumValidation: AWSChecksumCalculationMode { get set }
 }

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Middlewares/FlexibleChecksumsRequestMiddleware.swift

@@ -9,6 +9,7 @@
 import enum SmithyChecksums.ChecksumMismatchException
 import enum Smithy.ClientError
 import class Smithy.Context
+import struct Foundation.Data
 import AwsCommonRuntimeKit
 import AWSSDKChecksums
 import ClientRuntime
@@ -18,9 +19,11 @@
 
     public let id: String = "FlexibleChecksumsRequestMiddleware"
 
+    let requestChecksumRequired: Bool
     let checksumAlgorithm: String?
 
-    public init(checksumAlgorithm: String?) {
+    public init(requestChecksumRequired: Bool, checksumAlgorithm: String?) {
+        self.requestChecksumRequired = requestChecksumRequired
         self.checksumAlgorithm = checksumAlgorithm
     }
 
@@ -37,20 +40,6 @@
             throw ClientError.unknownError("No logger found!")
         }
 
-        guard let checksumString = checksumAlgorithm else {
-            logger.info("No checksum provided! Skipping flexible checksums workflow...")
-            return
-        }
-
-        guard let checksumHashFunction = ChecksumAlgorithm.from(string: checksumString) else {
-            logger.info("Found no supported checksums! Skipping flexible checksums workflow...")
-            return
-        }
-
-        // Determine the header name
-        let headerName = "x-amz-checksum-\(checksumHashFunction)"
-        logger.debug("Resolved checksum header name: \(headerName)")
-
         // Check if any checksum header is already provided by the user
         let checksumHeaderPrefix = "x-amz-checksum-"
         if builder.headers.headers.contains(where: {
@@ -61,33 +50,78 @@
             return
         }
 
+        var checksumHashFunction: ChecksumAlgorithm
+        if let checksumAlgorithm {
+            if let hashFunction = ChecksumAlgorithm.from(string: checksumAlgorithm) {
+                // If user chose supported algorithm, continue
+                checksumHashFunction = hashFunction
+            } else {
+                // If user chose unsupported algorithm, throw error
+                throw ClientError.invalidValue("Error: Checksum algorithm \(checksumAlgorithm) is not supported!")
+            }
+        } else {
+            // If user didn't choose an algorithm, then:
+            if requestChecksumRequired || (attributes.requestChecksumCalculation == .whenSupported) {
+                // If requestChecksumRequired == true OR RequestChecksumCalculation == when_supported, use CRC32 as default algorithm.
+                checksumHashFunction = ChecksumAlgorithm.from(string: "crc32")!
+                logger.info("No algorithm chosen by user. Defaulting to CRC32 checksum algorithm.")
+            } else {
+                // If requestChecksumRequired == false AND RequestChecksumCalculation == when_required, skip calculation.
+                logger.info("Checksum not required for the operation.")
+                logger.info("Client config `requestChecksumCalculation` set to `.whenRequired`")
+                logger.info("No checksum algorithm chosen by the user. Skipping checksum calculation...")
+                return
+            }
+        }
+
+        // Determine the header name
+        let headerName = "x-amz-checksum-\(checksumHashFunction)"
+        logger.debug("Resolved checksum header name: \(headerName)")
+
         // Handle body vs handle stream
         switch builder.body {
         case .data(let data):
-            guard let data else {
-                throw ClientError.dataNotFound("Cannot calculate checksum of empty body!")
+            try await calculateAndAddChecksumHeader(data: data)
+        case .stream(let stream):
+            if stream.isEligibleForChunkedStreaming {
+                // Handle calculating and adding checksum header in ChunkedStream
+                attributes.checksum = checksumHashFunction
+                builder.updateHeader(name: "x-amz-trailer", value: [headerName])
+            } else {
+                // If not eligible for chunked streaming, calculate and add checksum to request header now
+                //   instead of in a trailing header.
+                let streamBytes: Data?
+                let currentPosition = stream.position
+                if stream.isSeekable {
+                    // Explicit seek to beginning for correct behavior of FileHandle
+                    try stream.seek(toOffset: 0)
+                    streamBytes = try await stream.readToEnd()
+                    try stream.seek(toOffset: currentPosition)
+                } else {
+                    streamBytes = try await stream.readToEndAsync()
+                    builder.withBody(.data(streamBytes))
+                }
+                try await calculateAndAddChecksumHeader(data: streamBytes)
             }
+        case .noStream:
+            logger.info("Request body is empty. Skipping request checksum calculation...")
+        }
 
+        func calculateAndAddChecksumHeader(data: Data?) async throws {
+            guard let data else {
+                logger.info("Request body is empty. Skipping request checksum calculation...")
+                return
+            }
             if builder.headers.value(for: headerName) == nil {
                 logger.debug("Calculating checksum")
             }
-
             // Create checksum instance
             let checksum = checksumHashFunction.createChecksum()
-
             // Pass data to hash
             try checksum.update(chunk: data)
-
             // Retrieve the hash
             let hash = try checksum.digest().toBase64String()
-
             builder.updateHeader(name: headerName, value: [hash])
-        case .stream:
-            // Will handle calculating checksum and setting header later
-            attributes.checksum = checksumHashFunction
-            builder.updateHeader(name: "x-amz-trailer", value: [headerName])
-        case .noStream:
-            throw ClientError.dataNotFound("Cannot calculate the checksum of an empty body!")
         }
     }
 }

Sources/Core/AWSClientRuntime/Sources/AWSClientRuntime/Middlewares/FlexibleChecksumsResponseMiddleware.swift

@@ -15,14 +15,15 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
     let CHECKSUM_HEADER_VALIDATION_PRIORITY_LIST: [String] = [
         ChecksumAlgorithm.crc32c,
         .crc32,
+        .crc64nvme,
         .sha1,
         .sha256
     ].sorted().map { $0.toString() }
 
-    let validationMode: Bool
+    let validationMode: String
     let priorityList: [String]
 
-    public init(validationMode: Bool, priorityList: [String] = []) {
+    public init(validationMode: String, priorityList: [String] = []) {
         self.validationMode = validationMode
         self.priorityList = !priorityList.isEmpty
             ? withPriority(checksums: priorityList)
@@ -31,13 +32,15 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
 
     private func validateChecksum(response: HTTPResponse, logger: any LogAgent, attributes: Context) async throws {
         // Exit if validation should not be performed
-        if !validationMode {
+        if validationMode != "ENABLED" && attributes.responseChecksumValidation == .whenRequired {
             logger.info("Checksum validation should not be performed! Skipping workflow...")
             return
         }
 
         let checksumHeaderIsPresent = priorityList.first {
-            response.headers.value(for: "x-amz-checksum-\($0)") != nil
+            response.headers.value(for: "x-amz-checksum-\($0)") != nil  &&
+            // Checksum of checksums has "-#" at the end and should be ignored.
+            !(response.headers.value(for: "x-amz-checksum-\($0)")!.hasSuffix("-#"))
         }
 
         guard let checksumHeader = checksumHeaderIsPresent else {
@@ -65,7 +68,8 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
         switch response.body {
         case .data(let data):
             guard let data else {
-                throw ClientError.dataNotFound("Cannot calculate checksum of empty body!")
+                logger.info("Response body is empty. Skipping response checksum validation...")
+                return
             }
 
             let responseChecksumHasher = responseChecksum.createChecksum()
@@ -87,7 +91,8 @@ public struct FlexibleChecksumsResponseMiddleware<OperationStackInput, Operation
             attributes.httpResponse = response
             attributes.httpResponse?.body = validatingStream
         case .noStream:
-            throw ClientError.dataNotFound("Cannot calculate the checksum of an empty body!")
+            logger.info("Response body is empty. Skipping response checksum validation...")
+            return
         }
     }
 }