diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs index 6f6b932d6ff5..6307d7f3be78 100644 --- a/object_store/src/aws/client.rs +++ b/object_store/src/aws/client.rs @@ -27,7 +27,10 @@ use crate::client::header::{get_etag, HeaderConfig}; use crate::client::header::{get_put_result, get_version}; use crate::client::list::ListClient; use crate::client::retry::RetryExt; -use crate::client::s3::{CompleteMultipartUpload, CompleteMultipartUploadResult, CopyPartResult, InitiateMultipartUploadResult, ListResponse, MultipartPart}; +use crate::client::s3::{ + CompleteMultipartUpload, CompleteMultipartUploadResult, CopyPartResult, + InitiateMultipartUploadResult, ListResponse, MultipartPart, +}; use crate::client::GetOptionsExt; use crate::path::DELIMITER; use crate::{ @@ -660,7 +663,6 @@ impl S3Client { .request_with_config(Method::PUT, path, &config) .query(&[("partNumber", &part), ("uploadId", upload_id)]) .idempotent(true); - let checksum_sha256 = request.payload_sha256.map(|digest| digest.as_ref().into()); request = match data { PutPartPayload::Part(payload) => request.with_payload(payload), @@ -680,6 +682,11 @@ impl S3Client { request = request.with_encryption_headers(); } let response = request.send().await?; + let checksum_sha256 = response + .headers() + .get("x-amz-checksum-sha256") + .map(|v| v.to_str().unwrap().to_string()); + println!("response={response:#?}"); let content_id = match is_copy { false => get_etag(response.headers()).context(MetadataSnafu)?, diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs index 13d3a54b1853..0ee77d66143d 100644 --- a/object_store/src/aws/mod.rs +++ b/object_store/src/aws/mod.rs @@ -40,7 +40,7 @@ use crate::aws::client::{CompleteMultipartMode, PutPartPayload, RequestError, S3 use crate::client::get::GetClientExt; use crate::client::list::ListClientExt; use crate::client::CredentialProvider; -use crate::multipart::{MultipartStore}; +use crate::multipart::MultipartStore; use crate::signer::Signer; use crate::util::STRICT_ENCODE_SET; use crate::{ @@ -74,8 +74,8 @@ const STORE: &str = "S3"; /// [`CredentialProvider`] for [`AmazonS3`] pub type AwsCredentialProvider = Arc>; use crate::client::parts::Parts; -pub use credential::{AwsAuthorizer, AwsCredential}; use crate::client::s3::MultipartPart; +pub use credential::{AwsAuthorizer, AwsCredential}; /// Interface for [Amazon S3](https://aws.amazon.com/s3/). #[derive(Debug, Clone)] diff --git a/object_store/src/azure/client.rs b/object_store/src/azure/client.rs index 23edd6e4fdb1..31ca28d55acd 100644 --- a/object_store/src/azure/client.rs +++ b/object_store/src/azure/client.rs @@ -22,6 +22,7 @@ use crate::client::get::GetClient; use crate::client::header::{get_put_result, HeaderConfig}; use crate::client::list::ListClient; use crate::client::retry::RetryExt; +use crate::client::s3::MultipartPart; use crate::client::GetOptionsExt; use crate::multipart::PartId; use crate::path::DELIMITER; @@ -46,7 +47,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use url::Url; -use crate::client::s3::MultipartPart; const VERSION_HEADER: &str = "x-ms-version-id"; const USER_DEFINED_METADATA_HEADER_PREFIX: &str = "x-ms-meta-"; diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs index 2b3cd7b13355..fd42f1ce75d2 100644 --- a/object_store/src/azure/mod.rs +++ b/object_store/src/azure/mod.rs @@ -23,11 +23,9 @@ //! //! Unused blocks will automatically be dropped after 7 days. use crate::{ - multipart::{MultipartStore}, - path::Path, - signer::Signer, - GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, - PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, UploadPart, + multipart::MultipartStore, path::Path, signer::Signer, GetOptions, GetResult, ListResult, + MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, + PutPayload, PutResult, Result, UploadPart, }; use async_trait::async_trait; use futures::stream::{BoxStream, StreamExt, TryStreamExt}; @@ -50,9 +48,9 @@ mod credential; pub type AzureCredentialProvider = Arc>; use crate::azure::client::AzureClient; use crate::client::parts::Parts; +use crate::client::s3::MultipartPart; pub use builder::{AzureConfigKey, MicrosoftAzureBuilder}; pub use credential::AzureCredential; -use crate::client::s3::MultipartPart; const STORE: &str = "MicrosoftAzure"; diff --git a/object_store/src/client/parts.rs b/object_store/src/client/parts.rs index c0895d9446a0..d41db959cbf5 100644 --- a/object_store/src/client/parts.rs +++ b/object_store/src/client/parts.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use parking_lot::Mutex; use crate::client::s3::MultipartPart; +use parking_lot::Mutex; /// An interior mutable collection of upload parts and their corresponding part index #[derive(Debug, Default)] diff --git a/object_store/src/client/s3.rs b/object_store/src/client/s3.rs index 5bfa20d4897d..c69f048fc532 100644 --- a/object_store/src/client/s3.rs +++ b/object_store/src/client/s3.rs @@ -143,7 +143,7 @@ pub struct MultipartPart { #[serde(rename = "PartNumber")] pub part_number: usize, #[serde(rename = "ChecksumSHA256")] - pub checksum_sha256: Option>, + pub checksum_sha256: Option, } #[derive(Debug, Deserialize)] diff --git a/object_store/src/gcp/client.rs b/object_store/src/gcp/client.rs index 403eaf39c7b7..58a04ddaab48 100644 --- a/object_store/src/gcp/client.rs +++ b/object_store/src/gcp/client.rs @@ -19,7 +19,10 @@ use crate::client::get::GetClient; use crate::client::header::{get_put_result, get_version, HeaderConfig}; use crate::client::list::ListClient; use crate::client::retry::RetryExt; -use crate::client::s3::{CompleteMultipartUpload, CompleteMultipartUploadResult, InitiateMultipartUploadResult, ListResponse, MultipartPart}; +use crate::client::s3::{ + CompleteMultipartUpload, CompleteMultipartUploadResult, InitiateMultipartUploadResult, + ListResponse, MultipartPart, +}; use crate::client::GetOptionsExt; use crate::gcp::{GcpCredential, GcpCredentialProvider, GcpSigningCredentialProvider, STORE}; use crate::multipart::PartId; diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs index e2732238e3e4..f75f99d62009 100644 --- a/object_store/src/gcp/mod.rs +++ b/object_store/src/gcp/mod.rs @@ -41,9 +41,8 @@ use crate::client::CredentialProvider; use crate::gcp::credential::GCSAuthorizer; use crate::signer::Signer; use crate::{ - path::Path, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, - ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, - UploadPart, + path::Path, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, + ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, UploadPart, }; use async_trait::async_trait; use client::GoogleCloudStorageClient; @@ -54,10 +53,10 @@ use url::Url; use crate::client::get::GetClientExt; use crate::client::list::ListClientExt; use crate::client::parts::Parts; +use crate::client::s3::MultipartPart; use crate::multipart::MultipartStore; pub use builder::{GoogleCloudStorageBuilder, GoogleConfigKey}; pub use credential::{GcpCredential, GcpSigningCredential, ServiceAccountKey}; -use crate::client::s3::MultipartPart; mod builder; mod client; diff --git a/object_store/src/memory.rs b/object_store/src/memory.rs index f383ec35813f..eb995c4df209 100644 --- a/object_store/src/memory.rs +++ b/object_store/src/memory.rs @@ -27,7 +27,8 @@ use futures::{stream::BoxStream, StreamExt}; use parking_lot::RwLock; use snafu::{OptionExt, ResultExt, Snafu}; -use crate::multipart::{MultipartStore}; +use crate::client::s3::MultipartPart; +use crate::multipart::MultipartStore; use crate::util::InvalidGetRange; use crate::{ path::Path, Attributes, GetRange, GetResult, GetResultPayload, ListResult, MultipartId, @@ -35,7 +36,6 @@ use crate::{ Result, UpdateVersion, UploadPart, }; use crate::{GetOptions, PutPayload}; -use crate::client::s3::MultipartPart; /// A specialized `Error` for in-memory object store-related errors #[derive(Debug, Snafu)] diff --git a/object_store/src/multipart.rs b/object_store/src/multipart.rs index 436d310ca0ce..e52daf2c7b9b 100644 --- a/object_store/src/multipart.rs +++ b/object_store/src/multipart.rs @@ -23,9 +23,9 @@ use async_trait::async_trait; +use crate::client::s3::MultipartPart; use crate::path::Path; use crate::{MultipartId, PutPayload, PutResult, Result}; -use crate::client::s3::MultipartPart; /// Represents a part of a file that has been successfully uploaded in a multipart upload process. #[derive(Debug, Clone)] @@ -36,7 +36,9 @@ pub struct PartId { impl From for PartId { fn from(value: MultipartPart) -> Self { - PartId { content_id: value.e_tag.clone() } + PartId { + content_id: value.e_tag.clone(), + } } } diff --git a/object_store/src/throttle.rs b/object_store/src/throttle.rs index 3f3aff48aded..5b2b218ce8de 100644 --- a/object_store/src/throttle.rs +++ b/object_store/src/throttle.rs @@ -20,7 +20,8 @@ use parking_lot::Mutex; use std::ops::Range; use std::{convert::TryInto, sync::Arc}; -use crate::multipart::{MultipartStore}; +use crate::client::s3::MultipartPart; +use crate::multipart::MultipartStore; use crate::{ path::Path, GetResult, GetResultPayload, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, @@ -30,7 +31,6 @@ use async_trait::async_trait; use bytes::Bytes; use futures::{stream::BoxStream, FutureExt, StreamExt}; use std::time::Duration; -use crate::client::s3::MultipartPart; /// Configuration settings for throttled store #[derive(Debug, Default, Clone, Copy)]