From c3e42897f07ff066da492adb9d1acbb8eaf8dca0 Mon Sep 17 00:00:00 2001 From: Frank Wang <1454884738@qq.com> Date: Wed, 4 Dec 2024 21:54:43 +0000 Subject: [PATCH 1/3] feat(core): add `content_encoding` to `OpWrite` --- core/src/raw/ops.rs | 12 ++++++++ core/src/services/s3/backend.rs | 1 + core/src/services/s3/core.rs | 5 ++++ core/src/types/capability.rs | 2 ++ core/src/types/operator/operator.rs | 33 +++++++++++++++++++-- core/src/types/operator/operator_futures.rs | 5 ++++ core/tests/behavior/async_write.rs | 19 ++++++++++++ 7 files changed, 75 insertions(+), 2 deletions(-) diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs index 7cdac88a588e..3b196d7b20ee 100644 --- a/core/src/raw/ops.rs +++ b/core/src/raw/ops.rs @@ -541,6 +541,7 @@ pub struct OpWrite { concurrent: usize, content_type: Option, content_disposition: Option, + content_encoding: Option, cache_control: Option, executor: Option, if_match: Option, @@ -598,6 +599,17 @@ impl OpWrite { self } + /// Get the content encoding from option + pub fn content_encoding(&self) -> Option<&str> { + self.content_encoding.as_deref() + } + + /// Set the content encoding of option + pub fn with_content_encoding(mut self, content_encoding: &str) -> Self { + self.content_encoding = Some(content_encoding.to_string()); + self + } + /// Get the cache control from option pub fn cache_control(&self) -> Option<&str> { self.cache_control.as_deref() diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 8ffd6b7ab7b7..eb9a4370b089 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -944,6 +944,7 @@ impl Access for S3Backend { write_can_multi: true, write_with_cache_control: true, write_with_content_type: true, + write_with_content_encoding: true, write_with_if_match: !self.core.disable_write_with_if_match, write_with_if_not_exists: true, write_with_user_metadata: true, diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs index b69ef327ff69..ba6e1e3c08eb 100644 --- a/core/src/services/s3/core.rs +++ b/core/src/services/s3/core.rs @@ -31,6 +31,7 @@ use constants::X_AMZ_META_PREFIX; use http::header::HeaderName; use http::header::CACHE_CONTROL; use http::header::CONTENT_DISPOSITION; +use http::header::CONTENT_ENCODING; use http::header::CONTENT_LENGTH; use http::header::CONTENT_TYPE; use http::header::HOST; @@ -448,6 +449,10 @@ impl S3Core { req = req.header(CONTENT_TYPE, mime) } + if let Some(encoding) = args.content_encoding() { + req = req.header(CONTENT_ENCODING, encoding); + } + if let Some(pos) = args.content_disposition() { req = req.header(CONTENT_DISPOSITION, pos) } diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index 42c064368180..76bbc09611a5 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -128,6 +128,8 @@ pub struct Capability { pub write_with_content_type: bool, /// Indicates if Content-Disposition can be specified during write operations. pub write_with_content_disposition: bool, + /// Indicates if Content-Encoding can be specified during write operations. + pub write_with_content_encoding: bool, /// Indicates if Cache-Control can be specified during write operations. pub write_with_cache_control: bool, /// Indicates if conditional write operations using If-Match are supported. diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index 24bba50d266c..ca7b067e979a 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -1407,9 +1407,38 @@ impl Operator { /// # } /// ``` /// - /// ## `if_none_match` + /// ## `content_encoding` + /// + /// Sets Content-Encoding header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_encoding`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Encoding as system metadata on the target file + /// - The value should follow HTTP Content-Encoding header format + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the content encoding for the written content. + /// + /// ## Example /// - /// Sets an `if none match` condition with specified ETag for this write request. + /// ```no_run + /// # use opendal::Result; + /// # use opendal::Operator; + /// use bytes::Bytes; + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", bs) + /// .content_encoding("br") + /// .await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_none_match` /// /// ### Capability /// diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index de8f277ff284..2d119055794f 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -318,6 +318,11 @@ impl>> FutureWrite { self.map(|(args, options, bs)| (args.with_content_disposition(v), options, bs)) } + /// Set the content encoding of option + pub fn content_encoding(self, v: &str) -> Self { + self.map(|(args, options, bs)| (args.with_content_encoding(v), options, bs)) + } + /// Set the executor for this operation. pub fn executor(self, executor: Executor) -> Self { self.map(|(args, options, bs)| (args.with_executor(executor), options, bs)) diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs index 4540925018d0..e2375e69ed5a 100644 --- a/core/tests/behavior/async_write.rs +++ b/core/tests/behavior/async_write.rs @@ -44,6 +44,7 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_write_with_cache_control, test_write_with_content_type, test_write_with_content_disposition, + test_write_with_content_encoding, test_write_with_if_none_match, test_write_with_if_not_exists, test_write_with_if_match, @@ -211,6 +212,24 @@ pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> { Ok(()) } +/// Write a single file with content encoding should succeed. +pub async fn test_write_with_content_encoding(op: Operator) -> Result<()> { + if !op.info().full_capability().write_with_content_encoding { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + let target_content_encoding = "gzip"; + op.write_with(&path, content) + .content_encoding(target_content_encoding) + .await?; + + // TODO: check the content encoding in the stat op response? + + Ok(()) +} + /// write a single file with user defined metadata should succeed. pub async fn test_write_with_user_metadata(op: Operator) -> Result<()> { if !op.info().full_capability().write_with_user_metadata { From 7f8a00d86c435388e686bfb901cda2f68a95e381 Mon Sep 17 00:00:00 2001 From: Frank Wang <1454884738@qq.com> Date: Wed, 4 Dec 2024 19:21:06 -0600 Subject: [PATCH 2/3] chore: some docs fix --- core/src/services/s3/core.rs | 8 ++++---- core/src/types/operator/operator.rs | 11 +++++++---- core/tests/behavior/async_write.rs | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs index ba6e1e3c08eb..19bdbfdf7209 100644 --- a/core/src/services/s3/core.rs +++ b/core/src/services/s3/core.rs @@ -449,14 +449,14 @@ impl S3Core { req = req.header(CONTENT_TYPE, mime) } - if let Some(encoding) = args.content_encoding() { - req = req.header(CONTENT_ENCODING, encoding); - } - if let Some(pos) = args.content_disposition() { req = req.header(CONTENT_DISPOSITION, pos) } + if let Some(encoding) = args.content_encoding() { + req = req.header(CONTENT_ENCODING, encoding); + } + if let Some(cache_control) = args.cache_control() { req = req.header(CACHE_CONTROL, cache_control) } diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index ca7b067e979a..de4fffe66822 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -1430,16 +1430,19 @@ impl Operator { /// # use opendal::Operator; /// use bytes::Bytes; /// # async fn test(op: Operator) -> Result<()> { - /// let _ = op - /// .write_with("path/to/file", bs) - /// .content_encoding("br") - /// .await?; + /// let bs = b"hello, world!".to_vec(); + /// let _ = op + /// .write_with("path/to/file", bs) + /// .content_encoding("br") + /// .await?; /// # Ok(()) /// # } /// ``` /// /// ## `if_none_match` /// + /// Sets an `if none match` condition with specified ETag for this write request. + /// /// ### Capability /// /// Check [`Capability::write_with_if_none_match`] before using this feature. diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs index e2375e69ed5a..5b0a08b89bad 100644 --- a/core/tests/behavior/async_write.rs +++ b/core/tests/behavior/async_write.rs @@ -218,7 +218,7 @@ pub async fn test_write_with_content_encoding(op: Operator) -> Result<()> { return Ok(()); } - let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); let target_content_encoding = "gzip"; op.write_with(&path, content) From a39a7f29bff82707653ac260a7f5029a5a24f1a7 Mon Sep 17 00:00:00 2001 From: Frank Wang <1454884738@qq.com> Date: Mon, 9 Dec 2024 15:22:19 +0000 Subject: [PATCH 3/3] chore: complete test --- core/tests/behavior/async_write.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs index 5b0a08b89bad..8e577450f232 100644 --- a/core/tests/behavior/async_write.rs +++ b/core/tests/behavior/async_write.rs @@ -225,8 +225,12 @@ pub async fn test_write_with_content_encoding(op: Operator) -> Result<()> { .content_encoding(target_content_encoding) .await?; - // TODO: check the content encoding in the stat op response? - + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!( + meta.content_encoding() + .expect("content encoding must exist"), + target_content_encoding + ); Ok(()) }