Skip to content

Commit

Permalink
feat: upgrade to opendal 0.47 in object store (#17198)
Browse files Browse the repository at this point in the history
  • Loading branch information
hzxa21 authored Jun 18, 2024
1 parent 3c4daa8 commit b786edc
Show file tree
Hide file tree
Showing 16 changed files with 349 additions and 112 deletions.
140 changes: 110 additions & 30 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 0 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -319,8 +319,6 @@ tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev =
futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" }
# patch: unlimit 4MB message size for grpc client
etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" }
# todo(wcy-fdu): remove this patch fork after opendal release a new version to apply azure workload identity change.
reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "c7dd668" }
# patch to remove preserve_order from serde_json
deno_core = { git = "https://github.com/bakjos/deno_core", rev = "9b241c6" }
# patch to user reqwest 0.12.2
Expand Down
13 changes: 13 additions & 0 deletions src/common/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1045,6 +1045,14 @@ pub struct ObjectStoreConfig {
/// Some special configuration of S3 Backend
#[serde(default)]
pub s3: S3ObjectStoreConfig,

// TODO: the following field will be deprecated after opendal is stablized
#[serde(default = "default::object_store_config::opendal_upload_concurrency")]
pub opendal_upload_concurrency: usize,

// TODO: the following field will be deprecated after opendal is stablized
#[serde(default)]
pub opendal_writer_abort_on_err: bool,
}

impl ObjectStoreConfig {
Expand Down Expand Up @@ -1103,6 +1111,7 @@ pub struct S3ObjectStoreDeveloperConfig {
)]
pub retryable_service_error_codes: Vec<String>,

// TODO: the following field will be deprecated after opendal is stablized
#[serde(default = "default::object_store_config::s3::developer::use_opendal")]
pub use_opendal: bool,
}
Expand Down Expand Up @@ -2004,6 +2013,10 @@ pub mod default {
DEFAULT_REQ_MAX_RETRY_ATTEMPTS
}

pub fn opendal_upload_concurrency() -> usize {
8
}

pub mod s3 {
const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;

Expand Down
Loading

0 comments on commit b786edc

Please sign in to comment.