Skip to content

Commit

Permalink
resolve comment
Browse files Browse the repository at this point in the history
  • Loading branch information
wcy-fdu committed Apr 10, 2024
1 parent c0970e0 commit 5f272c9
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 20 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions src/object_store/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ thiserror-ext = { workspace = true }
tokio = { version = "0.2", package = "madsim-tokio", features = ["fs"] }
tokio-retry = "0.3"
tracing = "0.1"
url = "2"
# This crate is excluded from hakari (see hakari.toml) after hdfs is introduced...## [target.'cfg(not(madsim))'.dependencies]
# workspace-hack = { path = "../workspace-hack" }
#
Expand Down
24 changes: 14 additions & 10 deletions src/object_store/src/object/opendal_engine/opendal_s3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use opendal::raw::HttpClient;
use opendal::services::S3;
use opendal::Operator;
use risingwave_common::config::ObjectStoreConfig;
use url::Url;

use super::{EngineType, OpendalObjectStore};
use crate::object::ObjectResult;
Expand Down Expand Up @@ -69,17 +70,20 @@ impl OpendalObjectStore {
pub fn with_minio(server: &str, object_store_config: ObjectStoreConfig) -> ObjectResult<Self> {
let server = server.strip_prefix("minio://").unwrap();
let (access_key_id, rest) = server.split_once(':').unwrap();
let (secret_access_key, mut rest) = rest.split_once('@').unwrap();
let endpoint_prefix = if let Some(rest_stripped) = rest.strip_prefix("https://") {
rest = rest_stripped;
"https://"
} else if let Some(rest_stripped) = rest.strip_prefix("http://") {
rest = rest_stripped;
"http://"
} else {
"http://"
let (secret_access_key, rest) = rest.split_once('@').unwrap();
let mut parsed_url = Url::parse(rest).unwrap();
let endpoint_prefix = match parsed_url.scheme() {
"https" => {
parsed_url.set_scheme("http").unwrap();
"https://"
}
"http" => "http://",
_ => "http://",
};
let (address, bucket) = rest.split_once('/').unwrap();

let modified_rest = parsed_url.as_str();

let (address, bucket) = modified_rest.split_once('/').unwrap();
let mut builder = S3::default();
builder
.bucket(bucket)
Expand Down
24 changes: 14 additions & 10 deletions src/object_store/src/object/s3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ use risingwave_common::range::RangeBoundsExt;
use thiserror_ext::AsReport;
use tokio::task::JoinHandle;
use tokio_retry::strategy::{jitter, ExponentialBackoff};
use url::Url;

use super::object_metrics::ObjectStoreMetrics;
use super::{
Expand Down Expand Up @@ -644,17 +645,20 @@ impl S3ObjectStore {
) -> Self {
let server = server.strip_prefix("minio://").unwrap();
let (access_key_id, rest) = server.split_once(':').unwrap();
let (secret_access_key, mut rest) = rest.split_once('@').unwrap();
let endpoint_prefix = if let Some(rest_stripped) = rest.strip_prefix("https://") {
rest = rest_stripped;
"https://"
} else if let Some(rest_stripped) = rest.strip_prefix("http://") {
rest = rest_stripped;
"http://"
} else {
"http://"
let (secret_access_key, rest) = rest.split_once('@').unwrap();
let mut parsed_url = Url::parse(rest).unwrap();
let endpoint_prefix = match parsed_url.scheme() {
"https" => {
parsed_url.set_scheme("http").unwrap();
"https://"
}
"http" => "http://",
_ => "http://",
};
let (address, bucket) = rest.split_once('/').unwrap();

let modified_rest = parsed_url.as_str();

let (address, bucket) = modified_rest.split_once('/').unwrap();

#[cfg(madsim)]
let builder = aws_sdk_s3::config::Builder::new().credentials_provider(
Expand Down

0 comments on commit 5f272c9

Please sign in to comment.