From 8c933ea9d48aa3c2282571318e22da68749c10b5 Mon Sep 17 00:00:00 2001 From: evenyag Date: Wed, 21 Feb 2024 19:52:23 +0800 Subject: [PATCH 1/5] fix: always converts path to slash --- Cargo.lock | 7 +++++++ src/operator/Cargo.toml | 1 + src/operator/src/statement/copy_database.rs | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 522c9790bec2..3bfb67f063bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6078,6 +6078,7 @@ dependencies = [ "meter-macros", "object-store", "partition", + "path-slash", "prometheus", "query", "regex", @@ -6352,6 +6353,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "pathdiff" version = "0.2.1" diff --git a/src/operator/Cargo.toml b/src/operator/Cargo.toml index 0543604528f9..88effdebfff3 100644 --- a/src/operator/Cargo.toml +++ b/src/operator/Cargo.toml @@ -58,3 +58,4 @@ tonic.workspace = true [dev-dependencies] common-test-util.workspace = true +path-slash = "0.2" diff --git a/src/operator/src/statement/copy_database.rs b/src/operator/src/statement/copy_database.rs index 0ab4d09cf752..b5e16c106148 100644 --- a/src/operator/src/statement/copy_database.rs +++ b/src/operator/src/statement/copy_database.rs @@ -206,6 +206,7 @@ mod tests { use object_store::services::Fs; use object_store::util::normalize_dir; use object_store::ObjectStore; + use path_slash::PathExt; use table::requests::CopyDatabaseRequest; use crate::statement::copy_database::{list_files_to_copy, parse_file_name_to_copy}; @@ -223,10 +224,11 @@ mod tests { object_store.write("d", "").await.unwrap(); object_store.write("e.f.parquet", "").await.unwrap(); + let location = dir.path().to_slash().unwrap().to_string(); let request = CopyDatabaseRequest { catalog_name: "catalog_0".to_string(), schema_name: "schema_0".to_string(), - location: store_dir, + location, with: [("FORMAT".to_string(), "parquet".to_string())] .into_iter() .collect(), From c7fe18ca6a39ee96dfc8abe4655d6480bbaac093 Mon Sep 17 00:00:00 2001 From: evenyag Date: Wed, 21 Feb 2024 20:18:28 +0800 Subject: [PATCH 2/5] chore: print --- src/common/datasource/src/object_store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/datasource/src/object_store.rs b/src/common/datasource/src/object_store.rs index c9e36018c22b..393b82eaf07c 100644 --- a/src/common/datasource/src/object_store.rs +++ b/src/common/datasource/src/object_store.rs @@ -56,6 +56,7 @@ pub fn parse_url(url: &str) -> Result<(String, Option, String)> { pub fn build_backend(url: &str, connection: &HashMap) -> Result { let (schema, host, path) = parse_url(url)?; let (root, _) = find_dir_and_filename(&path); + println!("schema {schema}, host {host:?} path {path} root {root}"); match schema.to_uppercase().as_str() { S3_SCHEMA => { From 9f5079382d271ea3cd84110fd01318d76676127a Mon Sep 17 00:00:00 2001 From: evenyag Date: Wed, 21 Feb 2024 20:37:53 +0800 Subject: [PATCH 3/5] chore: normalize dir --- src/operator/src/statement/copy_database.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/operator/src/statement/copy_database.rs b/src/operator/src/statement/copy_database.rs index b5e16c106148..64ae69aa4563 100644 --- a/src/operator/src/statement/copy_database.rs +++ b/src/operator/src/statement/copy_database.rs @@ -224,7 +224,8 @@ mod tests { object_store.write("d", "").await.unwrap(); object_store.write("e.f.parquet", "").await.unwrap(); - let location = dir.path().to_slash().unwrap().to_string(); + let location = normalize_dir(dir.path().to_slash().unwrap()); + println!("location is {}", location); let request = CopyDatabaseRequest { catalog_name: "catalog_0".to_string(), schema_name: "schema_0".to_string(), From 171dbdd59f2c8faaeec461ae9d13f8e79cd30929 Mon Sep 17 00:00:00 2001 From: evenyag Date: Wed, 21 Feb 2024 20:40:26 +0800 Subject: [PATCH 4/5] chore: compile --- src/operator/src/statement/copy_database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/src/statement/copy_database.rs b/src/operator/src/statement/copy_database.rs index 64ae69aa4563..a962075a7769 100644 --- a/src/operator/src/statement/copy_database.rs +++ b/src/operator/src/statement/copy_database.rs @@ -224,7 +224,7 @@ mod tests { object_store.write("d", "").await.unwrap(); object_store.write("e.f.parquet", "").await.unwrap(); - let location = normalize_dir(dir.path().to_slash().unwrap()); + let location = normalize_dir(&dir.path().to_slash().unwrap()); println!("location is {}", location); let request = CopyDatabaseRequest { catalog_name: "catalog_0".to_string(), From c22aadaa33e022a6e003b6c6e1b3faf7df0275da Mon Sep 17 00:00:00 2001 From: evenyag Date: Wed, 21 Feb 2024 20:44:14 +0800 Subject: [PATCH 5/5] chore: rm print --- src/common/datasource/src/object_store.rs | 1 - src/operator/src/statement/copy_database.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/common/datasource/src/object_store.rs b/src/common/datasource/src/object_store.rs index 393b82eaf07c..c9e36018c22b 100644 --- a/src/common/datasource/src/object_store.rs +++ b/src/common/datasource/src/object_store.rs @@ -56,7 +56,6 @@ pub fn parse_url(url: &str) -> Result<(String, Option, String)> { pub fn build_backend(url: &str, connection: &HashMap) -> Result { let (schema, host, path) = parse_url(url)?; let (root, _) = find_dir_and_filename(&path); - println!("schema {schema}, host {host:?} path {path} root {root}"); match schema.to_uppercase().as_str() { S3_SCHEMA => { diff --git a/src/operator/src/statement/copy_database.rs b/src/operator/src/statement/copy_database.rs index a962075a7769..63236e3eb9f6 100644 --- a/src/operator/src/statement/copy_database.rs +++ b/src/operator/src/statement/copy_database.rs @@ -225,7 +225,6 @@ mod tests { object_store.write("e.f.parquet", "").await.unwrap(); let location = normalize_dir(&dir.path().to_slash().unwrap()); - println!("location is {}", location); let request = CopyDatabaseRequest { catalog_name: "catalog_0".to_string(), schema_name: "schema_0".to_string(),