From d9925b13c651fdffe8e737a5b7c7487f6c708914 Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 14 Oct 2024 21:01:08 +0800 Subject: [PATCH] feat(nimtabe): change nimtable iceberg bucket and data directory (#18899) --- risedev.yml | 4 ++-- src/common/src/session_config/mod.rs | 2 +- .../src/connector_common/iceberg/mod.rs | 18 ++++++++++-------- src/frontend/src/handler/create_table.rs | 10 ++++++---- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/risedev.yml b/risedev.yml index 599da58ba400b..3378afb21f21b 100644 --- a/risedev.yml +++ b/risedev.yml @@ -138,8 +138,8 @@ profile: NIMTABLE_ENABLE_CONFIG_LOAD: "false" AWS_ENDPOINT_URL: "http://127.0.0.1:9301" AWS_REGION: "ap-southeast-2" - AWS_S3_BUCKET: "hummock001" - RW_DATA_DIRECTORY: "data-dir" + NIMTABLE_S3_BUCKET: "hummock001" + NIMTABLE_DATA_DIRECTORY: "data-dir" AWS_ACCESS_KEY_ID: "hummockadmin" AWS_SECRET_ACCESS_KEY: "hummockadmin" RW_BACKEND: "postgres" diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs index 9a4ef1c7da3c5..4c7ab40fbcaa1 100644 --- a/src/common/src/session_config/mod.rs +++ b/src/common/src/session_config/mod.rs @@ -65,7 +65,7 @@ pub struct SessionConfig { /// If `RW_IMPLICIT_FLUSH` is on, then every INSERT/UPDATE/DELETE statement will block /// until the entire dataflow is refreshed. In other words, every related table & MV will /// be able to see the write. - #[parameter(default = true, rename = "rw_implicit_flush")] + #[parameter(default = true, alias = "rw_implicit_flush")] implicit_flush: bool, /// If `CREATE_COMPACTION_GROUP_FOR_MV` is on, dedicated compaction groups will be created in diff --git a/src/connector/src/connector_common/iceberg/mod.rs b/src/connector/src/connector_common/iceberg/mod.rs index fd8db214bb2e9..9d8377a540dc6 100644 --- a/src/connector/src/connector_common/iceberg/mod.rs +++ b/src/connector/src/connector_common/iceberg/mod.rs @@ -117,12 +117,14 @@ impl IcebergCommon { s3_region.clone().to_string(), ); - let Ok(s3_bucket) = std::env::var("AWS_S3_BUCKET") else { - bail!("To create an iceberg engine table, AWS_S3_BUCKET needed to be set"); + let Ok(s3_bucket) = std::env::var("NIMTABLE_S3_BUCKET") else { + bail!("To create an iceberg engine table, NIMTABLE_S3_BUCKET needed to be set"); }; - let Ok(data_directory) = std::env::var("RW_DATA_DIRECTORY") else { - bail!("To create an iceberg engine table, RW_DATA_DIRECTORY needed to be set"); + let Ok(data_directory) = std::env::var("NIMTABLE_DATA_DIRECTORY") else { + bail!( + "To create an iceberg engine table, NIMTABLE_DATA_DIRECTORY needed to be set" + ); }; let warehouse_path = format!("s3://{}/{}/nimtable", s3_bucket, data_directory); @@ -830,13 +832,13 @@ mod v2 { }; file_io_props.insert(S3_REGION.to_string(), s3_region); - let Ok(s3_bucket) = std::env::var("AWS_S3_BUCKET") else { - bail!("To create an iceberg engine table, AWS_S3_BUCKET needed to be set"); + let Ok(s3_bucket) = std::env::var("NIMTABLE_S3_BUCKET") else { + bail!("To create an iceberg engine table, NIMTABLE_S3_BUCKET needed to be set"); }; - let Ok(data_directory) = std::env::var("RW_DATA_DIRECTORY") else { + let Ok(data_directory) = std::env::var("NIMTABLE_DATA_DIRECTORY") else { bail!( - "To create an iceberg engine table, RW_DATA_DIRECTORY needed to be set" + "To create an iceberg engine table, NIMTABLE_DATA_DIRECTORY needed to be set" ); }; diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs index 3bbb1f31babb8..8c657c9bbe8b4 100644 --- a/src/frontend/src/handler/create_table.rs +++ b/src/frontend/src/handler/create_table.rs @@ -1344,12 +1344,14 @@ pub async fn handle_create_table( bail!("To create an iceberg engine table, AWS_REGION needed to be set"); }; - let Ok(s3_bucket) = std::env::var("AWS_S3_BUCKET") else { - bail!("To create an iceberg engine table, AWS_BUCKET needed to be set"); + let Ok(s3_bucket) = std::env::var("NIMTABLE_S3_BUCKET") else { + bail!("To create an iceberg engine table, NIMTABLE_S3_BUCKET needed to be set"); }; - let Ok(data_directory) = std::env::var("RW_DATA_DIRECTORY") else { - bail!("To create an iceberg engine table, RW_DATA_DIRECTORY needed to be set"); + let Ok(data_directory) = std::env::var("NIMTABLE_DATA_DIRECTORY") else { + bail!( + "To create an iceberg engine table, NIMTABLE_DATA_DIRECTORY needed to be set" + ); }; let s3_ak = if let Ok(s3_ak) = std::env::var("AWS_ACCESS_KEY_ID") {