From 0b04c14db26a6c79c674b3e2a91ab589c03fde98 Mon Sep 17 00:00:00 2001 From: Yuhao Su Date: Wed, 10 Jul 2024 20:07:37 -0500 Subject: [PATCH] improve --- src/common/secret/src/secret_manager.rs | 9 +++++---- src/compute/src/lib.rs | 9 +++++++-- src/compute/src/server.rs | 1 + src/frontend/src/lib.rs | 9 +++++++-- src/frontend/src/session.rs | 1 + src/meta/node/src/lib.rs | 9 +++++++-- src/meta/node/src/server.rs | 10 ++++++++-- src/meta/src/manager/env.rs | 4 ++-- src/tests/simulation/src/cluster.rs | 6 ++++++ 9 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/common/secret/src/secret_manager.rs b/src/common/secret/src/secret_manager.rs index b8364888dcad0..7483ac4d25089 100644 --- a/src/common/secret/src/secret_manager.rs +++ b/src/common/secret/src/secret_manager.rs @@ -41,11 +41,12 @@ impl LocalSecretManager { /// Initialize the secret manager with the given temp file path, cluster id, and encryption key. /// # Panics /// Panics if fail to create the secret file directory. - pub fn init(temp_file_dir: Option, cluster_id: String) { + pub fn init(temp_file_dir: String, cluster_id: String, worker_id: u32) { // use `get_or_init` to handle concurrent initialization in single node mode. INSTANCE.get_or_init(|| { - let temp_file_dir = temp_file_dir.unwrap_or_else(|| "./secrets".to_string()); - let secret_file_dir = PathBuf::from(temp_file_dir).join(cluster_id); + let secret_file_dir = PathBuf::from(temp_file_dir) + .join(cluster_id) + .join(worker_id.to_string()); std::fs::remove_dir_all(&secret_file_dir).ok(); std::fs::create_dir_all(&secret_file_dir).unwrap(); @@ -62,7 +63,7 @@ impl LocalSecretManager { pub fn global() -> &'static LocalSecretManager { // Initialize the secret manager for unit tests. #[cfg(debug_assertions)] - LocalSecretManager::init(None, "test".to_string()); + LocalSecretManager::init("./tmp".to_string(), "test_cluster".to_string(), 0); INSTANCE.get().unwrap() } diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs index 36668eae87452..cee364ceb26de 100644 --- a/src/compute/src/lib.rs +++ b/src/compute/src/lib.rs @@ -138,8 +138,13 @@ pub struct ComputeNodeOpts { pub connector_rpc_endpoint: Option, /// The path of the temp secret file directory. - #[clap(long, hide = true, env = "RW_TEMP_SECRET_FILE_DIR")] - pub temp_secret_file_dir: Option, + #[clap( + long, + hide = true, + env = "RW_TEMP_SECRET_FILE_DIR", + default_value = "./secrets" + )] + pub temp_secret_file_dir: String, } impl risingwave_common::opts::Opts for ComputeNodeOpts { diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index d22de8640d74d..8b1ea60cc9b13 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -219,6 +219,7 @@ pub async fn compute_node_serve( LocalSecretManager::init( opts.temp_secret_file_dir, meta_client.cluster_id().to_string(), + worker_id, ); // Initialize observer manager. diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs index 28cb01d4b63f0..3563b8458b358 100644 --- a/src/frontend/src/lib.rs +++ b/src/frontend/src/lib.rs @@ -144,8 +144,13 @@ pub struct FrontendOpts { pub enable_barrier_read: Option, /// The path of the temp secret file directory. - #[clap(long, hide = true, env = "RW_TEMP_SECRET_FILE_DIR")] - pub temp_secret_file_dir: Option, + #[clap( + long, + hide = true, + env = "RW_TEMP_SECRET_FILE_DIR", + default_value = "./secrets" + )] + pub temp_secret_file_dir: String, } impl risingwave_common::opts::Opts for FrontendOpts { diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs index f3a6349027710..7c694ba002662 100644 --- a/src/frontend/src/session.rs +++ b/src/frontend/src/session.rs @@ -322,6 +322,7 @@ impl FrontendEnv { LocalSecretManager::init( opts.temp_secret_file_dir, meta_client.cluster_id().to_string(), + worker_id, ); // This `session_params` should be initialized during the initial notification in `observer_manager` diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs index 3878263259bee..7cc76087d480f 100644 --- a/src/meta/node/src/lib.rs +++ b/src/meta/node/src/lib.rs @@ -193,8 +193,13 @@ pub struct MetaNodeOpts { pub secret_store_private_key_hex: Option, /// The path of the temp secret file directory. - #[clap(long, hide = true, env = "RW_TEMP_SECRET_FILE_DIR")] - pub temp_secret_file_dir: Option, + #[clap( + long, + hide = true, + env = "RW_TEMP_SECRET_FILE_DIR", + default_value = "./secrets" + )] + pub temp_secret_file_dir: String, } impl risingwave_common::opts::Opts for MetaNodeOpts { diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs index 0abd53026586b..41097d4b7e166 100644 --- a/src/meta/node/src/server.rs +++ b/src/meta/node/src/server.rs @@ -30,7 +30,9 @@ use risingwave_common_service::{MetricsManager, TracingExtractLayer}; use risingwave_meta::barrier::StreamRpcManager; use risingwave_meta::controller::catalog::CatalogController; use risingwave_meta::controller::cluster::ClusterController; -use risingwave_meta::manager::{MetaStoreImpl, MetadataManager, SystemParamsManagerImpl}; +use risingwave_meta::manager::{ + MetaStoreImpl, MetadataManager, SystemParamsManagerImpl, META_NODE_ID, +}; use risingwave_meta::rpc::election::dummy::DummyElectionClient; use risingwave_meta::rpc::intercept::MetricsMiddlewareLayer; use risingwave_meta::rpc::ElectionClientRef; @@ -519,7 +521,11 @@ pub async fn start_service_as_election_leader( ) .await?; - LocalSecretManager::init(opts.temp_secret_file_dir, env.cluster_id().to_string()); + LocalSecretManager::init( + opts.temp_secret_file_dir, + env.cluster_id().to_string(), + META_NODE_ID, + ); let notification_srv = NotificationServiceImpl::new( env.clone(), diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs index caefb840b5bd6..ffcb413a05ed8 100644 --- a/src/meta/src/manager/env.rs +++ b/src/meta/src/manager/env.rs @@ -285,7 +285,7 @@ pub struct MetaOpts { // The private key for the secret store, used when the secret is stored in the meta. pub secret_store_private_key: Option>, /// The path of the temp secret file directory. - pub temp_secret_file_dir: Option, + pub temp_secret_file_dir: String, pub table_info_statistic_history_times: usize, } @@ -349,7 +349,7 @@ impl MetaOpts { max_trivial_move_task_count_per_loop: 256, max_get_task_probe_times: 5, secret_store_private_key: Some("0123456789abcdef".as_bytes().to_vec()), - temp_secret_file_dir: None, + temp_secret_file_dir: "./secrets".to_string(), table_info_statistic_history_times: 240, } } diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs index 8e2ffece2fed9..6c9db8c48170f 100644 --- a/src/tests/simulation/src/cluster.rs +++ b/src/tests/simulation/src/cluster.rs @@ -450,6 +450,8 @@ impl Cluster { "hummock+sim://hummockadmin:hummockadmin@192.168.12.1:9301/hummock001", "--data-directory", "hummock_001", + "--temp-secret-file-dir", + &format!("./secrets/meta-{i}"), ]); handle .create_node() @@ -477,6 +479,8 @@ impl Cluster { "0.0.0.0:4566", "--advertise-addr", &format!("192.168.2.{i}:4566"), + "--temp-secret-file-dir", + &format!("./secrets/frontend-{i}"), ]); handle .create_node() @@ -505,6 +509,8 @@ impl Cluster { "6979321856", "--parallelism", &conf.compute_node_cores.to_string(), + "--temp-secret-file-dir", + &format!("./secrets/compute-{i}"), ]); handle .create_node()