refactor!: unify FrontendOptions and DatanodeOptions by using `Gr…

…pcOptions` (#4088) * refactor: move GrpcOptions to servers/grpc * fix: optimize code * fix: docs * refactor: move DatanodeOptions.rpc_hostname to grpc.hostname * fix: merge main * refactor code impl test: add test_depreacted_cli_options unit test * Update src/servers/src/grpc.rs Co-authored-by: Yingwen <[email protected]> --------- Co-authored-by: Yingwen <[email protected]>
GreptimeTeam · Jun 18, 2024 · 22d1268 · 22d1268
1 parent fe74efd
commit 22d1268
Show file tree

Hide file tree

Showing 17 changed files with 210 additions and 103 deletions.
diff --git a/config/config.md b/config/config.md
@@ -305,13 +305,24 @@
 | `node_id` | Integer | `None` | The datanode identifier and should be unique in the cluster. |
 | `require_lease_before_startup` | Bool | `false` | Start services after regions have obtained leases.<br/>It will block the datanode start if it can't receive leases in the heartbeat from metasrv. |
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
-| `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
-| `rpc_addr` | String | `127.0.0.1:3001` | The gRPC address of the datanode. |
-| `rpc_hostname` | String | `None` | The hostname of the datanode. |
-| `rpc_runtime_size` | Integer | `8` | The number of gRPC server worker threads. |
-| `rpc_max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
-| `rpc_max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. |
+| `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
+| `rpc_addr` | String | `None` | Deprecated, use `grpc.addr` instead. |
+| `rpc_hostname` | String | `None` | Deprecated, use `grpc.hostname` instead. |
+| `rpc_runtime_size` | Integer | `None` | Deprecated, use `grpc.runtime_size` instead. |
+| `rpc_max_recv_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_recv_message_size` instead. |
+| `rpc_max_send_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_send_message_size` instead. |
+| `grpc` | -- | -- | The gRPC server options. |
+| `grpc.addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. |
+| `grpc.hostname` | String | `127.0.0.1` | The hostname to advertise to the metasrv. |
+| `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
+| `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
+| `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
+| `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
+| `grpc.tls.mode` | String | `disable` | TLS mode. |
+| `grpc.tls.cert_path` | String | `None` | Certificate file path. |
+| `grpc.tls.key_path` | String | `None` | Private key file path. |
+| `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.read_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.write_rt_size` | Integer | `8` | The number of threads to execute the runtime for global write operations. |

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
@@ -13,27 +13,61 @@ require_lease_before_startup = false
 ## By default, it provides services after all regions have been initialized.
 init_regions_in_background = false
 
+## Enable telemetry to collect anonymous usage data.
+enable_telemetry = true
+
 ## Parallelism of initializing regions.
 init_regions_parallelism = 16
 
-## The gRPC address of the datanode.
+## Deprecated, use `grpc.addr` instead.
+## +toml2docs:none-default
 rpc_addr = "127.0.0.1:3001"
 
-## The hostname of the datanode.
+## Deprecated, use `grpc.hostname` instead.
 ## +toml2docs:none-default
 rpc_hostname = "127.0.0.1"
 
-## The number of gRPC server worker threads.
+## Deprecated, use `grpc.runtime_size` instead.
+## +toml2docs:none-default
 rpc_runtime_size = 8
 
-## The maximum receive message size for gRPC server.
+## Deprecated, use `grpc.rpc_max_recv_message_size` instead.
+## +toml2docs:none-default
 rpc_max_recv_message_size = "512MB"
 
-## The maximum send message size for gRPC server.
+## Deprecated, use `grpc.rpc_max_send_message_size` instead.
+## +toml2docs:none-default
 rpc_max_send_message_size = "512MB"
 
-## Enable telemetry to collect anonymous usage data.
-enable_telemetry = true
+## The gRPC server options.
+[grpc]
+## The address to bind the gRPC server.
+addr = "127.0.0.1:3001"
+## The hostname to advertise to the metasrv.
+hostname = "127.0.0.1"
+## The number of server worker threads.
+runtime_size = 8
+## The maximum receive message size for gRPC server.
+max_recv_message_size = "512MB"
+## The maximum send message size for gRPC server.
+max_send_message_size = "512MB"
+
+## gRPC server TLS options, see `mysql.tls` section.
+[grpc.tls]
+## TLS mode.
+mode = "disable"
+
+## Certificate file path.
+## +toml2docs:none-default
+cert_path = ""
+
+## Private key file path.
+## +toml2docs:none-default
+key_path = ""
+
+## Watch for Certificate and key file change and auto reload.
+## For now, gRPC tls config does not support auto reload.
+watch = false
 
 ## The runtime options.
 [runtime]

diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
@@ -19,8 +19,8 @@ use async_trait::async_trait;
 use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
 use common_config::Configurable;
-use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
+use common_telemetry::{info, warn};
 use common_version::{short_version, version};
 use common_wal::config::DatanodeWalConfig;
 use datanode::datanode::{Datanode, DatanodeBuilder};
@@ -155,6 +155,7 @@ impl StartCommand {
     }
 
     // The precedence order is: cli > config file > environment variables > default values.
+    #[allow(deprecated)]
     fn merge_with_cli_options(
         &self,
         global_options: &GlobalOptions,
@@ -176,11 +177,32 @@ impl StartCommand {
         };
 
         if let Some(addr) = &self.rpc_addr {
-            opts.rpc_addr.clone_from(addr);
+            opts.grpc.addr.clone_from(addr);
+        } else if let Some(addr) = &opts.rpc_addr {
+            warn!("Use the deprecated attribute `DatanodeOptions.rpc_addr`, please use `grpc.addr` instead.");
+            opts.grpc.addr.clone_from(addr);
         }
 
-        if self.rpc_hostname.is_some() {
-            opts.rpc_hostname.clone_from(&self.rpc_hostname);
+        if let Some(hostname) = &self.rpc_hostname {
+            opts.grpc.hostname.clone_from(hostname);
+        } else if let Some(hostname) = &opts.rpc_hostname {
+            warn!("Use the deprecated attribute `DatanodeOptions.rpc_hostname`, please use `grpc.hostname` instead.");
+            opts.grpc.hostname.clone_from(hostname);
+        }
+
+        if let Some(runtime_size) = opts.rpc_runtime_size {
+            warn!("Use the deprecated attribute `DatanodeOptions.rpc_runtime_size`, please use `grpc.runtime_size` instead.");
+            opts.grpc.runtime_size = runtime_size;
+        }
+
+        if let Some(max_recv_message_size) = opts.rpc_max_recv_message_size {
+            warn!("Use the deprecated attribute `DatanodeOptions.rpc_max_recv_message_size`, please use `grpc.max_recv_message_size` instead.");
+            opts.grpc.max_recv_message_size = max_recv_message_size;
+        }
+
+        if let Some(max_send_message_size) = opts.rpc_max_send_message_size {
+            warn!("Use the deprecated attribute `DatanodeOptions.rpc_max_send_message_size`, please use `grpc.max_send_message_size` instead.");
+            opts.grpc.max_send_message_size = max_send_message_size;
         }
 
         if let Some(node_id) = self.node_id {
@@ -302,16 +324,46 @@ mod tests {
     use super::*;
     use crate::options::GlobalOptions;
 
+    #[test]
+    fn test_deprecated_cli_options() {
+        common_telemetry::init_default_ut_logging();
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+            enable_memory_catalog = false
+            node_id = 42
+            
+            rpc_addr = "127.0.0.1:4001"
+            rpc_hostname = "192.168.0.1"
+            [grpc]
+            addr = "127.0.0.1:3001"
+            hostname = "127.0.0.1"
+            runtime_size = 8
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
+        let cmd = StartCommand {
+            config_file: Some(file.path().to_str().unwrap().to_string()),
+            ..Default::default()
+        };
+
+        let options = cmd.load_options(&Default::default()).unwrap().component;
+        assert_eq!("127.0.0.1:4001".to_string(), options.grpc.addr);
+        assert_eq!("192.168.0.1".to_string(), options.grpc.hostname);
+    }
+
     #[test]
     fn test_read_from_config_file() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
             mode = "distributed"
             enable_memory_catalog = false
             node_id = 42
-            rpc_addr = "127.0.0.1:3001"
-            rpc_hostname = "127.0.0.1"
-            rpc_runtime_size = 8
+            
+            [grpc]
+            addr = "127.0.0.1:3001"
+            hostname = "127.0.0.1"
+            runtime_size = 8
 
             [heartbeat]
             interval = "300ms"
@@ -358,7 +410,7 @@ mod tests {
 
         let options = cmd.load_options(&Default::default()).unwrap().component;
 
-        assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
+        assert_eq!("127.0.0.1:3001".to_string(), options.grpc.addr);
         assert_eq!(Some(42), options.node_id);
 
         let DatanodeWalConfig::RaftEngine(raft_engine_config) = options.wal else {
@@ -475,8 +527,8 @@ mod tests {
             enable_memory_catalog = false
             node_id = 42
             rpc_addr = "127.0.0.1:3001"
-            rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
+            rpc_hostname = "10.103.174.219"
 
             [meta_client]
             timeout = "3s"
@@ -572,6 +624,7 @@ mod tests {
                     opts.http.addr,
                     DatanodeOptions::default().component.http.addr
                 );
+                assert_eq!(opts.grpc.hostname, "10.103.174.219");
             },
         );
     }

diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
@@ -370,7 +370,7 @@ mod tests {
     use common_base::readable_size::ReadableSize;
     use common_config::ENV_VAR_SEP;
     use common_test_util::temp_dir::create_named_temp_file;
-    use frontend::service_config::GrpcOptions;
+    use servers::grpc::GrpcOptions;
     use servers::http::HttpOptions;
 
     use super::*;

diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
@@ -49,12 +49,13 @@ use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::server::Services;
 use frontend::service_config::{
-    GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
+    InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
 use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
 use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::export_metrics::ExportMetricsOption;
+use servers::grpc::GrpcOptions;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
@@ -203,7 +204,7 @@ impl StandaloneOptions {
             wal: cloned_opts.wal.into(),
             storage: cloned_opts.storage,
             region_engine: cloned_opts.region_engine,
-            rpc_addr: cloned_opts.grpc.addr,
+            grpc: cloned_opts.grpc,
             ..Default::default()
         }
     }
@@ -350,7 +351,7 @@ impl StartCommand {
 
         if let Some(addr) = &self.rpc_addr {
             // frontend grpc addr conflict with datanode default grpc addr
-            let datanode_grpc_addr = DatanodeOptions::default().rpc_addr;
+            let datanode_grpc_addr = DatanodeOptions::default().grpc.addr;
             if addr.eq(&datanode_grpc_addr) {
                 return IllegalConfigSnafu {
                     msg: format!(

diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
@@ -18,6 +18,9 @@ use cmd::options::GreptimeOptions;
 use cmd::standalone::StandaloneOptions;
 use common_base::readable_size::ReadableSize;
 use common_config::Configurable;
+use common_grpc::channel_manager::{
+    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+};
 use common_runtime::global::RuntimeOptions;
 use common_telemetry::logging::LoggingOptions;
 use common_wal::config::raft_engine::RaftEngineConfig;
@@ -30,7 +33,9 @@ use meta_srv::metasrv::MetasrvOptions;
 use meta_srv::selector::SelectorType;
 use mito2::config::MitoConfig;
 use servers::export_metrics::ExportMetricsOption;
+use servers::grpc::GrpcOptions;
 
+#[allow(deprecated)]
 #[test]
 fn test_load_datanode_example_config() {
     let example_config = common_test_util::find_workspace_path("config/datanode.example.toml");
@@ -46,7 +51,6 @@ fn test_load_datanode_example_config() {
         },
         component: DatanodeOptions {
             node_id: Some(42),
-            rpc_hostname: Some("127.0.0.1".to_string()),
             meta_client: Some(MetaClientOptions {
                 metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
                 timeout: Duration::from_secs(3),
@@ -90,6 +94,12 @@ fn test_load_datanode_example_config() {
                 remote_write: Some(Default::default()),
                 ..Default::default()
             },
+            grpc: GrpcOptions::default().with_addr("127.0.0.1:3001"),
+            rpc_addr: Some("127.0.0.1:3001".to_string()),
+            rpc_hostname: Some("127.0.0.1".to_string()),
+            rpc_runtime_size: Some(8),
+            rpc_max_recv_message_size: Some(DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE),
+            rpc_max_send_message_size: Some(DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE),
             ..Default::default()
         },
     };

diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
@@ -17,9 +17,6 @@
 use common_base::readable_size::ReadableSize;
 use common_base::secrets::{ExposeSecret, SecretString};
 use common_config::Configurable;
-use common_grpc::channel_manager::{
-    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
-};
 pub use common_procedure::options::ProcedureConfig;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use common_wal::config::DatanodeWalConfig;
@@ -28,6 +25,7 @@ use meta_client::MetaClientOptions;
 use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::export_metrics::ExportMetricsOption;
+use servers::grpc::GrpcOptions;
 use servers::heartbeat_options::HeartbeatOptions;
 use servers::http::HttpOptions;
 use servers::Mode;
@@ -265,13 +263,7 @@ pub struct DatanodeOptions {
     pub require_lease_before_startup: bool,
     pub init_regions_in_background: bool,
     pub init_regions_parallelism: usize,
-    pub rpc_addr: String,
-    pub rpc_hostname: Option<String>,
-    pub rpc_runtime_size: usize,
-    // Max gRPC receiving(decoding) message size
-    pub rpc_max_recv_message_size: ReadableSize,
-    // Max gRPC sending(encoding) message size
-    pub rpc_max_send_message_size: ReadableSize,
+    pub grpc: GrpcOptions,
     pub heartbeat: HeartbeatOptions,
     pub http: HttpOptions,
     pub meta_client: Option<MetaClientOptions>,
@@ -283,21 +275,30 @@ pub struct DatanodeOptions {
     pub enable_telemetry: bool,
     pub export_metrics: ExportMetricsOption,
     pub tracing: TracingOptions,
+
+    /// Deprecated options, please use the new options instead.
+    #[deprecated(note = "Please use `grpc.addr` instead.")]
+    pub rpc_addr: Option<String>,
+    #[deprecated(note = "Please use `grpc.hostname` instead.")]
+    pub rpc_hostname: Option<String>,
+    #[deprecated(note = "Please use `grpc.runtime_size` instead.")]
+    pub rpc_runtime_size: Option<usize>,
+    #[deprecated(note = "Please use `grpc.max_recv_message_size` instead.")]
+    pub rpc_max_recv_message_size: Option<ReadableSize>,
+    #[deprecated(note = "Please use `grpc.max_send_message_size` instead.")]
+    pub rpc_max_send_message_size: Option<ReadableSize>,
 }
 
 impl Default for DatanodeOptions {
+    #[allow(deprecated)]
     fn default() -> Self {
         Self {
             mode: Mode::Standalone,
             node_id: None,
             require_lease_before_startup: false,
             init_regions_in_background: false,
             init_regions_parallelism: 16,
-            rpc_addr: "127.0.0.1:3001".to_string(),
-            rpc_hostname: None,
-            rpc_runtime_size: 8,
-            rpc_max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
-            rpc_max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+            grpc: GrpcOptions::default().with_addr("127.0.0.1:3001"),
             http: HttpOptions::default(),
             meta_client: None,
             wal: DatanodeWalConfig::default(),
@@ -311,6 +312,13 @@ impl Default for DatanodeOptions {
             enable_telemetry: true,
             export_metrics: ExportMetricsOption::default(),
             tracing: TracingOptions::default(),
+
+            // Deprecated options
+            rpc_addr: None,
+            rpc_hostname: None,
+            rpc_runtime_size: None,
+            rpc_max_recv_message_size: None,
+            rpc_max_send_message_size: None,
         }
     }
 }