refactor: start datanode more flexibly (#2800)

* refactor: start datanode more flexibly * Update src/datanode/src/datanode.rs Co-authored-by: Weny Xu <[email protected]> * fix: resolve PR comments * Apply suggestions from code review Co-authored-by: JeremyHi <[email protected]> --------- Co-authored-by: Weny Xu <[email protected]> Co-authored-by: JeremyHi <[email protected]>
GreptimeTeam · Nov 24, 2023 · 64a36e9 · 64a36e9
1 parent 33566ea
commit 64a36e9
Show file tree

Hide file tree

Showing 15 changed files with 297 additions and 254 deletions.
diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
@@ -12,15 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
 use std::time::Duration;
 
+use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
 use common_telemetry::logging;
 use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use meta_client::MetaClientOptions;
 use servers::Mode;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
 use crate::error::{MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu};
 use crate::options::{Options, TopLevelOptions};
@@ -177,7 +179,27 @@ impl StartCommand {
         logging::info!("Datanode start command: {:#?}", self);
         logging::info!("Datanode options: {:#?}", opts);
 
-        let datanode = DatanodeBuilder::new(opts, None, plugins)
+        let node_id = opts
+            .node_id
+            .context(MissingConfigSnafu { msg: "'node_id'" })?;
+
+        let meta_config = opts.meta_client.as_ref().context(MissingConfigSnafu {
+            msg: "'meta_client_options'",
+        })?;
+
+        let meta_client = datanode::heartbeat::new_metasrv_client(node_id, meta_config)
+            .await
+            .context(StartDatanodeSnafu)?;
+
+        let meta_backend = Arc::new(MetaKvBackend {
+            client: Arc::new(meta_client.clone()),
+        });
+
+        let datanode = DatanodeBuilder::new(opts, plugins)
+            .with_meta_client(meta_client)
+            .with_kv_backend(meta_backend)
+            .enable_region_server_service()
+            .enable_http_service()
             .build()
             .await
             .context(StartDatanodeSnafu)?;

diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs
@@ -225,6 +225,13 @@ pub enum Error {
         #[snafu(source)]
         error: std::io::Error,
     },
+
+    #[snafu(display("Failed to parse address {}", addr))]
+    ParseAddr {
+        addr: String,
+        #[snafu(source)]
+        error: std::net::AddrParseError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -252,7 +259,9 @@ impl ErrorExt for Error {
             | Error::NotDataFromOutput { .. }
             | Error::CreateDir { .. }
             | Error::EmptyResult { .. }
-            | Error::InvalidDatabaseName { .. } => StatusCode::InvalidArguments,
+            | Error::InvalidDatabaseName { .. }
+            | Error::ParseAddr { .. } => StatusCode::InvalidArguments,
+
             Error::StartProcedureManager { source, .. }
             | Error::StopProcedureManager { source, .. } => source.status_code(),
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,

diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
@@ -28,7 +28,7 @@ pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
 /// Options mixed up from datanode, frontend and metasrv.
-#[derive(Serialize)]
+#[derive(Serialize, Debug)]
 pub struct MixOptions {
     pub data_home: String,
     pub procedure: ProcedureConfig,

diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
@@ -169,9 +169,7 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        // Start datanode instance before starting services, to avoid requests come in before internal components are started.
-        self.datanode.start().await.context(StartDatanodeSnafu)?;
-        info!("Datanode instance started");
+        self.datanode.start_telemetry();
 
         self.procedure_manager
             .start()
@@ -325,10 +323,8 @@ impl StartCommand {
         let dn_opts = opts.datanode.clone();
 
         info!("Standalone start command: {:#?}", self);
-        info!(
-            "Standalone frontend options: {:#?}, datanode options: {:#?}",
-            fe_opts, dn_opts
-        );
+
+        info!("Building standalone instance with {opts:#?}");
 
         // Ensure the data_home directory exists.
         fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
@@ -344,14 +340,12 @@ impl StartCommand {
         .await
         .context(StartFrontendSnafu)?;
 
-        let datanode = DatanodeBuilder::new(
-            dn_opts.clone(),
-            Some(kv_backend.clone()),
-            Default::default(),
-        )
-        .build()
-        .await
-        .context(StartDatanodeSnafu)?;
+        let datanode = DatanodeBuilder::new(dn_opts, fe_plugins.clone())
+            .with_kv_backend(kv_backend.clone())
+            .build()
+            .await
+            .context(StartDatanodeSnafu)?;
+
         let region_server = datanode.region_server();
 
         let catalog_manager = KvBackendCatalogManager::new(

diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml
@@ -17,6 +17,7 @@ axum = "0.6"
 axum-macros = "0.3"
 bytes = "1.1"
 catalog.workspace = true
+client.workspace = true
 common-base.workspace = true
 common-catalog.workspace = true
 common-config.workspace = true