feat: reopen corresponding regions on starting datanode (#2399)

* separate config and datanode impl Signed-off-by: Ruihang Xia <[email protected]> * partial implement of fetching region id list Signed-off-by: Ruihang Xia <[email protected]> * reopen all regions on starting region server Signed-off-by: Ruihang Xia <[email protected]> * update sqlness & assign default datanode id Signed-off-by: Ruihang Xia <[email protected]> * set writable on lease Signed-off-by: Ruihang Xia <[email protected]> * apply cr suggs. Signed-off-by: Ruihang Xia <[email protected]> * Update src/datanode/src/datanode.rs Co-authored-by: Yingwen <[email protected]> --------- Signed-off-by: Ruihang Xia <[email protected]> Co-authored-by: dennis zhuang <[email protected]> Co-authored-by: Yingwen <[email protected]>
GreptimeTeam · Sep 15, 2023 · c149c12 · c149c12
1 parent 0bd6b9b
commit c149c12
Show file tree

Hide file tree

Showing 22 changed files with 678 additions and 557 deletions.
diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
@@ -16,8 +16,8 @@ use std::time::Duration;
 
 use clap::Parser;
 use common_telemetry::logging;
-use datanode::datanode::builder::DatanodeBuilder;
-use datanode::datanode::{Datanode, DatanodeOptions};
+use datanode::config::DatanodeOptions;
+use datanode::datanode::{Datanode, DatanodeBuilder};
 use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;
@@ -163,7 +163,7 @@ impl StartCommand {
         logging::info!("Datanode start command: {:#?}", self);
         logging::info!("Datanode options: {:#?}", opts);
 
-        let datanode = DatanodeBuilder::new(opts, Default::default())
+        let datanode = DatanodeBuilder::new(opts, None, Default::default())
             .build()
             .await
             .context(StartDatanodeSnafu)?;
@@ -179,9 +179,7 @@ mod tests {
 
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::datanode::{
-        CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig,
-    };
+    use datanode::config::{CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig};
     use servers::Mode;
 
     use super::*;

diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
@@ -15,7 +15,7 @@
 use common_config::KvStoreConfig;
 use common_telemetry::logging::LoggingOptions;
 use config::{Config, Environment, File, FileFormat};
-use datanode::datanode::{DatanodeOptions, ProcedureConfig};
+use datanode::config::{DatanodeOptions, ProcedureConfig};
 use frontend::frontend::FrontendOptions;
 use meta_srv::metasrv::MetaSrvOptions;
 use serde::{Deserialize, Serialize};
@@ -26,6 +26,7 @@ use crate::error::{LoadLayeredConfigSnafu, Result};
 pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
+/// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
     pub data_home: String,
     pub procedure_cfg: ProcedureConfig,
@@ -119,7 +120,7 @@ mod tests {
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::datanode::{DatanodeOptions, ObjectStoreConfig};
+    use datanode::config::{DatanodeOptions, ObjectStoreConfig};
 
     use super::*;
 

diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
@@ -23,8 +23,8 @@ use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::datanode::builder::DatanodeBuilder;
-use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig};
+use datanode::config::{DatanodeOptions, ProcedureConfig, StorageConfig};
+use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::region_server::RegionServer;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
@@ -138,6 +138,7 @@ impl StandaloneOptions {
 
     fn datanode_options(self) -> DatanodeOptions {
         DatanodeOptions {
+            node_id: Some(0),
             enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
@@ -306,10 +307,11 @@ impl StartCommand {
         .await
         .context(StartFrontendSnafu)?;
 
-        let datanode = DatanodeBuilder::new(dn_opts.clone(), plugins.clone())
-            .build()
-            .await
-            .context(StartDatanodeSnafu)?;
+        let datanode =
+            DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), plugins.clone())
+                .build()
+                .await
+                .context(StartDatanodeSnafu)?;
         let region_server = datanode.region_server();
 
         let catalog_manager = KvBackendCatalogManager::new(
@@ -468,7 +470,7 @@ mod tests {
         assert!(fe_opts.influxdb_options.enable);
 
         match &dn_opts.storage.store {
-            datanode::datanode::ObjectStoreConfig::S3(s3_config) => {
+            datanode::config::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
                     "Secret([REDACTED alloc::string::String])".to_string(),
                     format!("{:?}", s3_config.access_key_id)

diff --git a/src/datanode/src/alive_keeper.rs b/src/datanode/src/alive_keeper.rs
@@ -42,10 +42,14 @@ const MAX_CLOSE_RETRY_TIMES: usize = 10;
 ///
 /// [RegionAliveKeeper] starts a [CountdownTask] for each region. When deadline is reached,
 /// the region will be closed.
+///
 /// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
 /// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
 /// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
 /// countdown.
+///
+/// On each lease extension, [RegionAliveKeeper] will reset the deadline to the corresponding time, and
+/// set region's status to "writable".
 pub struct RegionAliveKeeper {
     region_server: RegionServer,
     tasks: Arc<Mutex<HashMap<RegionId, Arc<CountdownTaskHandle>>>>,
@@ -313,6 +317,7 @@ impl CountdownTask {
                                     "Reset deadline of region {region_id} to approximately {} seconds later",
                                     (deadline - Instant::now()).as_secs_f32(),
                                 );
+                                let _ = self.region_server.set_writable(self.region_id, true);
                                 countdown.set(tokio::time::sleep_until(deadline));
                             }
                             // Else the countdown could be either: