refactor: introduce common-wal to aggregate wal stuff (#3171)

* refactor: aggregate wal configs * refactor: move wal options to common-wal * chore: slim Cargo.toml * fix: add missing crates * fix: format * chore: update comments * chore: add testing feature gate for test_util * fix: apply suggestions from code review Co-authored-by: JeremyHi <[email protected]> * fix: apply suggestions from code review * fix: compiling --------- Co-authored-by: JeremyHi <[email protected]>
GreptimeTeam · Jan 18, 2024 · 6320590 · 6320590
1 parent 3d7d2fd
commit 6320590
Show file tree

Hide file tree

Showing 62 changed files with 737 additions and 574 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -29,6 +29,7 @@ members = [
     "src/common/time",
     "src/common/decimal",
     "src/common/version",
+    "src/common/wal",
     "src/datanode",
     "src/datatypes",
     "src/file-engine",
@@ -124,6 +125,7 @@ rskafka = "0.5"
 rust_decimal = "1.33"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+serde_with = "3"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.7"
 sysinfo = "0.30"
@@ -169,6 +171,7 @@ common-telemetry = { path = "src/common/telemetry" }
 common-test-util = { path = "src/common/test-util" }
 common-time = { path = "src/common/time" }
 common-version = { path = "src/common/version" }
+common-wal = { path = "src/common/wal" }
 datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
 file-engine = { path = "src/file-engine" }

diff --git a/config/standalone.example.toml b/config/standalone.example.toml
@@ -88,7 +88,25 @@ enable = true
 # - "kafka"
 provider = "raft_engine"
 
-# There're none raft-engine wal config since meta srv only involves in remote wal currently.
+# Raft-engine wal options.
+# WAL data directory
+# dir = "/tmp/greptimedb/wal"
+# WAL file size in bytes.
+file_size = "256MB"
+# WAL purge threshold.
+purge_threshold = "4GB"
+# WAL purge interval in seconds.
+purge_interval = "10m"
+# WAL read batch size.
+read_batch_size = 128
+# Whether to sync log file after every write.
+sync_write = false
+# Whether to reuse logically truncated log files.
+enable_log_recycle = true
+# Whether to pre-create log files on start up
+prefill_log_files = false
+# Duration for fsyncing log files.
+sync_period = "1000ms"
 
 # Kafka wal options.
 # The broker endpoints of the Kafka cluster. ["127.0.0.1:9092"] by default.
@@ -125,25 +143,6 @@ provider = "raft_engine"
 # The deadline of retries.
 # backoff_deadline = "5mins"
 
-# WAL data directory
-# dir = "/tmp/greptimedb/wal"
-# WAL file size in bytes.
-file_size = "256MB"
-# WAL purge threshold.
-purge_threshold = "4GB"
-# WAL purge interval in seconds.
-purge_interval = "10m"
-# WAL read batch size.
-read_batch_size = 128
-# Whether to sync log file after every write.
-sync_write = false
-# Whether to reuse logically truncated log files.
-enable_log_recycle = true
-# Whether to pre-create log files on start up
-prefill_log_files = false
-# Duration for fsyncing log files.
-sync_period = "1000ms"
-
 # Metadata storage options.
 [metadata_store]
 # Kv file size in bytes.

diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
@@ -34,6 +34,7 @@ common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
 common-time.workspace = true
+common-wal.workspace = true
 config = "0.13"
 datanode.workspace = true
 datatypes.workspace = true

diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
@@ -18,8 +18,8 @@ use std::time::Duration;
 use async_trait::async_trait;
 use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
-use common_config::WalConfig;
 use common_telemetry::{info, logging};
+use common_wal::config::DatanodeWalConfig;
 use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::service::DatanodeServiceBuilder;
@@ -174,7 +174,7 @@ impl StartCommand {
 
         // `wal_dir` only affects raft-engine config.
         if let Some(wal_dir) = &self.wal_dir
-            && let WalConfig::RaftEngine(raft_engine_config) = &mut opts.wal
+            && let DatanodeWalConfig::RaftEngine(raft_engine_config) = &mut opts.wal
         {
             if raft_engine_config
                 .dir
@@ -316,7 +316,7 @@ mod tests {
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
 
-        let WalConfig::RaftEngine(raft_engine_config) = options.wal else {
+        let DatanodeWalConfig::RaftEngine(raft_engine_config) = options.wal else {
             unreachable!()
         };
         assert_eq!("/other/wal", raft_engine_config.dir.unwrap());
@@ -504,7 +504,7 @@ mod tests {
                 };
 
                 // Should be read from env, env > default values.
-                let WalConfig::RaftEngine(raft_engine_config) = opts.wal else {
+                let DatanodeWalConfig::RaftEngine(raft_engine_config) = opts.wal else {
                     unreachable!()
                 };
                 assert_eq!(raft_engine_config.read_batch_size, 100);

diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
@@ -14,8 +14,8 @@
 
 use clap::ArgMatches;
 use common_config::KvBackendConfig;
-use common_meta::wal::WalConfig as MetaSrvWalConfig;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
+use common_wal::config::MetaSrvWalConfig;
 use config::{Config, Environment, File, FileFormat};
 use datanode::config::{DatanodeOptions, ProcedureConfig};
 use frontend::error::{Result as FeResult, TomlFormatSnafu};
@@ -173,8 +173,8 @@ impl Options {
 mod tests {
     use std::io::Write;
 
-    use common_config::WalConfig;
     use common_test_util::temp_dir::create_named_temp_file;
+    use common_wal::config::DatanodeWalConfig;
     use datanode::config::{DatanodeOptions, ObjectStoreConfig};
 
     use super::*;
@@ -281,7 +281,7 @@ mod tests {
                 );
 
                 // Should be the values from config file, not environment variables.
-                let WalConfig::RaftEngine(raft_engine_config) = opts.wal else {
+                let DatanodeWalConfig::RaftEngine(raft_engine_config) = opts.wal else {
                     unreachable!()
                 };
                 assert_eq!(raft_engine_config.dir.unwrap(), "/tmp/greptimedb/wal");

diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
@@ -18,7 +18,6 @@ use std::{fs, path};
 use async_trait::async_trait;
 use clap::Parser;
 use common_catalog::consts::MIN_USER_TABLE_ID;
-use common_config::wal::StandaloneWalConfig;
 use common_config::{metadata_store_dir, KvBackendConfig};
 use common_meta::cache_invalidator::DummyCacheInvalidator;
 use common_meta::datanode_manager::DatanodeManagerRef;
@@ -29,11 +28,12 @@ use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::region_keeper::MemoryRegionKeeper;
 use common_meta::sequence::SequenceBuilder;
-use common_meta::wal::{WalOptionsAllocator, WalOptionsAllocatorRef};
+use common_meta::wal_options_allocator::{WalOptionsAllocator, WalOptionsAllocatorRef};
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
 use common_time::timezone::set_default_timezone;
+use common_wal::config::StandaloneWalConfig;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use file_engine::config::EngineConfig as FileEngineConfig;
@@ -497,8 +497,8 @@ mod tests {
 
     use auth::{Identity, Password, UserProviderRef};
     use common_base::readable_size::ReadableSize;
-    use common_config::WalConfig;
     use common_test_util::temp_dir::create_named_temp_file;
+    use common_wal::config::DatanodeWalConfig;
     use datanode::config::{FileConfig, GcsConfig};
     use servers::Mode;
 
@@ -605,7 +605,7 @@ mod tests {
         assert_eq!(None, fe_opts.mysql.reject_no_database);
         assert!(fe_opts.influxdb.enable);
 
-        let WalConfig::RaftEngine(raft_engine_config) = dn_opts.wal else {
+        let DatanodeWalConfig::RaftEngine(raft_engine_config) = dn_opts.wal else {
             unreachable!()
         };
         assert_eq!("/tmp/greptimedb/test/wal", raft_engine_config.dir.unwrap());

diff --git a/src/common/config/Cargo.toml b/src/common/config/Cargo.toml
@@ -8,9 +8,5 @@ license.workspace = true
 common-base.workspace = true
 humantime-serde.workspace = true
 num_cpus.workspace = true
-rskafka.workspace = true
 serde.workspace = true
-serde_json.workspace = true
-serde_with = "3"
 sysinfo.workspace = true
-toml.workspace = true
diff --git a/src/common/config/src/lib.rs b/src/common/config/src/lib.rs
@@ -13,13 +13,10 @@
 // limitations under the License.
 
 pub mod utils;
-pub mod wal;
 
 use common_base::readable_size::ReadableSize;
 use serde::{Deserialize, Serialize};
 
-pub use crate::wal::{KafkaWalOptions, WalConfig, WalOptions, WAL_OPTIONS_KEY};
-
 pub fn metadata_store_dir(store_dir: &str) -> String {
     format!("{store_dir}/metadata")
 }