fix configure

Signed-off-by: Little-Wallace <[email protected]>

src/common/src/config.rs

@@ -336,6 +336,12 @@ pub struct MetaConfig {
 
     #[serde(default = "default::meta::hybird_partition_vnode_count")]
     pub hybird_partition_vnode_count: u32,
+
+    #[serde(default = "default::meta::hybrid_few_partition_threshold")]
+    pub hybrid_few_partition_threshold: u64,
+    #[serde(default = "default::meta::hybrid_more_partition_threshold")]
+    pub hybrid_more_partition_threshold: u64,
+
     #[serde(default = "default::meta::event_log_enabled")]
     pub event_log_enabled: bool,
     /// Keeps the latest N events per channel.
@@ -1194,6 +1200,14 @@ pub mod default {
             4
         }
 
+        pub fn hybrid_few_partition_threshold() -> u64 {
+            128 * 1024 * 1024 // 128MB
+        }
+
+        pub fn hybrid_more_partition_threshold() -> u64 {
+            512 * 1024 * 1024 // 512MB
+        }
+
         pub fn event_log_enabled() -> bool {
             true
         }

src/config/docs.md

@@ -37,6 +37,8 @@ This page is automatically generated by `./risedev generate-example-config`
 | full_gc_interval_sec | Interval of automatic hummock full GC. | 86400 |
 | hummock_version_checkpoint_interval_sec | Interval of hummock version checkpoint. | 30 |
 | hybird_partition_vnode_count |  | 4 |
+| hybrid_few_partition_threshold |  | 134217728 |
+| hybrid_more_partition_threshold |  | 536870912 |
 | max_heartbeat_interval_secs | Maximum allowed heartbeat interval in seconds. | 60 |
 | meta_leader_lease_secs |  | 30 |
 | min_delta_log_num_for_hummock_version_checkpoint | The minimum delta log number a new checkpoint should compact, otherwise the checkpoint attempt is rejected. | 10 |

src/config/example.toml

@@ -49,6 +49,8 @@ min_table_split_write_throughput = 4194304
 compaction_task_max_heartbeat_interval_secs = 30
 compaction_task_max_progress_interval_secs = 600
 hybird_partition_vnode_count = 4
+hybrid_few_partition_threshold = 134217728
+hybrid_more_partition_threshold = 536870912
 event_log_enabled = true
 event_log_channel_max_size = 10
 enable_dropped_column_reclaim = false

src/meta/node/src/lib.rs

@@ -338,6 +338,8 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                 table_write_throughput_threshold: config.meta.table_write_throughput_threshold,
                 min_table_split_write_throughput: config.meta.min_table_split_write_throughput,
                 partition_vnode_count: config.meta.partition_vnode_count,
+                hybrid_few_partition_threshold: config.meta.hybrid_few_partition_threshold,
+                hybrid_more_partition_threshold: config.meta.hybrid_more_partition_threshold,
                 do_not_config_object_storage_lifecycle: config
                     .meta
                     .do_not_config_object_storage_lifecycle,
@@ -347,7 +349,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                 compaction_task_max_progress_interval_secs,
                 compaction_config: Some(config.meta.compaction_config),
                 cut_table_size_limit: config.meta.cut_table_size_limit,
-                hybird_partition_vnode_count: config.meta.hybird_partition_vnode_count,
+                hybrid_partition_node_count: config.meta.hybird_partition_vnode_count,
                 event_log_enabled: config.meta.event_log_enabled,
                 event_log_channel_max_size: config.meta.event_log_channel_max_size,
                 advertise_addr: opts.advertise_addr,

src/meta/src/hummock/manager/compaction.rs

@@ -294,13 +294,11 @@ impl HummockManager {
                 .existing_table_ids
                 .retain(|table_id| existing_table_ids.contains(table_id));
 
-            let hybrid_vnode_count = self.env.opts.hybird_partition_vnode_count;
+            let hybrid_vnode_count = self.env.opts.hybrid_partition_node_count;
             let default_partition_count = self.env.opts.partition_vnode_count;
             // We must ensure the partition threshold large enough to avoid too many small files.
-            let less_partition_threshold =
-                (hybrid_vnode_count as u64) * compaction_config.target_file_size_base;
-            let several_partition_threshold =
-                (default_partition_count as u64) * compaction_config.target_file_size_base;
+            let less_partition_threshold = self.env.opts.hybrid_few_partition_threshold;
+            let several_partition_threshold = self.env.opts.hybrid_more_partition_threshold;
             let params = self.env.system_params_reader().await;
             let barrier_interval_ms = params.barrier_interval_ms() as u64;
             let checkpoint_secs = std::cmp::max(

src/meta/src/manager/env.rs

@@ -252,7 +252,7 @@ pub struct MetaOpts {
     /// - Tables with high write throughput will be split at vnode granularity
     /// - Tables with high size tables will be split by table granularity
     /// When `hybird_partition_vnode_count` = 0,no longer be special alignment operations for the hybird compaction group
-    pub hybird_partition_vnode_count: u32,
+    pub hybrid_partition_node_count: u32,
 
     pub event_log_enabled: bool,
     pub event_log_channel_max_size: u32,
@@ -277,6 +277,9 @@ pub struct MetaOpts {
 
     /// The maximum number of times to probe for PullTaskEvent
     pub max_get_task_probe_times: usize,
+
+    pub hybrid_few_partition_threshold: u64,
+    pub hybrid_more_partition_threshold: u64,
 }
 
 impl MetaOpts {
@@ -323,7 +326,7 @@ impl MetaOpts {
             compaction_task_max_progress_interval_secs: 1,
             compaction_config: None,
             cut_table_size_limit: 1024 * 1024 * 1024,
-            hybird_partition_vnode_count: 4,
+            hybrid_partition_node_count: 4,
             event_log_enabled: false,
             event_log_channel_max_size: 1,
             advertise_addr: "".to_string(),