feat(config): add config for calculate max_overlapping_level_size

proto/hummock.proto

@@ -660,6 +660,7 @@ message RiseCtlUpdateCompactionConfigRequest {
       uint64 sst_allowed_trivial_move_min_size = 19;
       uint32 split_weight_by_vnode = 20;
       bool disable_auto_group_scheduling = 21;
+      uint32 max_overlapping_level_size_ratio = 22;
     }
   }
   repeated uint64 compaction_group_ids = 1;
@@ -858,6 +859,11 @@ message CompactionConfig {
 
   // The limitation of auto group scheduling
   optional bool disable_auto_group_scheduling = 23;
+
+  // The limitation of the max size of the overlapping-level for the compaction
+  // e.g. max_overlapping_level_size = sub_level_max_compaction_bytes * max_overlapping_level_size_ratio
+  // hummock will reorg the commit-sstables to the multi overlapping-level if the size of the commit-sstables is larger than max_overlapping_level_size
+  optional uint32 max_overlapping_level_size_ratio = 24;
 }
 
 message TableStats {

src/common/src/config.rs

@@ -2230,6 +2230,10 @@ pub mod default {
         pub fn disable_auto_group_scheduling() -> bool {
             false
         }
+
+        pub fn max_overlapping_level_size_ratio() -> u32 {
+            4
+        }
     }
 
     pub mod object_store_config {

src/ctl/src/cmd_impl/hummock/compaction_group.rs

@@ -68,6 +68,7 @@ pub fn build_compaction_config_vec(
     max_l0_compact_level: Option<u32>,
     sst_allowed_trivial_move_min_size: Option<u64>,
     disable_auto_group_scheduling: Option<bool>,
+    max_overlapping_level_size_ratio: Option<u32>,
 ) -> Vec<MutableConfig> {
     let mut configs = vec![];
     if let Some(c) = max_bytes_for_level_base {
@@ -127,6 +128,9 @@ pub fn build_compaction_config_vec(
     if let Some(c) = disable_auto_group_scheduling {
         configs.push(MutableConfig::DisableAutoGroupScheduling(c))
     }
+    if let Some(c) = max_overlapping_level_size_ratio {
+        configs.push(MutableConfig::MaxOverlappingLevelSizeRatio(c))
+    }
 
     configs
 }

src/ctl/src/lib.rs

@@ -191,6 +191,8 @@ enum HummockCommands {
         sst_allowed_trivial_move_min_size: Option<u64>,
         #[clap(long)]
         disable_auto_group_scheduling: Option<bool>,
+        #[clap(long)]
+        max_overlapping_level_size_ratio: Option<u32>,
     },
     /// Split given compaction group into two. Moves the given tables to the new group.
     SplitCompactionGroup {
@@ -578,6 +580,7 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
             max_l0_compact_level,
             sst_allowed_trivial_move_min_size,
             disable_auto_group_scheduling,
+            max_overlapping_level_size_ratio,
         }) => {
             cmd_impl::hummock::update_compaction_config(
                 context,
@@ -610,6 +613,7 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
                     max_l0_compact_level,
                     sst_allowed_trivial_move_min_size,
                     disable_auto_group_scheduling,
+                    max_overlapping_level_size_ratio,
                 ),
             )
             .await?

src/meta/src/hummock/compaction/compaction_config.rs

@@ -71,6 +71,9 @@ impl CompactionConfigBuilder {
                 disable_auto_group_scheduling: Some(
                     compaction_config::disable_auto_group_scheduling(),
                 ),
+                max_overlapping_level_size_ratio: Some(
+                    compaction_config::max_overlapping_level_size_ratio(),
+                ),
             },
         }
     }

src/meta/src/hummock/manager/commit_epoch.rs

@@ -17,6 +17,7 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
+use risingwave_common::config::default::compaction_config;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_hummock_sdk::change_log::ChangeLogDelta;
 use risingwave_hummock_sdk::compaction_group::group_split::split_sst_with_table_ids;
@@ -457,7 +458,11 @@ fn rewrite_commit_sstables_to_sub_level(
 
         let mut accumulated_size = 0;
         let mut ssts = vec![];
-        let sub_level_size_limit = config.sub_level_max_compaction_bytes * 2; // TODO: use config instead of magic number
+        let sub_level_size_limit = config.sub_level_max_compaction_bytes
+            * config
+                .max_overlapping_level_size_ratio
+                .unwrap_or(compaction_config::max_overlapping_level_size_ratio())
+                as u64;
 
         let level = overlapping_sstables.entry(group_id).or_default();
 

src/meta/src/hummock/manager/compaction/compaction_group_manager.rs

@@ -588,6 +588,9 @@ fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfi
             MutableConfig::DisableAutoGroupScheduling(c) => {
                 target.disable_auto_group_scheduling = Some(*c);
             }
+            MutableConfig::MaxOverlappingLevelSizeRatio(c) => {
+                target.max_overlapping_level_size_ratio = Some(*c);
+            }
         }
     }
 }