From 145875ebeb5c24f84f4b926f5cd65289b74cbf70 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Wed, 24 Apr 2024 16:18:07 +0800 Subject: [PATCH] fix ut Signed-off-by: Little-Wallace --- src/common/src/config.rs | 2 +- src/config/docs.md | 2 +- src/config/example.toml | 2 +- .../picker/base_level_compaction_picker.rs | 2 +- .../compaction/selector/level_selector.rs | 30 +++++++------------ 5 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/common/src/config.rs b/src/common/src/config.rs index e1e8ad8cfcd87..27ce99343fe9a 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -1613,7 +1613,7 @@ pub mod default { const DEFAULT_LEVEL_MULTIPLIER: u64 = 5; const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * 1024 * 1024; // 512MB; const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 300; - const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100; + const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 300; const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3; const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12; const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40; diff --git a/src/config/docs.md b/src/config/docs.md index eacd8039a5d75..d7ad5290f66fb 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -65,7 +65,7 @@ This page is automatically generated by `./risedev generate-example-config` |--------|-------------|---------| | compaction_filter_mask | | 6 | | enable_emergency_picker | | true | -| level0_max_compact_file_number | | 100 | +| level0_max_compact_file_number | | 300 | | level0_overlapping_sub_level_compact_level_count | | 12 | | level0_stop_write_threshold_sub_level_number | | 300 | | level0_sub_level_compact_level_count | | 3 | diff --git a/src/config/example.toml b/src/config/example.toml index 397bf9a89f69b..723e9dfcb7dd1 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -66,7 +66,7 @@ level0_stop_write_threshold_sub_level_number = 300 level0_sub_level_compact_level_count = 3 level0_overlapping_sub_level_compact_level_count = 12 max_space_reclaim_bytes = 536870912 -level0_max_compact_file_number = 100 +level0_max_compact_file_number = 300 tombstone_reclaim_ratio = 40 enable_emergency_picker = true diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index 17af43774240c..d46c316189504 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -159,7 +159,7 @@ impl LevelCompactionPicker { // divide by 2 because we need to select files of base level and it need use the other // half quota. std::cmp::max( - self.config.max_bytes_for_level_base, + target_level.total_file_size, self.config.max_compaction_bytes / 2, ) }; diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index 87141be587d4b..8ef64772f28eb 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -180,7 +180,7 @@ impl DynamicLevelSelectorCore { base_bytes_min + 1 } else { ctx.base_level = first_non_empty_level; - while ctx.base_level > 1 && cur_level_size > base_bytes_max * 2 { + while ctx.base_level > 1 && cur_level_size > base_bytes_max { ctx.base_level -= 1; cur_level_size /= self.config.max_bytes_for_level_multiplier; } @@ -485,6 +485,7 @@ pub mod tests { use risingwave_common::constants::hummock::CompactionFilterFlag; use risingwave_pb::hummock::compaction_config::CompactionMode; use risingwave_pb::hummock::hummock_version::Levels; + use risingwave_pb::hummock::OverlappingLevel; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; use crate::hummock::compaction::selector::tests::{ @@ -739,17 +740,17 @@ pub mod tests { assert_eq!(15000, levels.levels.get(2).unwrap().total_file_size); // l3 assert_eq!(10000, levels.levels.get(3).unwrap().total_file_size); // l4 - assert_eq!(100, ctx.level_max_bytes[1]); // l1 - assert_eq!(500, ctx.level_max_bytes[2]); // l2 - assert_eq!(2500, ctx.level_max_bytes[3]); // l3 - assert_eq!(12500, ctx.level_max_bytes[4]); // l4 + assert_eq!(200, ctx.level_max_bytes[1]); // l1 + assert_eq!(1000, ctx.level_max_bytes[2]); // l2 + assert_eq!(5000, ctx.level_max_bytes[3]); // l3 + assert_eq!(25000, ctx.level_max_bytes[4]); // l4 - // l1 pending = (0 + 1000 - 100) * ((25000 / 1000) + 1) + 1000 = 24400 - // l2 pending = (25000 + 900 - 500) * ((15000 / (25000 + 900)) + 1) = 40110 - // l3 pending = (15000 + 25400 - 2500) * ((10000 / (15000 + 25400) + 1)) = 47281 + // l1 pending = (0 + 1000 - 200) * ((25000 / 1000) + 1) + 1000 = 21800 + // l2 pending = (25000 + 800 - 1000) * ((15000 / (25000 + 800)) + 1) = 39218 + // l3 pending = (15000 + 24800 - 5000) * ((10000 / (15000 + 24800) + 1)) = 43543 let compact_pending_bytes = dynamic_level_core.compact_pending_bytes_needed(&levels); - assert_eq!(24400 + 40110 + 47281, compact_pending_bytes); + assert_eq!(21800 + 39218 + 43543, compact_pending_bytes); } #[test] @@ -771,12 +772,7 @@ pub mod tests { levels[5].total_file_size = 3125 * 2048 * 1024 * 1024; let levels = Levels { levels, - l0: Some(generate_l0_nonoverlapping_sublevels(generate_tables( - 15..25, - 0..600, - 3, - 100, - ))), + l0: Some(OverlappingLevel::default()), ..Default::default() }; let config = Arc::new(config); @@ -786,14 +782,10 @@ pub mod tests { ); let ctx = dynamic_level_core.calculate_level_base_size(&levels); assert!(ctx.level_max_bytes[0] > config.max_bytes_for_level_base); - for (idx, sz) in ctx.level_max_bytes.iter().enumerate() { - println!("level[{}]: {}MB", idx, *sz / 1024 / 1024); - } let levels_handlers = (0..7).map(LevelHandler::new).collect_vec(); let ctx = dynamic_level_core.get_priority_levels(&levels, &levels_handlers); for info in &ctx.score_levels { assert_eq!(info.score, 100); } - } }