From 85a4db66a6d9f1da4876ba5707d720ba10b754c6 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Wed, 10 Jan 2024 18:56:41 +0800 Subject: [PATCH 1/2] fix compatible Signed-off-by: Little-Wallace --- proto/hummock.proto | 1 - .../compaction/picker/base_level_compaction_picker.rs | 4 ++-- .../compaction/picker/intra_compaction_picker.rs | 2 +- .../picker/min_overlap_compaction_picker.rs | 11 +++++++---- .../compaction/picker/tier_compaction_picker.rs | 7 ++++++- .../src/hummock/compaction/selector/level_selector.rs | 3 ++- src/meta/src/hummock/manager/mod.rs | 6 +----- .../src/compaction_group/hummock_version_ext.rs | 1 - 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/proto/hummock.proto b/proto/hummock.proto index ae197caeab29e..1ff11077645d9 100644 --- a/proto/hummock.proto +++ b/proto/hummock.proto @@ -144,7 +144,6 @@ message HummockVersion { uint64 group_id = 3; uint64 parent_group_id = 4; repeated uint32 member_table_ids = 5; - uint32 vnode_partition_count = 6; } uint64 id = 1; // Levels of each compaction group diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index c5c09f1544ce7..ba3c08b8a90fc 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -66,7 +66,7 @@ impl CompactionPicker for LevelCompactionPicker { level_handlers, stats, ) { - ret.vnode_partition_count = levels.vnode_partition_count; + ret.vnode_partition_count = self.config.split_weight_by_vnode; return Some(ret); } @@ -74,7 +74,7 @@ impl CompactionPicker for LevelCompactionPicker { if let Some(ret) = self.pick_multi_level_to_base( l0, levels.get_level(self.target_level), - levels.vnode_partition_count, + self.config.split_weight_by_vnode, level_handlers, stats, ) { diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 034db2b127963..00ad9a74744a1 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -57,7 +57,7 @@ impl CompactionPicker for IntraCompactionPicker { return None; } - let vnode_partition_count = levels.vnode_partition_count; + let vnode_partition_count = self.config.split_weight_by_vnode; if let Some(ret) = self.pick_whole_level(l0, &level_handlers[0], vnode_partition_count, stats) diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 3065d108e700f..14c1ec3fd1640 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -28,6 +28,7 @@ pub struct MinOverlappingPicker { level: usize, target_level: usize, max_select_bytes: u64, + vnode_partition_count: u32, overlap_strategy: Arc, } @@ -36,6 +37,7 @@ impl MinOverlappingPicker { level: usize, target_level: usize, max_select_bytes: u64, + vnode_partition_count: u32, overlap_strategy: Arc, ) -> MinOverlappingPicker { MinOverlappingPicker { @@ -43,6 +45,7 @@ impl MinOverlappingPicker { target_level, max_select_bytes, overlap_strategy, + vnode_partition_count, } } @@ -163,7 +166,7 @@ impl CompactionPicker for MinOverlappingPicker { }, ], target_level: self.target_level, - vnode_partition_count: levels.vnode_partition_count, + vnode_partition_count: self.vnode_partition_count, ..Default::default() }) } @@ -425,7 +428,7 @@ pub mod tests { #[test] fn test_compact_l1() { let mut picker = - MinOverlappingPicker::new(1, 2, 10000, Arc::new(RangeOverlapStrategy::default())); + MinOverlappingPicker::new(1, 2, 10000, 0, Arc::new(RangeOverlapStrategy::default())); let levels = vec![ Level { level_idx: 1, @@ -497,7 +500,7 @@ pub mod tests { #[test] fn test_expand_l1_files() { let mut picker = - MinOverlappingPicker::new(1, 2, 10000, Arc::new(RangeOverlapStrategy::default())); + MinOverlappingPicker::new(1, 2, 10000, 0, Arc::new(RangeOverlapStrategy::default())); let levels = vec![ Level { level_idx: 1, @@ -822,7 +825,7 @@ pub mod tests { ]; // no limit let picker = - MinOverlappingPicker::new(2, 3, 1000, Arc::new(RangeOverlapStrategy::default())); + MinOverlappingPicker::new(2, 3, 1000, 0, Arc::new(RangeOverlapStrategy::default())); let (select_files, target_files) = picker.pick_tables( &levels[1].table_infos, &levels[2].table_infos, diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index 20b7d3b38c8e7..9ed22ba551fcc 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -155,7 +155,12 @@ impl CompactionPicker for TierCompactionPicker { return None; } - self.pick_overlapping_level(l0, &level_handlers[0], levels.vnode_partition_count, stats) + self.pick_overlapping_level( + l0, + &level_handlers[0], + self.config.split_weight_by_vnode, + stats, + ) } } diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index 587bd95743296..39f3a1acce875 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -121,6 +121,7 @@ impl DynamicLevelSelectorCore { picker_info.select_level, picker_info.target_level, self.config.max_bytes_for_level_base / 2, + self.config.split_weight_by_vnode, overlap_strategy, )) } @@ -245,7 +246,7 @@ impl DynamicLevelSelectorCore { .sub_levels .iter() .filter(|level| { - level.vnode_partition_count == levels.vnode_partition_count + level.vnode_partition_count == self.config.split_weight_by_vnode && level.level_type() == LevelType::Nonoverlapping }) .map(|level| level.total_file_size) diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index a64d6fab3e668..56d95c345cbaa 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -1058,11 +1058,7 @@ impl HummockManager { } else { table_to_vnode_partition .retain(|table_id, _| compact_task.existing_table_ids.contains(table_id)); - if current_version - .get_compaction_group_levels(compaction_group_id) - .vnode_partition_count - > 0 - { + if group_config.compaction_config.split_weight_by_vnode > 0 { for table_id in &compact_task.existing_table_ids { table_to_vnode_partition .entry(*table_id) diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 064201eb2a7df..a6950c909e958 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -831,7 +831,6 @@ pub fn build_initial_compaction_group_levels( group_id, parent_group_id: StaticCompactionGroupId::NewCompactionGroup as _, member_table_ids: vec![], - vnode_partition_count: compaction_config.split_weight_by_vnode, } } From 46ac5e2fec32c1ab197bf2c188baa8b73e201997 Mon Sep 17 00:00:00 2001 From: Little-Wallace Date: Thu, 11 Jan 2024 10:37:38 +0800 Subject: [PATCH 2/2] fix format check Signed-off-by: Little-Wallace --- .../hummock/compaction/picker/min_overlap_compaction_picker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 14c1ec3fd1640..8a590a44c7fb9 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -44,8 +44,8 @@ impl MinOverlappingPicker { level, target_level, max_select_bytes, - overlap_strategy, vnode_partition_count, + overlap_strategy, } }