From 42e50d866d478f04e764e1c98d43a6c72b6c61c8 Mon Sep 17 00:00:00 2001 From: Li0k Date: Tue, 5 Sep 2023 17:04:07 +0800 Subject: [PATCH 1/3] feat(storage): remove group deltas for l0 task --- src/meta/src/hummock/manager/mod.rs | 59 +++++++++++++++++++---------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index ce8767295456c..d318123b26952 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::borrow::BorrowMut; +use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::ops::{Deref, DerefMut}; use std::sync::atomic::AtomicBool; @@ -2775,34 +2776,52 @@ fn gen_version_delta<'a>( .or_default() .group_deltas; let mut gc_object_ids = vec![]; + let mut removed_table_ids_map = BTreeMap::default(); + for level in &compact_task.input_ssts { + let level_idx = level.level_idx; + let mut removed_table_ids = level + .table_infos + .iter() + .map(|sst| { + let object_id = sst.get_object_id(); + let sst_id = sst.get_sst_id(); + if !trivial_move + && drop_sst( + branched_ssts, + compact_task.compaction_group_id, + object_id, + sst_id, + ) + { + gc_object_ids.push(object_id); + } + sst_id + }) + .collect_vec(); + + match removed_table_ids_map.entry(level_idx) { + Vacant(entry) => { + entry.insert(removed_table_ids); + } + + Occupied(mut entry) => { + entry.get_mut().append(&mut removed_table_ids); + } + }; + } + + for (level_idx, removed_table_ids) in removed_table_ids_map { let group_delta = GroupDelta { delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { - level_idx: level.level_idx, - removed_table_ids: level - .table_infos - .iter() - .map(|sst| { - let object_id = sst.get_object_id(); - let sst_id = sst.get_sst_id(); - if !trivial_move - && drop_sst( - branched_ssts, - compact_task.compaction_group_id, - object_id, - sst_id, - ) - { - gc_object_ids.push(object_id); - } - sst_id - }) - .collect_vec(), + level_idx, + removed_table_ids, ..Default::default() })), }; group_deltas.push(group_delta); } + let group_delta = GroupDelta { delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { level_idx: compact_task.target_level, From 739b2c362d711108bf6abb60a78c3ee3675be496 Mon Sep 17 00:00:00 2001 From: Li0k Date: Tue, 5 Sep 2023 17:13:31 +0800 Subject: [PATCH 2/3] fix(storage): fix typo --- .../src/compaction_group/hummock_version_ext.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 80da4ca57e2b3..c2f4bde36ce26 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -348,7 +348,7 @@ impl HummockVersionUpdateExt for HummockVersion { } } } - for (z, level) in parent_levels.levels.iter_mut().enumerate() { + for (idx, level) in parent_levels.levels.iter_mut().enumerate() { let insert_table_infos = split_sst_info_for_level( &member_table_ids, allow_trivial_split, @@ -356,21 +356,23 @@ impl HummockVersionUpdateExt for HummockVersion { &mut split_id_vers, &mut new_sst_id, ); - cur_levels.levels[z].total_file_size += insert_table_infos + cur_levels.levels[idx].total_file_size += insert_table_infos .iter() .map(|sst| sst.file_size) .sum::(); - cur_levels.levels[z].uncompressed_file_size += insert_table_infos + cur_levels.levels[idx].uncompressed_file_size += insert_table_infos .iter() .map(|sst| sst.uncompressed_file_size) .sum::(); - cur_levels.levels[z].table_infos.extend(insert_table_infos); - cur_levels.levels[z].table_infos.sort_by(|sst1, sst2| { + cur_levels.levels[idx] + .table_infos + .extend(insert_table_infos); + cur_levels.levels[idx].table_infos.sort_by(|sst1, sst2| { let a = sst1.key_range.as_ref().unwrap(); let b = sst2.key_range.as_ref().unwrap(); a.compare(b) }); - assert!(can_concat(&cur_levels.levels[z].table_infos)); + assert!(can_concat(&cur_levels.levels[idx].table_infos)); level .table_infos .drain_filter(|sst_info| sst_info.table_ids.is_empty()) From db5c915eeb1d2de75429709d7999ebd8db666d1b Mon Sep 17 00:00:00 2001 From: Li0k Date: Wed, 6 Sep 2023 15:49:01 +0800 Subject: [PATCH 3/3] fix(storage): fix typo --- src/meta/src/hummock/manager/mod.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index d318123b26952..12fe85ad48f9d 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::borrow::BorrowMut; -use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::ops::{Deref, DerefMut}; use std::sync::atomic::AtomicBool; @@ -2776,7 +2775,7 @@ fn gen_version_delta<'a>( .or_default() .group_deltas; let mut gc_object_ids = vec![]; - let mut removed_table_ids_map = BTreeMap::default(); + let mut removed_table_ids_map: BTreeMap> = BTreeMap::default(); for level in &compact_task.input_ssts { let level_idx = level.level_idx; @@ -2800,15 +2799,10 @@ fn gen_version_delta<'a>( }) .collect_vec(); - match removed_table_ids_map.entry(level_idx) { - Vacant(entry) => { - entry.insert(removed_table_ids); - } - - Occupied(mut entry) => { - entry.get_mut().append(&mut removed_table_ids); - } - }; + removed_table_ids_map + .entry(level_idx) + .or_default() + .append(&mut removed_table_ids); } for (level_idx, removed_table_ids) in removed_table_ids_map {