From 4e3b9ff081d59c28668bdca9817930eb3fd013c3 Mon Sep 17 00:00:00 2001 From: Li0k Date: Mon, 26 Aug 2024 15:55:29 +0800 Subject: [PATCH] feat(storage): introduce new field `sst_size` for SstableInfo (#18005) --- proto/hummock.proto | 5 +++ .../rw_catalog/rw_hummock_version.rs | 2 + .../picker/base_level_compaction_picker.rs | 2 +- .../picker/compaction_task_validator.rs | 2 +- .../picker/intra_compaction_picker.rs | 4 +- .../picker/manual_compaction_picker.rs | 4 +- .../picker/min_overlap_compaction_picker.rs | 18 ++++----- .../picker/space_reclaim_compaction_picker.rs | 6 +-- .../tombstone_reclaim_compaction_picker.rs | 4 +- .../picker/trivial_move_compaction_picker.rs | 2 +- .../picker/ttl_reclaim_compaction_picker.rs | 2 +- .../compaction/selector/level_selector.rs | 6 +-- .../src/hummock/compaction/selector/mod.rs | 27 ++++++++------ src/meta/src/hummock/level_handler.rs | 2 +- src/meta/src/hummock/manager/compaction.rs | 2 +- src/meta/src/hummock/manager/tests.rs | 37 ++++++++++++++++++- src/meta/src/hummock/test_utils.rs | 12 ++++-- src/storage/hummock_sdk/src/compact.rs | 9 +++-- .../compaction_group/hummock_version_ext.rs | 20 +++++----- src/storage/hummock_sdk/src/lib.rs | 1 + src/storage/hummock_sdk/src/sstable_info.rs | 18 ++++++++- src/storage/hummock_sdk/src/time_travel.rs | 1 + .../src/hummock_read_version_tests.rs | 2 + .../hummock_test/src/hummock_storage_tests.rs | 10 +++++ src/storage/src/hummock/sstable/builder.rs | 1 + src/storage/src/hummock/test_utils.rs | 1 + 26 files changed, 143 insertions(+), 57 deletions(-) diff --git a/proto/hummock.proto b/proto/hummock.proto index 26cb374a9fc4..60dbe176958f 100644 --- a/proto/hummock.proto +++ b/proto/hummock.proto @@ -18,6 +18,7 @@ message SstableInfo { uint64 object_id = 1; uint64 sst_id = 2; KeyRange key_range = 3; + // represents the physical object size, which is usually used in the builder. uint64 file_size = 4; repeated uint32 table_ids = 5; uint64 meta_offset = 6; @@ -28,6 +29,10 @@ message SstableInfo { uint64 uncompressed_file_size = 11; uint64 range_tombstone_count = 12; BloomFilterType bloom_filter_kind = 13; + + // In order to calculate more finely in the compaction strategy, we need to re-calculate the sst_size after split sst + // `sst_size` represents the size of the sst instead of the object size(usually used in the meta). + uint64 sst_size = 14; } enum LevelType { diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs index 23087274c3b2..f239142661fb 100644 --- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs @@ -55,6 +55,7 @@ struct RwHummockSstable { range_tombstone_count: i64, bloom_filter_kind: i32, table_ids: JsonbVal, + sst_size: i64, } #[system_catalog(table, "rw_catalog.rw_hummock_current_version")] @@ -134,6 +135,7 @@ fn version_to_sstable_rows(version: HummockVersion) -> Vec { range_tombstone_count: sst.range_tombstone_count as _, bloom_filter_kind: sst.bloom_filter_kind as _, table_ids: json!(sst.table_ids).into(), + sst_size: sst.sst_size as _, }); } } diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index 97f1bd02cf1c..374cd999edd8 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -218,7 +218,7 @@ impl LevelCompactionPicker { break; } - target_level_size += sst.file_size; + target_level_size += sst.sst_size; } if pending_compact { diff --git a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs index 1e41500f547d..acf803d1ce57 100644 --- a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs +++ b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs @@ -157,7 +157,7 @@ impl CompactionTaskValidationRule for IntraCompactionTaskValidationRule { let level_select_size = select_level .table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); max_level_size = std::cmp::max(max_level_size, level_select_size); diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 3a085ac4e71b..644b70485d0f 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -191,7 +191,7 @@ impl IntraCompactionPicker { for level_select_table in &input.sstable_infos { let level_select_size = level_select_table .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); max_level_size = std::cmp::max(max_level_size, level_select_size); @@ -291,7 +291,7 @@ impl IntraCompactionPicker { .check_multiple_overlap(&l0.sub_levels[idx].table_infos) .is_empty()); - let select_input_size = select_sst.file_size; + let select_input_size = select_sst.sst_size; let input_levels = vec![ InputLevel { level_idx: 0, diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index 98f27254ab46..597d18cce4fd 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -301,8 +301,8 @@ impl CompactionPicker for ManualCompactionPicker { } Some(CompactionInput { - select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(), - target_input_size: target_input_ssts.iter().map(|sst| sst.file_size).sum(), + select_input_size: select_input_ssts.iter().map(|sst| sst.sst_size).sum(), + target_input_size: target_input_ssts.iter().map(|sst| sst.sst_size).sum(), total_file_count: (select_input_ssts.len() + target_input_ssts.len()) as u64, input_levels: vec![ InputLevel { diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 4291ffc15071..4d7b5ac1e72f 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -88,7 +88,7 @@ impl MinOverlappingPicker { let mut target_level_overlap_range = select_file_ranges[left].1.clone(); let mut total_file_size = 0; for other in &target_tables[target_level_overlap_range.clone()] { - total_file_size += other.file_size; + total_file_size += other.sst_size; } let start_idx = select_file_ranges[left].0; let mut end_idx = start_idx + 1; @@ -99,10 +99,10 @@ impl MinOverlappingPicker { { break; } - select_file_size += select_tables[*idx].file_size; + select_file_size += select_tables[*idx].sst_size; if range.end > target_level_overlap_range.end { for other in &target_tables[target_level_overlap_range.end..range.end] { - total_file_size += other.file_size; + total_file_size += other.sst_size; } target_level_overlap_range.end = range.end; } @@ -149,8 +149,8 @@ impl CompactionPicker for MinOverlappingPicker { return None; } Some(CompactionInput { - select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(), - target_input_size: target_input_ssts.iter().map(|sst| sst.file_size).sum(), + select_input_size: select_input_ssts.iter().map(|sst| sst.sst_size).sum(), + target_input_size: target_input_ssts.iter().map(|sst| sst.sst_size).sum(), total_file_count: (select_input_ssts.len() + target_input_ssts.len()) as u64, input_levels: vec![ InputLevel { @@ -310,7 +310,7 @@ impl NonOverlapSubLevelPicker { } basic_overlap_info.update(other); - add_files_size += other.file_size; + add_files_size += other.sst_size; add_files_count += 1; } @@ -339,7 +339,7 @@ impl NonOverlapSubLevelPicker { ret.total_file_count += ret.sstable_infos[reverse_index].len(); ret.total_file_size += ret.sstable_infos[reverse_index] .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); } @@ -349,7 +349,7 @@ impl NonOverlapSubLevelPicker { }); } else { ret.total_file_count = 1; - ret.total_file_size = sst.file_size; + ret.total_file_size = sst.sst_size; ret.sstable_infos[0].extend(vec![sst.clone()]); } @@ -370,7 +370,7 @@ impl NonOverlapSubLevelPicker { let mut total_level_count = 0; for (index, sstables) in ret.sstable_infos.iter().enumerate() { total_file_count += sstables.len(); - total_file_size += sstables.iter().map(|sst| sst.file_size).sum::(); + total_file_size += sstables.iter().map(|sst| sst.sst_size).sum::(); total_level_count += 1; // Atleast `min_expected_level_count`` level should be selected diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 7a4eb86831c3..b3e9eb75321a 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -84,7 +84,7 @@ impl SpaceReclaimCompactionPicker { } if !select_input_ssts.is_empty() { return Some(CompactionInput { - select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(), + select_input_size: select_input_ssts.iter().map(|sst| sst.sst_size).sum(), total_file_count: select_input_ssts.len() as u64, input_levels: vec![ InputLevel { @@ -140,7 +140,7 @@ impl SpaceReclaimCompactionPicker { // turn to next_round if !select_input_ssts.is_empty() { return Some(CompactionInput { - select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(), + select_input_size: select_input_ssts.iter().map(|sst| sst.sst_size).sum(), total_file_count: select_input_ssts.len() as u64, input_levels: vec![ InputLevel { @@ -311,7 +311,7 @@ mod test { let select_file_size: u64 = task.input.input_levels[0] .table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum(); assert!(select_file_size > max_space_reclaim_bytes); } diff --git a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs index ab6bfd1ed849..b4ab933931bd 100644 --- a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs @@ -103,11 +103,11 @@ impl TombstoneReclaimCompactionPicker { } }; return Some(CompactionInput { - select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(), + select_input_size: select_input_ssts.iter().map(|sst| sst.sst_size).sum(), target_input_size: target_level .table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum(), total_file_count: (select_input_ssts.len() + target_level.table_infos.len()) as u64, diff --git a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs index 0144a2170c75..6e172f15d380 100644 --- a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs @@ -90,7 +90,7 @@ impl TrivialMovePicker { self.pick_trivial_move_sst(select_tables, target_tables, level_handlers, stats) { return Some(CompactionInput { - select_input_size: trivial_move_sst.file_size, + select_input_size: trivial_move_sst.sst_size, total_file_count: 1, input_levels: vec![ InputLevel { diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index 5d3f17e31b6b..36ee06a3bb79 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -176,7 +176,7 @@ impl TtlReclaimCompactionPicker { }); Some(CompactionInput { - select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(), + select_input_size: select_input_ssts.iter().map(|sst| sst.sst_size).sum(), total_file_count: select_input_ssts.len() as _, input_levels: vec![ InputLevel { diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index 7076096fcfe1..321a3a65f713 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -254,7 +254,7 @@ impl DynamicLevelSelectorCore { }) .map(|level| level.total_file_size) .sum::() - - handlers[0].get_pending_output_file_size(ctx.base_level as u32); + .saturating_sub(handlers[0].get_pending_output_file_size(ctx.base_level as u32)); let base_level_size = levels.get_level(ctx.base_level).total_file_size; let base_level_sst_count = levels.get_level(ctx.base_level).table_infos.len() as u64; @@ -528,7 +528,7 @@ pub mod tests { levels.levels[3].total_file_size = levels.levels[3] .table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); let ctx = selector.calculate_level_base_size(&levels); @@ -555,7 +555,7 @@ pub mod tests { levels.levels[0].total_file_size = levels.levels[0] .table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); let ctx = selector.calculate_level_base_size(&levels); diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index 9c813efa6c86..02b50ff2ff67 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -136,11 +136,11 @@ pub mod tests { use crate::hummock::test_utils::iterator_test_key_of_epoch; pub fn push_table_level0_overlapping(levels: &mut Levels, sst: SstableInfo) { - levels.l0.total_file_size += sst.file_size; + levels.l0.total_file_size += sst.sst_size; levels.l0.sub_levels.push(Level { level_idx: 0, level_type: LevelType::Overlapping, - total_file_size: sst.file_size, + total_file_size: sst.sst_size, uncompressed_file_size: sst.uncompressed_file_size, sub_level_id: sst.sst_id, table_infos: vec![sst], @@ -154,7 +154,7 @@ pub mod tests { } pub fn push_tables_level0_nonoverlapping(levels: &mut Levels, table_infos: Vec) { - let total_file_size = table_infos.iter().map(|table| table.file_size).sum::(); + let total_file_size = table_infos.iter().map(|table| table.sst_size).sum::(); let uncompressed_file_size = table_infos .iter() .map(|table| table.uncompressed_file_size) @@ -179,6 +179,7 @@ pub mod tests { right: usize, epoch: u64, ) -> SstableInfo { + let object_size = (right - left + 1) as u64; SstableInfo { object_id: id, sst_id: id, @@ -187,10 +188,11 @@ pub mod tests { right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(), right_exclusive: false, }, - file_size: (right - left + 1) as u64, + file_size: object_size, table_ids: vec![table_prefix as u32], uncompressed_file_size: (right - left + 1) as u64, total_key_count: (right - left + 1) as u64, + sst_size: object_size, ..Default::default() } } @@ -206,6 +208,7 @@ pub mod tests { min_epoch: u64, max_epoch: u64, ) -> SstableInfo { + let object_size = (right - left + 1) as u64; SstableInfo { object_id: id, sst_id: id, @@ -214,11 +217,12 @@ pub mod tests { right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(), right_exclusive: false, }, - file_size: (right - left + 1) as u64, + file_size: object_size, table_ids, - uncompressed_file_size: (right - left + 1) as u64, + uncompressed_file_size: object_size, min_epoch, max_epoch, + sst_size: object_size, ..Default::default() } } @@ -235,6 +239,7 @@ pub mod tests { for id in ids { let mut table = generate_table(id, 1, start, start + step - 1, epoch); table.file_size = file_size; + table.sst_size = file_size; tables.push(table); start += step; } @@ -242,7 +247,7 @@ pub mod tests { } pub fn generate_level(level_idx: u32, table_infos: Vec) -> Level { - let total_file_size = table_infos.iter().map(|sst| sst.file_size).sum(); + let total_file_size = table_infos.iter().map(|sst| sst.sst_size).sum(); let uncompressed_file_size = table_infos .iter() .map(|sst| sst.uncompressed_file_size) @@ -261,7 +266,7 @@ pub mod tests { /// Returns a `OverlappingLevel`, with each `table_infos`'s element placed in a nonoverlapping /// sub-level. pub fn generate_l0_nonoverlapping_sublevels(table_infos: Vec) -> OverlappingLevel { - let total_file_size = table_infos.iter().map(|table| table.file_size).sum::(); + let total_file_size = table_infos.iter().map(|table| table.sst_size).sum::(); let uncompressed_file_size = table_infos .iter() .map(|table| table.uncompressed_file_size) @@ -273,7 +278,7 @@ pub mod tests { .map(|(idx, table)| Level { level_idx: 0, level_type: LevelType::Nonoverlapping, - total_file_size: table.file_size, + total_file_size: table.sst_size, uncompressed_file_size: table.uncompressed_file_size, sub_level_id: idx as u64, table_infos: vec![table], @@ -295,7 +300,7 @@ pub mod tests { .map(|(idx, table)| Level { level_idx: 0, level_type: LevelType::Nonoverlapping, - total_file_size: table.iter().map(|table| table.file_size).sum::(), + total_file_size: table.iter().map(|table| table.sst_size).sum::(), uncompressed_file_size: table .iter() .map(|sst| sst.uncompressed_file_size) @@ -330,7 +335,7 @@ pub mod tests { .map(|(idx, table)| Level { level_idx: 0, level_type: LevelType::Overlapping, - total_file_size: table.iter().map(|table| table.file_size).sum::(), + total_file_size: table.iter().map(|table| table.sst_size).sum::(), sub_level_id: idx as u64, table_infos: table.clone(), uncompressed_file_size: table diff --git a/src/meta/src/hummock/level_handler.rs b/src/meta/src/hummock/level_handler.rs index 2f729b7123d5..7ebc55eb1d3a 100644 --- a/src/meta/src/hummock/level_handler.rs +++ b/src/meta/src/hummock/level_handler.rs @@ -92,7 +92,7 @@ impl LevelHandler { let mut total_file_size = 0; for sst in ssts { self.compacting_files.insert(sst.sst_id, task_id); - total_file_size += sst.file_size; + total_file_size += sst.sst_size; table_ids.push(sst.sst_id); } diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs index 8655df136774..4696c0745201 100644 --- a/src/meta/src/hummock/manager/compaction.rs +++ b/src/meta/src/hummock/manager/compaction.rs @@ -1508,7 +1508,7 @@ impl HummockManager { existing_table_ids.extend(sst.table_ids.iter()); for table_id in &sst.table_ids { *table_size_info.entry(*table_id).or_default() += - sst.file_size / (sst.table_ids.len() as u64); + sst.sst_size / (sst.table_ids.len() as u64); } } } diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs index 2d25f196a60b..73fb98c03e0d 100644 --- a/src/meta/src/hummock/manager/tests.rs +++ b/src/meta/src/hummock/manager/tests.rs @@ -75,6 +75,8 @@ fn gen_sstable_info(sst_id: u64, idx: usize, table_ids: Vec) -> SstableInfo object_id: sst_id, min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() } } @@ -1215,6 +1217,7 @@ async fn test_version_stats() { }, file_size: 1024 * 1024 * 1024, table_ids: table_ids.clone(), + sst_size: 1024 * 1024 * 1024, ..Default::default() }, table_stats: table_ids @@ -1321,6 +1324,8 @@ async fn test_split_compaction_group_on_commit() { table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1413,6 +1418,8 @@ async fn test_split_compaction_group_on_demand_basic() { table_ids: vec![100], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1429,6 +1436,8 @@ async fn test_split_compaction_group_on_demand_basic() { table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1507,6 +1516,8 @@ async fn test_split_compaction_group_on_demand_non_trivial() { table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1596,6 +1607,8 @@ async fn test_split_compaction_group_trivial_expired() { table_ids: vec![100], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1612,6 +1625,8 @@ async fn test_split_compaction_group_trivial_expired() { right: iterator_test_key_of_epoch(101, 100, 20).into(), right_exclusive: false, }, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1703,6 +1718,8 @@ async fn test_split_compaction_group_trivial_expired() { table_ids: vec![100], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }], None, @@ -1762,6 +1779,8 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1790,6 +1809,8 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, }, + file_size: 100, + sst_size: 100, ..Default::default() }, SstableInfo { @@ -1801,6 +1822,8 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { right: iterator_test_key_of_epoch(1, 2, 2).into(), right_exclusive: false, }, + file_size: 100, + sst_size: 100, ..Default::default() }, ], @@ -1895,6 +1918,8 @@ async fn test_compaction_task_expiration_due_to_split_group() { table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -1911,6 +1936,8 @@ async fn test_compaction_task_expiration_due_to_split_group() { table_ids: vec![101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -2174,6 +2201,7 @@ async fn test_partition_level() { .await .unwrap(); let current_version = hummock_manager.get_current_version().await; + let new_group_id = current_version.levels.keys().max().cloned().unwrap(); assert_eq!( current_version @@ -2190,7 +2218,9 @@ async fn test_partition_level() { for epoch in 31..100 { let mut sst = gen_local_sstable_info(global_sst_id, 10, vec![100]); sst.sst_info.file_size = 10 * MB; + sst.sst_info.sst_size = 10 * MB; sst.sst_info.uncompressed_file_size = 10 * MB; + hummock_manager .commit_epoch_for_test( epoch, @@ -2213,10 +2243,11 @@ async fn test_partition_level() { level .table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::() }) .sum::(); + sst.sst_size = sst.file_size; global_sst_id += 1; let ret = hummock_manager .report_compact_task(task.task_id, TaskStatus::Success, vec![sst], None) @@ -2274,6 +2305,8 @@ async fn test_unregister_moved_table() { table_ids: vec![100], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), @@ -2290,6 +2323,8 @@ async fn test_unregister_moved_table() { table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, + file_size: 100, + sst_size: 100, ..Default::default() }, table_stats: Default::default(), diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs index 886af7bddc62..2188d9b53932 100644 --- a/src/meta/src/hummock/test_utils.rs +++ b/src/meta/src/hummock/test_utils.rs @@ -166,6 +166,7 @@ pub fn generate_test_sstables_with_table_id( ) -> Vec { let mut sst_info = vec![]; for (i, sst_id) in sst_ids.into_iter().enumerate() { + let object_size = 2; sst_info.push(SstableInfo { object_id: sst_id, sst_id, @@ -184,10 +185,11 @@ pub fn generate_test_sstables_with_table_id( )), right_exclusive: false, }, - file_size: 2, + file_size: object_size, table_ids: vec![table_id], - uncompressed_file_size: 2, + uncompressed_file_size: object_size, max_epoch: epoch, + sst_size: object_size, ..Default::default() }); } @@ -197,6 +199,7 @@ pub fn generate_test_sstables_with_table_id( pub fn generate_test_tables(epoch: u64, sst_ids: Vec) -> Vec { let mut sst_info = vec![]; for (i, sst_id) in sst_ids.into_iter().enumerate() { + let object_size = 2; sst_info.push(SstableInfo { object_id: sst_id, sst_id, @@ -205,10 +208,11 @@ pub fn generate_test_tables(epoch: u64, sst_ids: Vec) -> right: Bytes::from(iterator_test_key_of_epoch(sst_id, (i + 1) * 10, epoch)), right_exclusive: false, }, - file_size: 2, + file_size: object_size, table_ids: vec![sst_id as u32, sst_id as u32 * 10000], - uncompressed_file_size: 2, + uncompressed_file_size: object_size, max_epoch: epoch, + sst_size: object_size, ..Default::default() }); } diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index ff1241447791..d268da405b7a 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -83,18 +83,20 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { } if table.total_key_count != 0 { format!( - "[id: {}, obj_id: {} {}KB stale_ratio {}]", + "[id: {}, obj_id: {} object_size {}KB sst_size {}KB stale_ratio {}]", table.sst_id, table.object_id, table.file_size / 1024, + table.sst_size / 1024, (table.stale_key_count * 100 / table.total_key_count), ) } else { format!( - "[id: {}, obj_id: {} {}KB]", + "[id: {}, obj_id: {} object_size {}KB sst_size {}KB]", table.sst_id, table.object_id, table.file_size / 1024, + table.sst_size / 1024, ) } }) @@ -138,13 +140,14 @@ pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo) .unwrap_or(0); writeln!( s, - "SstableInfo: object id={}, SST id={}, KeyRange=[{:?},{:?}], table_ids: {:?}, size={}KB, stale_ratio={}%, bloom_filter_kind {:?}", + "SstableInfo: object id={}, SST id={}, KeyRange=[{:?},{:?}], table_ids: {:?}, object_size={}KB, sst_size={}KB stale_ratio={}%, bloom_filter_kind {:?}", sstable_info.object_id, sstable_info.sst_id, left_str, right_str, sstable_info.table_ids, sstable_info.file_size / 1024, + sstable_info.sst_size / 1024, stale_ratio, sstable_info.bloom_filter_kind, ) diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 13a0bcc08adf..bfd6402e21a8 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -411,9 +411,9 @@ impl HummockVersion { .table_infos .extract_if(|sst_info| sst_info.table_ids.is_empty()) .for_each(|sst_info| { - sub_level.total_file_size -= sst_info.file_size; + sub_level.total_file_size -= sst_info.sst_size; sub_level.uncompressed_file_size -= sst_info.uncompressed_file_size; - l0.total_file_size -= sst_info.file_size; + l0.total_file_size -= sst_info.sst_size; l0.uncompressed_file_size -= sst_info.uncompressed_file_size; }); if insert_table_infos.is_empty() { @@ -440,7 +440,7 @@ impl HummockVersion { split_sst_info_for_level(&member_table_ids, level, &mut new_sst_id); cur_levels.levels[idx].total_file_size += insert_table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); cur_levels.levels[idx].uncompressed_file_size += insert_table_infos .iter() @@ -457,7 +457,7 @@ impl HummockVersion { .table_infos .extract_if(|sst_info| sst_info.table_ids.is_empty()) .for_each(|sst_info| { - level.total_file_size -= sst_info.file_size; + level.total_file_size -= sst_info.sst_size; level.uncompressed_file_size -= sst_info.uncompressed_file_size; }); } @@ -1070,7 +1070,7 @@ pub fn new_sub_level( table_infos ); } - let total_file_size = table_infos.iter().map(|table| table.file_size).sum(); + let total_file_size = table_infos.iter().map(|table| table.sst_size).sum(); let uncompressed_file_size = table_infos .iter() .map(|table| table.uncompressed_file_size) @@ -1092,9 +1092,9 @@ pub fn add_ssts_to_sub_level( insert_table_infos: Vec, ) { insert_table_infos.iter().for_each(|sst| { - l0.sub_levels[sub_level_idx].total_file_size += sst.file_size; + l0.sub_levels[sub_level_idx].total_file_size += sst.sst_size; l0.sub_levels[sub_level_idx].uncompressed_file_size += sst.uncompressed_file_size; - l0.total_file_size += sst.file_size; + l0.total_file_size += sst.sst_size; l0.uncompressed_file_size += sst.uncompressed_file_size; }); l0.sub_levels[sub_level_idx] @@ -1174,7 +1174,7 @@ fn level_delete_ssts( operand.total_file_size = operand .table_infos .iter() - .map(|table| table.file_size) + .map(|table| table.sst_size) .sum::(); operand.uncompressed_file_size = operand .table_infos @@ -1187,7 +1187,7 @@ fn level_delete_ssts( fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec) { operand.total_file_size += insert_table_infos .iter() - .map(|sst| sst.file_size) + .map(|sst| sst.sst_size) .sum::(); operand.uncompressed_file_size += insert_table_infos .iter() @@ -1220,7 +1220,7 @@ pub fn object_size_map(version: &HummockVersion) -> HashMap u64 { + assert_eq!(self.sst_info.file_size, self.sst_info.sst_size); self.sst_info.file_size } } diff --git a/src/storage/hummock_sdk/src/sstable_info.rs b/src/storage/hummock_sdk/src/sstable_info.rs index 917c0d4835c3..2f64508e5731 100644 --- a/src/storage/hummock_sdk/src/sstable_info.rs +++ b/src/storage/hummock_sdk/src/sstable_info.rs @@ -33,6 +33,7 @@ pub struct SstableInfo { pub uncompressed_file_size: u64, pub range_tombstone_count: u64, pub bloom_filter_kind: PbBloomFilterType, + pub sst_size: u64, } impl SstableInfo { @@ -48,7 +49,8 @@ impl SstableInfo { + size_of::() // max_epoch + size_of::() // uncompressed_file_size + size_of::() // range_tombstone_count - + size_of::(); // bloom_filter_kind + + size_of::() // bloom_filter_kind + + size_of::(); // sst_size basic += self.key_range.left.len() + self.key_range.right.len() + size_of::(); basic @@ -87,6 +89,11 @@ impl From for SstableInfo { range_tombstone_count: pb_sstable_info.range_tombstone_count, bloom_filter_kind: PbBloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) .unwrap(), + sst_size: if pb_sstable_info.sst_size == 0 { + pb_sstable_info.file_size + } else { + pb_sstable_info.sst_size + }, } } } @@ -118,12 +125,18 @@ impl From<&PbSstableInfo> for SstableInfo { range_tombstone_count: pb_sstable_info.range_tombstone_count, bloom_filter_kind: PbBloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) .unwrap(), + sst_size: if pb_sstable_info.sst_size == 0 { + pb_sstable_info.file_size + } else { + pb_sstable_info.sst_size + }, } } } impl From for PbSstableInfo { fn from(sstable_info: SstableInfo) -> Self { + assert_ne!(0, sstable_info.sst_size); PbSstableInfo { object_id: sstable_info.object_id, sst_id: sstable_info.sst_id, @@ -154,12 +167,14 @@ impl From for PbSstableInfo { uncompressed_file_size: sstable_info.uncompressed_file_size, range_tombstone_count: sstable_info.range_tombstone_count, bloom_filter_kind: sstable_info.bloom_filter_kind.into(), + sst_size: sstable_info.sst_size, } } } impl From<&SstableInfo> for PbSstableInfo { fn from(sstable_info: &SstableInfo) -> Self { + assert_ne!(0, sstable_info.sst_size); PbSstableInfo { object_id: sstable_info.object_id, sst_id: sstable_info.sst_id, @@ -187,6 +202,7 @@ impl From<&SstableInfo> for PbSstableInfo { uncompressed_file_size: sstable_info.uncompressed_file_size, range_tombstone_count: sstable_info.range_tombstone_count, bloom_filter_kind: sstable_info.bloom_filter_kind.into(), + sst_size: sstable_info.sst_size, } } } diff --git a/src/storage/hummock_sdk/src/time_travel.rs b/src/storage/hummock_sdk/src/time_travel.rs index 5894ed3e4a6e..380d75340df2 100644 --- a/src/storage/hummock_sdk/src/time_travel.rs +++ b/src/storage/hummock_sdk/src/time_travel.rs @@ -60,6 +60,7 @@ fn stripped_sstable_info(origin: &SstableInfo) -> SstableInfo { uncompressed_file_size: Default::default(), range_tombstone_count: Default::default(), bloom_filter_kind: Default::default(), + sst_size: Default::default(), } } diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index 67c3aa8b059b..e9721dd8a319 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -167,6 +167,7 @@ async fn test_read_version_basic() { stale_key_count: 1, total_key_count: 1, uncompressed_file_size: 1, + sst_size: 1, ..Default::default() }), LocalSstableInfo::for_test(SstableInfo { @@ -185,6 +186,7 @@ async fn test_read_version_basic() { stale_key_count: 1, total_key_count: 1, uncompressed_file_size: 1, + sst_size: 1, ..Default::default() }), ], diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs index a1f74949b731..c53ab48b567d 100644 --- a/src/storage/hummock_test/src/hummock_storage_tests.rs +++ b/src/storage/hummock_test/src/hummock_storage_tests.rs @@ -2530,6 +2530,8 @@ async fn test_commit_multi_epoch() { sst_id: 11, object_id: 1, table_ids: vec![existing_table_id.table_id], + file_size: 100, + sst_size: 100, ..Default::default() }; @@ -2578,6 +2580,8 @@ async fn test_commit_multi_epoch() { sst_id: 22, object_id: 2, table_ids: vec![existing_table_id.table_id], + file_size: 100, + sst_size: 100, ..Default::default() }; @@ -2629,6 +2633,8 @@ async fn test_commit_multi_epoch() { sst_id: 33, object_id: 3, table_ids: vec![new_table_id.table_id], + file_size: 100, + sst_size: 100, ..Default::default() }; @@ -2671,6 +2677,8 @@ async fn test_commit_multi_epoch() { sst_id: 44, object_id: 4, table_ids: vec![new_table_id.table_id], + file_size: 100, + sst_size: 100, ..Default::default() }; @@ -2711,6 +2719,8 @@ async fn test_commit_multi_epoch() { sst_id: 55, object_id: 5, table_ids: vec![existing_table_id.table_id, new_table_id.table_id], + file_size: 100, + sst_size: 100, ..Default::default() }; diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index 77399eb3e3d0..268e62aa9c2b 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -515,6 +515,7 @@ impl SstableBuilder { min_epoch: cmp::min(min_epoch, tombstone_min_epoch), max_epoch: cmp::max(max_epoch, tombstone_max_epoch), range_tombstone_count: meta.monotonic_tombstone_events.len() as u64, + sst_size: meta.estimated_size as u64, }; tracing::trace!( "meta_size {} bloom_filter_size {} add_key_counts {} stale_key_count {} min_epoch {} max_epoch {} epoch_count {}", diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs index 53b58d895c80..03faa196a08b 100644 --- a/src/storage/src/hummock/test_utils.rs +++ b/src/storage/src/hummock/test_utils.rs @@ -123,6 +123,7 @@ pub fn gen_dummy_sst_info( uncompressed_file_size: file_size, min_epoch: epoch, max_epoch: epoch, + sst_size: file_size, ..Default::default() } }