From 1f30ff779d1f6ee185dc9c02bf370ef9b039653a Mon Sep 17 00:00:00 2001 From: Li0k Date: Fri, 1 Mar 2024 19:41:12 +0800 Subject: [PATCH 01/18] feat(storage): replace pb with struct --- Cargo.lock | 2 + src/compute/src/rpc/service/stream_service.rs | 2 +- src/ctl/src/cmd_impl/hummock/sst_dump.rs | 2 +- .../src/cmd_impl/hummock/validate_version.rs | 7 +- .../rw_catalog/rw_hummock_version.rs | 4 +- src/meta/service/Cargo.toml | 1 + src/meta/service/src/hummock_service.rs | 10 +- src/meta/src/barrier/mod.rs | 3 +- src/meta/src/hummock/compaction/mod.rs | 6 +- .../hummock/compaction/overlap_strategy.rs | 4 +- .../picker/base_level_compaction_picker.rs | 5 +- .../picker/emergency_compaction_picker.rs | 2 +- .../picker/intra_compaction_picker.rs | 7 +- .../picker/manual_compaction_picker.rs | 84 +-- .../picker/min_overlap_compaction_picker.rs | 8 +- src/meta/src/hummock/compaction/picker/mod.rs | 3 +- .../picker/space_reclaim_compaction_picker.rs | 6 +- .../picker/tier_compaction_picker.rs | 8 +- .../tombstone_reclaim_compaction_picker.rs | 5 +- .../picker/trivial_move_compaction_picker.rs | 3 +- .../picker/ttl_reclaim_compaction_picker.rs | 17 +- .../compaction/selector/emergency_selector.rs | 2 +- .../compaction/selector/level_selector.rs | 4 +- .../compaction/selector/manual_selector.rs | 10 +- .../src/hummock/compaction/selector/mod.rs | 14 +- .../selector/space_reclaim_selector.rs | 2 +- .../selector/tombstone_compaction_selector.rs | 2 +- .../compaction/selector/ttl_selector.rs | 2 +- src/meta/src/hummock/compactor_manager.rs | 8 +- src/meta/src/hummock/level_handler.rs | 2 +- src/meta/src/hummock/manager/context.rs | 2 +- src/meta/src/hummock/manager/mod.rs | 43 +- src/meta/src/hummock/manager/tests.rs | 52 +- src/meta/src/hummock/manager/versioning.rs | 23 +- src/meta/src/hummock/metrics_utils.rs | 3 +- .../src/hummock/mock_hummock_meta_client.rs | 12 +- src/meta/src/hummock/test_utils.rs | 32 +- src/meta/src/manager/diagnose.rs | 2 +- src/storage/benches/bench_compactor.rs | 3 +- src/storage/hummock_sdk/Cargo.toml | 1 + src/storage/hummock_sdk/src/compact.rs | 32 +- .../compaction_group/hummock_version_ext.rs | 66 +- src/storage/hummock_sdk/src/key_range.rs | 3 +- src/storage/hummock_sdk/src/lib.rs | 2 +- .../hummock_sdk/src/table_watermark.rs | 4 +- src/storage/hummock_sdk/src/version.rs | 658 +++++++++++++++++- .../hummock_test/src/compactor_tests.rs | 5 +- .../src/hummock_read_version_tests.rs | 11 +- .../src/hummock/compactor/compaction_utils.rs | 22 +- .../src/hummock/compactor/compactor_runner.rs | 34 +- .../compactor/fast_compactor_runner.rs | 7 +- src/storage/src/hummock/compactor/iterator.rs | 6 +- src/storage/src/hummock/compactor/mod.rs | 18 +- .../event_handler/hummock_event_handler.rs | 2 +- .../src/hummock/event_handler/uploader.rs | 8 +- .../iterator/concat_delete_range_iterator.rs | 2 +- .../src/hummock/iterator/concat_inner.rs | 2 +- .../hummock/iterator/delete_range_iterator.rs | 2 +- .../src/hummock/iterator/skip_watermark.rs | 27 +- .../src/hummock/iterator/test_utils.rs | 2 +- .../hummock/local_version/pinned_version.rs | 6 +- src/storage/src/hummock/mod.rs | 2 +- src/storage/src/hummock/sstable/builder.rs | 10 +- src/storage/src/hummock/sstable/mod.rs | 6 +- src/storage/src/hummock/sstable_store.rs | 6 +- .../src/hummock/store/hummock_storage.rs | 2 +- src/storage/src/hummock/store/version.rs | 14 +- src/storage/src/hummock/test_utils.rs | 11 +- src/storage/src/hummock/utils.rs | 14 +- src/storage/src/hummock/validator.rs | 24 +- 70 files changed, 1049 insertions(+), 367 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f60a6052a4d..b071da6a9ebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9500,6 +9500,7 @@ dependencies = [ "prost 0.12.1", "risingwave_common", "risingwave_pb", + "serde", "tracing", "workspace-hack", ] @@ -9751,6 +9752,7 @@ version = "1.7.0-alpha" dependencies = [ "anyhow", "async-trait", + "bytes", "either", "futures", "itertools 0.12.0", diff --git a/src/compute/src/rpc/service/stream_service.rs b/src/compute/src/rpc/service/stream_service.rs index 6e96406743f2..194fbf7b655d 100644 --- a/src/compute/src/rpc/service/stream_service.rs +++ b/src/compute/src/rpc/service/stream_service.rs @@ -177,7 +177,7 @@ impl StreamService for StreamServiceImpl { table_stats, }| GroupedSstableInfo { compaction_group_id, - sst: Some(sst_info), + sst: Some(sst_info.to_protobuf()), table_stats_map: to_prost_table_stats_map(table_stats), }, ) diff --git a/src/ctl/src/cmd_impl/hummock/sst_dump.rs b/src/ctl/src/cmd_impl/hummock/sst_dump.rs index 0fc65054b51e..f1ec2fb34e0d 100644 --- a/src/ctl/src/cmd_impl/hummock/sst_dump.rs +++ b/src/ctl/src/cmd_impl/hummock/sst_dump.rs @@ -29,9 +29,9 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAw use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, ValueRowDeserializer}; use risingwave_frontend::TableCatalog; use risingwave_hummock_sdk::key::FullKey; +use risingwave_hummock_sdk::version::{Level, SstableInfo}; use risingwave_hummock_sdk::HummockSstableObjectId; use risingwave_object_store::object::{ObjectMetadata, ObjectStoreImpl}; -use risingwave_pb::hummock::{Level, SstableInfo}; use risingwave_rpc_client::MetaClient; use risingwave_storage::hummock::value::HummockValue; use risingwave_storage::hummock::{ diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs index b2ae1c22f66c..17266e4a903b 100644 --- a/src/ctl/src/cmd_impl/hummock/validate_version.rs +++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs @@ -20,11 +20,11 @@ use itertools::Itertools; use risingwave_common::util::epoch::Epoch; use risingwave_hummock_sdk::compaction_group::hummock_version_ext; use risingwave_hummock_sdk::key::{FullKey, UserKey}; -use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta, SstableInfo}; use risingwave_hummock_sdk::{version_archive_dir, HummockSstableObjectId, HummockVersionId}; use risingwave_object_store::object::ObjectStoreRef; use risingwave_pb::hummock::group_delta::DeltaType; -use risingwave_pb::hummock::{HummockVersionArchive, SstableInfo}; +use risingwave_pb::hummock::HummockVersionArchive; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::hummock::value::HummockValue; use risingwave_storage::hummock::{Block, BlockHolder, BlockIterator, SstableStoreRef}; @@ -107,8 +107,7 @@ async fn print_user_key_in_version( .chain(cg.levels.iter()) { for sstable_info in &level.table_infos { - use risingwave_hummock_sdk::key_range::KeyRange; - let key_range: KeyRange = sstable_info.key_range.as_ref().unwrap().into(); + let key_range = sstable_info.key_range.as_ref().unwrap(); let left_user_key = FullKey::decode(&key_range.left); let right_user_key = FullKey::decode(&key_range.right); if left_user_key.user_key > *target_key || *target_key > right_user_key.user_key { diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs index 48e2dcfd5361..9e0c9a5b4cfb 100644 --- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs @@ -125,8 +125,8 @@ fn version_to_sstable_rows(version: HummockVersion) -> Vec { level_id: level.level_idx as _, sub_level_id: (level.level_idx > 0).then_some(level.sub_level_id as _), level_type: level.level_type as _, - key_range_left: key_range.left, - key_range_right: key_range.right, + key_range_left: key_range.left.to_vec(), + key_range_right: key_range.right.to_vec(), right_exclusive: key_range.right_exclusive, file_size: sst.file_size as _, meta_offset: sst.meta_offset as _, diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index a91e570ccd6a..814ebc81e89f 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -16,6 +16,7 @@ normal = ["workspace-hack"] [dependencies] anyhow = "1" async-trait = "0.1" +bytes = { version = "1" } either = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = "0.12" diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs index 953bd93b1e75..eb303c8bf978 100644 --- a/src/meta/service/src/hummock_service.rs +++ b/src/meta/service/src/hummock_service.rs @@ -18,6 +18,7 @@ use std::time::Duration; use futures::StreamExt; use itertools::Itertools; use risingwave_common::catalog::{TableId, NON_RESERVED_SYS_CATALOG_ID}; +use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::HummockVersionDelta; use risingwave_meta::manager::MetadataManager; use risingwave_pb::hummock::get_compaction_score_response::PickerInfo; @@ -224,6 +225,7 @@ impl HummockManagerService for HummockServiceImpl { &self, request: Request, ) -> Result, Status> { + use bytes::Bytes; let request = request.into_inner(); let compaction_group_id = request.compaction_group_id; let mut option = ManualCompactionOption { @@ -234,8 +236,12 @@ impl HummockManagerService for HummockServiceImpl { // rewrite the key_range match request.key_range { - Some(key_range) => { - option.key_range = key_range; + Some(pb_key_range) => { + option.key_range = KeyRange { + left: Bytes::from(pb_key_range.left), + right: Bytes::from(pb_key_range.right), + right_exclusive: pb_key_range.right_exclusive, + }; } None => { diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index b64829c9005c..7562e106689f 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -31,6 +31,7 @@ use risingwave_common::util::epoch::{Epoch, INVALID_EPOCH}; use risingwave_hummock_sdk::table_watermark::{ merge_multiple_new_table_watermarks, TableWatermarks, }; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId}; use risingwave_pb::catalog::table::TableType; use risingwave_pb::common::WorkerNode; @@ -965,7 +966,7 @@ fn collect_commit_epoch_info(resps: &mut [BarrierCompleteResponse]) -> CommitEpo sst_to_worker.insert(sst_info.get_object_id(), resp.worker_id); ExtendedSstableInfo::new( grouped.compaction_group_id, - sst_info, + SstableInfo::from_protobuf(&sst_info), std::mem::take(&mut grouped.table_stats_map), ) }) diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs index 197a565090a3..3f82e2767c4b 100644 --- a/src/meta/src/hummock/compaction/mod.rs +++ b/src/meta/src/hummock/compaction/mod.rs @@ -17,6 +17,7 @@ pub mod compaction_config; mod overlap_strategy; use risingwave_common::catalog::TableOption; +use risingwave_hummock_sdk::version::{CompactTask, Levels}; use risingwave_pb::hummock::compact_task::{self, TaskType}; mod picker; @@ -28,8 +29,7 @@ use std::sync::Arc; use picker::{LevelCompactionPicker, TierCompactionPicker}; use risingwave_hummock_sdk::{can_concat, CompactionGroupId, HummockCompactionTaskId}; use risingwave_pb::hummock::compaction_config::CompactionMode; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{CompactTask, CompactionConfig, LevelType}; +use risingwave_pb::hummock::{CompactionConfig, LevelType}; pub use selector::CompactionSelector; use self::selector::LocalSelectorStatistic; @@ -69,7 +69,7 @@ pub struct CompactionTask { pub compaction_task_type: compact_task::TaskType, } -pub fn create_overlap_strategy(compaction_mode: CompactionMode) -> Arc { +pub fn create_overlap_strategy(compaction_mode: CompactionMode) -> Arc { match compaction_mode { CompactionMode::Range => Arc::new(RangeOverlapStrategy::default()), CompactionMode::Unspecified => unreachable!(), diff --git a/src/meta/src/hummock/compaction/overlap_strategy.rs b/src/meta/src/hummock/compaction/overlap_strategy.rs index 1a713cc40117..ecd89d8a27eb 100644 --- a/src/meta/src/hummock/compaction/overlap_strategy.rs +++ b/src/meta/src/hummock/compaction/overlap_strategy.rs @@ -17,9 +17,9 @@ use std::fmt::Debug; use std::ops::Range; use itertools::Itertools; -use risingwave_hummock_sdk::key_range::KeyRangeCommon; +use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::KeyComparator; -use risingwave_pb::hummock::{KeyRange, SstableInfo}; pub trait OverlapInfo: Debug { fn check_overlap(&self, a: &SstableInfo) -> bool; diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index 6617b9496fe1..216d848989d8 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -16,14 +16,15 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{CompactionConfig, InputLevel, Level, LevelType, OverlappingLevel}; +use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, OverlappingLevel}; +use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker; use super::{ CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic, ValidationRuleType, }; +use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::compaction::picker::TrivialMovePicker; use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig}; use crate::hummock::level_handler::LevelHandler; diff --git a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs index 4efcca28a981..50679558d81d 100644 --- a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs @@ -14,7 +14,7 @@ use std::sync::Arc; -use risingwave_pb::hummock::hummock_version::Levels; +use risingwave_hummock_sdk::version::Levels; use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::{ diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 15cf7e9d01f1..7b8a28d8c8f8 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -14,14 +14,15 @@ use std::sync::Arc; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel}; +use risingwave_hummock_sdk::version::{InputLevel, Levels, OverlappingLevel}; +use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker; use super::{ CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic, ValidationRuleType, }; +use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::compaction::picker::TrivialMovePicker; use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig}; use crate::hummock::level_handler::LevelHandler; @@ -411,7 +412,7 @@ impl WholeLevelCompactionPicker { #[cfg(test)] pub mod tests { - use risingwave_pb::hummock::Level; + use risingwave_hummock_sdk::version::{Level, Levels}; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index f46a99bc80c0..c5318e4ec396 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{InputLevel, Level, LevelType, OverlappingLevel, SstableInfo}; +use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, OverlappingLevel, SstableInfo}; +use risingwave_pb::hummock::LevelType; use super::{CompactionInput, CompactionPicker, LocalPickerStatistic}; use crate::hummock::compaction::overlap_strategy::{ @@ -328,8 +328,10 @@ impl CompactionPicker for ManualCompactionPicker { pub mod tests { use std::collections::{HashMap, HashSet}; + use bytes::Bytes; + use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_pb::hummock::compact_task; - pub use risingwave_pb::hummock::{KeyRange, Level, LevelType}; + pub use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; @@ -416,8 +418,8 @@ pub mod tests { let option = ManualCompactionOption { level: 1, key_range: KeyRange { - left: iterator_test_key_of_epoch(1, 0, 1), - right: iterator_test_key_of_epoch(1, 201, 1), + left: Bytes::from(iterator_test_key_of_epoch(1, 0, 1)), + right: Bytes::from(iterator_test_key_of_epoch(1, 201, 1)), right_exclusive: false, }, ..Default::default() @@ -509,8 +511,8 @@ pub mod tests { sst_ids: vec![], level: 1, key_range: KeyRange { - left: iterator_test_key_of_epoch(1, 101, 1), - right: iterator_test_key_of_epoch(1, 199, 1), + left: Bytes::from(iterator_test_key_of_epoch(1, 101, 1)), + right: Bytes::from(iterator_test_key_of_epoch(1, 199, 1)), right_exclusive: false, }, internal_table_id: HashSet::from([2]), @@ -643,8 +645,8 @@ pub mod tests { sst_ids: vec![1], level: 0, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -669,8 +671,8 @@ pub mod tests { sst_ids: vec![], level: 0, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -716,8 +718,8 @@ pub mod tests { sst_ids: vec![], level: 0, key_range: KeyRange { - left: iterator_test_key_of_epoch(1, 0, 2), - right: iterator_test_key_of_epoch(1, 200, 2), + left: Bytes::from(iterator_test_key_of_epoch(1, 0, 2)), + right: Bytes::from(iterator_test_key_of_epoch(1, 200, 2)), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -767,8 +769,8 @@ pub mod tests { sst_ids: sst_id_filter.clone(), level: *input_level as _, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -808,8 +810,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // No matching internal table id. @@ -830,8 +832,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // Include all sub level's table ids @@ -874,8 +876,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // Only include bottom sub level's table id @@ -917,8 +919,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // Only include partial top sub level's table id, but the whole top sub level is @@ -961,8 +963,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // Only include bottom sub level's table id @@ -994,8 +996,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // No matching internal table id. @@ -1017,8 +1019,8 @@ pub mod tests { sst_ids: vec![], level: input_level, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, // Only include partial input level's table id @@ -1069,8 +1071,8 @@ pub mod tests { sst_ids: sst_id_filter.clone(), level: *input_level as _, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -1115,8 +1117,8 @@ pub mod tests { sst_ids: sst_id_filter.clone(), level: *input_level as _, key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -1185,8 +1187,8 @@ pub mod tests { let option = ManualCompactionOption { sst_ids: vec![0, 1], key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -1222,8 +1224,8 @@ pub mod tests { let option = ManualCompactionOption { sst_ids: vec![], key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -1295,8 +1297,8 @@ pub mod tests { let option = ManualCompactionOption { sst_ids: vec![0, 1], key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), @@ -1334,8 +1336,8 @@ pub mod tests { let option = ManualCompactionOption { sst_ids: vec![], key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 01c3c050e574..4ae1bae5a836 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use risingwave_hummock_sdk::append_sstable_info_to_string; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{InputLevel, Level, LevelType, SstableInfo}; +use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, SstableInfo}; +use risingwave_pb::hummock::LevelType; use super::{CompactionInput, CompactionPicker, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT}; use crate::hummock::compaction::overlap_strategy::OverlapStrategy; @@ -326,7 +326,7 @@ impl NonOverlapSubLevelPicker { level_ssts.sort_by(|sst1, sst2| { let a = sst1.key_range.as_ref().unwrap(); let b = sst2.key_range.as_ref().unwrap(); - a.compare(b) + a.cmp(b) }); }); @@ -477,7 +477,7 @@ impl NonOverlapSubLevelPicker { pub mod tests { use std::collections::BTreeSet; - pub use risingwave_pb::hummock::{Level, LevelType}; + pub use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::compaction::overlap_strategy::RangeOverlapStrategy; diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs index 7e3312327268..e18bf6165126 100644 --- a/src/meta/src/hummock/compaction/picker/mod.rs +++ b/src/meta/src/hummock/compaction/picker/mod.rs @@ -31,8 +31,7 @@ pub use emergency_compaction_picker::EmergencyCompactionPicker; pub use intra_compaction_picker::IntraCompactionPicker; pub use manual_compaction_picker::ManualCompactionPicker; pub use min_overlap_compaction_picker::MinOverlappingPicker; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::InputLevel; +use risingwave_hummock_sdk::version::{InputLevel, Levels}; pub use space_reclaim_compaction_picker::{SpaceReclaimCompactionPicker, SpaceReclaimPickerState}; pub use tier_compaction_picker::TierCompactionPicker; pub use tombstone_reclaim_compaction_picker::{ diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 5d05fedbe533..b7886c136a3d 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -14,8 +14,7 @@ use std::collections::HashSet; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{InputLevel, SstableInfo}; +use risingwave_hummock_sdk::version::{InputLevel, Levels, SstableInfo}; use super::CompactionInput; use crate::hummock::level_handler::LevelHandler; @@ -172,8 +171,9 @@ mod test { use std::sync::Arc; use itertools::Itertools; + use risingwave_hummock_sdk::version::Level; use risingwave_pb::hummock::compact_task; - pub use risingwave_pb::hummock::{Level, LevelType}; + pub use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index 9ed22ba551fc..32f83f0cc0f2 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -14,8 +14,8 @@ use std::sync::Arc; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel}; +use risingwave_hummock_sdk::version::{InputLevel, Levels, OverlappingLevel}; +use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::{ CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic, @@ -169,8 +169,8 @@ pub mod tests { use std::sync::Arc; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::new_sub_level; - use risingwave_pb::hummock::hummock_version::Levels; - use risingwave_pb::hummock::{LevelType, OverlappingLevel}; + use risingwave_hummock_sdk::version::{Levels, OverlappingLevel}; + use risingwave_pb::hummock::LevelType; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; use crate::hummock::compaction::picker::{ diff --git a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs index f07a8872c9b0..c3b6cb4e169b 100644 --- a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs @@ -14,8 +14,7 @@ use std::sync::Arc; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::InputLevel; +use risingwave_hummock_sdk::version::{InputLevel, Levels}; use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::compaction::picker::CompactionInput; @@ -133,7 +132,7 @@ impl TombstoneReclaimCompactionPicker { #[cfg(test)] pub mod tests { - use risingwave_pb::hummock::OverlappingLevel; + use risingwave_hummock_sdk::version::OverlappingLevel; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs index bab0dca0c324..faf37140b660 100644 --- a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs @@ -14,7 +14,8 @@ use std::sync::Arc; -use risingwave_pb::hummock::{InputLevel, LevelType, SstableInfo}; +use risingwave_hummock_sdk::version::{InputLevel, SstableInfo}; +use risingwave_pb::hummock::LevelType; use super::{CompactionInput, LocalPickerStatistic}; use crate::hummock::compaction::overlap_strategy::OverlapStrategy; diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index f690b0f80112..240309fef082 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -14,12 +14,12 @@ use std::collections::{HashMap, HashSet}; +use bytes::Bytes; use risingwave_common::catalog::TableOption; use risingwave_common::util::epoch::Epoch; use risingwave_hummock_sdk::compaction_group::StateTableId; -use risingwave_hummock_sdk::key_range::KeyRangeCommon; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{InputLevel, KeyRange, SstableInfo}; +use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; +use risingwave_hummock_sdk::version::{InputLevel, Levels, SstableInfo}; use super::CompactionInput; use crate::hummock::level_handler::LevelHandler; @@ -46,8 +46,8 @@ impl TtlPickerState { pub fn init(&mut self, key_range: KeyRange) { self.last_select_end_bound = KeyRange { - left: vec![], - right: key_range.left.clone(), + left: Bytes::default(), + right: Bytes::from(key_range.left.clone()), right_exclusive: true, }; self.end_bound_in_round = key_range; @@ -173,8 +173,8 @@ impl TtlReclaimCompactionPicker { let select_last_sst = select_input_ssts.last().unwrap(); state.last_select_end_bound.full_key_extend(&KeyRange { - left: vec![], - right: select_last_sst.key_range.as_ref().unwrap().right.clone(), + left: Bytes::default(), + right: Bytes::from(select_last_sst.key_range.as_ref().unwrap().right.clone()), right_exclusive: select_last_sst.key_range.as_ref().unwrap().right_exclusive, }); @@ -204,8 +204,9 @@ mod test { use std::sync::Arc; use itertools::Itertools; + use risingwave_hummock_sdk::version::Level; use risingwave_pb::hummock::compact_task; - pub use risingwave_pb::hummock::{Level, LevelType}; + pub use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/selector/emergency_selector.rs b/src/meta/src/hummock/compaction/selector/emergency_selector.rs index c386aee5c864..7873b8725ea8 100644 --- a/src/meta/src/hummock/compaction/selector/emergency_selector.rs +++ b/src/meta/src/hummock/compaction/selector/emergency_selector.rs @@ -16,9 +16,9 @@ use std::collections::HashMap; use std::sync::Arc; use risingwave_common::catalog::TableOption; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task; -use risingwave_pb::hummock::hummock_version::Levels; use super::{CompactionSelector, DynamicLevelSelectorCore, LocalSelectorStatistic}; use crate::hummock::compaction::picker::{EmergencyCompactionPicker, LocalPickerStatistic}; diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index 5c118269cfee..a5fcfd19df3c 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use risingwave_common::catalog::TableOption; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; -use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::{compact_task, CompactionConfig, LevelType}; use super::{ @@ -488,8 +488,8 @@ pub mod tests { use itertools::Itertools; use risingwave_common::constants::hummock::CompactionFilterFlag; + use risingwave_hummock_sdk::version::Levels; use risingwave_pb::hummock::compaction_config::CompactionMode; - use risingwave_pb::hummock::hummock_version::Levels; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; use crate::hummock::compaction::selector::tests::{ diff --git a/src/meta/src/hummock/compaction/selector/manual_selector.rs b/src/meta/src/hummock/compaction/selector/manual_selector.rs index 427efadf3914..855ea02e47c4 100644 --- a/src/meta/src/hummock/compaction/selector/manual_selector.rs +++ b/src/meta/src/hummock/compaction/selector/manual_selector.rs @@ -20,11 +20,13 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use bytes::Bytes; use risingwave_common::catalog::TableOption; use risingwave_hummock_sdk::compaction_group::StateTableId; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockSstableId}; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::{compact_task, KeyRange}; +use risingwave_pb::hummock::compact_task; use super::{CompactionSelector, DynamicLevelSelectorCore, LocalSelectorStatistic}; use crate::hummock::compaction::picker::{ @@ -53,8 +55,8 @@ impl Default for ManualCompactionOption { Self { sst_ids: vec![], key_range: KeyRange { - left: vec![], - right: vec![], + left: Bytes::default(), + right: Bytes::default(), right_exclusive: false, }, internal_table_id: HashSet::default(), diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index a342a661ecb7..dcbb6e7636ae 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -31,9 +31,9 @@ pub use emergency_selector::EmergencySelector; pub use level_selector::{DynamicLevelSelector, DynamicLevelSelectorCore}; pub use manual_selector::{ManualCompactionOption, ManualCompactionSelector}; use risingwave_common::catalog::TableOption; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task; -use risingwave_pb::hummock::hummock_version::Levels; pub use space_reclaim_selector::SpaceReclaimCompactionSelector; pub use tombstone_compaction_selector::TombstoneCompactionSelector; pub use ttl_selector::TtlCompactionSelector; @@ -116,7 +116,9 @@ pub mod tests { use std::ops::Range; use itertools::Itertools; - use risingwave_pb::hummock::{KeyRange, Level, LevelType, OverlappingLevel, SstableInfo}; + use risingwave_hummock_sdk::key_range::KeyRange; + use risingwave_hummock_sdk::version::{Level, OverlappingLevel, SstableInfo}; + use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::test_utils::iterator_test_key_of_epoch; @@ -176,8 +178,8 @@ pub mod tests { object_id: id, sst_id: id, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(table_prefix, left, epoch), - right: iterator_test_key_of_epoch(table_prefix, right, epoch), + left: iterator_test_key_of_epoch(table_prefix, left, epoch).into(), + right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(), right_exclusive: false, }), file_size: (right - left + 1) as u64, @@ -203,8 +205,8 @@ pub mod tests { object_id: id, sst_id: id, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(table_prefix, left, epoch), - right: iterator_test_key_of_epoch(table_prefix, right, epoch), + left: iterator_test_key_of_epoch(table_prefix, left, epoch).into(), + right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(), right_exclusive: false, }), file_size: (right - left + 1) as u64, diff --git a/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs b/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs index c48cb0fe605c..1aa348fff5a7 100644 --- a/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs +++ b/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs @@ -21,9 +21,9 @@ use std::collections::HashMap; use std::sync::Arc; use risingwave_common::catalog::TableOption; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task; -use risingwave_pb::hummock::hummock_version::Levels; use super::{CompactionSelector, DynamicLevelSelectorCore}; use crate::hummock::compaction::picker::{SpaceReclaimCompactionPicker, SpaceReclaimPickerState}; diff --git a/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs b/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs index 5cbae609caa8..fbcc15dafa2f 100644 --- a/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs +++ b/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs @@ -16,9 +16,9 @@ use std::collections::HashMap; use std::sync::Arc; use risingwave_common::catalog::TableOption; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task; -use risingwave_pb::hummock::hummock_version::Levels; use super::{CompactionSelector, DynamicLevelSelectorCore}; use crate::hummock::compaction::picker::{ diff --git a/src/meta/src/hummock/compaction/selector/ttl_selector.rs b/src/meta/src/hummock/compaction/selector/ttl_selector.rs index ed099ede4158..5eb314db4e88 100644 --- a/src/meta/src/hummock/compaction/selector/ttl_selector.rs +++ b/src/meta/src/hummock/compaction/selector/ttl_selector.rs @@ -21,9 +21,9 @@ use std::collections::HashMap; use std::sync::Arc; use risingwave_common::catalog::TableOption; +use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task; -use risingwave_pb::hummock::hummock_version::Levels; use super::{CompactionSelector, DynamicLevelSelectorCore}; use crate::hummock::compaction::picker::{TtlPickerState, TtlReclaimCompactionPicker}; diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs index 8a548d2522e8..708077fe1c3d 100644 --- a/src/meta/src/hummock/compactor_manager.rs +++ b/src/meta/src/hummock/compactor_manager.rs @@ -19,11 +19,11 @@ use std::time::{Duration, Instant, SystemTime}; use fail::fail_point; use parking_lot::RwLock; use risingwave_hummock_sdk::compact::statistics_compact_task; +use risingwave_hummock_sdk::version::CompactTask; use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId}; use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent; use risingwave_pb::hummock::{ - CancelCompactTask, CompactTask, CompactTaskAssignment, CompactTaskProgress, - SubscribeCompactionEventResponse, + CancelCompactTask, CompactTaskAssignment, CompactTaskProgress, SubscribeCompactionEventResponse, }; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; @@ -149,7 +149,9 @@ impl CompactorManagerInner { }; // Initialize heartbeat for existing tasks. task_assignment.into_iter().for_each(|assignment| { - manager.initiate_task_heartbeat(assignment.compact_task.unwrap()); + manager.initiate_task_heartbeat(CompactTask::from_protobuf( + assignment.compact_task.as_ref().unwrap(), + )); }); Ok(manager) } diff --git a/src/meta/src/hummock/level_handler.rs b/src/meta/src/hummock/level_handler.rs index 25bc8af8e66d..921b80fb87b5 100644 --- a/src/meta/src/hummock/level_handler.rs +++ b/src/meta/src/hummock/level_handler.rs @@ -15,9 +15,9 @@ use std::collections::HashMap; use itertools::Itertools; +use risingwave_hummock_sdk::version::{Level, SstableInfo}; use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockSstableId}; use risingwave_pb::hummock::level_handler::RunningCompactTask; -use risingwave_pb::hummock::{Level, SstableInfo}; #[derive(Clone, Debug, PartialEq)] pub struct LevelHandler { diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs index 98588579137b..59d671503dba 100644 --- a/src/meta/src/hummock/manager/context.rs +++ b/src/meta/src/hummock/manager/context.rs @@ -172,7 +172,7 @@ impl HummockManager { .collect_vec(); if compactor .send_event(ResponseEvent::ValidationTask(ValidationTask { - sst_infos, + sst_infos: sst_infos.iter().map(|sst| sst.to_protobuf()).collect_vec(), sst_id_to_worker_id: sst_to_context.clone(), epoch, })) diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index 8984aff7af04..b0ed6f2d54e0 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -40,7 +40,8 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_table_compaction_group_id_mapping, try_get_compaction_group_id_by_table_id, BranchedSstInfo, HummockLevelsExt, }; -use risingwave_hummock_sdk::version::HummockVersionDelta; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::{CompactTask, HummockVersionDelta, SstableInfo}; use risingwave_hummock_sdk::{ version_archive_dir, version_checkpoint_path, CompactionGroupId, ExtendedSstableInfo, HummockCompactionTaskId, HummockContextId, HummockEpoch, HummockSstableId, @@ -60,9 +61,9 @@ use risingwave_pb::hummock::subscribe_compaction_event_response::{ Event as ResponseEvent, PullTaskAck, }; use risingwave_pb::hummock::{ - CompactTask, CompactTaskAssignment, CompactionConfig, GroupDelta, HummockPinnedSnapshot, + CompactTaskAssignment, CompactionConfig, GroupDelta, HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, HummockVersionStats, IntraLevelDelta, - PbCompactionGroupInfo, SstableInfo, SubscribeCompactionEventRequest, TableOption, + PbCompactionGroupInfo, SubscribeCompactionEventRequest, TableOption, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use rw_futures_util::{pending_on_none, select_all}; @@ -971,7 +972,7 @@ impl HummockManager { let mut compact_task = CompactTask { input_ssts: compact_task.input.input_levels, - splits: vec![risingwave_pb::hummock::KeyRange::inf()], + splits: vec![KeyRange::inf()], watermark, sorted_output_ssts: vec![], task_id, @@ -1075,7 +1076,7 @@ impl HummockManager { compact_task_assignment.insert( compact_task.task_id, CompactTaskAssignment { - compact_task: Some(compact_task.clone()), + compact_task: Some(compact_task.to_protobuf()), context_id: META_NODE_ID, // deprecated }, ); @@ -1329,7 +1330,9 @@ impl HummockManager { input_task } else { match compact_task_assignment.remove(task_id) { - Some(compact_task) => compact_task.compact_task.unwrap(), + Some(compact_task) => { + CompactTask::from_protobuf(&compact_task.compact_task.unwrap()) + } None => { tracing::warn!("{}", format!("compact task {} not found", task_id)); return Ok(false); @@ -1590,7 +1593,7 @@ impl HummockManager { }; if !is_sst_belong_to_group_declared { let mut group_table_ids: BTreeMap<_, Vec> = BTreeMap::new(); - for table_id in sst.get_table_ids() { + for table_id in &sst.table_ids { match try_get_compaction_group_id_by_table_id( &versioning.current_version, *table_id, @@ -1605,14 +1608,13 @@ impl HummockManager { tracing::warn!( "table {} in SST {} doesn't belong to any compaction group", table_id, - sst.get_object_id(), + sst.object_id, ); } } } let is_trivial_adjust = group_table_ids.len() == 1 - && group_table_ids.first_key_value().unwrap().1.len() - == sst.get_table_ids().len(); + && group_table_ids.first_key_value().unwrap().1.len() == sst.table_ids.len(); if is_trivial_adjust { *compaction_group_id = *group_table_ids.first_key_value().unwrap().0; // is_sst_belong_to_group_declared = true; @@ -1649,7 +1651,7 @@ impl HummockManager { new_sst_id += 1; } if !branch_groups.is_empty() { - branched_ssts.insert(sst.get_object_id(), branch_groups); + branched_ssts.insert(sst.object_id, branch_groups); } } else { sstables.push(original_sstable); @@ -1690,7 +1692,10 @@ impl HummockManager { let group_delta = GroupDelta { delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { level_idx: 0, - inserted_table_infos: group_sstables.clone(), + inserted_table_infos: group_sstables + .iter() + .map(|sst| sst.to_protobuf()) + .collect_vec(), l0_sub_level_id, ..Default::default() })), @@ -2089,7 +2094,7 @@ impl HummockManager { let compact_task_string = compact_task_to_string(&compact_task); // TODO: shall we need to cancel on meta ? compactor - .send_event(ResponseEvent::CompactTask(compact_task)) + .send_event(ResponseEvent::CompactTask(compact_task.to_protobuf())) .with_context(|| { format!( "Failed to trigger compaction task for compaction_group {}", @@ -3152,7 +3157,7 @@ impl HummockManager { Ok(Some(compact_task)) => { let task_id = compact_task.task_id; if let Err(e) = compactor.send_event( - ResponseEvent::CompactTask(compact_task), + ResponseEvent::CompactTask(compact_task.to_protobuf()), ) { tracing::warn!( error = %e.as_report(), @@ -3204,7 +3209,7 @@ impl HummockManager { .report_compact_task( task_id, TaskStatus::try_from(task_status).unwrap(), - sorted_output_ssts, + sorted_output_ssts.iter().map(|pb_sst| SstableInfo::from_protobuf(pb_sst)).collect_vec(), Some(table_stats_change), ) .await @@ -3287,7 +3292,7 @@ fn gen_version_delta<'a>( .table_infos .iter() .map(|sst| { - let object_id = sst.get_object_id(); + let object_id = sst.object_id; let sst_id = sst.get_sst_id(); if !trivial_move && drop_sst( @@ -3323,7 +3328,11 @@ fn gen_version_delta<'a>( let group_delta = GroupDelta { delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { level_idx: compact_task.target_level, - inserted_table_infos: compact_task.sorted_output_ssts.clone(), + inserted_table_infos: compact_task + .sorted_output_ssts + .iter() + .map(|sst| sst.to_protobuf()) + .collect_vec(), l0_sub_level_id: compact_task.target_sub_level_id, vnode_partition_count: compact_task.split_weight_by_vnode, ..Default::default() diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs index 877be8919262..71d4e120c4fb 100644 --- a/src/meta/src/hummock/manager/tests.rs +++ b/src/meta/src/hummock/manager/tests.rs @@ -24,18 +24,16 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_compaction_group_ssts, BranchedSstInfo, }; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; +use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::table_stats::{to_prost_table_stats_map, TableStats, TableStatsMap}; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{ CompactionGroupId, ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, FIRST_VERSION_ID, }; use risingwave_pb::common::{HostAddress, WorkerType}; use risingwave_pb::hummock::compact_task::TaskStatus; -use risingwave_pb::hummock::{ - CompactTask, HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, KeyRange, - SstableInfo, -}; +use risingwave_pb::hummock::{HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot}; use risingwave_pb::meta::add_worker_node_request::Property; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; @@ -66,8 +64,8 @@ fn gen_sstable_info(sst_id: u64, idx: usize, table_ids: Vec) -> SstableInfo SstableInfo { sst_id, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, idx, 1), - right: iterator_test_key_of_epoch(1, idx, 1), + left: iterator_test_key_of_epoch(1, idx, 1).into(), + right: iterator_test_key_of_epoch(1, idx, 1).into(), right_exclusive: false, }), table_ids, @@ -1176,8 +1174,8 @@ async fn test_version_stats() { object_id: sst_ids[idx], sst_id: sst_ids[idx], key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, 1, 1), - right: iterator_test_key_of_epoch(1, 1, 1), + left: iterator_test_key_of_epoch(1, 1, 1).into(), + right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, }), file_size: 1024 * 1024 * 1024, @@ -1407,8 +1405,8 @@ async fn test_split_compaction_group_on_demand_basic() { object_id: 10, sst_id: 10, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(100, 1, 20), - right: iterator_test_key_of_epoch(100, 100, 20), + left: iterator_test_key_of_epoch(100, 1, 20).into(), + right: iterator_test_key_of_epoch(100, 100, 20).into(), right_exclusive: false, }), table_ids: vec![100], @@ -1424,8 +1422,8 @@ async fn test_split_compaction_group_on_demand_basic() { object_id: 11, sst_id: 11, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(100, 101, 20), - right: iterator_test_key_of_epoch(101, 100, 20), + left: iterator_test_key_of_epoch(100, 101, 20).into(), + right: iterator_test_key_of_epoch(101, 100, 20).into(), right_exclusive: false, }), table_ids: vec![100, 101], @@ -1614,8 +1612,8 @@ async fn test_split_compaction_group_trivial_expired() { object_id: 10, sst_id: 10, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(100, 1, 20), - right: iterator_test_key_of_epoch(100, 100, 20), + left: iterator_test_key_of_epoch(100, 1, 20).into(), + right: iterator_test_key_of_epoch(100, 100, 20).into(), right_exclusive: false, }), table_ids: vec![100], @@ -1634,8 +1632,8 @@ async fn test_split_compaction_group_trivial_expired() { min_epoch: 20, max_epoch: 20, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(101, 1, 20), - right: iterator_test_key_of_epoch(101, 100, 20), + left: iterator_test_key_of_epoch(101, 1, 20).into(), + right: iterator_test_key_of_epoch(101, 100, 20).into(), right_exclusive: false, }), ..Default::default() @@ -1776,8 +1774,8 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { object_id: 10, sst_id: 10, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, 1, 1), - right: iterator_test_key_of_epoch(1, 1, 1), + left: iterator_test_key_of_epoch(1, 1, 1).into(), + right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, }), table_ids: vec![100, 101], @@ -1810,8 +1808,8 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { sst_id: 11, table_ids: vec![100, 101], key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, 1, 1), - right: iterator_test_key_of_epoch(1, 1, 1), + left: iterator_test_key_of_epoch(1, 1, 1).into(), + right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, }), ..Default::default() @@ -1821,8 +1819,8 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { sst_id: 12, table_ids: vec![100], key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, 2, 2), - right: iterator_test_key_of_epoch(1, 2, 2), + left: iterator_test_key_of_epoch(1, 2, 2).into(), + right: iterator_test_key_of_epoch(1, 2, 2).into(), right_exclusive: false, }), ..Default::default() @@ -1923,8 +1921,8 @@ async fn test_compaction_task_expiration_due_to_split_group() { object_id: 10, sst_id: 10, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, 1, 1), - right: iterator_test_key_of_epoch(1, 1, 1), + left: iterator_test_key_of_epoch(1, 1, 1).into(), + right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, }), table_ids: vec![100, 101], @@ -1940,8 +1938,8 @@ async fn test_compaction_task_expiration_due_to_split_group() { object_id: 11, sst_id: 11, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(1, 1, 1), - right: iterator_test_key_of_epoch(1, 1, 1), + left: iterator_test_key_of_epoch(1, 1, 1).into(), + right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, }), table_ids: vec![101], diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs index 5cf270076e6c..0eb04f603b60 100644 --- a/src/meta/src/hummock/manager/versioning.rs +++ b/src/meta/src/hummock/manager/versioning.rs @@ -23,15 +23,14 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ }; use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId}; use risingwave_hummock_sdk::table_stats::add_prost_table_stats_map; -use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta, SstableInfo}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID, }; use risingwave_pb::common::WorkerNode; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{ - CompactionConfig, HummockPinnedSnapshot, HummockPinnedVersion, HummockVersionStats, - SstableInfo, TableStats, + CompactionConfig, HummockPinnedSnapshot, HummockPinnedVersion, HummockVersionStats, TableStats, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; @@ -408,13 +407,13 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use risingwave_hummock_sdk::version::HummockVersion; + use risingwave_hummock_sdk::key_range::KeyRange; + use risingwave_hummock_sdk::version::{ + HummockVersion, Level, Levels, OverlappingLevel, SstableInfo, + }; use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId}; - use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::write_limits::WriteLimit; - use risingwave_pb::hummock::{ - HummockPinnedVersion, HummockVersionStats, KeyRange, Level, OverlappingLevel, SstableInfo, - }; + use risingwave_pb::hummock::{HummockPinnedVersion, HummockVersionStats}; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; use crate::hummock::manager::versioning::{ @@ -545,8 +544,8 @@ mod tests { fn test_estimate_table_stats() { let sst = SstableInfo { key_range: Some(KeyRange { - left: vec![1; 10], - right: vec![1; 20], + left: vec![1; 10].into(), + right: vec![1; 20].into(), ..Default::default() }), table_ids: vec![1, 2, 3], @@ -611,8 +610,8 @@ mod tests { fn test_estimate_table_stats_large_key_range() { let sst = SstableInfo { key_range: Some(KeyRange { - left: vec![1; 1000], - right: vec![1; 2000], + left: vec![1; 1000].into(), + right: vec![1; 2000].into(), ..Default::default() }), table_ids: vec![1, 2, 3], diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs index cd0fd437c379..94270853f87d 100644 --- a/src/meta/src/hummock/metrics_utils.rs +++ b/src/meta/src/hummock/metrics_utils.rs @@ -21,11 +21,10 @@ use itertools::{enumerate, Itertools}; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ object_size_map, BranchedSstInfo, }; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{HummockVersion, Levels}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, }; -use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{ CompactionConfig, HummockPinnedSnapshot, HummockPinnedVersion, HummockVersionStats, LevelType, diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index bdc674830458..01f1c08dfb86 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -22,10 +22,11 @@ use async_trait::async_trait; use fail::fail_point; use futures::stream::BoxStream; use futures::{Stream, StreamExt}; +use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::table_watermark::TableWatermarks; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{ HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, SstObjectIdRange, @@ -35,7 +36,7 @@ use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_pb::hummock::subscribe_compaction_event_request::{Event, ReportTask}; use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent; use risingwave_pb::hummock::{ - compact_task, CompactTask, HummockSnapshot, SubscribeCompactionEventRequest, + compact_task, HummockSnapshot, SubscribeCompactionEventRequest, SubscribeCompactionEventResponse, VacuumTask, }; use risingwave_rpc_client::error::{Result, RpcError}; @@ -291,7 +292,7 @@ impl HummockMetaClient for MockHummockMetaClient { .unwrap() { let resp = SubscribeCompactionEventResponse { - event: Some(ResponseEvent::CompactTask(task)), + event: Some(ResponseEvent::CompactTask(task.to_protobuf())), create_at: SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("Clock may have gone backwards") @@ -322,7 +323,10 @@ impl HummockMetaClient for MockHummockMetaClient { .report_compact_task( task_id, TaskStatus::try_from(task_status).unwrap(), - sorted_output_ssts, + sorted_output_ssts + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .collect_vec(), Some(table_stats_change), ) .await diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs index 5aa7e3172442..f8169b0627d6 100644 --- a/src/meta/src/hummock/test_utils.rs +++ b/src/meta/src/hummock/test_utils.rs @@ -15,17 +15,19 @@ use std::sync::Arc; use std::time::Duration; +use bytes::Bytes; use itertools::Itertools; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::key_with_epoch; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, LocalSstableInfo, }; use risingwave_pb::common::{HostAddress, WorkerNode, WorkerType}; #[cfg(test)] use risingwave_pb::hummock::compact_task::TaskStatus; -use risingwave_pb::hummock::{CompactionConfig, HummockSnapshot, KeyRange, SstableInfo}; +use risingwave_pb::hummock::{CompactionConfig, HummockSnapshot}; use risingwave_pb::meta::add_worker_node_request::Property; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; @@ -54,6 +56,8 @@ pub async fn add_test_tables( context_id: HummockContextId, ) -> Vec> { // Increase version by 2. + + use risingwave_hummock_sdk::version::SstableInfo; let mut epoch: u64 = 1; let sstable_ids = get_sst_ids(hummock_manager, 3).await; let test_tables = generate_test_sstables_with_table_id(epoch, 1, sstable_ids); @@ -66,7 +70,7 @@ pub async fn add_test_tables( let ssts = to_local_sstable_info(&test_tables); let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id)) .collect(); hummock_manager .commit_epoch(epoch, CommitEpochInfo::for_test(ssts, sst_to_worker)) @@ -142,7 +146,7 @@ pub async fn add_test_tables( let ssts = to_local_sstable_info(&test_tables_3); let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id)) .collect(); hummock_manager .commit_epoch(epoch, CommitEpochInfo::for_test(ssts, sst_to_worker)) @@ -162,18 +166,18 @@ pub fn generate_test_sstables_with_table_id( object_id: sst_id, sst_id, key_range: Some(KeyRange { - left: key_with_epoch( + left: Bytes::from(key_with_epoch( format!("{:03}\0\0_key_test_{:05}", table_id, i + 1) .as_bytes() .to_vec(), epoch, - ), - right: key_with_epoch( + )), + right: Bytes::from(key_with_epoch( format!("{:03}\0\0_key_test_{:05}", table_id, (i + 1) * 10) .as_bytes() .to_vec(), epoch, - ), + )), right_exclusive: false, }), file_size: 2, @@ -193,8 +197,8 @@ pub fn generate_test_tables(epoch: u64, sst_ids: Vec) -> object_id: sst_id, sst_id, key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(sst_id, i + 1, epoch), - right: iterator_test_key_of_epoch(sst_id, (i + 1) * 10, epoch), + left: Bytes::from(iterator_test_key_of_epoch(sst_id, i + 1, epoch)), + right: Bytes::from(iterator_test_key_of_epoch(sst_id, (i + 1) * 10, epoch)), right_exclusive: false, }), file_size: 2, @@ -270,7 +274,7 @@ pub fn iterator_test_key_of_epoch( pub fn get_sorted_object_ids(sstables: &[SstableInfo]) -> Vec { sstables .iter() - .map(|table| table.get_object_id()) + .map(|table| table.object_id) .sorted() .collect_vec() } @@ -289,7 +293,7 @@ pub fn get_sorted_committed_object_ids( .levels .iter() .chain(levels.l0.as_ref().unwrap().sub_levels.iter()) - .flat_map(|levels| levels.table_infos.iter().map(|info| info.get_object_id())) + .flat_map(|levels| levels.table_infos.iter().map(|info| info.object_id)) .sorted() .collect_vec() } @@ -378,7 +382,7 @@ pub async fn commit_from_meta_node( ) -> crate::hummock::error::Result> { let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), META_NODE_ID)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, META_NODE_ID)) .collect(); hummock_manager_ref .commit_epoch(epoch, CommitEpochInfo::for_test(ssts, sst_to_worker)) @@ -395,7 +399,7 @@ pub async fn add_ssts( let ssts = to_local_sstable_info(&test_tables); let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id)) .collect(); hummock_manager .commit_epoch(epoch, CommitEpochInfo::for_test(ssts, sst_to_worker)) diff --git a/src/meta/src/manager/diagnose.rs b/src/meta/src/manager/diagnose.rs index 76184de9eb7c..d0f5167225b9 100644 --- a/src/meta/src/manager/diagnose.rs +++ b/src/meta/src/manager/diagnose.rs @@ -20,8 +20,8 @@ use std::sync::Arc; use itertools::Itertools; use prometheus_http_query::response::Data::Vector; use risingwave_common::types::Timestamptz; +use risingwave_hummock_sdk::version::Level; use risingwave_pb::common::WorkerType; -use risingwave_pb::hummock::Level; use risingwave_pb::meta::event_log::Event; use risingwave_pb::meta::EventLog; use risingwave_pb::monitor_service::StackTraceResponse; diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index 94ea463eb5af..4490b8bd16d8 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -23,10 +23,11 @@ use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl}; -use risingwave_pb::hummock::{compact_task, SstableInfo}; +use risingwave_pb::hummock::compact_task; use risingwave_storage::hummock::compactor::compactor_runner::compact_and_build_sst; use risingwave_storage::hummock::compactor::{ ConcatSstableIterator, DummyCompactionFilter, TaskConfig, TaskProgress, diff --git a/src/storage/hummock_sdk/Cargo.toml b/src/storage/hummock_sdk/Cargo.toml index da4416200dd4..c188f80a4ed2 100644 --- a/src/storage/hummock_sdk/Cargo.toml +++ b/src/storage/hummock_sdk/Cargo.toml @@ -22,6 +22,7 @@ parse-display = "0.9" prost = { workspace = true } risingwave_common = { workspace = true } risingwave_pb = { workspace = true } +serde = { version = "1", features = ["derive"] } tracing = "0.1" [target.'cfg(not(madsim))'.dependencies] diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index deb0b90bfcd2..a5720e6f9499 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_pb::hummock::{CompactTask, LevelType, SstableInfo}; +use risingwave_pb::hummock::compact_task::TaskType; +use risingwave_pb::hummock::LevelType; + +use crate::version::{CompactTask, SstableInfo}; pub fn compact_task_to_string(compact_task: &CompactTask) -> String { use std::fmt::Write; @@ -23,7 +26,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { "Compaction task id: {:?}, group-id: {:?}, task type: {:?}, target level: {:?}, target sub level: {:?}", compact_task.task_id, compact_task.compaction_group_id, - compact_task.task_type(), + TaskType::try_from(compact_task.task_type).unwrap(), compact_task.target_level, compact_task.target_sub_level_id ) @@ -36,12 +39,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { ) .unwrap(); writeln!(s, "Compaction # splits: {:?} ", compact_task.splits.len()).unwrap(); - writeln!( - s, - "Compaction task status: {:?} ", - compact_task.task_status() - ) - .unwrap(); + writeln!(s, "Compaction task status: {:?} ", compact_task.task_status).unwrap(); writeln!( s, "Compaction task table_ids: {:?} ", @@ -57,7 +55,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { if table.total_key_count != 0 { format!( "[id: {}, obj_id: {} {}KB stale_ratio {} delete_range_ratio {}]", - table.get_sst_id(), + table.sst_id, table.object_id, table.file_size / 1024, (table.stale_key_count * 100 / table.total_key_count), @@ -66,7 +64,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { } else { format!( "[id: {}, obj_id: {} {}KB]", - table.get_sst_id(), + table.sst_id, table.object_id, table.file_size / 1024, ) @@ -89,12 +87,12 @@ pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo) let left_str = if key_range.left.is_empty() { "-inf".to_string() } else { - hex::encode(key_range.left.as_slice()) + hex::encode(&key_range.left) }; let right_str = if key_range.right.is_empty() { "+inf".to_string() } else { - hex::encode(key_range.right.as_slice()) + hex::encode(&key_range.right) }; let stale_ratio = (sstable_info.stale_key_count * 100) @@ -106,8 +104,8 @@ pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo) writeln!( s, "SstableInfo: object id={}, SST id={}, KeyRange=[{:?},{:?}], table_ids: {:?}, size={}KB, stale_ratio={}%, range_tombstone_count={} range_tombstone_ratio={}% bloom_filter_kind {:?}", - sstable_info.get_object_id(), - sstable_info.get_sst_id(), + sstable_info.object_id, + sstable_info.sst_id, left_str, right_str, sstable_info.table_ids, @@ -130,8 +128,8 @@ pub fn statistics_compact_task(task: &CompactTask) -> CompactTaskStatistics { total_file_count += level.table_infos.len() as u64; level.table_infos.iter().for_each(|sst| { - total_file_size += sst.get_file_size(); - total_uncompressed_file_size += sst.get_uncompressed_file_size(); + total_file_size += sst.file_size; + total_uncompressed_file_size += sst.uncompressed_file_size; total_key_count += sst.total_key_count; }); } @@ -173,7 +171,7 @@ pub fn estimate_memory_for_compact_task( // input for level in &task.input_ssts { - if level.level_type() == LevelType::Nonoverlapping { + if level.level_type == LevelType::Nonoverlapping as i32 { let mut cur_level_max_sst_meta_size = 0; for sst in &level.table_infos { let meta_size = sst.file_size - sst.meta_offset; diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 9e07598d0792..05f8d5d07135 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -19,13 +19,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_pb::hummock::group_delta::DeltaType; -use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; use risingwave_pb::hummock::table_watermarks::PbEpochNewWatermarks; use risingwave_pb::hummock::{ CompactionConfig, CompatibilityVersion, GroupConstruct, GroupDestroy, GroupMetaChange, - GroupTableChange, Level, LevelType, OverlappingLevel, PbLevelType, PbTableWatermarks, - SstableInfo, + GroupTableChange, LevelType, PbLevelType, PbTableWatermarks, }; use tracing::warn; @@ -34,7 +32,9 @@ use crate::compaction_group::StaticCompactionGroupId; use crate::key_range::KeyRangeCommon; use crate::prost_key_range::KeyRangeExt; use crate::table_watermark::{TableWatermarks, TableWatermarksIndex, VnodeWatermark}; -use crate::version::{HummockVersion, HummockVersionDelta}; +use crate::version::{ + HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, +}; use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId}; pub struct GroupDeltasSummary { @@ -72,7 +72,12 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary if !intra_level.inserted_table_infos.is_empty() { insert_sst_level_id = intra_level.level_idx; insert_sub_level_id = intra_level.l0_sub_level_id; - insert_table_infos.extend(intra_level.inserted_table_infos.iter().cloned()); + insert_table_infos.extend( + intra_level + .inserted_table_infos + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)), + ); } new_vnode_partition_count = intra_level.vnode_partition_count; } @@ -163,7 +168,7 @@ impl HummockVersion { level .table_infos .iter() - .map(|table_info| table_info.get_object_id()) + .map(|table_info| table_info.object_id) }) .collect_vec() } @@ -210,11 +215,11 @@ impl HummockVersion { pub fn safe_epoch_table_watermarks( &self, existing_table_ids: &[u32], - ) -> BTreeMap { + ) -> BTreeMap { fn extract_single_table_watermark( table_watermarks: &TableWatermarks, safe_epoch: u64, - ) -> Option { + ) -> Option { if let Some((first_epoch, first_epoch_watermark)) = table_watermarks.watermarks.first() { assert!( @@ -224,15 +229,9 @@ impl HummockVersion { safe_epoch ); if *first_epoch == safe_epoch { - Some(PbTableWatermarks { - epoch_watermarks: vec![PbEpochNewWatermarks { - watermarks: first_epoch_watermark - .iter() - .map(VnodeWatermark::to_protobuf) - .collect(), - epoch: *first_epoch, - }], - is_ascending: table_watermarks.direction.is_ascending(), + Some(TableWatermarks { + watermarks: vec![(*first_epoch, first_epoch_watermark.clone())], + direction: table_watermarks.direction, }) } else { None @@ -284,7 +283,7 @@ impl HummockVersion { .flat_map(|level| level.get_table_infos()) .map(|sst_info| { // `sst_info.table_ids` will never be empty. - for table_id in sst_info.get_table_ids() { + for table_id in &sst_info.table_ids { if member_table_ids.contains(table_id) { return 2; } @@ -393,7 +392,7 @@ impl HummockVersion { cur_levels.levels[idx].table_infos.sort_by(|sst1, sst2| { let a = sst1.key_range.as_ref().unwrap(); let b = sst2.key_range.as_ref().unwrap(); - a.compare(b) + a.cmp(b) }); assert!(can_concat(&cur_levels.levels[idx].table_infos)); level @@ -432,8 +431,12 @@ impl HummockVersion { if let DeltaType::IntraLevel(delta) = group_delta.get_delta_type().unwrap() { if !delta.inserted_table_infos.is_empty() { info.insert_sst_level = delta.level_idx; - info.insert_sst_infos - .extend(delta.inserted_table_infos.iter().cloned()); + info.insert_sst_infos.extend( + delta + .inserted_table_infos + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)), + ); } if !delta.removed_table_ids.is_empty() { for id in &delta.removed_table_ids { @@ -641,7 +644,7 @@ impl HummockVersion { if table_info.sst_id == table_info.object_id { continue; } - let object_id = table_info.get_object_id(); + let object_id = table_info.object_id; let entry: &mut BranchedSstInfo = ret.entry(object_id).or_default(); if let Some(exist_sst_id) = entry.get(compaction_group_id) { panic!("we do not allow more than one sst with the same object id in one grou. object-id: {}, duplicated sst id: {:?} and {}", object_id, exist_sst_id, table_info.sst_id); @@ -864,7 +867,7 @@ fn split_sst_info_for_level( let mut branch_table_info = sst_info.clone(); branch_table_info.sst_id = *new_sst_id; *new_sst_id += 1; - let parent_old_sst_id = sst_info.get_sst_id(); + let parent_old_sst_id = sst_info.sst_id; if is_trivial { // This is a compatibility design. we only clear the table-ids for files which would // be removed in later code. In the version-delta generated by new @@ -948,7 +951,7 @@ pub fn get_compaction_group_ssts( level .table_infos .iter() - .map(|table_info| (table_info.get_object_id(), table_info.get_sst_id())) + .map(|table_info| (table_info.object_id, table_info.sst_id)) }) } @@ -1000,7 +1003,7 @@ pub fn add_ssts_to_sub_level( .sort_by(|sst1, sst2| { let a = sst1.key_range.as_ref().unwrap(); let b = sst2.key_range.as_ref().unwrap(); - a.compare(b) + a.cmp(b) }); assert!( can_concat(&l0.sub_levels[sub_level_idx].table_infos), @@ -1109,7 +1112,7 @@ fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec) operand.table_infos.sort_by(|sst1, sst2| { let a = sst1.key_range.as_ref().unwrap(); let b = sst2.key_range.as_ref().unwrap(); - a.compare(b) + a.cmp(b) }); if operand.level_type == LevelType::Overlapping as i32 { operand.level_type = LevelType::Nonoverlapping as i32; @@ -1288,15 +1291,15 @@ mod tests { use std::collections::HashMap; use risingwave_pb::hummock::group_delta::DeltaType; - use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; use risingwave_pb::hummock::{ - CompactionConfig, GroupConstruct, GroupDelta, GroupDestroy, IntraLevelDelta, Level, - LevelType, OverlappingLevel, SstableInfo, + CompactionConfig, GroupConstruct, GroupDelta, GroupDestroy, IntraLevelDelta, LevelType, }; use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels; - use crate::version::{HummockVersion, HummockVersionDelta}; + use crate::version::{ + HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, + }; #[test] fn test_get_sst_object_ids() { @@ -1416,7 +1419,8 @@ mod tests { object_id: 1, sst_id: 1, ..Default::default() - }], + } + .to_protobuf()], ..Default::default() })), }], diff --git a/src/storage/hummock_sdk/src/key_range.rs b/src/storage/hummock_sdk/src/key_range.rs index 95359e8a5497..24c5fc386ede 100644 --- a/src/storage/hummock_sdk/src/key_range.rs +++ b/src/storage/hummock_sdk/src/key_range.rs @@ -15,11 +15,12 @@ use std::cmp; use bytes::Bytes; +use serde::Serialize; use super::key_cmp::KeyComparator; use crate::key::{FullKey, UserKey}; -#[derive(PartialEq, Eq, Clone, Debug, Default)] +#[derive(PartialEq, Eq, Clone, Debug, Default, Serialize)] pub struct KeyRange { pub left: Bytes, pub right: Bytes, diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index ad5fe438e94f..1697423a9722 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -31,7 +31,7 @@ use std::cmp::Ordering; pub use key_cmp::*; use risingwave_common::util::epoch::EPOCH_SPILL_TIME_MASK; use risingwave_pb::common::{batch_query_epoch, BatchQueryEpoch}; -use risingwave_pb::hummock::SstableInfo; +use version::SstableInfo; use crate::compaction_group::StaticCompactionGroupId; use crate::key_range::KeyRangeCommon; diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index 44bde5882310..50494893230b 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -311,8 +311,8 @@ impl WatermarkDirection { #[derive(Clone, Debug, PartialEq, EstimateSize)] pub struct VnodeWatermark { - vnode_bitmap: Arc, - watermark: Bytes, + pub vnode_bitmap: Arc, + pub watermark: Bytes, } impl VnodeWatermark { diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 0c6680d7cee8..2ceb66556853 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -12,22 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::mem::size_of; +use std::ops::Deref; -use prost::Message; +use bytes::Bytes; +use itertools::Itertools; use risingwave_common::catalog::TableId; +use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType}; use risingwave_pb::hummock::hummock_version::PbLevels; use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas; -use risingwave_pb::hummock::{PbHummockVersion, PbHummockVersionDelta}; +use risingwave_pb::hummock::{ + LevelType, PbCompactTask, PbHummockVersion, PbHummockVersionDelta, PbInputLevel, PbKeyRange, + PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, PbValidationTask, TableOption, +}; +use risingwave_pb::java_binding::key_range; +use serde::Serialize; +use crate::key_range::KeyRange; use crate::table_watermark::TableWatermarks; use crate::{CompactionGroupId, HummockSstableObjectId}; #[derive(Debug, Clone, PartialEq)] pub struct HummockVersion { pub id: u64, - pub levels: HashMap, + pub levels: HashMap, pub max_committed_epoch: u64, pub safe_epoch: u64, pub table_watermarks: HashMap, @@ -39,6 +48,218 @@ impl Default for HummockVersion { } } +#[derive(Debug, Clone, PartialEq, Default, Serialize)] +pub struct OverlappingLevel { + pub sub_levels: Vec, + pub total_file_size: u64, + pub uncompressed_file_size: u64, +} + +impl OverlappingLevel { + pub fn from_rpc_protobuf(pb_overlapping_level: &PbOverlappingLevel) -> Self { + Self::from_protobuf_inner(pb_overlapping_level) + } + + pub fn from_persisted_protobuf(pb_overlapping_level: &PbOverlappingLevel) -> Self { + Self::from_protobuf_inner(pb_overlapping_level) + } + + fn from_protobuf_inner(pb_overlapping_level: &PbOverlappingLevel) -> Self { + Self { + sub_levels: pb_overlapping_level + .sub_levels + .iter() + .map(|level| Level::from_protobuf(level)) + .collect_vec(), + total_file_size: pb_overlapping_level.total_file_size, + uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, + } + } + + pub fn to_protobuf(&self) -> PbOverlappingLevel { + PbOverlappingLevel { + sub_levels: self + .sub_levels + .iter() + .map(|pb_level| pb_level.to_protobuf()) + .collect_vec(), + total_file_size: self.total_file_size, + uncompressed_file_size: self.uncompressed_file_size, + } + } + + pub fn estimated_encode_len(&self) -> usize { + self.sub_levels + .iter() + .map(|level| level.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + } +} + +impl OverlappingLevel { + pub fn get_sub_levels(&self) -> &Vec { + &self.sub_levels + } +} + +#[derive(Debug, Clone, PartialEq, Default, Serialize)] +pub struct Level { + pub level_idx: u32, + pub level_type: i32, + pub table_infos: Vec, + pub total_file_size: u64, + pub sub_level_id: u64, + pub uncompressed_file_size: u64, + pub vnode_partition_count: u32, +} + +impl Level { + fn from_protobuf(pb_level: &PbLevel) -> Self { + Self { + level_idx: pb_level.level_idx, + level_type: pb_level.level_type, + table_infos: pb_level + .table_infos + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .collect_vec(), + total_file_size: pb_level.total_file_size, + sub_level_id: pb_level.sub_level_id, + uncompressed_file_size: pb_level.uncompressed_file_size, + vnode_partition_count: pb_level.vnode_partition_count, + } + } + + pub fn to_protobuf(&self) -> PbLevel { + PbLevel { + level_idx: self.level_idx, + level_type: self.level_type, + table_infos: self + .table_infos + .iter() + .map(|sst| sst.to_protobuf()) + .collect_vec(), + total_file_size: self.total_file_size, + sub_level_id: self.sub_level_id, + uncompressed_file_size: self.uncompressed_file_size, + vnode_partition_count: self.vnode_partition_count, + } + } + + pub fn estimated_encode_len(&self) -> usize { + size_of::() + + size_of::() + + self + .table_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + } +} + +impl Level { + pub fn get_table_infos(&self) -> &Vec { + &self.table_infos + } + + pub fn level_type(&self) -> LevelType { + LevelType::try_from(self.level_type).unwrap() + } + + pub fn get_sub_level_id(&self) -> u64 { + self.sub_level_id + } + + pub fn get_level_idx(&self) -> u32 { + self.level_idx + } + + pub fn get_total_file_size(&self) -> u64 { + self.total_file_size + } + + pub fn get_uncompressed_file_size(&self) -> u64 { + self.uncompressed_file_size + } +} + +#[derive(Debug, Clone, PartialEq, Default, Serialize)] +pub struct Levels { + pub levels: Vec, + pub l0: Option, + pub group_id: u64, + pub parent_group_id: u64, + pub member_table_ids: Vec, +} + +impl Levels { + fn from_protobuf(pb_levels: &PbLevels) -> Self { + Self { + l0: if pb_levels.l0.is_some() { + Some(OverlappingLevel::from_rpc_protobuf( + pb_levels.l0.as_ref().unwrap(), + )) + } else { + None + }, + levels: pb_levels + .levels + .iter() + .map(|pb_level| Level::from_protobuf(pb_level)) + .collect_vec(), + group_id: pb_levels.group_id, + parent_group_id: pb_levels.parent_group_id, + member_table_ids: pb_levels.member_table_ids.clone(), + } + } + + pub fn to_protobuf(&self) -> PbLevels { + PbLevels { + l0: if self.l0.is_some() { + Some(self.l0.as_ref().unwrap().to_protobuf()) + } else { + None + }, + levels: self + .levels + .iter() + .map(|level| level.to_protobuf()) + .collect_vec(), + group_id: self.group_id, + parent_group_id: self.parent_group_id, + member_table_ids: self.member_table_ids.clone(), + } + } + + pub fn estimated_encode_len(&self) -> usize { + let mut basic = self + .levels + .iter() + .map(|level| level.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::(); + if let Some(l0) = self.l0.as_ref() { + basic += l0.estimated_encode_len(); + } + + basic + } +} + +impl Levels { + pub fn get_levels(&self) -> &Vec { + &self.levels + } +} + impl HummockVersion { /// Convert the `PbHummockVersion` received from rpc to `HummockVersion`. No need to /// maintain backward compatibility. @@ -58,7 +279,12 @@ impl HummockVersion { levels: pb_version .levels .iter() - .map(|(group_id, levels)| (*group_id as CompactionGroupId, levels.clone())) + .map(|(group_id, levels)| { + ( + *group_id as CompactionGroupId, + Levels::from_protobuf(levels), + ) + }) .collect(), max_committed_epoch: pb_version.max_committed_epoch, safe_epoch: pb_version.safe_epoch, @@ -81,7 +307,7 @@ impl HummockVersion { levels: self .levels .iter() - .map(|(group_id, levels)| (*group_id as _, levels.clone())) + .map(|(group_id, levels)| (*group_id as _, levels.to_protobuf())) .collect(), max_committed_epoch: self.max_committed_epoch, safe_epoch: self.safe_epoch, @@ -98,7 +324,7 @@ impl HummockVersion { + self .levels .values() - .map(|level| level.encoded_len()) + .map(|level| level.estimated_encode_len()) .sum::() + self.table_watermarks.len() * size_of::() + self @@ -190,3 +416,421 @@ impl HummockVersionDelta { } } } + +#[derive(Debug, PartialEq, Clone, Default, Serialize)] +pub struct SstableInfo { + pub object_id: u64, + pub sst_id: u64, + pub key_range: Option, + pub file_size: u64, + pub table_ids: Vec, + pub meta_offset: u64, + pub stale_key_count: u64, + pub total_key_count: u64, + pub min_epoch: u64, + pub max_epoch: u64, + pub uncompressed_file_size: u64, + pub range_tombstone_count: u64, + pub bloom_filter_kind: i32, +} + +impl SstableInfo { + pub fn from_protobuf(pb_sstable_info: &PbSstableInfo) -> Self { + Self { + object_id: pb_sstable_info.object_id, + sst_id: pb_sstable_info.sst_id, + key_range: if pb_sstable_info.key_range.is_some() { + let pb_keyrange = pb_sstable_info.key_range.as_ref().unwrap(); + let key_range = KeyRange { + left: Bytes::from(pb_keyrange.left.clone()), + right: Bytes::from(pb_keyrange.right.clone()), + right_exclusive: pb_keyrange.right_exclusive, + }; + Some(key_range) + } else { + None + }, + + file_size: pb_sstable_info.file_size, + table_ids: pb_sstable_info.table_ids.clone(), + meta_offset: pb_sstable_info.meta_offset, + stale_key_count: pb_sstable_info.stale_key_count, + total_key_count: pb_sstable_info.total_key_count, + min_epoch: pb_sstable_info.min_epoch, + max_epoch: pb_sstable_info.max_epoch, + uncompressed_file_size: pb_sstable_info.uncompressed_file_size, + range_tombstone_count: pb_sstable_info.range_tombstone_count, + bloom_filter_kind: pb_sstable_info.bloom_filter_kind, + } + } + + pub fn to_protobuf(&self) -> PbSstableInfo { + PbSstableInfo { + object_id: self.object_id, + sst_id: self.sst_id, + key_range: if self.key_range.is_some() { + let keyrange = self.key_range.as_ref().unwrap(); + let pb_key_range = PbKeyRange { + left: keyrange.left.to_vec(), + right: keyrange.right.to_vec(), + right_exclusive: keyrange.right_exclusive, + }; + Some(pb_key_range) + } else { + None + }, + + file_size: self.file_size, + table_ids: self.table_ids.clone(), + meta_offset: self.meta_offset, + stale_key_count: self.stale_key_count, + total_key_count: self.total_key_count, + min_epoch: self.min_epoch, + max_epoch: self.max_epoch, + uncompressed_file_size: self.uncompressed_file_size, + range_tombstone_count: self.range_tombstone_count, + bloom_filter_kind: self.bloom_filter_kind, + } + } + + pub fn estimated_encode_len(&self) -> usize { + let mut basic = size_of::() // object_id + + size_of::() // sstable_id + + size_of::() // file_size + + self.table_ids.len() * size_of::() // table_ids + + size_of::() // meta_offset + + size_of::() // stale_key_count + + size_of::() // total_key_count + + size_of::() // min_epoch + + size_of::() // max_epoch + + size_of::() // uncompressed_file_size + + size_of::() // range_tombstone_count + + size_of::(); // bloom_filter_kind + + if let Some(key_range) = &self.key_range { + basic += key_range.left.len() + key_range.right.len() + size_of::(); + } + + basic + } +} + +impl SstableInfo { + pub fn get_sst_id(&self) -> u64 { + self.sst_id + } + + pub fn get_object_id(&self) -> u64 { + self.sst_id + } + + pub fn get_file_size(&self) -> u64 { + self.file_size + } +} + +#[derive(Clone, PartialEq, Default, Debug)] +pub struct InputLevel { + pub level_idx: u32, + pub level_type: i32, + pub table_infos: Vec, +} + +impl InputLevel { + pub fn from_protobuf(pb_input_level: &PbInputLevel) -> Self { + Self { + level_idx: pb_input_level.level_idx, + level_type: pb_input_level.level_type, + table_infos: pb_input_level + .table_infos + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .collect_vec(), + } + } + + pub fn to_protobuf(&self) -> PbInputLevel { + PbInputLevel { + level_idx: self.level_idx, + level_type: self.level_type, + table_infos: self + .table_infos + .iter() + .map(|sst| sst.to_protobuf()) + .collect_vec(), + } + } + + pub fn estimated_encode_len(&self) -> usize { + size_of::() + + size_of::() + + self + .table_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + } +} + +impl InputLevel { + pub fn get_table_infos(&self) -> &Vec { + &self.table_infos + } + + pub fn level_type(&self) -> PbLevelType { + LevelType::try_from(self.level_type).unwrap() + } + + pub fn get_level_idx(&self) -> u32 { + self.level_idx + } +} + +#[derive(Clone, PartialEq, Default)] +pub struct CompactTask { + /// SSTs to be compacted, which will be removed from LSM after compaction + pub input_ssts: Vec, + /// In ideal case, the compaction will generate splits.len() tables which have key range + /// corresponding to that in \[splits\], respectively + pub splits: Vec, + /// low watermark in 'ts-aware compaction' + pub watermark: u64, + /// compaction output, which will be added to \[target_level\] of LSM after compaction + pub sorted_output_ssts: Vec, + /// task id assigned by hummock storage service + pub task_id: u64, + /// compaction output will be added to \[target_level\] of LSM after compaction + pub target_level: u32, + pub gc_delete_keys: bool, + /// Lbase in LSM + pub base_level: u32, + pub task_status: i32, + /// compaction group the task belongs to + pub compaction_group_id: u64, + /// existing_table_ids for compaction drop key + pub existing_table_ids: Vec, + pub compression_algorithm: u32, + pub target_file_size: u64, + pub compaction_filter_mask: u32, + pub table_options: BTreeMap, + pub current_epoch_time: u64, + pub target_sub_level_id: u64, + /// Identifies whether the task is space_reclaim, if the compact_task_type increases, it will be refactored to enum + pub task_type: i32, + /// Deprecated. use table_vnode_partition instead; + pub split_by_state_table: bool, + /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed. + /// Deprecated. use table_vnode_partition instead; + pub split_weight_by_vnode: u32, + pub table_vnode_partition: BTreeMap, + /// The table watermark of any table id. In compaction we only use the table watermarks on safe epoch, + /// so we only need to include the table watermarks on safe epoch to reduce the size of metadata. + pub table_watermarks: BTreeMap, +} + +impl CompactTask { + pub fn from_protobuf(pb_compact_task: &PbCompactTask) -> Self { + Self { + input_ssts: pb_compact_task + .input_ssts + .iter() + .map(|pb_input_level| InputLevel::from_protobuf(pb_input_level)) + .collect_vec(), + splits: pb_compact_task + .splits + .iter() + .map(|pb_keyrange| KeyRange { + left: Bytes::from(pb_keyrange.left.clone()), + right: Bytes::from(pb_keyrange.right.clone()), + right_exclusive: pb_keyrange.right_exclusive, + }) + .collect_vec(), + watermark: pb_compact_task.watermark, + sorted_output_ssts: pb_compact_task + .sorted_output_ssts + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .collect_vec(), + task_id: pb_compact_task.task_id, + target_level: pb_compact_task.target_level, + gc_delete_keys: pb_compact_task.gc_delete_keys, + base_level: pb_compact_task.base_level, + task_status: pb_compact_task.task_status, + compaction_group_id: pb_compact_task.compaction_group_id, + existing_table_ids: pb_compact_task.existing_table_ids.clone(), + compression_algorithm: pb_compact_task.compression_algorithm, + target_file_size: pb_compact_task.target_file_size, + compaction_filter_mask: pb_compact_task.compaction_filter_mask, + table_options: pb_compact_task.table_options.clone(), + current_epoch_time: pb_compact_task.current_epoch_time, + target_sub_level_id: pb_compact_task.target_sub_level_id, + task_type: pb_compact_task.task_type, + #[allow(deprecated)] + split_by_state_table: pb_compact_task.split_by_state_table, + split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, + table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), + table_watermarks: pb_compact_task + .table_watermarks + .iter() + .map(|(table_id, pb_table_watermark)| { + ( + *table_id, + TableWatermarks::from_protobuf(pb_table_watermark), + ) + }) + .collect(), + } + } + + pub fn to_protobuf(&self) -> PbCompactTask { + PbCompactTask { + input_ssts: self + .input_ssts + .iter() + .map(|input_level| input_level.to_protobuf()) + .collect_vec(), + splits: self + .splits + .iter() + .map(|keyrange| PbKeyRange { + left: keyrange.left.to_vec(), + right: keyrange.right.to_vec(), + right_exclusive: keyrange.right_exclusive, + }) + .collect_vec(), + watermark: self.watermark, + sorted_output_ssts: self + .sorted_output_ssts + .iter() + .map(|sst| sst.to_protobuf()) + .collect_vec(), + task_id: self.task_id, + target_level: self.target_level, + gc_delete_keys: self.gc_delete_keys, + base_level: self.base_level, + task_status: self.task_status, + compaction_group_id: self.compaction_group_id, + existing_table_ids: self.existing_table_ids.clone(), + compression_algorithm: self.compression_algorithm, + target_file_size: self.target_file_size, + compaction_filter_mask: self.compaction_filter_mask, + table_options: self.table_options.clone(), + current_epoch_time: self.current_epoch_time, + target_sub_level_id: self.target_sub_level_id, + task_type: self.task_type, + #[allow(deprecated)] + split_by_state_table: self.split_by_state_table, + split_weight_by_vnode: self.split_weight_by_vnode, + table_vnode_partition: self.table_vnode_partition.clone(), + table_watermarks: self + .table_watermarks + .iter() + .map(|(table_id, table_watermark)| (*table_id, table_watermark.to_protobuf())) + .collect(), + } + } + + pub fn estimated_encode_len(&self) -> usize { + self.input_ssts + .iter() + .map(|input_level| input_level.estimated_encode_len()) + .sum::() + + self + .splits + .iter() + .map(|split| split.left.len() + split.right.len() + size_of::()) + .sum::() + + size_of::() + + self + .sorted_output_ssts + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.existing_table_ids.len() * size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.table_options.len() * size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.table_vnode_partition.len() * size_of::() + + self + .table_watermarks + .values() + .map(|table_watermark| size_of::() + table_watermark.estimated_encode_len()) + .sum::() + } +} + +impl CompactTask { + pub fn task_type(&self) -> PbTaskType { + PbTaskType::try_from(self.task_type).unwrap() + } + + pub fn task_status(&self) -> PbTaskStatus { + PbTaskStatus::try_from(self.task_status).unwrap() + } + + pub fn set_task_status(&mut self, s: PbTaskStatus) { + self.task_status = s as i32; + } + + pub fn get_input_ssts(&self) -> &Vec { + &self.input_ssts + } + + pub fn get_task_id(&self) -> u64 { + self.task_id + } +} + +#[derive(Clone, PartialEq, Default)] +pub struct ValidationTask { + pub sst_infos: Vec, + pub sst_id_to_worker_id: HashMap, + pub epoch: u64, +} + +impl ValidationTask { + pub fn from_protobuf(pb_validation_task: &PbValidationTask) -> Self { + Self { + sst_infos: pb_validation_task + .sst_infos + .iter() + .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .collect_vec(), + sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(), + epoch: pb_validation_task.epoch, + } + } + + pub fn to_protobuf(&self) -> PbValidationTask { + PbValidationTask { + sst_infos: self + .sst_infos + .iter() + .map(|sst| sst.to_protobuf()) + .collect_vec(), + sst_id_to_worker_id: self.sst_id_to_worker_id.clone(), + epoch: self.epoch, + } + } + + pub fn estimated_encode_len(&self) -> usize { + self.sst_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) + + size_of::() + } +} diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index fc6debe8183c..3df81bab956e 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -31,9 +31,10 @@ pub(crate) mod tests { use risingwave_hummock_sdk::can_concat; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::{next_key, FullKey, TableKey, TABLE_PREFIX_LEN}; + use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; - use risingwave_hummock_sdk::version::HummockVersion; + use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, InputLevel, SstableInfo}; use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder; use risingwave_meta::hummock::compaction::selector::{ default_compaction_selector, ManualCompactionOption, @@ -44,7 +45,7 @@ pub(crate) mod tests { }; use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient}; use risingwave_pb::common::{HostAddress, WorkerType}; - use risingwave_pb::hummock::{CompactTask, InputLevel, KeyRange, SstableInfo, TableOption}; + use risingwave_pb::hummock::TableOption; use risingwave_pb::meta::add_worker_node_request::Property; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::filter_key_extractor::{ diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index bf841b5e4956..33f66f6c6a4f 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -20,9 +20,10 @@ use itertools::Itertools; use parking_lot::RwLock; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::{key_with_epoch, map_table_key_range}; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; use risingwave_meta::hummock::test_utils::setup_compute_env; -use risingwave_pb::hummock::{KeyRange, SstableInfo}; use risingwave_storage::hummock::iterator::test_utils::{ iterator_test_table_key_of, iterator_test_user_key_of, }; @@ -147,8 +148,8 @@ async fn test_read_version_basic() { object_id: 1, sst_id: 1, key_range: Some(KeyRange { - left: key_with_epoch(iterator_test_user_key_of(1).encode(), 1), - right: key_with_epoch(iterator_test_user_key_of(2).encode(), 2), + left: key_with_epoch(iterator_test_user_key_of(1).encode(), 1).into(), + right: key_with_epoch(iterator_test_user_key_of(2).encode(), 2).into(), right_exclusive: false, }), file_size: 1, @@ -163,8 +164,8 @@ async fn test_read_version_basic() { object_id: 2, sst_id: 2, key_range: Some(KeyRange { - left: key_with_epoch(iterator_test_user_key_of(3).encode(), 3), - right: key_with_epoch(iterator_test_user_key_of(3).encode(), 3), + left: key_with_epoch(iterator_test_user_key_of(3).encode(), 3).into(), + right: key_with_epoch(iterator_test_user_key_of(3).encode(), 3).into(), right_exclusive: false, }), file_size: 1, diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index 758a6d125345..1e2ad1e8a265 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -18,6 +18,7 @@ use std::ops::Bound; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use bytes::Bytes; use itertools::Itertools; use risingwave_common::constants::hummock::CompactionFilterFlag; use risingwave_hummock_sdk::compaction_group::StateTableId; @@ -25,10 +26,9 @@ use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::TableStatsMap; +use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, EpochWithGap, KeyComparator}; -use risingwave_pb::hummock::{ - compact_task, CompactTask, KeyRange as KeyRange_vec, LevelType, SstableInfo, -}; +use risingwave_pb::hummock::{compact_task, LevelType}; use tokio::time::Instant; pub use super::context::CompactorContext; @@ -172,7 +172,7 @@ fn generate_splits_fast( sstable_infos: &Vec, compaction_size: u64, context: CompactorContext, -) -> HummockResult> { +) -> HummockResult> { let worker_num = context.compaction_executor.worker_num(); let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20; @@ -209,13 +209,13 @@ fn generate_splits_fast( return Ok(vec![]); } let mut splits = vec![]; - splits.push(KeyRange_vec::new(vec![], vec![])); + splits.push(KeyRange::default()); let parallel_key_count = indexes.len() / parallelism; let mut last_split_key_count = 0; for key in indexes { if last_split_key_count >= parallel_key_count { - splits.last_mut().unwrap().right = key.clone(); - splits.push(KeyRange_vec::new(key.clone(), vec![])); + splits.last_mut().unwrap().right = Bytes::from(key.clone()); + splits.push(KeyRange::new(Bytes::from(key.clone()), Bytes::default())); last_split_key_count = 0; } last_split_key_count += 1; @@ -227,7 +227,7 @@ pub async fn generate_splits( sstable_infos: &Vec, compaction_size: u64, context: CompactorContext, -) -> HummockResult> { +) -> HummockResult> { let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20; if compaction_size > parallel_compact_size { if sstable_infos.len() > MAX_FILE_COUNT { @@ -259,7 +259,7 @@ pub async fn generate_splits( // sort by key, as for every data block has the same size; indexes.sort_by(|a, b| KeyComparator::compare_encoded_full_key(a.1.as_ref(), b.1.as_ref())); let mut splits = vec![]; - splits.push(KeyRange_vec::new(vec![], vec![])); + splits.push(KeyRange::default()); let worker_num = context.compaction_executor.worker_num(); @@ -283,8 +283,8 @@ pub async fn generate_splits( && !last_key.eq(&key) && remaining_size > parallel_compact_size { - splits.last_mut().unwrap().right = key.clone(); - splits.push(KeyRange_vec::new(key.clone(), vec![])); + splits.last_mut().unwrap().right = Bytes::from(key.clone()); + splits.push(KeyRange::new(Bytes::from(key.clone()), Bytes::default())); last_buffer_size = data_size; } else { last_buffer_size += data_size; diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index 441480d66a6b..b40d6d43d720 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -27,9 +27,10 @@ use risingwave_hummock_sdk::compact::{ use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, PointRange}; use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, TableStatsMap}; +use risingwave_hummock_sdk::version::CompactTask; use risingwave_hummock_sdk::{can_concat, EpochWithGap, HummockEpoch}; use risingwave_pb::hummock::compact_task::{TaskStatus, TaskType}; -use risingwave_pb::hummock::{BloomFilterType, CompactTask, LevelType}; +use risingwave_pb::hummock::{BloomFilterType, LevelType}; use thiserror_ext::AsReport; use tokio::sync::oneshot::Receiver; @@ -90,8 +91,8 @@ impl CompactorRunner { BlockedXor16FilterBuilder::is_kv_count_too_large(kv_count) || task.target_level > 0; let key_range = KeyRange { - left: Bytes::copy_from_slice(task.splits[split_index].get_left()), - right: Bytes::copy_from_slice(task.splits[split_index].get_right()), + left: task.splits[split_index].left.clone(), + right: task.splits[split_index].right.clone(), right_exclusive: true, }; @@ -104,7 +105,7 @@ impl CompactorRunner { gc_delete_keys: task.gc_delete_keys, watermark: task.watermark, stats_target_table_ids: Some(HashSet::from_iter(task.existing_table_ids.clone())), - task_type: task.task_type(), + task_type: TaskType::try_from(task.task_type).unwrap(), is_target_l0_or_lbase: task.target_level == 0 || task.target_level == task.base_level, use_block_based_filter, @@ -173,13 +174,15 @@ impl CompactorRunner { .table_infos .iter() .filter(|table_info| { - let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap()); + // let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap()); let table_ids = &table_info.table_ids; let exist_table = table_ids.iter().any(|table_id| { self.compact_task.existing_table_ids.contains(table_id) }); - self.key_range.full_key_overlap(&key_range) && exist_table + self.key_range + .full_key_overlap(table_info.key_range.as_ref().unwrap()) + && exist_table }) .cloned() .collect_vec(); @@ -201,13 +204,17 @@ impl CompactorRunner { )); } else { for table_info in &level.table_infos { - let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap()); + // let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap()); let table_ids = &table_info.table_ids; let exist_table = table_ids .iter() .any(|table_id| self.compact_task.existing_table_ids.contains(table_id)); - if !self.key_range.full_key_overlap(&key_range) || !exist_table { + if !self + .key_range + .full_key_overlap(table_info.key_range.as_ref().unwrap()) + || !exist_table + { continue; } if table_info.range_tombstone_count > 0 { @@ -388,7 +395,7 @@ pub async fn compact( .sum::(); let all_ssts_are_blocked_filter = sstable_infos .iter() - .all(|table_info| table_info.bloom_filter_kind() == BloomFilterType::Blocked); + .all(|table_info| table_info.bloom_filter_kind == BloomFilterType::Blocked as i32); let delete_key_count = sstable_infos .iter() @@ -408,7 +415,7 @@ pub async fn compact( && compaction_size < context.storage_opts.compactor_fast_max_compact_task_size && delete_key_count * 100 < context.storage_opts.compactor_fast_max_compact_delete_ratio as u64 * total_key_count - && compact_task.task_type() == TaskType::Dynamic; + && compact_task.task_type == TaskType::Dynamic as i32; if !optimize_by_copy_block { match generate_splits(&sstable_infos, compaction_size, context.clone()).await { @@ -669,7 +676,7 @@ fn compact_done( task_status: TaskStatus, ) -> (CompactTask, HashMap) { let mut table_stats_map = TableStatsMap::default(); - compact_task.set_task_status(task_status); + compact_task.task_status = task_status as i32; compact_task .sorted_output_ssts .reserve(compact_task.splits.len()); @@ -938,10 +945,11 @@ where mod tests { use std::collections::{HashSet, VecDeque}; + use foyer::common::code::Key; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::{TableKey, UserKey}; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; - use risingwave_pb::hummock::{InputLevel, PbKeyRange}; + use risingwave_hummock_sdk::version::InputLevel; use super::*; use crate::filter_key_extractor::FullKeyFilterKeyExtractor; @@ -1013,7 +1021,7 @@ mod tests { table_infos: vec![sstable_info_1, sstable_info_2], }], existing_table_ids: vec![2], - splits: vec![PbKeyRange::inf()], + splits: vec![KeyRange::inf()], watermark: 10, ..Default::default() }; diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs index ab4ddc216b1a..5831121f42a4 100644 --- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs +++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs @@ -25,8 +25,9 @@ use itertools::Itertools; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::table_stats::TableStats; +use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, compact_task_to_string, EpochWithGap, LocalSstableInfo}; -use risingwave_pb::hummock::{CompactTask, SstableInfo}; +use risingwave_pb::hummock::compact_task::TaskType; use crate::filter_key_extractor::FilterKeyExtractorImpl; use crate::hummock::block_stream::BlockDataStream; @@ -291,7 +292,7 @@ impl CompactorRunner { gc_delete_keys: task.gc_delete_keys, watermark: task.watermark, stats_target_table_ids: Some(HashSet::from_iter(task.existing_table_ids.clone())), - task_type: task.task_type(), + task_type: TaskType::try_from(task.task_type).unwrap(), is_target_l0_or_lbase: task.target_level == 0 || task.target_level == task.base_level, table_vnode_partition: task.table_vnode_partition.clone(), use_block_based_filter: true, @@ -515,7 +516,7 @@ impl CompactorRunner { ) .await?; let sst_infos = ssts.iter().map(|sst| sst.sst_info.clone()).collect_vec(); - assert!(can_concat(&sst_infos)); + assert!(can_concat(sst_infos.as_slice())); Ok((ssts, statistic)) } } diff --git a/src/storage/src/hummock/compactor/iterator.rs b/src/storage/src/hummock/compactor/iterator.rs index 9d0a1047dcaf..63effbcc4cf1 100644 --- a/src/storage/src/hummock/compactor/iterator.rs +++ b/src/storage/src/hummock/compactor/iterator.rs @@ -23,8 +23,8 @@ use fail::fail_point; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::KeyComparator; -use risingwave_pb::hummock::SstableInfo; use crate::hummock::block_stream::BlockDataStream; use crate::hummock::compactor::task_progress::TaskProgress; @@ -244,8 +244,8 @@ impl SstableStreamIterator { fn sst_debug_info(&self) -> String { format!( "object_id={}, sst_id={}, meta_offset={}, table_ids={:?}", - self.sstable_info.get_object_id(), - self.sstable_info.get_sst_id(), + self.sstable_info.object_id, + self.sstable_info.sst_id, self.sstable_info.meta_offset, self.sstable_info.table_ids ) diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index b71231ed162a..42bce4aaeeef 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -15,6 +15,8 @@ mod compaction_executor; mod compaction_filter; pub mod compaction_utils; +use itertools::Itertools; +use risingwave_hummock_sdk::version::{CompactTask, ValidationTask}; use risingwave_pb::compactor::{dispatch_compaction_task_request, DispatchCompactionTaskRequest}; use risingwave_pb::hummock::report_compaction_task_request::{ Event as ReportCompactionTaskEvent, HeartBeat as SharedHeartBeat, @@ -191,7 +193,7 @@ impl Compactor { debug_assert!(ssts .iter() - .all(|table_info| table_info.sst_info.get_table_ids().is_sorted())); + .all(|table_info| table_info.sst_info.table_ids.is_sorted())); if task_id.is_some() { // skip shared buffer compaction @@ -461,6 +463,7 @@ pub fn start_compactor( executor.spawn(async move { match event { ResponseEvent::CompactTask(compact_task) => { + let compact_task = CompactTask::from_protobuf(&compact_task); let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); @@ -479,20 +482,20 @@ pub fn start_compactor( Err(err) => { tracing::warn!(error = %err.as_report(), "Failed to track pending SST object id"); let mut compact_task = compact_task; - compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed); + compact_task.task_status = TaskStatus::TrackSstObjectIdFailed as i32; ((compact_task, HashMap::default()),None) } }; shutdown.lock().unwrap().remove(&task_id); let enable_check_compaction_result = context.storage_opts.check_compaction_result; - let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success as i32; if let Err(e) = request_sender.send(SubscribeCompactionEventRequest { event: Some(RequestEvent::ReportTask( ReportTask { task_id: compact_task.task_id, task_status: compact_task.task_status, - sorted_output_ssts: compact_task.sorted_output_ssts.clone(), + sorted_output_ssts: compact_task.sorted_output_ssts.iter().map(|sst| sst.to_protobuf()).collect_vec(), table_stats_change:to_prost_table_stats_map(table_stats), } )), @@ -540,6 +543,7 @@ pub fn start_compactor( } } ResponseEvent::ValidationTask(validation_task) => { + let validation_task = ValidationTask::from_protobuf(&validation_task); validate_ssts( validation_task, context.sstable_store.clone(), @@ -665,6 +669,7 @@ pub fn start_shared_compactor( SharedComapctorObjectIdManager::new(output_object_ids_deque, cloned_grpc_proxy_client.clone(), context.storage_opts.sstable_id_remote_fetch_number); match dispatch_task.unwrap() { dispatch_compaction_task_request::Task::CompactTask(compact_task) => { + let compact_task = CompactTask::from_protobuf(&compact_task); let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); @@ -680,7 +685,7 @@ pub fn start_shared_compactor( shutdown.lock().unwrap().remove(&task_id); let report_compaction_task_request = ReportCompactionTaskRequest { event: Some(ReportCompactionTaskEvent::ReportTask(ReportSharedTask { - compact_task: Some(compact_task.clone()), + compact_task: Some(compact_task.to_protobuf()), table_stats_change: to_prost_table_stats_map(table_stats), })), }; @@ -692,7 +697,7 @@ pub fn start_shared_compactor( Ok(_) => { // TODO: remove this method after we have running risingwave cluster with fast compact algorithm stably for a long time. let enable_check_compaction_result = context.storage_opts.check_compaction_result; - let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success as i32; if enable_check_compaction_result && need_check_task { match check_compaction_result(&compact_task, context.clone()).await { Err(e) => { @@ -754,6 +759,7 @@ pub fn start_shared_compactor( } } dispatch_compaction_task_request::Task::ValidationTask(validation_task) => { + let validation_task = ValidationTask::from_protobuf(&validation_task); validate_ssts(validation_task, context.sstable_store.clone()).await; } dispatch_compaction_task_request::Task::CancelCompactTask(cancel_compact_task) => { diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index f1209775e154..ae1389b46c45 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -219,7 +219,7 @@ impl HummockEventHandler { ); let write_conflict_detector = ConflictDetector::new_from_config(storage_opts); - let uploader = HummockUploader::new( + let uploader = HummockUploader::new( state_store_metrics, pinned_version.clone(), spawn_upload_task, diff --git a/src/storage/src/hummock/event_handler/uploader.rs b/src/storage/src/hummock/event_handler/uploader.rs index 7ffa3956c234..98adf1ca8380 100644 --- a/src/storage/src/hummock/event_handler/uploader.rs +++ b/src/storage/src/hummock/event_handler/uploader.rs @@ -1180,9 +1180,9 @@ mod tests { use prometheus::core::GenericGauge; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::{FullKey, TableKey}; - use risingwave_hummock_sdk::version::HummockVersion; + use risingwave_hummock_sdk::key_range::KeyRange; + use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; - use risingwave_pb::hummock::{KeyRange, SstableInfo}; use spin::Mutex; use tokio::spawn; use tokio::sync::mpsc::unbounded_channel; @@ -1273,8 +1273,8 @@ mod tests { object_id: gen_sst_object_id, sst_id: gen_sst_object_id, key_range: Some(KeyRange { - left: start_full_key.encode(), - right: end_full_key.encode(), + left: Bytes::from(start_full_key.encode()), + right: Bytes::from(end_full_key.encode()), right_exclusive: true, }), table_ids: vec![TEST_TABLE_ID.table_id], diff --git a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs index 32e77fd8e91e..b6cae2cc527d 100644 --- a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs +++ b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs @@ -15,8 +15,8 @@ use std::future::Future; use risingwave_hummock_sdk::key::{FullKey, PointRange, UserKey}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; -use risingwave_pb::hummock::SstableInfo; use crate::hummock::iterator::DeleteRangeIterator; use crate::hummock::sstable_store::SstableStoreRef; diff --git a/src/storage/src/hummock/iterator/concat_inner.rs b/src/storage/src/hummock/iterator/concat_inner.rs index d624068f3694..833e36da9408 100644 --- a/src/storage/src/hummock/iterator/concat_inner.rs +++ b/src/storage/src/hummock/iterator/concat_inner.rs @@ -16,7 +16,7 @@ use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::Arc; use risingwave_hummock_sdk::key::FullKey; -use risingwave_pb::hummock::SstableInfo; +use risingwave_hummock_sdk::version::SstableInfo; use crate::hummock::iterator::{DirectionEnum, HummockIterator, HummockIteratorDirection}; use crate::hummock::sstable::SstableIteratorReadOptions; diff --git a/src/storage/src/hummock/iterator/delete_range_iterator.rs b/src/storage/src/hummock/iterator/delete_range_iterator.rs index 62bc98422ac2..87b823e6be24 100644 --- a/src/storage/src/hummock/iterator/delete_range_iterator.rs +++ b/src/storage/src/hummock/iterator/delete_range_iterator.rs @@ -17,8 +17,8 @@ use std::future::Future; use risingwave_common::util::epoch::is_max_epoch; use risingwave_hummock_sdk::key::{PointRange, UserKey}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; -use risingwave_pb::hummock::SstableInfo; use crate::hummock::iterator::concat_delete_range_iterator::ConcatDeleteRangeIterator; use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferDeleteRangeIterator; diff --git a/src/storage/src/hummock/iterator/skip_watermark.rs b/src/storage/src/hummock/iterator/skip_watermark.rs index 01be9c44a8db..e1e8f3c3d6a9 100644 --- a/src/storage/src/hummock/iterator/skip_watermark.rs +++ b/src/storage/src/hummock/iterator/skip_watermark.rs @@ -20,9 +20,11 @@ use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::TableId; use risingwave_common::hash::{VirtualNode, VnodeBitmapExt}; use risingwave_hummock_sdk::key::FullKey; -use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, WatermarkDirection}; -use risingwave_pb::hummock::PbTableWatermarks; +use risingwave_hummock_sdk::table_watermark::{ + ReadTableWatermark, TableWatermarks, WatermarkDirection, +}; +// use risingwave_pb::hummock::PbTableWatermarks; use crate::hummock::iterator::{Forward, HummockIterator}; use crate::hummock::value::HummockValue; use crate::hummock::HummockResult; @@ -45,24 +47,17 @@ impl> SkipWatermarkIterator { pub fn from_safe_epoch_watermarks( inner: I, - safe_epoch_watermarks: &BTreeMap, + safe_epoch_watermarks: &BTreeMap, ) -> Self { let watermarks = safe_epoch_watermarks .iter() .map(|(table_id, watermarks)| { - assert_eq!(watermarks.epoch_watermarks.len(), 1); - let vnode_watermarks = &watermarks - .epoch_watermarks - .first() - .expect("should exist") - .watermarks; + assert_eq!(watermarks.watermarks.len(), 1); + let vnode_watermarks = &watermarks.watermarks.first().expect("should exist").1; let mut vnode_watermark_map = BTreeMap::new(); for vnode_watermark in vnode_watermarks { let watermark = Bytes::copy_from_slice(&vnode_watermark.watermark); - for vnode in - Bitmap::from(vnode_watermark.vnode_bitmap.as_ref().expect("should exist")) - .iter_vnodes() - { + for vnode in vnode_watermark.vnode_bitmap.as_ref().iter_vnodes() { assert!( vnode_watermark_map .insert(vnode, watermark.clone()) @@ -75,11 +70,7 @@ impl> SkipWatermarkIterator { ( TableId::from(*table_id), ReadTableWatermark { - direction: if watermarks.is_ascending { - WatermarkDirection::Ascending - } else { - WatermarkDirection::Descending - }, + direction: watermarks.direction, vnode_watermarks: vnode_watermark_map, }, ) diff --git a/src/storage/src/hummock/iterator/test_utils.rs b/src/storage/src/hummock/iterator/test_utils.rs index 62602074c601..c032fe094228 100644 --- a/src/storage/src/hummock/iterator/test_utils.rs +++ b/src/storage/src/hummock/iterator/test_utils.rs @@ -20,11 +20,11 @@ use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKey}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId}; use risingwave_object_store::object::{ InMemObjectStore, ObjectStore, ObjectStoreImpl, ObjectStoreRef, }; -use risingwave_pb::hummock::SstableInfo; use crate::hummock::sstable::SstableIteratorReadOptions; use crate::hummock::sstable_store::SstableStore; diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs index 46ef8edc442b..16492321d1e0 100644 --- a/src/storage/src/hummock/local_version/pinned_version.rs +++ b/src/storage/src/hummock/local_version/pinned_version.rs @@ -20,10 +20,8 @@ use std::time::{Duration, Instant}; use auto_enums::auto_enum; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::table_watermark::TableWatermarksIndex; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{HummockVersion, Level, Levels}; use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId, INVALID_VERSION_ID}; -use risingwave_pb::hummock::hummock_version::Levels; -use risingwave_pb::hummock::PbLevel; use risingwave_rpc_client::HummockMetaClient; use thiserror_ext::AsReport; use tokio::sync::mpsc::error::TryRecvError; @@ -143,7 +141,7 @@ impl PinnedVersion { self.version.levels.get(&compaction_group_id).unwrap() } - pub fn levels(&self, table_id: TableId) -> impl Iterator { + pub fn levels(&self, table_id: TableId) -> impl Iterator { #[auto_enum(Iterator)] match self.compaction_group_index.get(&table_id) { Some(compaction_group_id) => { diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs index 9efb22675f07..1de670951881 100644 --- a/src/storage/src/hummock/mod.rs +++ b/src/storage/src/hummock/mod.rs @@ -19,8 +19,8 @@ use std::sync::Arc; use bytes::Bytes; use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKeyRangeRef}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{HummockEpoch, *}; -use risingwave_pb::hummock::SstableInfo; mod block_cache; pub use block_cache::*; diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index ebaa60e16705..371c2e9b38bd 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -19,9 +19,11 @@ use std::sync::Arc; use bytes::{Bytes, BytesMut}; use risingwave_common::util::epoch::{is_compatibility_max_epoch, is_max_epoch}; use risingwave_hummock_sdk::key::{user_key, FullKey, MAX_KEY_LEN}; +use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::table_stats::{TableStats, TableStatsMap}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{HummockEpoch, KeyComparator, LocalSstableInfo}; -use risingwave_pb::hummock::{BloomFilterType, SstableInfo}; +use risingwave_pb::hummock::BloomFilterType; use super::utils::CompressionAlgorithm; use super::{ @@ -629,9 +631,9 @@ impl SstableBuilder { object_id: self.sstable_id, sst_id: self.sstable_id, bloom_filter_kind: bloom_filter_kind as i32, - key_range: Some(risingwave_pb::hummock::KeyRange { - left: meta.smallest_key.clone(), - right: meta.largest_key.clone(), + key_range: Some(KeyRange { + left: Bytes::from(meta.smallest_key.clone()), + right: Bytes::from(meta.largest_key.clone()), right_exclusive, }), file_size: meta.estimated_size as u64, diff --git a/src/storage/src/hummock/sstable/mod.rs b/src/storage/src/hummock/sstable/mod.rs index 1988f7a29a8c..45df644b27ef 100644 --- a/src/storage/src/hummock/sstable/mod.rs +++ b/src/storage/src/hummock/sstable/mod.rs @@ -18,7 +18,9 @@ mod block; use std::fmt::{Debug, Display, Formatter}; +use std::mem::size_of; use std::ops::{BitXor, Bound, Range}; +use std::sync::Arc; pub use block::*; mod block_iterator; @@ -26,6 +28,8 @@ pub use block_iterator::*; mod bloom; mod xor_filter; pub use bloom::BloomFilterBuilder; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_pb::hummock::{PbKeyRange, PbSstableInfo}; pub use xor_filter::{ BlockedXor16FilterBuilder, Xor16FilterBuilder, Xor8FilterBuilder, XorFilterReader, }; @@ -36,7 +40,7 @@ use risingwave_common::catalog::TableId; pub use writer::*; mod forward_sstable_iterator; pub mod multi_builder; -use bytes::{Buf, BufMut}; +use bytes::{Buf, BufMut, Bytes}; pub use forward_sstable_iterator::*; mod backward_sstable_iterator; pub use backward_sstable_iterator::*; diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs index c603b7d8f503..8e26b304e154 100644 --- a/src/storage/src/hummock/sstable_store.rs +++ b/src/storage/src/hummock/sstable_store.rs @@ -27,12 +27,12 @@ use risingwave_common::cache::{ CachePriority, LookupResponse, LruCacheEventListener, LruKey, LruValue, }; use risingwave_common::config::StorageMemoryConfig; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{HummockSstableObjectId, OBJECT_SUFFIX}; use risingwave_hummock_trace::TracedCachePolicy; use risingwave_object_store::object::{ ObjectError, ObjectMetadataIter, ObjectStoreRef, ObjectStreamingUploader, }; -use risingwave_pb::hummock::SstableInfo; use thiserror_ext::AsReport; use tokio::task::JoinHandle; use tokio::time::Instant; @@ -611,7 +611,7 @@ impl SstableStore { sst: &SstableInfo, stats: &mut StoreLocalStatistic, ) -> impl Future> + Send + 'static { - let object_id = sst.get_object_id(); + let object_id = sst.object_id; let lookup_response = self .meta_cache .lookup_with_request_dedup::<_, HummockError, _>( @@ -1180,8 +1180,8 @@ mod tests { use std::ops::Range; use std::sync::Arc; + use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockSstableObjectId; - use risingwave_pb::hummock::SstableInfo; use super::{SstableStoreRef, SstableWriterOptions}; use crate::hummock::iterator::test_utils::{iterator_test_key_of, mock_sstable_store}; diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs index d2064ffd09de..fd19a620cc32 100644 --- a/src/storage/src/hummock/store/hummock_storage.rs +++ b/src/storage/src/hummock/store/hummock_storage.rs @@ -26,8 +26,8 @@ use risingwave_common::util::epoch::is_max_epoch; use risingwave_common_service::observer_manager::{NotificationClient, ObserverManager}; use risingwave_hummock_sdk::key::{is_empty_key_range, TableKey, TableKeyRange}; use risingwave_hummock_sdk::table_watermark::ReadTableWatermark; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockReadEpoch; -use risingwave_pb::hummock::SstableInfo; use risingwave_rpc_client::HummockMetaClient; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use tokio::sync::oneshot; diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs index 513451b84852..95b8887051ee 100644 --- a/src/storage/src/hummock/store/version.rs +++ b/src/storage/src/hummock/store/version.rs @@ -33,9 +33,9 @@ use risingwave_hummock_sdk::key_range::KeyRangeCommon; use risingwave_hummock_sdk::table_watermark::{ ReadTableWatermark, TableWatermarksIndex, VnodeWatermark, WatermarkDirection, }; -use risingwave_hummock_sdk::version::HummockVersionDelta; +use risingwave_hummock_sdk::version::{HummockVersionDelta, SstableInfo}; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, LocalSstableInfo}; -use risingwave_pb::hummock::{LevelType, SstableInfo}; +use risingwave_pb::hummock::LevelType; use sync_point::sync_point; use tracing::Instrument; @@ -577,9 +577,7 @@ pub fn read_filter_for_batch( })); sst_vec.extend(staging_ssts.into_iter().filter(|staging_sst| { - assert!( - staging_sst.get_max_epoch() <= min_epoch || staging_sst.get_min_epoch() > min_epoch - ); + assert!(staging_sst.max_epoch <= min_epoch || staging_sst.min_epoch > min_epoch); // Dedup staging SSTs in different shard. Duplicates can happen in the following case: // - Table 1 Shard 1 produces IMM 1 // - Table 1 Shard 2 produces IMM 2 @@ -961,7 +959,7 @@ impl HummockVersionReader { .binary_search(&read_options.table_id.table_id) .is_ok() }) - .cloned() + .map(|sst| sst.clone()) .collect_vec(); if sstables.is_empty() { continue; @@ -969,7 +967,7 @@ impl HummockVersionReader { if sstables.len() > 1 { let ssts_which_have_delete_range = sstables .iter() - .filter(|sst| sst.get_range_tombstone_count() > 0) + .filter(|sst| sst.range_tombstone_count > 0) .cloned() .collect_vec(); if !ssts_which_have_delete_range.is_empty() { @@ -1035,7 +1033,7 @@ impl HummockVersionReader { .sstable(sstable_info, &mut local_stats) .instrument(tracing::trace_span!("get_sstable")) .await?; - assert_eq!(sstable_info.get_object_id(), sstable.id); + assert_eq!(sstable_info.object_id, sstable.id); if !sstable.meta.monotonic_tombstone_events.is_empty() && !read_options.ignore_range_tombstone { diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs index d6c2ce901178..a1e539c5d737 100644 --- a/src/storage/src/hummock/test_utils.rs +++ b/src/storage/src/hummock/test_utils.rs @@ -23,8 +23,9 @@ use risingwave_common::cache::CachePriority; use risingwave_common::catalog::TableId; use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::key::{FullKey, PointRange, TableKey, UserKey}; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId}; -use risingwave_pb::hummock::{KeyRange, SstableInfo}; use super::iterator::test_utils::iterator_test_table_key_of; use super::{ @@ -107,8 +108,8 @@ pub fn gen_dummy_sst_info( object_id: id, sst_id: id, key_range: Some(KeyRange { - left: FullKey::for_test(table_id, min_table_key, epoch).encode(), - right: FullKey::for_test(table_id, max_table_key, epoch).encode(), + left: Bytes::from(FullKey::for_test(table_id, min_table_key, epoch).encode()), + right: Bytes::from(FullKey::for_test(table_id, max_table_key, epoch).encode()), right_exclusive: false, }), file_size, @@ -184,8 +185,8 @@ pub async fn put_sst( object_id: sst_object_id, sst_id: sst_object_id, key_range: Some(KeyRange { - left: meta.smallest_key.clone(), - right: meta.largest_key.clone(), + left: Bytes::from(meta.smallest_key.clone()), + right: Bytes::from(meta.largest_key.clone()), right_exclusive: false, }), file_size: meta.estimated_size as u64, diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs index 09c4a7eef0e4..7bd7caecfa75 100644 --- a/src/storage/src/hummock/utils.rs +++ b/src/storage/src/hummock/utils.rs @@ -26,9 +26,8 @@ use risingwave_common::catalog::{TableId, TableOption}; use risingwave_hummock_sdk::key::{ bound_table_key_range, EmptySliceRef, FullKey, TableKey, UserKey, }; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{can_concat, HummockEpoch}; -use risingwave_pb::hummock::SstableInfo; use tokio::sync::watch::Sender; use tokio::sync::Notify; @@ -91,7 +90,7 @@ pub fn validate_table_key_range(version: &HummockVersion) { assert!( t.key_range.is_some(), "key_range in table [{}] is none", - t.get_object_id() + t.object_id ); } } @@ -103,8 +102,8 @@ where B: AsRef<[u8]> + EmptySliceRef, { let table_range = info.key_range.as_ref().unwrap(); - let table_start = FullKey::decode(table_range.left.as_slice()).user_key; - let table_end = FullKey::decode(table_range.right.as_slice()).user_key; + let table_start = FullKey::decode(table_range.left.as_ref()).user_key; + let table_end = FullKey::decode(table_range.right.as_ref()).user_key; let (left, right) = bound_table_key_range(table_id, table_key_range); let left: Bound> = left.as_ref().map(|key| key.as_ref()); let right: Bound> = right.as_ref().map(|key| key.as_ref()); @@ -116,10 +115,7 @@ where } else { Bound::Included(&table_end) }, - ) && info - .get_table_ids() - .binary_search(&table_id.table_id()) - .is_ok() + ) && info.table_ids.binary_search(&table_id.table_id()).is_ok() } /// Search the SST containing the specified key within a level, using binary search. diff --git a/src/storage/src/hummock/validator.rs b/src/storage/src/hummock/validator.rs index 8c38a99f4324..03994c7beae2 100644 --- a/src/storage/src/hummock/validator.rs +++ b/src/storage/src/hummock/validator.rs @@ -18,7 +18,7 @@ use std::collections::HashMap; use std::sync::Arc; use risingwave_hummock_sdk::key::FullKey; -use risingwave_pb::hummock::ValidationTask; +use risingwave_hummock_sdk::version::ValidationTask; use crate::hummock::iterator::HummockIterator; use crate::hummock::sstable::SstableIteratorReadOptions; @@ -40,7 +40,7 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef) .expect("valid worker_id"); tracing::debug!( "Validating SST {} from worker {}, epoch {}", - sst.get_object_id(), + sst.object_id, worker_id, task.epoch ); @@ -48,7 +48,7 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef) Ok(holder) => holder, Err(_err) => { // One reasonable cause is the SST has been vacuumed. - tracing::info!("Skip sanity check for SST {}.", sst.get_object_id()); + tracing::info!("Skip sanity check for SST {}.", sst.object_id); continue; } }; @@ -67,7 +67,7 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef) ); let mut previous_key: Option>> = None; if let Err(_err) = iter.rewind().await { - tracing::info!("Skip sanity check for SST {}.", sst.get_object_id()); + tracing::info!("Skip sanity check for SST {}.", sst.object_id); } while iter.is_valid() { key_counts += 1; @@ -78,38 +78,32 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef) { panic!("SST sanity check failed: Duplicate key {:x?} in SST object {} from worker {} and SST object {} from worker {}", current_key, - sst.get_object_id(), + sst.object_id, worker_id, duplicate_sst_object_id, duplicate_worker_id) } - visited_keys.insert(current_key.to_owned(), (sst.get_object_id(), worker_id)); + visited_keys.insert(current_key.to_owned(), (sst.object_id, worker_id)); // Ordered and Locally unique if let Some(previous_key) = previous_key.take() { let cmp = previous_key.cmp(¤t_key); if cmp != cmp::Ordering::Less { panic!( "SST sanity check failed: For SST {}, expect {:x?} < {:x?}, got {:#?}", - sst.get_object_id(), - previous_key, - current_key, - cmp + sst.object_id, previous_key, current_key, cmp ) } } previous_key = Some(current_key); if let Err(_err) = iter.next().await { - tracing::info!( - "Skip remaining sanity check for SST {}", - sst.get_object_id(), - ); + tracing::info!("Skip remaining sanity check for SST {}", sst.object_id,); break; } } tracing::debug!( "Validated {} keys for SST {}, epoch {}", key_counts, - sst.get_object_id(), + sst.object_id, task.epoch ); iter.collect_local_statistic(&mut unused); From 163ff206ca49d8433f901b0b9a84c7c8563151c1 Mon Sep 17 00:00:00 2001 From: Li0k Date: Mon, 4 Mar 2024 15:59:26 +0800 Subject: [PATCH 02/18] fix(storage): fix check --- .../picker/min_overlap_compaction_picker.rs | 1 - .../picker/ttl_reclaim_compaction_picker.rs | 4 ++-- src/meta/src/hummock/manager/mod.rs | 4 +--- .../src/hummock/mock_hummock_meta_client.rs | 2 +- src/meta/src/hummock/test_utils.rs | 2 -- .../compaction_group/hummock_version_ext.rs | 10 ++++------ src/storage/hummock_sdk/src/version.rs | 20 +++++++++---------- .../hummock_test/src/compactor_tests.rs | 1 - .../src/hummock/compactor/compaction_utils.rs | 1 - .../src/hummock/compactor/compactor_runner.rs | 4 +--- .../event_handler/hummock_event_handler.rs | 2 +- .../src/hummock/iterator/skip_watermark.rs | 1 - src/storage/src/hummock/sstable/mod.rs | 6 +----- src/storage/src/hummock/store/version.rs | 2 +- 14 files changed, 21 insertions(+), 39 deletions(-) diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 4ae1bae5a836..9259930974d2 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -16,7 +16,6 @@ use std::sync::Arc; use risingwave_hummock_sdk::append_sstable_info_to_string; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; -use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, SstableInfo}; use risingwave_pb::hummock::LevelType; diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index 240309fef082..ff1c9ceb7782 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -47,7 +47,7 @@ impl TtlPickerState { pub fn init(&mut self, key_range: KeyRange) { self.last_select_end_bound = KeyRange { left: Bytes::default(), - right: Bytes::from(key_range.left.clone()), + right: key_range.left.clone(), right_exclusive: true, }; self.end_bound_in_round = key_range; @@ -174,7 +174,7 @@ impl TtlReclaimCompactionPicker { let select_last_sst = select_input_ssts.last().unwrap(); state.last_select_end_bound.full_key_extend(&KeyRange { left: Bytes::default(), - right: Bytes::from(select_last_sst.key_range.as_ref().unwrap().right.clone()), + right: select_last_sst.key_range.as_ref().unwrap().right.clone(), right_exclusive: select_last_sst.key_range.as_ref().unwrap().right_exclusive, }); diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index b0ed6f2d54e0..4cca418b9356 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -968,8 +968,6 @@ impl HummockManager { _ => 0, }; let vnode_partition_count = compact_task.input.vnode_partition_count; - use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; - let mut compact_task = CompactTask { input_ssts: compact_task.input.input_levels, splits: vec![KeyRange::inf()], @@ -3209,7 +3207,7 @@ impl HummockManager { .report_compact_task( task_id, TaskStatus::try_from(task_status).unwrap(), - sorted_output_ssts.iter().map(|pb_sst| SstableInfo::from_protobuf(pb_sst)).collect_vec(), + sorted_output_ssts.iter().map(SstableInfo::from_protobuf).collect_vec(), Some(table_stats_change), ) .await diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index 01f1c08dfb86..0568079f6e80 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -325,7 +325,7 @@ impl HummockMetaClient for MockHummockMetaClient { TaskStatus::try_from(task_status).unwrap(), sorted_output_ssts .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .map(SstableInfo::from_protobuf) .collect_vec(), Some(table_stats_change), ) diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs index f8169b0627d6..5e5464f0e99e 100644 --- a/src/meta/src/hummock/test_utils.rs +++ b/src/meta/src/hummock/test_utils.rs @@ -56,8 +56,6 @@ pub async fn add_test_tables( context_id: HummockContextId, ) -> Vec> { // Increase version by 2. - - use risingwave_hummock_sdk::version::SstableInfo; let mut epoch: u64 = 1; let sstable_ids = get_sst_ids(hummock_manager, 3).await; let test_tables = generate_test_sstables_with_table_id(epoch, 1, sstable_ids); diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 05f8d5d07135..a93241c285b7 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -20,18 +20,16 @@ use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_pb::hummock::group_delta::DeltaType; use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; -use risingwave_pb::hummock::table_watermarks::PbEpochNewWatermarks; use risingwave_pb::hummock::{ CompactionConfig, CompatibilityVersion, GroupConstruct, GroupDestroy, GroupMetaChange, - GroupTableChange, LevelType, PbLevelType, PbTableWatermarks, + GroupTableChange, LevelType, PbLevelType, }; use tracing::warn; use super::StateTableId; use crate::compaction_group::StaticCompactionGroupId; use crate::key_range::KeyRangeCommon; -use crate::prost_key_range::KeyRangeExt; -use crate::table_watermark::{TableWatermarks, TableWatermarksIndex, VnodeWatermark}; +use crate::table_watermark::{TableWatermarks, TableWatermarksIndex}; use crate::version::{ HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, }; @@ -76,7 +74,7 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary intra_level .inserted_table_infos .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)), + .map(SstableInfo::from_protobuf), ); } new_vnode_partition_count = intra_level.vnode_partition_count; @@ -435,7 +433,7 @@ impl HummockVersion { delta .inserted_table_infos .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)), + .map(SstableInfo::from_protobuf), ); } if !delta.removed_table_ids.is_empty() { diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 2ceb66556853..9e4d00fa4cc6 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -14,7 +14,6 @@ use std::collections::{BTreeMap, HashMap}; use std::mem::size_of; -use std::ops::Deref; use bytes::Bytes; use itertools::Itertools; @@ -26,7 +25,6 @@ use risingwave_pb::hummock::{ LevelType, PbCompactTask, PbHummockVersion, PbHummockVersionDelta, PbInputLevel, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, PbValidationTask, TableOption, }; -use risingwave_pb::java_binding::key_range; use serde::Serialize; use crate::key_range::KeyRange; @@ -69,7 +67,7 @@ impl OverlappingLevel { sub_levels: pb_overlapping_level .sub_levels .iter() - .map(|level| Level::from_protobuf(level)) + .map(Level::from_protobuf) .collect_vec(), total_file_size: pb_overlapping_level.total_file_size, uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, @@ -123,7 +121,7 @@ impl Level { table_infos: pb_level .table_infos .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .map(SstableInfo::from_protobuf) .collect_vec(), total_file_size: pb_level.total_file_size, sub_level_id: pb_level.sub_level_id, @@ -211,7 +209,7 @@ impl Levels { levels: pb_levels .levels .iter() - .map(|pb_level| Level::from_protobuf(pb_level)) + .map(Level::from_protobuf) .collect_vec(), group_id: pb_levels.group_id, parent_group_id: pb_levels.parent_group_id, @@ -544,7 +542,7 @@ impl InputLevel { table_infos: pb_input_level .table_infos .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .map(SstableInfo::from_protobuf) .collect_vec(), } } @@ -634,7 +632,7 @@ impl CompactTask { input_ssts: pb_compact_task .input_ssts .iter() - .map(|pb_input_level| InputLevel::from_protobuf(pb_input_level)) + .map(InputLevel::from_protobuf) .collect_vec(), splits: pb_compact_task .splits @@ -649,7 +647,7 @@ impl CompactTask { sorted_output_ssts: pb_compact_task .sorted_output_ssts .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .map(SstableInfo::from_protobuf) .collect_vec(), task_id: pb_compact_task.task_id, target_level: pb_compact_task.target_level, @@ -718,8 +716,7 @@ impl CompactTask { current_epoch_time: self.current_epoch_time, target_sub_level_id: self.target_sub_level_id, task_type: self.task_type, - #[allow(deprecated)] - split_by_state_table: self.split_by_state_table, + //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, split_weight_by_vnode: self.split_weight_by_vnode, table_vnode_partition: self.table_vnode_partition.clone(), table_watermarks: self @@ -727,6 +724,7 @@ impl CompactTask { .iter() .map(|(table_id, table_watermark)| (*table_id, table_watermark.to_protobuf())) .collect(), + ..Default::default() } } @@ -806,7 +804,7 @@ impl ValidationTask { sst_infos: pb_validation_task .sst_infos .iter() - .map(|pb_sst| SstableInfo::from_protobuf(pb_sst)) + .map(SstableInfo::from_protobuf) .collect_vec(), sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(), epoch: pb_validation_task.epoch, diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index 3df81bab956e..8679b9ccc010 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -32,7 +32,6 @@ pub(crate) mod tests { use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::{next_key, FullKey, TableKey, TABLE_PREFIX_LEN}; use risingwave_hummock_sdk::key_range::KeyRange; - use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, InputLevel, SstableInfo}; use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index 1e2ad1e8a265..53778b684c6f 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -24,7 +24,6 @@ use risingwave_common::constants::hummock::CompactionFilterFlag; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::TableStatsMap; use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, EpochWithGap, KeyComparator}; diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index b40d6d43d720..6d19b444499c 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -17,7 +17,6 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use await_tree::InstrumentAwait; -use bytes::Bytes; use futures::{stream, FutureExt, StreamExt}; use itertools::Itertools; use risingwave_common::util::epoch::is_max_epoch; @@ -945,10 +944,9 @@ where mod tests { use std::collections::{HashSet, VecDeque}; - use foyer::common::code::Key; + use bytes::Bytes; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::{TableKey, UserKey}; - use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::version::InputLevel; use super::*; diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index ae1389b46c45..f1209775e154 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -219,7 +219,7 @@ impl HummockEventHandler { ); let write_conflict_detector = ConflictDetector::new_from_config(storage_opts); - let uploader = HummockUploader::new( + let uploader = HummockUploader::new( state_store_metrics, pinned_version.clone(), spawn_upload_task, diff --git a/src/storage/src/hummock/iterator/skip_watermark.rs b/src/storage/src/hummock/iterator/skip_watermark.rs index e1e8f3c3d6a9..09563748cce1 100644 --- a/src/storage/src/hummock/iterator/skip_watermark.rs +++ b/src/storage/src/hummock/iterator/skip_watermark.rs @@ -16,7 +16,6 @@ use std::cmp::Ordering; use std::collections::{BTreeMap, VecDeque}; use bytes::Bytes; -use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::TableId; use risingwave_common::hash::{VirtualNode, VnodeBitmapExt}; use risingwave_hummock_sdk::key::FullKey; diff --git a/src/storage/src/hummock/sstable/mod.rs b/src/storage/src/hummock/sstable/mod.rs index 45df644b27ef..1988f7a29a8c 100644 --- a/src/storage/src/hummock/sstable/mod.rs +++ b/src/storage/src/hummock/sstable/mod.rs @@ -18,9 +18,7 @@ mod block; use std::fmt::{Debug, Display, Formatter}; -use std::mem::size_of; use std::ops::{BitXor, Bound, Range}; -use std::sync::Arc; pub use block::*; mod block_iterator; @@ -28,8 +26,6 @@ pub use block_iterator::*; mod bloom; mod xor_filter; pub use bloom::BloomFilterBuilder; -use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_pb::hummock::{PbKeyRange, PbSstableInfo}; pub use xor_filter::{ BlockedXor16FilterBuilder, Xor16FilterBuilder, Xor8FilterBuilder, XorFilterReader, }; @@ -40,7 +36,7 @@ use risingwave_common::catalog::TableId; pub use writer::*; mod forward_sstable_iterator; pub mod multi_builder; -use bytes::{Buf, BufMut, Bytes}; +use bytes::{Buf, BufMut}; pub use forward_sstable_iterator::*; mod backward_sstable_iterator; pub use backward_sstable_iterator::*; diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs index 95b8887051ee..ecc799815236 100644 --- a/src/storage/src/hummock/store/version.rs +++ b/src/storage/src/hummock/store/version.rs @@ -959,7 +959,7 @@ impl HummockVersionReader { .binary_search(&read_options.table_id.table_id) .is_ok() }) - .map(|sst| sst.clone()) + .cloned() .collect_vec(); if sstables.is_empty() { continue; From 940b233f81d10888bf2a47e39c712c15c12cfd9b Mon Sep 17 00:00:00 2001 From: Li0k Date: Mon, 4 Mar 2024 18:43:38 +0800 Subject: [PATCH 03/18] refactor(storage): refactor and reduce copy --- src/compute/src/rpc/service/stream_service.rs | 6 +- src/meta/service/src/hummock_service.rs | 1 + src/meta/service/src/notification_service.rs | 1 + src/meta/src/backup_restore/restore.rs | 2 +- src/meta/src/barrier/mod.rs | 6 +- src/meta/src/hummock/compactor_manager.rs | 6 +- src/meta/src/hummock/manager/checkpoint.rs | 2 +- src/meta/src/hummock/manager/context.rs | 6 +- src/meta/src/hummock/manager/mod.rs | 20 +- src/meta/src/hummock/metrics_utils.rs | 1 + .../src/hummock/mock_hummock_meta_client.rs | 8 +- src/meta/src/hummock/model/ext/hummock.rs | 1 + src/meta/src/hummock/model/version_delta.rs | 13 +- src/rpc_client/src/meta_client.rs | 2 +- src/storage/backup/src/meta_snapshot_v1.rs | 1 + src/storage/backup/src/meta_snapshot_v2.rs | 1 + .../compaction_group/hummock_version_ext.rs | 5 +- src/storage/hummock_sdk/src/lib.rs | 22 + .../hummock_sdk/src/table_watermark.rs | 209 +++++-- src/storage/hummock_sdk/src/version.rs | 586 ++++++++++++------ .../src/mock_notification_client.rs | 1 + src/storage/src/hummock/compactor/mod.rs | 11 +- src/storage/src/store.rs | 10 +- 23 files changed, 647 insertions(+), 274 deletions(-) diff --git a/src/compute/src/rpc/service/stream_service.rs b/src/compute/src/rpc/service/stream_service.rs index 194fbf7b655d..c8b79ea44146 100644 --- a/src/compute/src/rpc/service/stream_service.rs +++ b/src/compute/src/rpc/service/stream_service.rs @@ -15,7 +15,7 @@ use await_tree::InstrumentAwait; use itertools::Itertools; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; -use risingwave_hummock_sdk::LocalSstableInfo; +use risingwave_hummock_sdk::{LocalSstableInfo, ProtoSerializeOwnExt}; use risingwave_pb::stream_service::barrier_complete_response::GroupedSstableInfo; use risingwave_pb::stream_service::stream_service_server::StreamService; use risingwave_pb::stream_service::*; @@ -177,7 +177,7 @@ impl StreamService for StreamServiceImpl { table_stats, }| GroupedSstableInfo { compaction_group_id, - sst: Some(sst_info.to_protobuf()), + sst: Some(sst_info.to_protobuf_own()), table_stats_map: to_prost_table_stats_map(table_stats), }, ) @@ -185,7 +185,7 @@ impl StreamService for StreamServiceImpl { worker_id: self.env.worker_id(), table_watermarks: table_watermarks .into_iter() - .map(|(key, value)| (key.table_id, value.to_protobuf())) + .map(|(key, value)| (key.table_id, value.to_protobuf_own())) .collect(), })) } diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs index eb303c8bf978..15d222a9f34e 100644 --- a/src/meta/service/src/hummock_service.rs +++ b/src/meta/service/src/hummock_service.rs @@ -20,6 +20,7 @@ use itertools::Itertools; use risingwave_common::catalog::{TableId, NON_RESERVED_SYS_CATALOG_ID}; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::HummockVersionDelta; +use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta::manager::MetadataManager; use risingwave_pb::hummock::get_compaction_score_response::PickerInfo; use risingwave_pb::hummock::hummock_manager_service_server::HummockManagerService; diff --git a/src/meta/service/src/notification_service.rs b/src/meta/service/src/notification_service.rs index 06f61e5f26e5..290dc3d42e4d 100644 --- a/src/meta/service/src/notification_service.rs +++ b/src/meta/service/src/notification_service.rs @@ -13,6 +13,7 @@ // limitations under the License. use itertools::Itertools; +use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta::manager::MetadataManager; use risingwave_meta::MetaResult; use risingwave_pb::backup_service::MetaBackupManifestId; diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs index 1afe17b15beb..cbaf61c2a285 100644 --- a/src/meta/src/backup_restore/restore.rs +++ b/src/meta/src/backup_restore/restore.rs @@ -20,7 +20,7 @@ use risingwave_backup::storage::{MetaSnapshotStorage, MetaSnapshotStorageRef}; use risingwave_backup::MetaSnapshotId; use risingwave_common::config::{MetaBackend, ObjectStoreConfig}; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_hummock_sdk::version_checkpoint_path; +use risingwave_hummock_sdk::{version_checkpoint_path, ProtoSerializeExt}; use risingwave_object_store::object::build_remote_object_store; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_pb::hummock::PbHummockVersionCheckpoint; diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index 05129a40477b..6d1a7e07e945 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -32,7 +32,9 @@ use risingwave_hummock_sdk::table_watermark::{ merge_multiple_new_table_watermarks, TableWatermarks, }; use risingwave_hummock_sdk::version::SstableInfo; -use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId}; +use risingwave_hummock_sdk::{ + ExtendedSstableInfo, HummockSstableObjectId, ProtoSerializeExt, ProtoSerializeOwnExt, +}; use risingwave_pb::catalog::table::TableType; use risingwave_pb::common::WorkerNode; use risingwave_pb::ddl_service::DdlProgress; @@ -968,7 +970,7 @@ fn collect_commit_epoch_info(resps: &mut [BarrierCompleteResponse]) -> CommitEpo sst_to_worker.insert(sst_info.get_object_id(), resp.worker_id); ExtendedSstableInfo::new( grouped.compaction_group_id, - SstableInfo::from_protobuf(&sst_info), + SstableInfo::from_protobuf_own(sst_info), std::mem::take(&mut grouped.table_stats_map), ) }) diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs index 708077fe1c3d..d3698a33907d 100644 --- a/src/meta/src/hummock/compactor_manager.rs +++ b/src/meta/src/hummock/compactor_manager.rs @@ -20,7 +20,7 @@ use fail::fail_point; use parking_lot::RwLock; use risingwave_hummock_sdk::compact::statistics_compact_task; use risingwave_hummock_sdk::version::CompactTask; -use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId}; +use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId, ProtoSerializeOwnExt}; use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent; use risingwave_pb::hummock::{ CancelCompactTask, CompactTaskAssignment, CompactTaskProgress, SubscribeCompactionEventResponse, @@ -149,8 +149,8 @@ impl CompactorManagerInner { }; // Initialize heartbeat for existing tasks. task_assignment.into_iter().for_each(|assignment| { - manager.initiate_task_heartbeat(CompactTask::from_protobuf( - assignment.compact_task.as_ref().unwrap(), + manager.initiate_task_heartbeat(CompactTask::from_protobuf_own( + assignment.compact_task.unwrap(), )); }); Ok(manager) diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs index 8f3a7cdff695..07cce899b77b 100644 --- a/src/meta/src/hummock/manager/checkpoint.rs +++ b/src/meta/src/hummock/manager/checkpoint.rs @@ -22,7 +22,7 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ object_size_map, summarize_group_deltas, }; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_hummock_sdk::HummockVersionId; +use risingwave_hummock_sdk::{HummockVersionId, ProtoSerializeExt}; use risingwave_pb::hummock::hummock_version_checkpoint::{PbStaleObjects, StaleObjects}; use risingwave_pb::hummock::{PbHummockVersionArchive, PbHummockVersionCheckpoint}; use thiserror_ext::AsReport; diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs index 59d671503dba..3b9e5dab4d02 100644 --- a/src/meta/src/hummock/manager/context.rs +++ b/src/meta/src/hummock/manager/context.rs @@ -21,6 +21,7 @@ use itertools::Itertools; use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{ ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId, + ProtoSerializeOwnExt, }; use risingwave_pb::hummock::ValidationTask; @@ -172,7 +173,10 @@ impl HummockManager { .collect_vec(); if compactor .send_event(ResponseEvent::ValidationTask(ValidationTask { - sst_infos: sst_infos.iter().map(|sst| sst.to_protobuf()).collect_vec(), + sst_infos: sst_infos + .into_iter() + .map(|sst| sst.to_protobuf_own()) + .collect_vec(), sst_id_to_worker_id: sst_to_context.clone(), epoch, })) diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index 4cca418b9356..4d95ad1381a3 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -45,7 +45,8 @@ use risingwave_hummock_sdk::version::{CompactTask, HummockVersionDelta, SstableI use risingwave_hummock_sdk::{ version_archive_dir, version_checkpoint_path, CompactionGroupId, ExtendedSstableInfo, HummockCompactionTaskId, HummockContextId, HummockEpoch, HummockSstableId, - HummockSstableObjectId, HummockVersionId, SstObjectIdRange, INVALID_VERSION_ID, + HummockSstableObjectId, HummockVersionId, ProtoSerializeExt, ProtoSerializeOwnExt, + SstObjectIdRange, INVALID_VERSION_ID, }; use risingwave_meta_model_v2::{ compaction_status, compaction_task, hummock_pinned_snapshot, hummock_pinned_version, @@ -94,8 +95,8 @@ use crate::manager::{ClusterManagerRef, FragmentManagerRef}; use crate::manager::{MetaSrvEnv, MetadataManager, TableId, META_NODE_ID}; use crate::model::{ BTreeMapEntryTransaction, BTreeMapEntryTransactionWrapper, BTreeMapTransaction, - BTreeMapTransactionWrapper, ClusterId, MetadataModel, MetadataModelError, ValTransaction, - VarTransaction, VarTransactionWrapper, + BTreeMapTransactionWrapper, ClusterId, MetadataModelError, ValTransaction, VarTransaction, + VarTransactionWrapper, }; use crate::rpc::metrics::MetaMetrics; use crate::storage::MetaStore; @@ -443,6 +444,8 @@ impl HummockManager { versioning_guard: &mut RwLockWriteGuard<'_, Versioning>, ) -> Result<()> { use sea_orm::EntityTrait; + + use crate::model::MetadataModel; let sql_meta_store = self.sql_meta_store(); let compaction_statuses: BTreeMap = match &sql_meta_store { @@ -1691,8 +1694,8 @@ impl HummockManager { delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { level_idx: 0, inserted_table_infos: group_sstables - .iter() - .map(|sst| sst.to_protobuf()) + .into_iter() + .map(|sst| sst.to_protobuf_own()) .collect_vec(), l0_sub_level_id, ..Default::default() @@ -1921,6 +1924,7 @@ impl HummockManager { table_catalogs: Vec, compaction_groups: Vec, ) -> Result<()> { + use crate::model::MetadataModel; for table in &table_catalogs { table.insert(self.env.meta_store_checked()).await?; } @@ -2092,7 +2096,7 @@ impl HummockManager { let compact_task_string = compact_task_to_string(&compact_task); // TODO: shall we need to cancel on meta ? compactor - .send_event(ResponseEvent::CompactTask(compact_task.to_protobuf())) + .send_event(ResponseEvent::CompactTask(compact_task.to_protobuf_own())) .with_context(|| { format!( "Failed to trigger compaction task for compaction_group {}", @@ -3155,7 +3159,7 @@ impl HummockManager { Ok(Some(compact_task)) => { let task_id = compact_task.task_id; if let Err(e) = compactor.send_event( - ResponseEvent::CompactTask(compact_task.to_protobuf()), + ResponseEvent::CompactTask(compact_task.to_protobuf_own()), ) { tracing::warn!( error = %e.as_report(), @@ -3207,7 +3211,7 @@ impl HummockManager { .report_compact_task( task_id, TaskStatus::try_from(task_status).unwrap(), - sorted_output_ssts.iter().map(SstableInfo::from_protobuf).collect_vec(), + sorted_output_ssts.into_iter().map(SstableInfo::from_protobuf_own).collect_vec(), Some(table_stats_change), ) .await diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs index 94270853f87d..0c9795ef129e 100644 --- a/src/meta/src/hummock/metrics_utils.rs +++ b/src/meta/src/hummock/metrics_utils.rs @@ -24,6 +24,7 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ use risingwave_hummock_sdk::version::{HummockVersion, Levels}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, + ProtoSerializeSizeEstimatedExt, }; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{ diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index 0568079f6e80..47a407cb44fd 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -29,7 +29,7 @@ use risingwave_hummock_sdk::table_watermark::TableWatermarks; use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{ HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, - SstObjectIdRange, + ProtoSerializeOwnExt, SstObjectIdRange, }; use risingwave_pb::common::{HostAddress, WorkerType}; use risingwave_pb::hummock::compact_task::TaskStatus; @@ -292,7 +292,7 @@ impl HummockMetaClient for MockHummockMetaClient { .unwrap() { let resp = SubscribeCompactionEventResponse { - event: Some(ResponseEvent::CompactTask(task.to_protobuf())), + event: Some(ResponseEvent::CompactTask(task.to_protobuf_own())), create_at: SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("Clock may have gone backwards") @@ -324,8 +324,8 @@ impl HummockMetaClient for MockHummockMetaClient { task_id, TaskStatus::try_from(task_status).unwrap(), sorted_output_ssts - .iter() - .map(SstableInfo::from_protobuf) + .into_iter() + .map(SstableInfo::from_protobuf_own) .collect_vec(), Some(table_stats_change), ) diff --git a/src/meta/src/hummock/model/ext/hummock.rs b/src/meta/src/hummock/model/ext/hummock.rs index b7d136776a95..170c1e9f733e 100644 --- a/src/meta/src/hummock/model/ext/hummock.rs +++ b/src/meta/src/hummock/model/ext/hummock.rs @@ -14,6 +14,7 @@ use itertools::Itertools; use risingwave_hummock_sdk::version::HummockVersionDelta; +use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta_model_v2::compaction_config::CompactionConfig; use risingwave_meta_model_v2::compaction_status::LevelHandlers; use risingwave_meta_model_v2::compaction_task::CompactionTask; diff --git a/src/meta/src/hummock/model/version_delta.rs b/src/meta/src/hummock/model/version_delta.rs index 34cadc675da7..c4a36d0ffdf5 100644 --- a/src/meta/src/hummock/model/version_delta.rs +++ b/src/meta/src/hummock/model/version_delta.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use prost::Message; use risingwave_hummock_sdk::version::HummockVersionDelta; use risingwave_hummock_sdk::HummockVersionId; use risingwave_pb::hummock::PbHummockVersionDelta; @@ -30,11 +29,19 @@ impl MetadataModel for HummockVersionDelta { } fn to_protobuf(&self) -> Self::PbType { - self.to_protobuf() + use risingwave_hummock_sdk::ProtoSerializeExt; + ::to_protobuf( + self, + ) } fn to_protobuf_encoded_vec(&self) -> Vec { - self.to_protobuf().encode_to_vec() + use prost::Message; + use risingwave_hummock_sdk::ProtoSerializeExt; + ::to_protobuf( + self, + ) + .encode_to_vec() } fn from_protobuf(prost: Self::PbType) -> Self { diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs index 4d0b3b6b7a67..45db1a326e12 100644 --- a/src/rpc_client/src/meta_client.rs +++ b/src/rpc_client/src/meta_client.rs @@ -39,7 +39,7 @@ use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, - SstObjectIdRange, + ProtoSerializeExt, SstObjectIdRange, }; use risingwave_pb::backup_service::backup_service_client::BackupServiceClient; use risingwave_pb::backup_service::*; diff --git a/src/storage/backup/src/meta_snapshot_v1.rs b/src/storage/backup/src/meta_snapshot_v1.rs index 4cc67d87f656..8d9dbf153e9e 100644 --- a/src/storage/backup/src/meta_snapshot_v1.rs +++ b/src/storage/backup/src/meta_snapshot_v1.rs @@ -19,6 +19,7 @@ use bytes::{Buf, BufMut}; use itertools::Itertools; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_pb::catalog::{ Connection, Database, Function, Index, Schema, Sink, Source, Table, View, }; diff --git a/src/storage/backup/src/meta_snapshot_v2.rs b/src/storage/backup/src/meta_snapshot_v2.rs index 20f620f8b654..9f10ef300de0 100644 --- a/src/storage/backup/src/meta_snapshot_v2.rs +++ b/src/storage/backup/src/meta_snapshot_v2.rs @@ -16,6 +16,7 @@ use std::fmt::{Display, Formatter}; use bytes::{Buf, BufMut}; use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta_model_v2 as model_v2; use serde::{Deserialize, Serialize}; diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index a93241c285b7..16da62eaa58d 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -33,7 +33,9 @@ use crate::table_watermark::{TableWatermarks, TableWatermarksIndex}; use crate::version::{ HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, }; -use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId}; +use crate::{ + can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId, ProtoSerializeExt, +}; pub struct GroupDeltasSummary { pub delete_sst_levels: Vec, @@ -1298,6 +1300,7 @@ mod tests { use crate::version::{ HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, }; + use crate::ProtoSerializeExt; #[test] fn test_get_sst_object_ids() { diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index 77dce3983c26..dff8f37d10e2 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -355,3 +355,25 @@ impl EpochWithGap { self.0 & EPOCH_SPILL_TIME_MASK } } + +pub trait ProtoSerializeSizeEstimatedExt { + fn estimated_encode_len(&self) -> usize; +} + +pub trait ProtoSerializeExt { + type PB: Clone + PartialEq + ::prost::Message; + type T: Clone + PartialEq; + + fn from_protobuf(pb: &Self::PB) -> Self::T; + + fn to_protobuf(&self) -> Self::PB; +} + +pub trait ProtoSerializeOwnExt { + type PB: Clone + PartialEq + ::prost::Message; + type T: Clone + PartialEq; + + fn from_protobuf_own(pb: Self::PB) -> Self::T; + + fn to_protobuf_own(self) -> Self::PB; +} diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index 50494893230b..72d05a1beb54 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -29,7 +29,9 @@ use risingwave_pb::hummock::{PbTableWatermarks, PbVnodeWatermark}; use tracing::{debug, warn}; use crate::key::{prefix_slice_with_vnode, vnode_range, TableKey, TableKeyRange}; -use crate::HummockEpoch; +use crate::{ + HummockEpoch, ProtoSerializeExt, ProtoSerializeOwnExt, ProtoSerializeSizeEstimatedExt, +}; #[derive(Clone)] pub struct ReadTableWatermark { @@ -323,26 +325,50 @@ impl VnodeWatermark { } } - pub fn to_protobuf(&self) -> PbVnodeWatermark { - PbVnodeWatermark { + pub fn vnode_bitmap(&self) -> &Bitmap { + &self.vnode_bitmap + } + + pub fn watermark(&self) -> &Bytes { + &self.watermark + } +} + +impl ProtoSerializeExt for VnodeWatermark { + type PB = PbVnodeWatermark; + type T = VnodeWatermark; + + fn to_protobuf(&self) -> Self::PB { + Self::PB { watermark: self.watermark.to_vec(), vnode_bitmap: Some(self.vnode_bitmap.to_protobuf()), } } - pub fn from_protobuf(pb: &PbVnodeWatermark) -> Self { - Self { + fn from_protobuf(pb: &Self::PB) -> Self::T { + Self::T { vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), watermark: Bytes::from(pb.watermark.clone()), } } +} - pub fn vnode_bitmap(&self) -> &Bitmap { - &self.vnode_bitmap +impl ProtoSerializeOwnExt for VnodeWatermark { + type PB = PbVnodeWatermark; + type T = VnodeWatermark; + + fn to_protobuf_own(self) -> Self::PB { + Self::PB { + watermark: self.watermark.into(), + vnode_bitmap: Some(self.vnode_bitmap.to_protobuf()), // TODO: make Bitmao without copy + } } - pub fn watermark(&self) -> &Bytes { - &self.watermark + fn from_protobuf_own(pb: Self::PB) -> Self::T { + Self::T { + vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), + watermark: Bytes::from(pb.watermark), + } } } @@ -365,38 +391,6 @@ impl TableWatermarks { } } - pub fn to_protobuf(&self) -> PbTableWatermarks { - PbTableWatermarks { - epoch_watermarks: self - .watermarks - .iter() - .map(|(epoch, watermarks)| PbEpochNewWatermarks { - watermarks: watermarks.iter().map(VnodeWatermark::to_protobuf).collect(), - epoch: *epoch, - }) - .collect(), - is_ascending: match self.direction { - WatermarkDirection::Ascending => true, - WatermarkDirection::Descending => false, - }, - } - } - - pub fn estimated_encode_len(&self) -> usize { - self.watermarks.len() * size_of::() - + self - .watermarks - .iter() - .map(|(_, watermarks)| { - watermarks - .iter() - .map(|watermark| watermark.estimated_size()) - .sum::() - }) - .sum::() - + size_of::() // for direction - } - pub fn add_new_epoch_watermarks( &mut self, epoch: HummockEpoch, @@ -410,29 +404,6 @@ impl TableWatermarks { self.watermarks.push((epoch, watermarks)); } - pub fn from_protobuf(pb: &PbTableWatermarks) -> Self { - Self { - watermarks: pb - .epoch_watermarks - .iter() - .map(|epoch_watermark| { - let epoch = epoch_watermark.epoch; - let watermarks = epoch_watermark - .watermarks - .iter() - .map(VnodeWatermark::from_protobuf) - .collect(); - (epoch, watermarks) - }) - .collect(), - direction: if pb.is_ascending { - WatermarkDirection::Ascending - } else { - WatermarkDirection::Descending - }, - } - } - pub fn build_index(&self, committed_epoch: HummockEpoch) -> TableWatermarksIndex { let mut ret = TableWatermarksIndex { index: HashMap::new(), @@ -593,6 +564,116 @@ impl TableWatermarks { } } +impl ProtoSerializeSizeEstimatedExt for TableWatermarks { + fn estimated_encode_len(&self) -> usize { + self.watermarks.len() * size_of::() + + self + .watermarks + .iter() + .map(|(_, watermarks)| { + watermarks + .iter() + .map(|watermark| watermark.estimated_size()) + .sum::() + }) + .sum::() + + size_of::() // for direction + } +} + +impl ProtoSerializeExt for TableWatermarks { + type PB = PbTableWatermarks; + type T = TableWatermarks; + + fn to_protobuf(&self) -> Self::PB { + Self::PB { + epoch_watermarks: self + .watermarks + .iter() + .map(|(epoch, watermarks)| PbEpochNewWatermarks { + watermarks: watermarks.iter().map(VnodeWatermark::to_protobuf).collect(), + epoch: *epoch, + }) + .collect(), + is_ascending: match self.direction { + WatermarkDirection::Ascending => true, + WatermarkDirection::Descending => false, + }, + } + } + + fn from_protobuf(pb: &Self::PB) -> Self::T { + Self::T { + watermarks: pb + .epoch_watermarks + .iter() + .map(|epoch_watermark| { + let epoch = epoch_watermark.epoch; + let watermarks = epoch_watermark + .watermarks + .iter() + .map(VnodeWatermark::from_protobuf) + .collect(); + (epoch, watermarks) + }) + .collect(), + direction: if pb.is_ascending { + WatermarkDirection::Ascending + } else { + WatermarkDirection::Descending + }, + } + } +} + +impl ProtoSerializeOwnExt for TableWatermarks { + type PB = PbTableWatermarks; + type T = TableWatermarks; + + fn to_protobuf_own(self) -> Self::PB { + Self::PB { + epoch_watermarks: self + .watermarks + .into_iter() + .map(|(epoch, watermarks)| PbEpochNewWatermarks { + watermarks: watermarks + .into_iter() + .map(VnodeWatermark::to_protobuf_own) + .collect(), + epoch, + }) + .collect(), + is_ascending: match self.direction { + WatermarkDirection::Ascending => true, + WatermarkDirection::Descending => false, + }, + } + } + + fn from_protobuf_own(pb: Self::PB) -> Self::T { + Self::T { + watermarks: pb + .epoch_watermarks + .into_iter() + .map(|epoch_watermark| { + let epoch = epoch_watermark.epoch; + let watermarks = epoch_watermark + .watermarks + .into_iter() + .map(VnodeWatermark::from_protobuf_own) + .collect(); + (epoch, watermarks) + }) + .collect(), + direction: if pb.is_ascending { + WatermarkDirection::Ascending + } else { + WatermarkDirection::Descending + }, + } + } +} + #[cfg(test)] mod tests { use std::collections::Bound::Included; diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 9e4d00fa4cc6..6ab3a6fedbe7 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -29,22 +29,10 @@ use serde::Serialize; use crate::key_range::KeyRange; use crate::table_watermark::TableWatermarks; -use crate::{CompactionGroupId, HummockSstableObjectId}; - -#[derive(Debug, Clone, PartialEq)] -pub struct HummockVersion { - pub id: u64, - pub levels: HashMap, - pub max_committed_epoch: u64, - pub safe_epoch: u64, - pub table_watermarks: HashMap, -} - -impl Default for HummockVersion { - fn default() -> Self { - HummockVersion::from_protobuf_inner(&PbHummockVersion::default()) - } -} +use crate::{ + CompactionGroupId, HummockSstableObjectId, ProtoSerializeExt, ProtoSerializeOwnExt, + ProtoSerializeSizeEstimatedExt, +}; #[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct OverlappingLevel { @@ -53,17 +41,12 @@ pub struct OverlappingLevel { pub uncompressed_file_size: u64, } -impl OverlappingLevel { - pub fn from_rpc_protobuf(pb_overlapping_level: &PbOverlappingLevel) -> Self { - Self::from_protobuf_inner(pb_overlapping_level) - } - - pub fn from_persisted_protobuf(pb_overlapping_level: &PbOverlappingLevel) -> Self { - Self::from_protobuf_inner(pb_overlapping_level) - } +impl ProtoSerializeExt for OverlappingLevel { + type PB = PbOverlappingLevel; + type T = OverlappingLevel; - fn from_protobuf_inner(pb_overlapping_level: &PbOverlappingLevel) -> Self { - Self { + fn from_protobuf(pb_overlapping_level: &Self::PB) -> Self::T { + Self::T { sub_levels: pb_overlapping_level .sub_levels .iter() @@ -74,8 +57,8 @@ impl OverlappingLevel { } } - pub fn to_protobuf(&self) -> PbOverlappingLevel { - PbOverlappingLevel { + fn to_protobuf(&self) -> Self::PB { + Self::PB { sub_levels: self .sub_levels .iter() @@ -85,8 +68,10 @@ impl OverlappingLevel { uncompressed_file_size: self.uncompressed_file_size, } } +} - pub fn estimated_encode_len(&self) -> usize { +impl ProtoSerializeSizeEstimatedExt for OverlappingLevel { + fn estimated_encode_len(&self) -> usize { self.sub_levels .iter() .map(|level| level.estimated_encode_len()) @@ -113,8 +98,11 @@ pub struct Level { pub vnode_partition_count: u32, } -impl Level { - fn from_protobuf(pb_level: &PbLevel) -> Self { +impl ProtoSerializeExt for Level { + type PB = PbLevel; + type T = Level; + + fn from_protobuf(pb_level: &Self::PB) -> Self::T { Self { level_idx: pb_level.level_idx, level_type: pb_level.level_type, @@ -130,8 +118,8 @@ impl Level { } } - pub fn to_protobuf(&self) -> PbLevel { - PbLevel { + fn to_protobuf(&self) -> Self::PB { + Self::PB { level_idx: self.level_idx, level_type: self.level_type, table_infos: self @@ -145,8 +133,10 @@ impl Level { vnode_partition_count: self.vnode_partition_count, } } +} - pub fn estimated_encode_len(&self) -> usize { +impl ProtoSerializeSizeEstimatedExt for Level { + fn estimated_encode_len(&self) -> usize { size_of::() + size_of::() + self @@ -197,10 +187,37 @@ pub struct Levels { } impl Levels { - fn from_protobuf(pb_levels: &PbLevels) -> Self { + pub fn get_levels(&self) -> &Vec { + &self.levels + } +} + +impl ProtoSerializeSizeEstimatedExt for Levels { + fn estimated_encode_len(&self) -> usize { + let mut basic = self + .levels + .iter() + .map(|level| level.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::(); + if let Some(l0) = self.l0.as_ref() { + basic += l0.estimated_encode_len(); + } + + basic + } +} + +impl ProtoSerializeExt for Levels { + type PB = PbLevels; + type T = Levels; + + fn from_protobuf(pb_levels: &Self::PB) -> Self::T { Self { l0: if pb_levels.l0.is_some() { - Some(OverlappingLevel::from_rpc_protobuf( + Some(OverlappingLevel::from_protobuf( pb_levels.l0.as_ref().unwrap(), )) } else { @@ -217,8 +234,8 @@ impl Levels { } } - pub fn to_protobuf(&self) -> PbLevels { - PbLevels { + fn to_protobuf(&self) -> Self::PB { + Self::PB { l0: if self.l0.is_some() { Some(self.l0.as_ref().unwrap().to_protobuf()) } else { @@ -234,27 +251,20 @@ impl Levels { member_table_ids: self.member_table_ids.clone(), } } +} - pub fn estimated_encode_len(&self) -> usize { - let mut basic = self - .levels - .iter() - .map(|level| level.estimated_encode_len()) - .sum::() - + size_of::() - + size_of::() - + size_of::(); - if let Some(l0) = self.l0.as_ref() { - basic += l0.estimated_encode_len(); - } - - basic - } +#[derive(Debug, Clone, PartialEq)] +pub struct HummockVersion { + pub id: u64, + pub levels: HashMap, + pub max_committed_epoch: u64, + pub safe_epoch: u64, + pub table_watermarks: HashMap, } -impl Levels { - pub fn get_levels(&self) -> &Vec { - &self.levels +impl Default for HummockVersion { + fn default() -> Self { + HummockVersion::from_protobuf(&PbHummockVersion::default()) } } @@ -262,17 +272,39 @@ impl HummockVersion { /// Convert the `PbHummockVersion` received from rpc to `HummockVersion`. No need to /// maintain backward compatibility. pub fn from_rpc_protobuf(pb_version: &PbHummockVersion) -> Self { - Self::from_protobuf_inner(pb_version) + Self::from_protobuf(pb_version) } /// Convert the `PbHummockVersion` deserialized from persisted state to `HummockVersion`. /// We should maintain backward compatibility. pub fn from_persisted_protobuf(pb_version: &PbHummockVersion) -> Self { - Self::from_protobuf_inner(pb_version) + Self::from_protobuf(pb_version) } +} - fn from_protobuf_inner(pb_version: &PbHummockVersion) -> Self { - Self { +impl ProtoSerializeSizeEstimatedExt for HummockVersion { + fn estimated_encode_len(&self) -> usize { + self.levels.len() * size_of::() + + self + .levels + .values() + .map(|level| level.estimated_encode_len()) + .sum::() + + self.table_watermarks.len() * size_of::() + + self + .table_watermarks + .values() + .map(|table_watermark| table_watermark.estimated_encode_len()) + .sum::() + } +} + +impl ProtoSerializeExt for HummockVersion { + type PB = PbHummockVersion; + type T = HummockVersion; + + fn from_protobuf(pb_version: &Self::PB) -> Self::T { + Self::T { id: pb_version.id, levels: pb_version .levels @@ -299,8 +331,8 @@ impl HummockVersion { } } - pub fn to_protobuf(&self) -> PbHummockVersion { - PbHummockVersion { + fn to_protobuf(&self) -> Self::PB { + Self::PB { id: self.id, levels: self .levels @@ -316,21 +348,6 @@ impl HummockVersion { .collect(), } } - - pub fn estimated_encode_len(&self) -> usize { - self.levels.len() * size_of::() - + self - .levels - .values() - .map(|level| level.estimated_encode_len()) - .sum::() - + self.table_watermarks.len() * size_of::() - + self - .table_watermarks - .values() - .map(|table_watermark| table_watermark.estimated_encode_len()) - .sum::() - } } #[derive(Debug, PartialEq, Clone)] @@ -348,7 +365,7 @@ pub struct HummockVersionDelta { impl Default for HummockVersionDelta { fn default() -> Self { - HummockVersionDelta::from_protobuf_inner(&PbHummockVersionDelta::default()) + HummockVersionDelta::from_protobuf(&PbHummockVersionDelta::default()) } } @@ -356,17 +373,22 @@ impl HummockVersionDelta { /// Convert the `PbHummockVersionDelta` deserialized from persisted state to `HummockVersionDelta`. /// We should maintain backward compatibility. pub fn from_persisted_protobuf(delta: &PbHummockVersionDelta) -> Self { - Self::from_protobuf_inner(delta) + Self::from_protobuf(delta) } /// Convert the `PbHummockVersionDelta` received from rpc to `HummockVersionDelta`. No need to /// maintain backward compatibility. pub fn from_rpc_protobuf(delta: &PbHummockVersionDelta) -> Self { - Self::from_protobuf_inner(delta) + Self::from_protobuf(delta) } +} - fn from_protobuf_inner(delta: &PbHummockVersionDelta) -> Self { - Self { +impl ProtoSerializeExt for HummockVersionDelta { + type PB = PbHummockVersionDelta; + type T = HummockVersionDelta; + + fn from_protobuf(delta: &Self::PB) -> Self::T { + Self::T { id: delta.id, prev_id: delta.prev_id, group_deltas: delta.group_deltas.clone(), @@ -392,8 +414,8 @@ impl HummockVersionDelta { } } - pub fn to_protobuf(&self) -> PbHummockVersionDelta { - PbHummockVersionDelta { + fn to_protobuf(&self) -> Self::PB { + Self::PB { id: self.id, prev_id: self.prev_id, group_deltas: self.group_deltas.clone(), @@ -432,8 +454,34 @@ pub struct SstableInfo { pub bloom_filter_kind: i32, } -impl SstableInfo { - pub fn from_protobuf(pb_sstable_info: &PbSstableInfo) -> Self { +impl ProtoSerializeSizeEstimatedExt for SstableInfo { + fn estimated_encode_len(&self) -> usize { + let mut basic = size_of::() // object_id + + size_of::() // sstable_id + + size_of::() // file_size + + self.table_ids.len() * size_of::() // table_ids + + size_of::() // meta_offset + + size_of::() // stale_key_count + + size_of::() // total_key_count + + size_of::() // min_epoch + + size_of::() // max_epoch + + size_of::() // uncompressed_file_size + + size_of::() // range_tombstone_count + + size_of::(); // bloom_filter_kind + + if let Some(key_range) = &self.key_range { + basic += key_range.left.len() + key_range.right.len() + size_of::(); + } + + basic + } +} + +impl ProtoSerializeExt for SstableInfo { + type PB = PbSstableInfo; + type T = SstableInfo; + + fn from_protobuf(pb_sstable_info: &Self::PB) -> Self::T { Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, @@ -462,7 +510,7 @@ impl SstableInfo { } } - pub fn to_protobuf(&self) -> PbSstableInfo { + fn to_protobuf(&self) -> Self::PB { PbSstableInfo { object_id: self.object_id, sst_id: self.sst_id, @@ -490,26 +538,68 @@ impl SstableInfo { bloom_filter_kind: self.bloom_filter_kind, } } +} - pub fn estimated_encode_len(&self) -> usize { - let mut basic = size_of::() // object_id - + size_of::() // sstable_id - + size_of::() // file_size - + self.table_ids.len() * size_of::() // table_ids - + size_of::() // meta_offset - + size_of::() // stale_key_count - + size_of::() // total_key_count - + size_of::() // min_epoch - + size_of::() // max_epoch - + size_of::() // uncompressed_file_size - + size_of::() // range_tombstone_count - + size_of::(); // bloom_filter_kind +impl ProtoSerializeOwnExt for SstableInfo { + type PB = PbSstableInfo; + type T = SstableInfo; - if let Some(key_range) = &self.key_range { - basic += key_range.left.len() + key_range.right.len() + size_of::(); + fn from_protobuf_own(pb_sstable_info: Self::PB) -> Self::T { + Self::T { + object_id: pb_sstable_info.object_id, + sst_id: pb_sstable_info.sst_id, + key_range: if pb_sstable_info.key_range.is_some() { + let pb_keyrange = pb_sstable_info.key_range.unwrap(); + let key_range = KeyRange { + left: Bytes::from(pb_keyrange.left), + right: Bytes::from(pb_keyrange.right), + right_exclusive: pb_keyrange.right_exclusive, + }; + Some(key_range) + } else { + None + }, + + file_size: pb_sstable_info.file_size, + table_ids: pb_sstable_info.table_ids.clone(), + meta_offset: pb_sstable_info.meta_offset, + stale_key_count: pb_sstable_info.stale_key_count, + total_key_count: pb_sstable_info.total_key_count, + min_epoch: pb_sstable_info.min_epoch, + max_epoch: pb_sstable_info.max_epoch, + uncompressed_file_size: pb_sstable_info.uncompressed_file_size, + range_tombstone_count: pb_sstable_info.range_tombstone_count, + bloom_filter_kind: pb_sstable_info.bloom_filter_kind, } + } - basic + fn to_protobuf_own(self) -> Self::PB { + Self::PB { + object_id: self.object_id, + sst_id: self.sst_id, + key_range: if self.key_range.is_some() { + let keyrange = self.key_range.unwrap(); + let pb_key_range = PbKeyRange { + left: keyrange.left.into(), + right: keyrange.right.into(), + right_exclusive: keyrange.right_exclusive, + }; + Some(pb_key_range) + } else { + None + }, + + file_size: self.file_size, + table_ids: self.table_ids.clone(), + meta_offset: self.meta_offset, + stale_key_count: self.stale_key_count, + total_key_count: self.total_key_count, + min_epoch: self.min_epoch, + max_epoch: self.max_epoch, + uncompressed_file_size: self.uncompressed_file_size, + range_tombstone_count: self.range_tombstone_count, + bloom_filter_kind: self.bloom_filter_kind, + } } } @@ -527,15 +617,30 @@ impl SstableInfo { } } -#[derive(Clone, PartialEq, Default, Debug)] +#[derive(Clone, PartialEq, Default, Debug, Serialize)] pub struct InputLevel { pub level_idx: u32, pub level_type: i32, pub table_infos: Vec, } -impl InputLevel { - pub fn from_protobuf(pb_input_level: &PbInputLevel) -> Self { +impl ProtoSerializeSizeEstimatedExt for InputLevel { + fn estimated_encode_len(&self) -> usize { + size_of::() + + size_of::() + + self + .table_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + } +} + +impl ProtoSerializeExt for InputLevel { + type PB = PbInputLevel; + type T = InputLevel; + + fn from_protobuf(pb_input_level: &Self::PB) -> Self::T { Self { level_idx: pb_input_level.level_idx, level_type: pb_input_level.level_type, @@ -547,8 +652,8 @@ impl InputLevel { } } - pub fn to_protobuf(&self) -> PbInputLevel { - PbInputLevel { + fn to_protobuf(&self) -> Self::PB { + Self::PB { level_idx: self.level_idx, level_type: self.level_type, table_infos: self @@ -558,15 +663,34 @@ impl InputLevel { .collect_vec(), } } +} - pub fn estimated_encode_len(&self) -> usize { - size_of::() - + size_of::() - + self +impl ProtoSerializeOwnExt for InputLevel { + type PB = PbInputLevel; + type T = InputLevel; + + fn from_protobuf_own(pb_input_level: Self::PB) -> Self::T { + Self { + level_idx: pb_input_level.level_idx, + level_type: pb_input_level.level_type, + table_infos: pb_input_level .table_infos - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() + .into_iter() + .map(SstableInfo::from_protobuf_own) + .collect_vec(), + } + } + + fn to_protobuf_own(self) -> Self::PB { + Self::PB { + level_idx: self.level_idx, + level_type: self.level_type, + table_infos: self + .table_infos + .into_iter() + .map(|sst| sst.to_protobuf_own()) + .collect_vec(), + } } } @@ -626,9 +750,54 @@ pub struct CompactTask { pub table_watermarks: BTreeMap, } -impl CompactTask { - pub fn from_protobuf(pb_compact_task: &PbCompactTask) -> Self { - Self { +impl ProtoSerializeSizeEstimatedExt for CompactTask { + fn estimated_encode_len(&self) -> usize { + self.input_ssts + .iter() + .map(|input_level| input_level.estimated_encode_len()) + .sum::() + + self + .splits + .iter() + .map(|split| split.left.len() + split.right.len() + size_of::()) + .sum::() + + size_of::() + + self + .sorted_output_ssts + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.existing_table_ids.len() * size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.table_options.len() * size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.table_vnode_partition.len() * size_of::() + + self + .table_watermarks + .values() + .map(|table_watermark| size_of::() + table_watermark.estimated_encode_len()) + .sum::() + } +} + +impl ProtoSerializeExt for CompactTask { + type PB = PbCompactTask; + type T = CompactTask; + + fn from_protobuf(pb_compact_task: &Self::PB) -> Self::T { + Self::T { input_ssts: pb_compact_task .input_ssts .iter() @@ -680,8 +849,8 @@ impl CompactTask { } } - pub fn to_protobuf(&self) -> PbCompactTask { - PbCompactTask { + fn to_protobuf(&self) -> Self::PB { + Self::PB { input_ssts: self .input_ssts .iter() @@ -727,45 +896,111 @@ impl CompactTask { ..Default::default() } } +} - pub fn estimated_encode_len(&self) -> usize { - self.input_ssts - .iter() - .map(|input_level| input_level.estimated_encode_len()) - .sum::() - + self +impl ProtoSerializeOwnExt for CompactTask { + type PB = PbCompactTask; + type T = CompactTask; + + fn from_protobuf_own(pb_compact_task: Self::PB) -> Self::T { + Self::T { + input_ssts: pb_compact_task + .input_ssts + .into_iter() + .map(InputLevel::from_protobuf_own) + .collect_vec(), + splits: pb_compact_task .splits - .iter() - .map(|split| split.left.len() + split.right.len() + size_of::()) - .sum::() - + size_of::() - + self + .into_iter() + .map(|pb_keyrange| KeyRange { + left: Bytes::from(pb_keyrange.left), + right: Bytes::from(pb_keyrange.right), + right_exclusive: pb_keyrange.right_exclusive, + }) + .collect_vec(), + watermark: pb_compact_task.watermark, + sorted_output_ssts: pb_compact_task .sorted_output_ssts - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + self.existing_table_ids.len() * size_of::() - + size_of::() - + size_of::() - + size_of::() - + self.table_options.len() * size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + self.table_vnode_partition.len() * size_of::() - + self + .into_iter() + .map(SstableInfo::from_protobuf_own) + .collect_vec(), + task_id: pb_compact_task.task_id, + target_level: pb_compact_task.target_level, + gc_delete_keys: pb_compact_task.gc_delete_keys, + base_level: pb_compact_task.base_level, + task_status: pb_compact_task.task_status, + compaction_group_id: pb_compact_task.compaction_group_id, + existing_table_ids: pb_compact_task.existing_table_ids.clone(), + compression_algorithm: pb_compact_task.compression_algorithm, + target_file_size: pb_compact_task.target_file_size, + compaction_filter_mask: pb_compact_task.compaction_filter_mask, + table_options: pb_compact_task.table_options.clone(), + current_epoch_time: pb_compact_task.current_epoch_time, + target_sub_level_id: pb_compact_task.target_sub_level_id, + task_type: pb_compact_task.task_type, + #[allow(deprecated)] + split_by_state_table: pb_compact_task.split_by_state_table, + split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, + table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), + table_watermarks: pb_compact_task .table_watermarks - .values() - .map(|table_watermark| size_of::() + table_watermark.estimated_encode_len()) - .sum::() + .into_iter() + .map(|(table_id, pb_table_watermark)| { + ( + table_id, + TableWatermarks::from_protobuf_own(pb_table_watermark), + ) + }) + .collect(), + } + } + + fn to_protobuf_own(self) -> Self::PB { + Self::PB { + input_ssts: self + .input_ssts + .into_iter() + .map(|input_level| input_level.to_protobuf_own()) + .collect_vec(), + splits: self + .splits + .into_iter() + .map(|keyrange| PbKeyRange { + left: keyrange.left.into(), + right: keyrange.right.into(), + right_exclusive: keyrange.right_exclusive, + }) + .collect_vec(), + watermark: self.watermark, + sorted_output_ssts: self + .sorted_output_ssts + .into_iter() + .map(|sst| sst.to_protobuf_own()) + .collect_vec(), + task_id: self.task_id, + target_level: self.target_level, + gc_delete_keys: self.gc_delete_keys, + base_level: self.base_level, + task_status: self.task_status, + compaction_group_id: self.compaction_group_id, + existing_table_ids: self.existing_table_ids.clone(), + compression_algorithm: self.compression_algorithm, + target_file_size: self.target_file_size, + compaction_filter_mask: self.compaction_filter_mask, + table_options: self.table_options.clone(), + current_epoch_time: self.current_epoch_time, + target_sub_level_id: self.target_sub_level_id, + task_type: self.task_type, + //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, + split_weight_by_vnode: self.split_weight_by_vnode, + table_vnode_partition: self.table_vnode_partition.clone(), + table_watermarks: self + .table_watermarks + .into_iter() + .map(|(table_id, table_watermark)| (table_id, table_watermark.to_protobuf_own())) + .collect(), + ..Default::default() + } } } @@ -791,44 +1026,49 @@ impl CompactTask { } } -#[derive(Clone, PartialEq, Default)] +#[derive(Clone, PartialEq, Default, Serialize)] pub struct ValidationTask { pub sst_infos: Vec, pub sst_id_to_worker_id: HashMap, pub epoch: u64, } -impl ValidationTask { - pub fn from_protobuf(pb_validation_task: &PbValidationTask) -> Self { +impl ProtoSerializeSizeEstimatedExt for ValidationTask { + fn estimated_encode_len(&self) -> usize { + self.sst_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) + + size_of::() + } +} + +impl ProtoSerializeOwnExt for ValidationTask { + type PB = PbValidationTask; + type T = ValidationTask; + + fn from_protobuf_own(pb_validation_task: Self::PB) -> Self::T { Self { sst_infos: pb_validation_task .sst_infos - .iter() - .map(SstableInfo::from_protobuf) + .into_iter() + .map(SstableInfo::from_protobuf_own) .collect_vec(), sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(), epoch: pb_validation_task.epoch, } } - pub fn to_protobuf(&self) -> PbValidationTask { - PbValidationTask { + fn to_protobuf_own(self) -> Self::PB { + Self::PB { sst_infos: self .sst_infos - .iter() - .map(|sst| sst.to_protobuf()) + .into_iter() + .map(|sst| sst.to_protobuf_own()) .collect_vec(), sst_id_to_worker_id: self.sst_id_to_worker_id.clone(), epoch: self.epoch, } } - - pub fn estimated_encode_len(&self) -> usize { - self.sst_infos - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) - + size_of::() - } } diff --git a/src/storage/hummock_test/src/mock_notification_client.rs b/src/storage/hummock_test/src/mock_notification_client.rs index 68c75d57937a..e9e7095dde56 100644 --- a/src/storage/hummock_test/src/mock_notification_client.rs +++ b/src/storage/hummock_test/src/mock_notification_client.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use risingwave_common::util::addr::HostAddr; use risingwave_common_service::observer_manager::{Channel, NotificationClient, ObserverError}; +use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta::hummock::{HummockManager, HummockManagerRef}; use risingwave_meta::manager::{MessageStatus, MetaSrvEnv, NotificationManagerRef, WorkerKey}; use risingwave_pb::backup_service::MetaBackupManifestId; diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index 42bce4aaeeef..9c5857dd5925 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -52,7 +52,10 @@ use futures::{pin_mut, StreamExt}; pub use iterator::{ConcatSstableIterator, SstableStreamIterator}; use more_asserts::assert_ge; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; -use risingwave_hummock_sdk::{compact_task_to_string, HummockCompactionTaskId, LocalSstableInfo}; +use risingwave_hummock_sdk::{ + compact_task_to_string, HummockCompactionTaskId, LocalSstableInfo, ProtoSerializeExt, + ProtoSerializeOwnExt, +}; use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_pb::hummock::subscribe_compaction_event_request::{ Event as RequestEvent, HeartBeat, PullTask, ReportTask, @@ -463,7 +466,7 @@ pub fn start_compactor( executor.spawn(async move { match event { ResponseEvent::CompactTask(compact_task) => { - let compact_task = CompactTask::from_protobuf(&compact_task); + let compact_task = CompactTask::from_protobuf_own(compact_task); let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); @@ -543,7 +546,7 @@ pub fn start_compactor( } } ResponseEvent::ValidationTask(validation_task) => { - let validation_task = ValidationTask::from_protobuf(&validation_task); + let validation_task = ValidationTask::from_protobuf_own(validation_task); validate_ssts( validation_task, context.sstable_store.clone(), @@ -759,7 +762,7 @@ pub fn start_shared_compactor( } } dispatch_compaction_task_request::Task::ValidationTask(validation_task) => { - let validation_task = ValidationTask::from_protobuf(&validation_task); + let validation_task = ValidationTask::from_protobuf_own(validation_task); validate_ssts(validation_task, context.sstable_store.clone()).await; } dispatch_compaction_task_request::Task::CancelCompactTask(cancel_compact_task) => { diff --git a/src/storage/src/store.rs b/src/storage/src/store.rs index e5b096de4719..23e992284bb7 100644 --- a/src/storage/src/store.rs +++ b/src/storage/src/store.rs @@ -30,7 +30,7 @@ use risingwave_hummock_sdk::key::{FullKey, TableKey, TableKeyRange}; use risingwave_hummock_sdk::table_watermark::{ TableWatermarks, VnodeWatermark, WatermarkDirection, }; -use risingwave_hummock_sdk::{HummockReadEpoch, LocalSstableInfo}; +use risingwave_hummock_sdk::{HummockReadEpoch, LocalSstableInfo, ProtoSerializeOwnExt}; use risingwave_hummock_trace::{ TracedBitmap, TracedInitOptions, TracedNewLocalOptions, TracedOpConsistencyLevel, TracedPrefetchOptions, TracedReadOptions, TracedSealCurrentEpochOptions, TracedWriteOptions, @@ -581,8 +581,8 @@ impl From for TracedSealCurrentEpochOptions { ( direction == WatermarkDirection::Ascending, watermarks - .iter() - .map(|watermark| Message::encode_to_vec(&watermark.to_protobuf())) + .into_iter() + .map(|watermark| Message::encode_to_vec(&watermark.to_protobuf_own())) .collect(), ) }), @@ -604,10 +604,10 @@ impl From for SealCurrentEpochOptions { WatermarkDirection::Descending }, watermarks - .iter() + .into_iter() .map(|serialized_watermark| { Message::decode(serialized_watermark.as_slice()) - .map(|pb| VnodeWatermark::from_protobuf(&pb)) + .map(VnodeWatermark::from_protobuf_own) .expect("should not failed") }) .collect(), From afd746981ae9e529438e264e5c5aba72ee5d32c7 Mon Sep 17 00:00:00 2001 From: Li0k Date: Fri, 8 Mar 2024 15:37:42 +0800 Subject: [PATCH 04/18] fix(storage): address comments --- src/ctl/src/cmd_impl/hummock/sst_dump.rs | 2 +- src/meta/src/hummock/compaction/mod.rs | 2 +- .../picker/base_level_compaction_picker.rs | 13 +-- .../picker/emergency_compaction_picker.rs | 8 +- .../picker/intra_compaction_picker.rs | 32 +++-- .../picker/manual_compaction_picker.rs | 18 +-- .../picker/min_overlap_compaction_picker.rs | 34 +++--- .../picker/space_reclaim_compaction_picker.rs | 2 +- .../picker/trivial_move_compaction_picker.rs | 4 +- .../picker/ttl_reclaim_compaction_picker.rs | 2 +- .../src/hummock/compaction/selector/mod.rs | 14 +-- src/meta/src/hummock/compactor_manager.rs | 2 +- src/meta/src/hummock/manager/mod.rs | 4 +- src/storage/hummock_sdk/src/compact.rs | 5 +- .../compaction_group/hummock_version_ext.rs | 12 +- src/storage/hummock_sdk/src/lib.rs | 8 +- .../hummock_sdk/src/table_watermark.rs | 20 ++-- src/storage/hummock_sdk/src/version.rs | 109 ++++++++---------- .../hummock_test/src/compactor_tests.rs | 8 +- .../src/hummock/compactor/compaction_utils.rs | 2 +- .../src/hummock/compactor/compactor_runner.rs | 14 +-- .../compactor/fast_compactor_runner.rs | 3 +- src/storage/src/hummock/compactor/mod.rs | 8 +- src/storage/src/hummock/sstable/builder.rs | 2 +- src/storage/src/hummock/store/version.rs | 2 +- 25 files changed, 154 insertions(+), 176 deletions(-) diff --git a/src/ctl/src/cmd_impl/hummock/sst_dump.rs b/src/ctl/src/cmd_impl/hummock/sst_dump.rs index f1ec2fb34e0d..3156c6213ddd 100644 --- a/src/ctl/src/cmd_impl/hummock/sst_dump.rs +++ b/src/ctl/src/cmd_impl/hummock/sst_dump.rs @@ -156,7 +156,7 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result } fn print_level(level: &Level, sst_info: &SstableInfo) { - println!("Level Type: {}", level.level_type); + println!("Level Type: {}", level.level_type.as_str_name()); println!("Level Idx: {}", level.level_idx); if level.level_idx == 0 { println!("L0 Sub-Level Idx: {}", level.sub_level_id); diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs index 3f82e2767c4b..cf0d2915f68b 100644 --- a/src/meta/src/hummock/compaction/mod.rs +++ b/src/meta/src/hummock/compaction/mod.rs @@ -69,7 +69,7 @@ pub struct CompactionTask { pub compaction_task_type: compact_task::TaskType, } -pub fn create_overlap_strategy(compaction_mode: CompactionMode) -> Arc { +pub fn create_overlap_strategy(compaction_mode: CompactionMode) -> Arc { match compaction_mode { CompactionMode::Range => Arc::new(RangeOverlapStrategy::default()), CompactionMode::Unspecified => unreachable!(), diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index 216d848989d8..ba702ca3ad49 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -24,7 +24,6 @@ use super::{ CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic, ValidationRuleType, }; -use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::compaction::picker::TrivialMovePicker; use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig}; use crate::hummock::level_handler::LevelHandler; @@ -47,7 +46,7 @@ impl CompactionPicker for LevelCompactionPicker { if l0.sub_levels.is_empty() { return None; } - if l0.sub_levels[0].level_type != LevelType::Nonoverlapping as i32 + if l0.sub_levels[0].level_type != LevelType::Nonoverlapping && l0.sub_levels[0].table_infos.len() > 1 { stats.skip_by_overlapping += 1; @@ -226,7 +225,7 @@ impl LevelCompactionPicker { .into_iter() .map(|table_infos| InputLevel { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos, }) .collect_vec(); @@ -394,7 +393,7 @@ pub mod tests { let levels = vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(3, 1, 0, 50, 1), generate_table(4, 1, 150, 200, 1), @@ -458,7 +457,7 @@ pub mod tests { let mut picker = create_compaction_picker_for_test(); let levels = vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![], total_file_size: 0, sub_level_id: 0, @@ -523,7 +522,7 @@ pub mod tests { let mut levels = Levels { levels: vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(1, 1, 100, 399, 2), generate_table(2, 1, 400, 699, 2), @@ -571,7 +570,7 @@ pub mod tests { ]); // We can set level_type only because the input above is valid. for s in &mut l0.sub_levels { - s.level_type = LevelType::Nonoverlapping as i32; + s.level_type = LevelType::Nonoverlapping; } let levels = Levels { l0: Some(l0), diff --git a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs index 50679558d81d..8598ba837109 100644 --- a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs @@ -55,22 +55,20 @@ impl EmergencyCompactionPicker { let overlapping_count = l0 .sub_levels .iter() - .filter(|level| level.level_type == LevelType::Overlapping as i32) + .filter(|level| level.level_type == LevelType::Overlapping) .count(); let no_overlap_count = l0 .sub_levels .iter() .filter(|level| { - level.level_type == LevelType::Nonoverlapping as i32 - && level.vnode_partition_count == 0 + level.level_type == LevelType::Nonoverlapping && level.vnode_partition_count == 0 }) .count(); let partitioned_count = l0 .sub_levels .iter() .filter(|level| { - level.level_type == LevelType::Nonoverlapping as i32 - && level.vnode_partition_count > 0 + level.level_type == LevelType::Nonoverlapping && level.vnode_partition_count > 0 }) .count(); // We trigger `EmergencyCompactionPicker` only when some unexpected condition cause the number of l0 levels increase and the origin strategy diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 7b8a28d8c8f8..007432eb33fb 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -22,7 +22,6 @@ use super::{ CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic, ValidationRuleType, }; -use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::compaction::picker::TrivialMovePicker; use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig}; use crate::hummock::level_handler::LevelHandler; @@ -45,7 +44,7 @@ impl CompactionPicker for IntraCompactionPicker { if l0.sub_levels.is_empty() { return None; } - if l0.sub_levels[0].level_type != LevelType::Nonoverlapping as i32 + if l0.sub_levels[0].level_type != LevelType::Nonoverlapping && l0.sub_levels[0].table_infos.len() > 1 { stats.skip_by_overlapping += 1; @@ -191,7 +190,7 @@ impl IntraCompactionPicker { } select_level_inputs.push(InputLevel { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: level_select_sst, }); @@ -236,11 +235,11 @@ impl IntraCompactionPicker { let overlap_strategy = create_overlap_strategy(self.config.compaction_mode()); for (idx, level) in l0.sub_levels.iter().enumerate() { - if level.level_type == LevelType::Overlapping as i32 || idx + 1 >= l0.sub_levels.len() { + if level.level_type == LevelType::Overlapping || idx + 1 >= l0.sub_levels.len() { continue; } - if l0.sub_levels[idx + 1].level_type == LevelType::Overlapping as i32 { + if l0.sub_levels[idx + 1].level_type == LevelType::Overlapping { continue; } @@ -275,8 +274,7 @@ impl IntraCompactionPicker { .is_empty()); let mut target_level_idx = idx; while target_level_idx > 0 { - if l0.sub_levels[target_level_idx - 1].level_type - != LevelType::Nonoverlapping as i32 + if l0.sub_levels[target_level_idx - 1].level_type != LevelType::Nonoverlapping || !overlap .check_multiple_overlap(&l0.sub_levels[target_level_idx - 1].table_infos) .is_empty() @@ -290,12 +288,12 @@ impl IntraCompactionPicker { let input_levels = vec![ InputLevel { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![select_sst], }, InputLevel { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![], }, ]; @@ -439,7 +437,7 @@ pub mod tests { // compacting_key_range. let levels = vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![], ..Default::default() }]; @@ -487,7 +485,7 @@ pub mod tests { let mut levels = Levels { levels: vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![generate_table(3, 1, 200, 300, 2)], ..Default::default() }], @@ -753,20 +751,20 @@ pub mod tests { .is_none()); // Cannot trivial move because latter sub-level is overlapping - levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32; - levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Overlapping as i32; + levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping; + levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Overlapping; let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats); assert!(ret.is_none()); // Cannot trivial move because former sub-level is overlapping - levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Overlapping as i32; - levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32; + levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Overlapping; + levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping; let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats); assert!(ret.is_none()); // trivial move - levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32; - levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32; + levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping; + levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping; let ret = picker .pick_compaction(&levels, &levels_handler, &mut local_stats) .unwrap(); diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index c5318e4ec396..8a7549654436 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -163,7 +163,7 @@ impl ManualCompactionPicker { input_levels.reverse(); input_levels.push(InputLevel { level_idx: self.target_level as u32, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: target_input_ssts, }); @@ -380,7 +380,7 @@ pub mod tests { let levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(0, 1, 0, 100, 1), generate_table(1, 1, 101, 200, 1), @@ -390,7 +390,7 @@ pub mod tests { }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(4, 1, 0, 100, 1), generate_table(5, 1, 101, 150, 1), @@ -555,7 +555,7 @@ pub mod tests { let mut levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(3, 1, 0, 100, 1), generate_table(4, 2, 2000, 3000, 1), @@ -564,7 +564,7 @@ pub mod tests { }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(1, 1, 0, 100, 1), generate_table(2, 2, 2000, 3000, 1), @@ -604,7 +604,7 @@ pub mod tests { let l0 = generate_l0_overlapping_sublevels(vec![]); let levels = vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(1, 1, 0, 100, 1), generate_table(2, 2, 100, 200, 1), @@ -628,7 +628,7 @@ pub mod tests { let l0 = generate_l0_nonoverlapping_sublevels(vec![]); let levels = vec![Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![], total_file_size: 0, sub_level_id: 0, @@ -1164,7 +1164,7 @@ pub mod tests { generate_level(3, vec![]), Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(2, 1, 0, 100, 1), generate_table(3, 1, 101, 200, 1), @@ -1271,7 +1271,7 @@ pub mod tests { ), Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(2, 1, 0, 100, 1), generate_table(3, 1, 101, 200, 1), diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 9259930974d2..921d743199dd 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -155,12 +155,12 @@ impl CompactionPicker for MinOverlappingPicker { input_levels: vec![ InputLevel { level_idx: self.level as u32, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: select_input_ssts, }, InputLevel { level_idx: self.target_level as u32, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: target_input_ssts, }, ], @@ -491,7 +491,7 @@ pub mod tests { let levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(0, 1, 0, 100, 1), generate_table(1, 1, 101, 200, 1), @@ -501,7 +501,7 @@ pub mod tests { }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(4, 1, 0, 100, 1), generate_table(5, 1, 101, 150, 1), @@ -563,7 +563,7 @@ pub mod tests { let levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(0, 1, 50, 99, 2), generate_table(1, 1, 100, 149, 2), @@ -573,7 +573,7 @@ pub mod tests { }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(4, 1, 50, 199, 1), generate_table(5, 1, 200, 399, 1), @@ -617,7 +617,7 @@ pub mod tests { let levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(0, 1, 50, 99, 2), generate_table(1, 1, 100, 149, 2), @@ -631,7 +631,7 @@ pub mod tests { }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(4, 1, 50, 199, 1), generate_table(5, 1, 200, 249, 1), @@ -644,7 +644,7 @@ pub mod tests { }, Level { level_idx: 3, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(11, 1, 250, 300, 2), generate_table(12, 1, 350, 400, 2), @@ -655,7 +655,7 @@ pub mod tests { }, Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(14, 1, 250, 300, 2), generate_table(15, 1, 350, 400, 2), @@ -720,7 +720,7 @@ pub mod tests { let levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(0, 1, 50, 99, 2), generate_table(1, 1, 100, 149, 2), @@ -734,7 +734,7 @@ pub mod tests { }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(4, 1, 50, 99, 1), generate_table(5, 1, 150, 200, 1), @@ -747,7 +747,7 @@ pub mod tests { }, Level { level_idx: 3, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(11, 1, 250, 300, 2), generate_table(12, 1, 350, 400, 2), @@ -758,7 +758,7 @@ pub mod tests { }, Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(14, 1, 250, 300, 2), generate_table(15, 1, 350, 400, 2), @@ -856,14 +856,14 @@ pub mod tests { let levels = [ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![generate_table(0, 1, 400, 500, 2)], total_file_size: 100, ..Default::default() }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(1, 1, 100, 200, 1), generate_table(2, 1, 600, 700, 1), @@ -873,7 +873,7 @@ pub mod tests { }, Level { level_idx: 3, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(3, 1, 100, 300, 2), generate_table(4, 1, 600, 800, 1), diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index b7886c136a3d..eef293966843 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -210,7 +210,7 @@ mod test { ), Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table_with_ids_and_epochs(2, 1, 0, 100, 1, vec![2], 0, 0), generate_table_with_ids_and_epochs(3, 1, 101, 200, 1, vec![3], 0, 0), diff --git a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs index faf37140b660..8478602ff62d 100644 --- a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs @@ -85,12 +85,12 @@ impl TrivialMovePicker { input_levels: vec![ InputLevel { level_idx: self.level as u32, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![trivial_move_sst], }, InputLevel { level_idx: self.target_level as u32, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![], }, ], diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index ff1c9ceb7782..c90c92ee01b0 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -245,7 +245,7 @@ mod test { ), Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table_with_ids_and_epochs(2, 1, 0, 100, 1, vec![2], expired_epoch, 0), generate_table_with_ids_and_epochs( diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index dcbb6e7636ae..011720660d38 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -127,7 +127,7 @@ pub mod tests { levels.l0.as_mut().unwrap().total_file_size += sst.file_size; levels.l0.as_mut().unwrap().sub_levels.push(Level { level_idx: 0, - level_type: LevelType::Overlapping as i32, + level_type: LevelType::Overlapping, total_file_size: sst.file_size, uncompressed_file_size: sst.uncompressed_file_size, sub_level_id: sst.get_sst_id(), @@ -145,7 +145,7 @@ pub mod tests { .sub_levels .last_mut() .unwrap() - .level_type = LevelType::Nonoverlapping as i32; + .level_type = LevelType::Nonoverlapping; } pub fn push_tables_level0_nonoverlapping(levels: &mut Levels, table_infos: Vec) { @@ -158,7 +158,7 @@ pub mod tests { levels.l0.as_mut().unwrap().total_file_size += total_file_size; levels.l0.as_mut().unwrap().sub_levels.push(Level { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, total_file_size, sub_level_id, table_infos, @@ -244,7 +244,7 @@ pub mod tests { .sum(); Level { level_idx, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos, total_file_size, sub_level_id: 0, @@ -267,7 +267,7 @@ pub mod tests { .enumerate() .map(|(idx, table)| Level { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, total_file_size: table.file_size, uncompressed_file_size: table.uncompressed_file_size, sub_level_id: idx as u64, @@ -289,7 +289,7 @@ pub mod tests { .enumerate() .map(|(idx, table)| Level { level_idx: 0, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, total_file_size: table.iter().map(|table| table.file_size).sum::(), uncompressed_file_size: table .iter() @@ -324,7 +324,7 @@ pub mod tests { .enumerate() .map(|(idx, table)| Level { level_idx: 0, - level_type: LevelType::Overlapping as i32, + level_type: LevelType::Overlapping, total_file_size: table.iter().map(|table| table.file_size).sum::(), sub_level_id: idx as u64, table_infos: table.clone(), diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs index d3698a33907d..91dca871fdc4 100644 --- a/src/meta/src/hummock/compactor_manager.rs +++ b/src/meta/src/hummock/compactor_manager.rs @@ -297,7 +297,7 @@ impl CompactorManagerInner { task.target_level, task.base_level, task.target_sub_level_id, - task.task_type, + task.task_type.as_str_name(), compact_task_statistics ); } diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index 4d95ad1381a3..14bc0d949320 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -984,7 +984,7 @@ impl HummockManager { .get_compaction_group_levels(compaction_group_id) .is_last_level(target_level_id), base_level: compact_task.base_level as u32, - task_status: TaskStatus::Pending as i32, + task_status: TaskStatus::Pending, compaction_group_id: group_config.group_id, existing_table_ids: member_table_ids.clone(), compression_algorithm, @@ -1002,7 +1002,7 @@ impl HummockManager { current_epoch_time: Epoch::now().0, compaction_filter_mask: group_config.compaction_config.compaction_filter_mask, target_sub_level_id: compact_task.input.target_sub_level_id, - task_type: compact_task.compaction_task_type as i32, + task_type: compact_task.compaction_task_type, split_weight_by_vnode: compact_task.input.vnode_partition_count, ..Default::default() }; diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index a5720e6f9499..ee9e8b504a3e 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_pb::hummock::compact_task::TaskType; use risingwave_pb::hummock::LevelType; use crate::version::{CompactTask, SstableInfo}; @@ -26,7 +25,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { "Compaction task id: {:?}, group-id: {:?}, task type: {:?}, target level: {:?}, target sub level: {:?}", compact_task.task_id, compact_task.compaction_group_id, - TaskType::try_from(compact_task.task_type).unwrap(), + compact_task.task_type.as_str_name(), compact_task.target_level, compact_task.target_sub_level_id ) @@ -171,7 +170,7 @@ pub fn estimate_memory_for_compact_task( // input for level in &task.input_ssts { - if level.level_type == LevelType::Nonoverlapping as i32 { + if level.level_type == LevelType::Nonoverlapping { let mut cur_level_max_sst_meta_size = 0; for sst in &level.table_infos { let meta_size = sst.file_size - sst.meta_offset; diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 16da62eaa58d..bd0ccfaea861 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -823,7 +823,7 @@ pub fn build_initial_compaction_group_levels( for l in 0..compaction_config.get_max_level() { levels.push(Level { level_idx: (l + 1) as u32, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![], total_file_size: 0, sub_level_id: 0, @@ -974,7 +974,7 @@ pub fn new_sub_level( .sum(); Level { level_idx: 0, - level_type: level_type as i32, + level_type, table_infos, total_file_size, sub_level_id, @@ -997,7 +997,7 @@ pub fn add_ssts_to_sub_level( l0.sub_levels[sub_level_idx] .table_infos .extend(insert_table_infos); - if l0.sub_levels[sub_level_idx].level_type == LevelType::Nonoverlapping as i32 { + if l0.sub_levels[sub_level_idx].level_type == LevelType::Nonoverlapping { l0.sub_levels[sub_level_idx] .table_infos .sort_by(|sst1, sst2| { @@ -1114,8 +1114,8 @@ fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec) let b = sst2.key_range.as_ref().unwrap(); a.cmp(b) }); - if operand.level_type == LevelType::Overlapping as i32 { - operand.level_type = LevelType::Nonoverlapping as i32; + if operand.level_type == LevelType::Overlapping { + operand.level_type = LevelType::Nonoverlapping; } assert!( can_concat(&operand.table_infos), @@ -1440,7 +1440,7 @@ mod tests { ); cg1.levels[0] = Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![SstableInfo { object_id: 1, sst_id: 1, diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index dff8f37d10e2..f0d110847a48 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -360,20 +360,20 @@ pub trait ProtoSerializeSizeEstimatedExt { fn estimated_encode_len(&self) -> usize; } +// Define methods for converting Proto struct and rust struct to and from each other pub trait ProtoSerializeExt { type PB: Clone + PartialEq + ::prost::Message; - type T: Clone + PartialEq; - fn from_protobuf(pb: &Self::PB) -> Self::T; + fn from_protobuf(pb: &Self::PB) -> Self; fn to_protobuf(&self) -> Self::PB; } +// Similar to `ProtoSerializeExt``, we allow `ProtoSerializeOwnExt to provide a method to obtain ownership. Therefore, copying behavior in memory can be reduced pub trait ProtoSerializeOwnExt { type PB: Clone + PartialEq + ::prost::Message; - type T: Clone + PartialEq; - fn from_protobuf_own(pb: Self::PB) -> Self::T; + fn from_protobuf_own(pb: Self::PB) -> Self; fn to_protobuf_own(self) -> Self::PB; } diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index 72d05a1beb54..7592220b368a 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -336,7 +336,6 @@ impl VnodeWatermark { impl ProtoSerializeExt for VnodeWatermark { type PB = PbVnodeWatermark; - type T = VnodeWatermark; fn to_protobuf(&self) -> Self::PB { Self::PB { @@ -345,8 +344,8 @@ impl ProtoSerializeExt for VnodeWatermark { } } - fn from_protobuf(pb: &Self::PB) -> Self::T { - Self::T { + fn from_protobuf(pb: &Self::PB) -> Self { + Self { vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), watermark: Bytes::from(pb.watermark.clone()), } @@ -355,7 +354,6 @@ impl ProtoSerializeExt for VnodeWatermark { impl ProtoSerializeOwnExt for VnodeWatermark { type PB = PbVnodeWatermark; - type T = VnodeWatermark; fn to_protobuf_own(self) -> Self::PB { Self::PB { @@ -364,8 +362,8 @@ impl ProtoSerializeOwnExt for VnodeWatermark { } } - fn from_protobuf_own(pb: Self::PB) -> Self::T { - Self::T { + fn from_protobuf_own(pb: Self::PB) -> Self { + Self { vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), watermark: Bytes::from(pb.watermark), } @@ -583,7 +581,6 @@ impl ProtoSerializeSizeEstimatedExt for TableWatermarks { impl ProtoSerializeExt for TableWatermarks { type PB = PbTableWatermarks; - type T = TableWatermarks; fn to_protobuf(&self) -> Self::PB { Self::PB { @@ -602,8 +599,8 @@ impl ProtoSerializeExt for TableWatermarks { } } - fn from_protobuf(pb: &Self::PB) -> Self::T { - Self::T { + fn from_protobuf(pb: &Self::PB) -> Self { + Self { watermarks: pb .epoch_watermarks .iter() @@ -628,7 +625,6 @@ impl ProtoSerializeExt for TableWatermarks { impl ProtoSerializeOwnExt for TableWatermarks { type PB = PbTableWatermarks; - type T = TableWatermarks; fn to_protobuf_own(self) -> Self::PB { Self::PB { @@ -650,8 +646,8 @@ impl ProtoSerializeOwnExt for TableWatermarks { } } - fn from_protobuf_own(pb: Self::PB) -> Self::T { - Self::T { + fn from_protobuf_own(pb: Self::PB) -> Self { + Self { watermarks: pb .epoch_watermarks .into_iter() diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 6ab3a6fedbe7..a7ac18c58aba 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -18,12 +18,13 @@ use std::mem::size_of; use bytes::Bytes; use itertools::Itertools; use risingwave_common::catalog::TableId; -use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType}; +use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType, TaskStatus, TaskType}; use risingwave_pb::hummock::hummock_version::PbLevels; use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas; use risingwave_pb::hummock::{ - LevelType, PbCompactTask, PbHummockVersion, PbHummockVersionDelta, PbInputLevel, PbKeyRange, - PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, PbValidationTask, TableOption, + BloomFilterType, LevelType, PbCompactTask, PbHummockVersion, PbHummockVersionDelta, + PbInputLevel, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, + PbValidationTask, TableOption, }; use serde::Serialize; @@ -43,10 +44,9 @@ pub struct OverlappingLevel { impl ProtoSerializeExt for OverlappingLevel { type PB = PbOverlappingLevel; - type T = OverlappingLevel; - fn from_protobuf(pb_overlapping_level: &Self::PB) -> Self::T { - Self::T { + fn from_protobuf(pb_overlapping_level: &Self::PB) -> Self { + Self { sub_levels: pb_overlapping_level .sub_levels .iter() @@ -90,7 +90,7 @@ impl OverlappingLevel { #[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct Level { pub level_idx: u32, - pub level_type: i32, + pub level_type: LevelType, pub table_infos: Vec, pub total_file_size: u64, pub sub_level_id: u64, @@ -100,12 +100,11 @@ pub struct Level { impl ProtoSerializeExt for Level { type PB = PbLevel; - type T = Level; - fn from_protobuf(pb_level: &Self::PB) -> Self::T { + fn from_protobuf(pb_level: &Self::PB) -> Self { Self { level_idx: pb_level.level_idx, - level_type: pb_level.level_type, + level_type: LevelType::try_from(pb_level.level_type).unwrap(), table_infos: pb_level .table_infos .iter() @@ -121,7 +120,7 @@ impl ProtoSerializeExt for Level { fn to_protobuf(&self) -> Self::PB { Self::PB { level_idx: self.level_idx, - level_type: self.level_type, + level_type: self.level_type.into(), table_infos: self .table_infos .iter() @@ -157,7 +156,7 @@ impl Level { } pub fn level_type(&self) -> LevelType { - LevelType::try_from(self.level_type).unwrap() + self.level_type } pub fn get_sub_level_id(&self) -> u64 { @@ -212,9 +211,8 @@ impl ProtoSerializeSizeEstimatedExt for Levels { impl ProtoSerializeExt for Levels { type PB = PbLevels; - type T = Levels; - fn from_protobuf(pb_levels: &Self::PB) -> Self::T { + fn from_protobuf(pb_levels: &Self::PB) -> Self { Self { l0: if pb_levels.l0.is_some() { Some(OverlappingLevel::from_protobuf( @@ -301,10 +299,9 @@ impl ProtoSerializeSizeEstimatedExt for HummockVersion { impl ProtoSerializeExt for HummockVersion { type PB = PbHummockVersion; - type T = HummockVersion; - fn from_protobuf(pb_version: &Self::PB) -> Self::T { - Self::T { + fn from_protobuf(pb_version: &Self::PB) -> Self { + Self { id: pb_version.id, levels: pb_version .levels @@ -385,10 +382,9 @@ impl HummockVersionDelta { impl ProtoSerializeExt for HummockVersionDelta { type PB = PbHummockVersionDelta; - type T = HummockVersionDelta; - fn from_protobuf(delta: &Self::PB) -> Self::T { - Self::T { + fn from_protobuf(delta: &Self::PB) -> Self { + Self { id: delta.id, prev_id: delta.prev_id, group_deltas: delta.group_deltas.clone(), @@ -451,7 +447,7 @@ pub struct SstableInfo { pub max_epoch: u64, pub uncompressed_file_size: u64, pub range_tombstone_count: u64, - pub bloom_filter_kind: i32, + pub bloom_filter_kind: BloomFilterType, } impl ProtoSerializeSizeEstimatedExt for SstableInfo { @@ -479,9 +475,8 @@ impl ProtoSerializeSizeEstimatedExt for SstableInfo { impl ProtoSerializeExt for SstableInfo { type PB = PbSstableInfo; - type T = SstableInfo; - fn from_protobuf(pb_sstable_info: &Self::PB) -> Self::T { + fn from_protobuf(pb_sstable_info: &Self::PB) -> Self { Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, @@ -506,7 +501,8 @@ impl ProtoSerializeExt for SstableInfo { max_epoch: pb_sstable_info.max_epoch, uncompressed_file_size: pb_sstable_info.uncompressed_file_size, range_tombstone_count: pb_sstable_info.range_tombstone_count, - bloom_filter_kind: pb_sstable_info.bloom_filter_kind, + bloom_filter_kind: BloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) + .unwrap(), } } @@ -535,17 +531,16 @@ impl ProtoSerializeExt for SstableInfo { max_epoch: self.max_epoch, uncompressed_file_size: self.uncompressed_file_size, range_tombstone_count: self.range_tombstone_count, - bloom_filter_kind: self.bloom_filter_kind, + bloom_filter_kind: self.bloom_filter_kind.into(), } } } impl ProtoSerializeOwnExt for SstableInfo { type PB = PbSstableInfo; - type T = SstableInfo; - fn from_protobuf_own(pb_sstable_info: Self::PB) -> Self::T { - Self::T { + fn from_protobuf_own(pb_sstable_info: Self::PB) -> Self { + Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, key_range: if pb_sstable_info.key_range.is_some() { @@ -569,7 +564,8 @@ impl ProtoSerializeOwnExt for SstableInfo { max_epoch: pb_sstable_info.max_epoch, uncompressed_file_size: pb_sstable_info.uncompressed_file_size, range_tombstone_count: pb_sstable_info.range_tombstone_count, - bloom_filter_kind: pb_sstable_info.bloom_filter_kind, + bloom_filter_kind: BloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) + .unwrap(), } } @@ -598,7 +594,7 @@ impl ProtoSerializeOwnExt for SstableInfo { max_epoch: self.max_epoch, uncompressed_file_size: self.uncompressed_file_size, range_tombstone_count: self.range_tombstone_count, - bloom_filter_kind: self.bloom_filter_kind, + bloom_filter_kind: self.bloom_filter_kind.into(), } } } @@ -620,7 +616,7 @@ impl SstableInfo { #[derive(Clone, PartialEq, Default, Debug, Serialize)] pub struct InputLevel { pub level_idx: u32, - pub level_type: i32, + pub level_type: LevelType, pub table_infos: Vec, } @@ -638,12 +634,11 @@ impl ProtoSerializeSizeEstimatedExt for InputLevel { impl ProtoSerializeExt for InputLevel { type PB = PbInputLevel; - type T = InputLevel; - fn from_protobuf(pb_input_level: &Self::PB) -> Self::T { + fn from_protobuf(pb_input_level: &Self::PB) -> Self { Self { level_idx: pb_input_level.level_idx, - level_type: pb_input_level.level_type, + level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), table_infos: pb_input_level .table_infos .iter() @@ -655,7 +650,7 @@ impl ProtoSerializeExt for InputLevel { fn to_protobuf(&self) -> Self::PB { Self::PB { level_idx: self.level_idx, - level_type: self.level_type, + level_type: self.level_type.into(), table_infos: self .table_infos .iter() @@ -667,12 +662,11 @@ impl ProtoSerializeExt for InputLevel { impl ProtoSerializeOwnExt for InputLevel { type PB = PbInputLevel; - type T = InputLevel; - fn from_protobuf_own(pb_input_level: Self::PB) -> Self::T { + fn from_protobuf_own(pb_input_level: Self::PB) -> Self { Self { level_idx: pb_input_level.level_idx, - level_type: pb_input_level.level_type, + level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), table_infos: pb_input_level .table_infos .into_iter() @@ -684,7 +678,7 @@ impl ProtoSerializeOwnExt for InputLevel { fn to_protobuf_own(self) -> Self::PB { Self::PB { level_idx: self.level_idx, - level_type: self.level_type, + level_type: self.level_type.into(), table_infos: self .table_infos .into_iter() @@ -700,7 +694,7 @@ impl InputLevel { } pub fn level_type(&self) -> PbLevelType { - LevelType::try_from(self.level_type).unwrap() + self.level_type } pub fn get_level_idx(&self) -> u32 { @@ -726,7 +720,7 @@ pub struct CompactTask { pub gc_delete_keys: bool, /// Lbase in LSM pub base_level: u32, - pub task_status: i32, + pub task_status: TaskStatus, /// compaction group the task belongs to pub compaction_group_id: u64, /// existing_table_ids for compaction drop key @@ -738,7 +732,7 @@ pub struct CompactTask { pub current_epoch_time: u64, pub target_sub_level_id: u64, /// Identifies whether the task is space_reclaim, if the compact_task_type increases, it will be refactored to enum - pub task_type: i32, + pub task_type: TaskType, /// Deprecated. use table_vnode_partition instead; pub split_by_state_table: bool, /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed. @@ -794,10 +788,9 @@ impl ProtoSerializeSizeEstimatedExt for CompactTask { impl ProtoSerializeExt for CompactTask { type PB = PbCompactTask; - type T = CompactTask; - fn from_protobuf(pb_compact_task: &Self::PB) -> Self::T { - Self::T { + fn from_protobuf(pb_compact_task: &Self::PB) -> Self { + Self { input_ssts: pb_compact_task .input_ssts .iter() @@ -822,7 +815,7 @@ impl ProtoSerializeExt for CompactTask { target_level: pb_compact_task.target_level, gc_delete_keys: pb_compact_task.gc_delete_keys, base_level: pb_compact_task.base_level, - task_status: pb_compact_task.task_status, + task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(), compaction_group_id: pb_compact_task.compaction_group_id, existing_table_ids: pb_compact_task.existing_table_ids.clone(), compression_algorithm: pb_compact_task.compression_algorithm, @@ -831,7 +824,7 @@ impl ProtoSerializeExt for CompactTask { table_options: pb_compact_task.table_options.clone(), current_epoch_time: pb_compact_task.current_epoch_time, target_sub_level_id: pb_compact_task.target_sub_level_id, - task_type: pb_compact_task.task_type, + task_type: TaskType::try_from(pb_compact_task.task_type).unwrap(), #[allow(deprecated)] split_by_state_table: pb_compact_task.split_by_state_table, split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, @@ -875,7 +868,7 @@ impl ProtoSerializeExt for CompactTask { target_level: self.target_level, gc_delete_keys: self.gc_delete_keys, base_level: self.base_level, - task_status: self.task_status, + task_status: self.task_status.into(), compaction_group_id: self.compaction_group_id, existing_table_ids: self.existing_table_ids.clone(), compression_algorithm: self.compression_algorithm, @@ -884,7 +877,7 @@ impl ProtoSerializeExt for CompactTask { table_options: self.table_options.clone(), current_epoch_time: self.current_epoch_time, target_sub_level_id: self.target_sub_level_id, - task_type: self.task_type, + task_type: self.task_type.into(), //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, split_weight_by_vnode: self.split_weight_by_vnode, table_vnode_partition: self.table_vnode_partition.clone(), @@ -900,10 +893,9 @@ impl ProtoSerializeExt for CompactTask { impl ProtoSerializeOwnExt for CompactTask { type PB = PbCompactTask; - type T = CompactTask; - fn from_protobuf_own(pb_compact_task: Self::PB) -> Self::T { - Self::T { + fn from_protobuf_own(pb_compact_task: Self::PB) -> Self { + Self { input_ssts: pb_compact_task .input_ssts .into_iter() @@ -928,7 +920,7 @@ impl ProtoSerializeOwnExt for CompactTask { target_level: pb_compact_task.target_level, gc_delete_keys: pb_compact_task.gc_delete_keys, base_level: pb_compact_task.base_level, - task_status: pb_compact_task.task_status, + task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(), compaction_group_id: pb_compact_task.compaction_group_id, existing_table_ids: pb_compact_task.existing_table_ids.clone(), compression_algorithm: pb_compact_task.compression_algorithm, @@ -937,7 +929,7 @@ impl ProtoSerializeOwnExt for CompactTask { table_options: pb_compact_task.table_options.clone(), current_epoch_time: pb_compact_task.current_epoch_time, target_sub_level_id: pb_compact_task.target_sub_level_id, - task_type: pb_compact_task.task_type, + task_type: TaskType::try_from(pb_compact_task.task_type).unwrap(), #[allow(deprecated)] split_by_state_table: pb_compact_task.split_by_state_table, split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, @@ -981,7 +973,7 @@ impl ProtoSerializeOwnExt for CompactTask { target_level: self.target_level, gc_delete_keys: self.gc_delete_keys, base_level: self.base_level, - task_status: self.task_status, + task_status: self.task_status.into(), compaction_group_id: self.compaction_group_id, existing_table_ids: self.existing_table_ids.clone(), compression_algorithm: self.compression_algorithm, @@ -990,7 +982,7 @@ impl ProtoSerializeOwnExt for CompactTask { table_options: self.table_options.clone(), current_epoch_time: self.current_epoch_time, target_sub_level_id: self.target_sub_level_id, - task_type: self.task_type, + task_type: self.task_type.into(), //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, split_weight_by_vnode: self.split_weight_by_vnode, table_vnode_partition: self.table_vnode_partition.clone(), @@ -1014,7 +1006,7 @@ impl CompactTask { } pub fn set_task_status(&mut self, s: PbTaskStatus) { - self.task_status = s as i32; + self.task_status = s; } pub fn get_input_ssts(&self) -> &Vec { @@ -1046,9 +1038,8 @@ impl ProtoSerializeSizeEstimatedExt for ValidationTask { impl ProtoSerializeOwnExt for ValidationTask { type PB = PbValidationTask; - type T = ValidationTask; - fn from_protobuf_own(pb_validation_task: Self::PB) -> Self::T { + fn from_protobuf_own(pb_validation_task: Self::PB) -> Self { Self { sst_infos: pb_validation_task .sst_infos diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index 8679b9ccc010..3fc5b85b2d65 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -1487,12 +1487,12 @@ pub(crate) mod tests { input_ssts: vec![ InputLevel { level_idx: 5, - level_type: 1, + level_type: risingwave_pb::hummock::LevelType::Nonoverlapping, table_infos: ssts.drain(..select_file_count).collect_vec(), }, InputLevel { level_idx: 6, - level_type: 1, + level_type: risingwave_pb::hummock::LevelType::Nonoverlapping, table_infos: ssts, }, ], @@ -1736,12 +1736,12 @@ pub(crate) mod tests { input_ssts: vec![ InputLevel { level_idx: 5, - level_type: 1, + level_type: risingwave_pb::hummock::LevelType::Nonoverlapping, table_infos: sst_infos.drain(..1).collect_vec(), }, InputLevel { level_idx: 6, - level_type: 1, + level_type: risingwave_pb::hummock::LevelType::Nonoverlapping, table_infos: sst_infos, }, ], diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index 53778b684c6f..0414e54b2f38 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -348,7 +348,7 @@ pub async fn check_compaction_result( } // Do not need to filter the table because manager has done it. - if level.level_type == LevelType::Nonoverlapping as i32 { + if level.level_type == LevelType::Nonoverlapping { debug_assert!(can_concat(&level.table_infos)); del_iter.add_concat_iter(level.table_infos.clone(), context.sstable_store.clone()); diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index 6d19b444499c..da84274d72f1 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -104,7 +104,7 @@ impl CompactorRunner { gc_delete_keys: task.gc_delete_keys, watermark: task.watermark, stats_target_table_ids: Some(HashSet::from_iter(task.existing_table_ids.clone())), - task_type: TaskType::try_from(task.task_type).unwrap(), + task_type: task.task_type, is_target_l0_or_lbase: task.target_level == 0 || task.target_level == task.base_level, use_block_based_filter, @@ -167,13 +167,12 @@ impl CompactorRunner { } // Do not need to filter the table because manager has done it. - if level.level_type == LevelType::Nonoverlapping as i32 { + if level.level_type == LevelType::Nonoverlapping { debug_assert!(can_concat(&level.table_infos)); let tables = level .table_infos .iter() .filter(|table_info| { - // let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap()); let table_ids = &table_info.table_ids; let exist_table = table_ids.iter().any(|table_id| { self.compact_task.existing_table_ids.contains(table_id) @@ -203,7 +202,6 @@ impl CompactorRunner { )); } else { for table_info in &level.table_infos { - // let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap()); let table_ids = &table_info.table_ids; let exist_table = table_ids .iter() @@ -394,7 +392,7 @@ pub async fn compact( .sum::(); let all_ssts_are_blocked_filter = sstable_infos .iter() - .all(|table_info| table_info.bloom_filter_kind == BloomFilterType::Blocked as i32); + .all(|table_info| table_info.bloom_filter_kind == BloomFilterType::Blocked); let delete_key_count = sstable_infos .iter() @@ -414,7 +412,7 @@ pub async fn compact( && compaction_size < context.storage_opts.compactor_fast_max_compact_task_size && delete_key_count * 100 < context.storage_opts.compactor_fast_max_compact_delete_ratio as u64 * total_key_count - && compact_task.task_type == TaskType::Dynamic as i32; + && compact_task.task_type == TaskType::Dynamic; if !optimize_by_copy_block { match generate_splits(&sstable_infos, compaction_size, context.clone()).await { @@ -675,7 +673,7 @@ fn compact_done( task_status: TaskStatus, ) -> (CompactTask, HashMap) { let mut table_stats_map = TableStatsMap::default(); - compact_task.task_status = task_status as i32; + compact_task.task_status = task_status; compact_task .sorted_output_ssts .reserve(compact_task.splits.len()); @@ -1015,7 +1013,7 @@ mod tests { let compact_task = CompactTask { input_ssts: vec![InputLevel { level_idx: 0, - level_type: 0, + level_type: LevelType::Nonoverlapping, table_infos: vec![sstable_info_1, sstable_info_2], }], existing_table_ids: vec![2], diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs index 5831121f42a4..0e05025da007 100644 --- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs +++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs @@ -27,7 +27,6 @@ use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::table_stats::TableStats; use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, compact_task_to_string, EpochWithGap, LocalSstableInfo}; -use risingwave_pb::hummock::compact_task::TaskType; use crate::filter_key_extractor::FilterKeyExtractorImpl; use crate::hummock::block_stream::BlockDataStream; @@ -292,7 +291,7 @@ impl CompactorRunner { gc_delete_keys: task.gc_delete_keys, watermark: task.watermark, stats_target_table_ids: Some(HashSet::from_iter(task.existing_table_ids.clone())), - task_type: TaskType::try_from(task.task_type).unwrap(), + task_type: task.task_type, is_target_l0_or_lbase: task.target_level == 0 || task.target_level == task.base_level, table_vnode_partition: task.table_vnode_partition.clone(), use_block_based_filter: true, diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index 9c5857dd5925..17ffb072b568 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -485,19 +485,19 @@ pub fn start_compactor( Err(err) => { tracing::warn!(error = %err.as_report(), "Failed to track pending SST object id"); let mut compact_task = compact_task; - compact_task.task_status = TaskStatus::TrackSstObjectIdFailed as i32; + compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed); ((compact_task, HashMap::default()),None) } }; shutdown.lock().unwrap().remove(&task_id); let enable_check_compaction_result = context.storage_opts.check_compaction_result; - let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success as i32; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success; if let Err(e) = request_sender.send(SubscribeCompactionEventRequest { event: Some(RequestEvent::ReportTask( ReportTask { task_id: compact_task.task_id, - task_status: compact_task.task_status, + task_status: compact_task.task_status.into(), sorted_output_ssts: compact_task.sorted_output_ssts.iter().map(|sst| sst.to_protobuf()).collect_vec(), table_stats_change:to_prost_table_stats_map(table_stats), } @@ -700,7 +700,7 @@ pub fn start_shared_compactor( Ok(_) => { // TODO: remove this method after we have running risingwave cluster with fast compact algorithm stably for a long time. let enable_check_compaction_result = context.storage_opts.check_compaction_result; - let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success as i32; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success; if enable_check_compaction_result && need_check_task { match check_compaction_result(&compact_task, context.clone()).await { Err(e) => { diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index 371c2e9b38bd..0e375210c17c 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -630,7 +630,7 @@ impl SstableBuilder { let sst_info = SstableInfo { object_id: self.sstable_id, sst_id: self.sstable_id, - bloom_filter_kind: bloom_filter_kind as i32, + bloom_filter_kind, key_range: Some(KeyRange { left: Bytes::from(meta.smallest_key.clone()), right: Bytes::from(meta.largest_key.clone()), diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs index ecc799815236..4a16052c7214 100644 --- a/src/storage/src/hummock/store/version.rs +++ b/src/storage/src/hummock/store/version.rs @@ -950,7 +950,7 @@ impl HummockVersionReader { continue; } - if level.level_type == LevelType::Nonoverlapping as i32 { + if level.level_type == LevelType::Nonoverlapping { let table_infos = prune_nonoverlapping_ssts(&level.table_infos, user_key_range_ref); let sstables = table_infos .filter(|sstable_info| { From 3bf2bf3e291219acc02778583c855a6e563b9859 Mon Sep 17 00:00:00 2001 From: Li0k Date: Fri, 8 Mar 2024 18:19:17 +0800 Subject: [PATCH 05/18] fix(storage): fix compile --- src/storage/benches/bench_compactor.rs | 2 +- .../hummock_test/src/compactor_tests.rs | 49 +++++-------------- 2 files changed, 13 insertions(+), 38 deletions(-) diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index e5411dfa8914..3b5ddc64de22 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -31,7 +31,7 @@ use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl}; -use risingwave_pb::hummock::{compact_task, SstableInfo, TableSchema}; +use risingwave_pb::hummock::{compact_task, TableSchema}; use risingwave_storage::hummock::compactor::compactor_runner::compact_and_build_sst; use risingwave_storage::hummock::compactor::{ ConcatSstableIterator, DummyCompactionFilter, TaskConfig, TaskProgress, diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index d0cc3d511053..6443cdcb41e2 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -32,20 +32,17 @@ pub(crate) mod tests { use risingwave_common_service::observer_manager::NotificationClient; use risingwave_hummock_sdk::can_concat; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; - use risingwave_hummock_sdk::key::{next_key, FullKey, TableKey, TABLE_PREFIX_LEN}; - use risingwave_hummock_sdk::key_range::KeyRange; - use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; - use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, InputLevel, SstableInfo}; use risingwave_hummock_sdk::key::{ next_key, prefix_slice_with_vnode, prefixed_range_with_vnode, FullKey, TableKey, TABLE_PREFIX_LEN, }; + use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; use risingwave_hummock_sdk::table_watermark::{ - ReadTableWatermark, VnodeWatermark, WatermarkDirection, + ReadTableWatermark, TableWatermarks, VnodeWatermark, WatermarkDirection, }; - use risingwave_hummock_sdk::version::HummockVersion; + use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, InputLevel, SstableInfo}; use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder; use risingwave_meta::hummock::compaction::selector::{ default_compaction_selector, ManualCompactionOption, @@ -56,14 +53,7 @@ pub(crate) mod tests { }; use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient}; use risingwave_pb::common::{HostAddress, WorkerType}; -<<<<<<< HEAD use risingwave_pb::hummock::TableOption; -======= - use risingwave_pb::hummock::table_watermarks::PbEpochNewWatermarks; - use risingwave_pb::hummock::{ - CompactTask, InputLevel, KeyRange, SstableInfo, TableOption, TableWatermarks, - }; ->>>>>>> ef7eb1d48ef2f43701455e63b8aa2737a4166f10 use risingwave_pb::meta::add_worker_node_request::Property; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::filter_key_extractor::{ @@ -1416,10 +1406,6 @@ pub(crate) mod tests { normal_tables.push(sstable_store.sstable(sst_info, &mut stats).await.unwrap()); } assert!(fast_ret.iter().all(|f| f.file_size < capacity * 6 / 5)); - println!( - "fast sstables file size: {:?}", - fast_ret.iter().map(|f| f.file_size).collect_vec(), - ); assert!(can_concat(&ret)); assert!(can_concat(&fast_ret)); let read_options = Arc::new(SstableIteratorReadOptions::default()); @@ -1548,7 +1534,6 @@ pub(crate) mod tests { sstable_store.clone(), ) .await; - println!("generate ssts size: {}", sst.file_size); ssts.push(sst); } let select_file_count = ssts.len() / 2; @@ -1892,10 +1877,6 @@ pub(crate) mod tests { max_sst_file_size = std::cmp::max(max_sst_file_size, sst_info.file_size); sst_infos.push(sst_info); } - println!( - "input data: {}", - sst_infos.iter().map(|sst| sst.file_size).sum::(), - ); let target_file_size = max_sst_file_size / 4; let mut table_watermarks = BTreeMap::default(); @@ -1914,13 +1895,12 @@ pub(crate) mod tests { table_watermarks.insert( 1, TableWatermarks { - epoch_watermarks: vec![PbEpochNewWatermarks { - watermarks: vec![ - VnodeWatermark::new(bitmap.clone(), watermark_key.clone()).to_protobuf() - ], - epoch: 500, - }], - is_ascending: true, + watermarks: vec![( + 500, + vec![VnodeWatermark::new(bitmap.clone(), watermark_key.clone())], + )], + + direction: WatermarkDirection::Ascending, }, ); @@ -1928,12 +1908,12 @@ pub(crate) mod tests { input_ssts: vec![ InputLevel { level_idx: 5, - level_type: 1, + level_type: risingwave_pb::hummock::LevelType::Nonoverlapping, table_infos: sst_infos.drain(..1).collect_vec(), }, InputLevel { level_idx: 6, - level_type: 1, + level_type: risingwave_pb::hummock::LevelType::Nonoverlapping, table_infos: sst_infos, }, ], @@ -1950,12 +1930,7 @@ pub(crate) mod tests { ..Default::default() }; let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await; - println!( - "normal compact result data: {}, fast compact result data: {}", - ret.iter().map(|sst| sst.file_size).sum::(), - fast_ret.iter().map(|sst| sst.file_size).sum::(), - ); - // check_compaction_result(compact_ctx.sstable_store, ret.clone(), fast_ret, target_file_size).await; + let mut fast_tables = Vec::with_capacity(fast_ret.len()); let mut normal_tables = Vec::with_capacity(ret.len()); let mut stats = StoreLocalStatistic::default(); From 4f72cba300436e76b2b2e4391349ce8291a30142 Mon Sep 17 00:00:00 2001 From: Li0k Date: Fri, 8 Mar 2024 18:31:03 +0800 Subject: [PATCH 06/18] fix(storage): fix check --- .../hummock/compaction/selector/level_selector.rs | 7 ++++--- src/storage/benches/bench_compactor.rs | 15 ++++++++------- src/storage/hummock_test/src/compactor_tests.rs | 1 - .../src/hummock_read_version_tests.rs | 2 +- .../src/hummock/compactor/compaction_utils.rs | 9 +++++---- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index a5fcfd19df3c..a8d10bf12a63 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -23,7 +23,8 @@ use risingwave_common::catalog::TableOption; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; -use risingwave_pb::hummock::{compact_task, CompactionConfig, LevelType}; +use risingwave_pb::hummock::compact_task::PbTaskType; +use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::{ create_compaction_task, CompactionSelector, LevelCompactionPicker, TierCompactionPicker, @@ -476,8 +477,8 @@ impl CompactionSelector for DynamicLevelSelector { "DynamicLevelSelector" } - fn task_type(&self) -> compact_task::TaskType { - compact_task::TaskType::Dynamic + fn task_type(&self) -> PbTaskType { + PbTaskType::Dynamic } } diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index 3b5ddc64de22..5e4a80eeecf8 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -31,7 +31,8 @@ use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl}; -use risingwave_pb::hummock::{compact_task, TableSchema}; +use risingwave_pb::hummock::compact_task::PbTaskType; +use risingwave_pb::hummock::PbTableSchema; use risingwave_storage::hummock::compactor::compactor_runner::compact_and_build_sst; use risingwave_storage::hummock::compactor::{ ConcatSstableIterator, DummyCompactionFilter, TaskConfig, TaskProgress, @@ -266,7 +267,7 @@ async fn compact>( gc_delete_keys: false, watermark: 0, stats_target_table_ids: None, - task_type: compact_task::TaskType::Dynamic, + task_type: PbTaskType::Dynamic, use_block_based_filter: true, ..Default::default() }); @@ -417,7 +418,7 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) { gc_delete_keys: false, watermark: 0, stats_target_table_ids: None, - task_type: compact_task::TaskType::Dynamic, + task_type: PbTaskType::Dynamic, use_block_based_filter: true, table_schemas: vec![].into_iter().collect(), ..Default::default() @@ -426,13 +427,13 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) { let mut task_config_schema = task_config_no_schema.clone(); task_config_schema.table_schemas.insert( 10, - TableSchema { + PbTableSchema { column_ids: (0..column_num as i32).collect(), }, ); task_config_schema.table_schemas.insert( 11, - TableSchema { + PbTableSchema { column_ids: (0..column_num as i32).collect(), }, ); @@ -440,13 +441,13 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) { let mut task_config_schema_cause_drop = task_config_no_schema.clone(); task_config_schema_cause_drop.table_schemas.insert( 10, - TableSchema { + PbTableSchema { column_ids: (0..column_num as i32 / 2).collect(), }, ); task_config_schema_cause_drop.table_schemas.insert( 11, - TableSchema { + PbTableSchema { column_ids: (0..column_num as i32 / 2).collect(), }, ); diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index 6443cdcb41e2..41629905fa0e 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -37,7 +37,6 @@ pub(crate) mod tests { TABLE_PREFIX_LEN, }; use risingwave_hummock_sdk::key_range::KeyRange; - use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; use risingwave_hummock_sdk::table_watermark::{ ReadTableWatermark, TableWatermarks, VnodeWatermark, WatermarkDirection, diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index 1feb72409d7e..627d0d7fee05 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -24,7 +24,7 @@ use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::key::{key_with_epoch, map_table_key_range}; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::SstableInfo; -use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; +use risingwave_hummock_sdk::LocalSstableInfo; use risingwave_meta::hummock::test_utils::setup_compute_env; use risingwave_storage::hummock::iterator::test_utils::{ iterator_test_table_key_of, iterator_test_user_key_of, diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index d9dc1e5e4c68..d37f1ed4c895 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -27,7 +27,8 @@ use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::table_stats::TableStatsMap; use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, EpochWithGap, KeyComparator}; -use risingwave_pb::hummock::{compact_task, LevelType, TableSchema}; +use risingwave_pb::hummock::compact_task::PbTaskType; +use risingwave_pb::hummock::{PbLevelType, PbTableSchema}; use tokio::time::Instant; pub use super::context::CompactorContext; @@ -125,7 +126,7 @@ pub struct TaskConfig { /// change. For an divided SST as input, a dropped key shouldn't be counted if its table id /// doesn't belong to this divided SST. See `Compactor::compact_and_build_sst`. pub stats_target_table_ids: Option>, - pub task_type: compact_task::TaskType, + pub task_type: PbTaskType, pub is_target_l0_or_lbase: bool, pub use_block_based_filter: bool, @@ -133,7 +134,7 @@ pub struct TaskConfig { /// `TableId` -> `TableSchema` /// Schemas in `table_schemas` are at least as new as the one used to create `input_ssts`. /// For a table with schema existing in `table_schemas`, its columns not in `table_schemas` but in `input_ssts` can be safely dropped. - pub table_schemas: HashMap, + pub table_schemas: HashMap, /// `disable_drop_column_optimization` should only be set in benchmark. pub disable_drop_column_optimization: bool, } @@ -354,7 +355,7 @@ pub async fn check_compaction_result( } // Do not need to filter the table because manager has done it. - if level.level_type == LevelType::Nonoverlapping { + if level.level_type == PbLevelType::Nonoverlapping { debug_assert!(can_concat(&level.table_infos)); del_iter.add_concat_iter(level.table_infos.clone(), context.sstable_store.clone()); From 0a75e90b5be1a9c673a943b0503d06d4723ac8fb Mon Sep 17 00:00:00 2001 From: Li0k Date: Sat, 9 Mar 2024 02:04:15 +0800 Subject: [PATCH 07/18] fix(storage): fix check --- src/storage/hummock_sdk/src/compact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index ee9e8b504a3e..d79e2529982b 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -25,7 +25,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { "Compaction task id: {:?}, group-id: {:?}, task type: {:?}, target level: {:?}, target sub level: {:?}", compact_task.task_id, compact_task.compaction_group_id, - compact_task.task_type.as_str_name(), + compact_task.task_type, compact_task.target_level, compact_task.target_sub_level_id ) From 5672d61c896bdfb0d1d33b6ea0064e3b681b5ac5 Mon Sep 17 00:00:00 2001 From: Li0k Date: Tue, 7 May 2024 17:40:12 +0800 Subject: [PATCH 08/18] refactor(storage): refactor version --- src/compute/src/rpc/service/stream_service.rs | 6 +- src/meta/service/src/hummock_service.rs | 15 +- src/meta/service/src/notification_service.rs | 3 +- src/meta/src/backup_restore/restore.rs | 4 +- src/meta/src/barrier/mod.rs | 11 +- src/meta/src/hummock/compactor_manager.rs | 6 +- src/meta/src/hummock/manager/checkpoint.rs | 12 +- src/meta/src/hummock/manager/context.rs | 6 +- src/meta/src/hummock/manager/mod.rs | 23 +- .../src/hummock/mock_hummock_meta_client.rs | 6 +- src/meta/src/hummock/model/ext/hummock.rs | 3 +- src/meta/src/hummock/model/version_delta.rs | 12 +- src/rpc_client/src/meta_client.rs | 4 +- src/storage/backup/src/meta_snapshot_v1.rs | 5 +- src/storage/backup/src/meta_snapshot_v2.rs | 4 +- .../compaction_group/hummock_version_ext.rs | 17 +- src/storage/hummock_sdk/src/lib.rs | 18 - .../hummock_sdk/src/table_watermark.rs | 111 ++- src/storage/hummock_sdk/src/version.rs | 785 ++++++++++-------- .../src/mock_notification_client.rs | 3 +- src/storage/src/hummock/compactor/mod.rs | 25 +- src/storage/src/store.rs | 10 +- 22 files changed, 587 insertions(+), 502 deletions(-) diff --git a/src/compute/src/rpc/service/stream_service.rs b/src/compute/src/rpc/service/stream_service.rs index c8b79ea44146..96b21ecc80b2 100644 --- a/src/compute/src/rpc/service/stream_service.rs +++ b/src/compute/src/rpc/service/stream_service.rs @@ -15,7 +15,7 @@ use await_tree::InstrumentAwait; use itertools::Itertools; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; -use risingwave_hummock_sdk::{LocalSstableInfo, ProtoSerializeOwnExt}; +use risingwave_hummock_sdk::LocalSstableInfo; use risingwave_pb::stream_service::barrier_complete_response::GroupedSstableInfo; use risingwave_pb::stream_service::stream_service_server::StreamService; use risingwave_pb::stream_service::*; @@ -177,7 +177,7 @@ impl StreamService for StreamServiceImpl { table_stats, }| GroupedSstableInfo { compaction_group_id, - sst: Some(sst_info.to_protobuf_own()), + sst: Some(sst_info.into()), table_stats_map: to_prost_table_stats_map(table_stats), }, ) @@ -185,7 +185,7 @@ impl StreamService for StreamServiceImpl { worker_id: self.env.worker_id(), table_watermarks: table_watermarks .into_iter() - .map(|(key, value)| (key.table_id, value.to_protobuf_own())) + .map(|(key, value)| (key.table_id, value.into())) .collect(), })) } diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs index 15d222a9f34e..d0c787b54f6d 100644 --- a/src/meta/service/src/hummock_service.rs +++ b/src/meta/service/src/hummock_service.rs @@ -20,7 +20,6 @@ use itertools::Itertools; use risingwave_common::catalog::{TableId, NON_RESERVED_SYS_CATALOG_ID}; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::HummockVersionDelta; -use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta::manager::MetadataManager; use risingwave_pb::hummock::get_compaction_score_response::PickerInfo; use risingwave_pb::hummock::hummock_manager_service_server::HummockManagerService; @@ -87,7 +86,7 @@ impl HummockManagerService for HummockServiceImpl { let current_version = self.hummock_manager.get_current_version().await; Ok(Response::new(GetCurrentVersionResponse { status: None, - current_version: Some(current_version.to_protobuf()), + current_version: Some(current_version.into()), })) } @@ -103,7 +102,7 @@ impl HummockManagerService for HummockServiceImpl { )) .await?; Ok(Response::new(ReplayVersionDeltaResponse { - version: Some(version.to_protobuf()), + version: Some(version.into()), modified_compaction_groups: compaction_groups, })) } @@ -125,7 +124,7 @@ impl HummockManagerService for HummockServiceImpl { ) -> Result, Status> { let version = self.hummock_manager.disable_commit_epoch().await; Ok(Response::new(DisableCommitEpochResponse { - current_version: Some(version.to_protobuf()), + current_version: Some(version.into()), })) } @@ -141,8 +140,8 @@ impl HummockManagerService for HummockServiceImpl { let resp = ListVersionDeltasResponse { version_deltas: Some(PbHummockVersionDeltas { version_deltas: version_deltas - .iter() - .map(HummockVersionDelta::to_protobuf) + .into_iter() + .map(HummockVersionDelta::into) .collect(), }), }; @@ -433,7 +432,7 @@ impl HummockManagerService for HummockServiceImpl { let req = request.into_inner(); let version = self.hummock_manager.pin_version(req.context_id).await?; Ok(Response::new(PinVersionResponse { - pinned_version: Some(version.to_protobuf()), + pinned_version: Some(version.into()), })) } @@ -471,7 +470,7 @@ impl HummockManagerService for HummockServiceImpl { ) -> Result, Status> { let checkpoint_version = self.hummock_manager.get_checkpoint_version().await; Ok(Response::new(RiseCtlGetCheckpointVersionResponse { - checkpoint_version: Some(checkpoint_version.to_protobuf()), + checkpoint_version: Some(checkpoint_version.into()), })) } diff --git a/src/meta/service/src/notification_service.rs b/src/meta/service/src/notification_service.rs index 9e538ef61b73..e7a2d151e33a 100644 --- a/src/meta/service/src/notification_service.rs +++ b/src/meta/service/src/notification_service.rs @@ -13,7 +13,6 @@ // limitations under the License. use itertools::Itertools; -use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta::manager::MetadataManager; use risingwave_meta::MetaResult; use risingwave_pb::backup_service::MetaBackupManifestId; @@ -280,7 +279,7 @@ impl NotificationServiceImpl { Ok(MetaSnapshot { tables, - hummock_version: Some(hummock_version.to_protobuf()), + hummock_version: Some(hummock_version.into()), version: Some(SnapshotVersion { catalog_version, ..Default::default() diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs index cbaf61c2a285..dbb5a045e2f7 100644 --- a/src/meta/src/backup_restore/restore.rs +++ b/src/meta/src/backup_restore/restore.rs @@ -20,7 +20,7 @@ use risingwave_backup::storage::{MetaSnapshotStorage, MetaSnapshotStorageRef}; use risingwave_backup::MetaSnapshotId; use risingwave_common::config::{MetaBackend, ObjectStoreConfig}; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_hummock_sdk::{version_checkpoint_path, ProtoSerializeExt}; +use risingwave_hummock_sdk::version_checkpoint_path; use risingwave_object_store::object::build_remote_object_store; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_pb::hummock::PbHummockVersionCheckpoint; @@ -86,7 +86,7 @@ async fn restore_hummock_version( ); let checkpoint_path = version_checkpoint_path(hummock_storage_directory); let checkpoint = PbHummockVersionCheckpoint { - version: Some(hummock_version.to_protobuf()), + version: Some(hummock_version.into()), // Ignore stale objects. Full GC will clear them. stale_objects: Default::default(), }; diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index c2da6d0ed803..977b3c067359 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -34,9 +34,7 @@ use risingwave_hummock_sdk::table_watermark::{ merge_multiple_new_table_watermarks, TableWatermarks, }; use risingwave_hummock_sdk::version::SstableInfo; -use risingwave_hummock_sdk::{ - ExtendedSstableInfo, HummockSstableObjectId, ProtoSerializeExt, ProtoSerializeOwnExt, -}; +use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId}; use risingwave_pb::catalog::table::TableType; use risingwave_pb::common::WorkerNode; use risingwave_pb::ddl_service::DdlProgress; @@ -1095,7 +1093,7 @@ fn collect_commit_epoch_info( sst_to_worker.insert(sst_info.get_object_id(), resp.worker_id); ExtendedSstableInfo::new( grouped.compaction_group_id, - SstableInfo::from_protobuf_own(sst_info), + SstableInfo::from(sst_info), grouped.table_stats_map, ) }); @@ -1112,10 +1110,7 @@ fn collect_commit_epoch_info( watermarks .into_iter() .map(|(table_id, watermarks)| { - ( - TableId::new(table_id), - TableWatermarks::from_protobuf(&watermarks), - ) + (TableId::new(table_id), TableWatermarks::from(&watermarks)) }) .collect() }) diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs index 91dca871fdc4..56499491c65d 100644 --- a/src/meta/src/hummock/compactor_manager.rs +++ b/src/meta/src/hummock/compactor_manager.rs @@ -20,7 +20,7 @@ use fail::fail_point; use parking_lot::RwLock; use risingwave_hummock_sdk::compact::statistics_compact_task; use risingwave_hummock_sdk::version::CompactTask; -use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId, ProtoSerializeOwnExt}; +use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId}; use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent; use risingwave_pb::hummock::{ CancelCompactTask, CompactTaskAssignment, CompactTaskProgress, SubscribeCompactionEventResponse, @@ -149,9 +149,7 @@ impl CompactorManagerInner { }; // Initialize heartbeat for existing tasks. task_assignment.into_iter().for_each(|assignment| { - manager.initiate_task_heartbeat(CompactTask::from_protobuf_own( - assignment.compact_task.unwrap(), - )); + manager.initiate_task_heartbeat(CompactTask::from(assignment.compact_task.unwrap())); }); Ok(manager) } diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs index 07cce899b77b..6c8a40b58864 100644 --- a/src/meta/src/hummock/manager/checkpoint.rs +++ b/src/meta/src/hummock/manager/checkpoint.rs @@ -22,9 +22,11 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ object_size_map, summarize_group_deltas, }; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_hummock_sdk::{HummockVersionId, ProtoSerializeExt}; +use risingwave_hummock_sdk::HummockVersionId; use risingwave_pb::hummock::hummock_version_checkpoint::{PbStaleObjects, StaleObjects}; -use risingwave_pb::hummock::{PbHummockVersionArchive, PbHummockVersionCheckpoint}; +use risingwave_pb::hummock::{ + PbHummockVersion, PbHummockVersionArchive, PbHummockVersionCheckpoint, +}; use thiserror_ext::AsReport; use crate::hummock::error::Result; @@ -52,7 +54,7 @@ impl HummockVersionCheckpoint { pub fn to_protobuf(&self) -> PbHummockVersionCheckpoint { PbHummockVersionCheckpoint { - version: Some(self.version.to_protobuf()), + version: Some(PbHummockVersion::from(&self.version)), stale_objects: self.stale_objects.clone(), } } @@ -161,11 +163,11 @@ impl HummockManager { } } else { archive = Some(PbHummockVersionArchive { - version: Some(old_checkpoint.version.to_protobuf()), + version: Some(PbHummockVersion::from(&old_checkpoint.version)), version_deltas: versioning .hummock_version_deltas .range((Excluded(old_checkpoint_id), Included(new_checkpoint_id))) - .map(|(_, version_delta)| version_delta.to_protobuf()) + .map(|(_, version_delta)| version_delta.into()) .collect(), }); } diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs index 3b9e5dab4d02..d34337f7bd04 100644 --- a/src/meta/src/hummock/manager/context.rs +++ b/src/meta/src/hummock/manager/context.rs @@ -21,7 +21,6 @@ use itertools::Itertools; use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{ ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId, - ProtoSerializeOwnExt, }; use risingwave_pb::hummock::ValidationTask; @@ -173,10 +172,7 @@ impl HummockManager { .collect_vec(); if compactor .send_event(ResponseEvent::ValidationTask(ValidationTask { - sst_infos: sst_infos - .into_iter() - .map(|sst| sst.to_protobuf_own()) - .collect_vec(), + sst_infos: sst_infos.into_iter().map(|sst| sst.into()).collect_vec(), sst_id_to_worker_id: sst_to_context.clone(), epoch, })) diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index bb81c8a84cf6..a9ff787cbcda 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -45,8 +45,7 @@ use risingwave_hummock_sdk::version::{CompactTask, HummockVersionDelta, SstableI use risingwave_hummock_sdk::{ version_archive_dir, version_checkpoint_path, CompactionGroupId, ExtendedSstableInfo, HummockCompactionTaskId, HummockContextId, HummockEpoch, HummockSstableId, - HummockSstableObjectId, HummockVersionId, ProtoSerializeExt, ProtoSerializeOwnExt, - SstObjectIdRange, INVALID_VERSION_ID, + HummockSstableObjectId, HummockVersionId, SstObjectIdRange, INVALID_VERSION_ID, }; use risingwave_meta_model_v2::{ compaction_status, compaction_task, hummock_pinned_snapshot, hummock_pinned_version, @@ -63,7 +62,7 @@ use risingwave_pb::hummock::subscribe_compaction_event_response::{ }; use risingwave_pb::hummock::{ CompactTaskAssignment, CompactionConfig, GroupDelta, HummockPinnedSnapshot, - HummockPinnedVersion, HummockSnapshot, HummockVersionStats, IntraLevelDelta, + HummockPinnedVersion, HummockSnapshot, HummockVersionStats, IntraLevelDelta, PbCompactTask, PbCompactionGroupInfo, SubscribeCompactionEventRequest, TableOption, TableSchema, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; @@ -1078,7 +1077,7 @@ impl HummockManager { compact_task_assignment.insert( compact_task.task_id, CompactTaskAssignment { - compact_task: Some(compact_task.to_protobuf()), + compact_task: Some(PbCompactTask::from(&compact_task)), context_id: META_NODE_ID, // deprecated }, ); @@ -1347,9 +1346,7 @@ impl HummockManager { input_task } else { match compact_task_assignment.remove(task_id) { - Some(compact_task) => { - CompactTask::from_protobuf(&compact_task.compact_task.unwrap()) - } + Some(compact_task) => CompactTask::from(&compact_task.compact_task.unwrap()), None => { tracing::warn!("{}", format!("compact task {} not found", task_id)); return Ok(false); @@ -1721,7 +1718,7 @@ impl HummockManager { level_idx: 0, inserted_table_infos: group_sstables .into_iter() - .map(|sst| sst.to_protobuf_own()) + .map(|sst| sst.into()) .collect_vec(), l0_sub_level_id, ..Default::default() @@ -2119,7 +2116,7 @@ impl HummockManager { let compact_task_string = compact_task_to_string(&compact_task); // TODO: shall we need to cancel on meta ? compactor - .send_event(ResponseEvent::CompactTask(compact_task.to_protobuf_own())) + .send_event(ResponseEvent::CompactTask(compact_task.into())) .with_context(|| { format!( "Failed to trigger compaction task for compaction_group {}", @@ -2192,7 +2189,7 @@ impl HummockManager { .last_key_value() .unwrap() .1 - .to_protobuf()], + .into()], }), ); } @@ -3182,7 +3179,7 @@ impl HummockManager { Ok(Some(compact_task)) => { let task_id = compact_task.task_id; if let Err(e) = compactor.send_event( - ResponseEvent::CompactTask(compact_task.to_protobuf_own()), + ResponseEvent::CompactTask(compact_task.into()), ) { tracing::warn!( error = %e.as_report(), @@ -3234,7 +3231,7 @@ impl HummockManager { .report_compact_task( task_id, TaskStatus::try_from(task_status).unwrap(), - sorted_output_ssts.into_iter().map(SstableInfo::from_protobuf_own).collect_vec(), + sorted_output_ssts.into_iter().map(SstableInfo::from).collect_vec(), Some(table_stats_change), ) .await @@ -3356,7 +3353,7 @@ fn gen_version_delta<'a>( inserted_table_infos: compact_task .sorted_output_ssts .iter() - .map(|sst| sst.to_protobuf()) + .map(|sst| sst.into()) .collect_vec(), l0_sub_level_id: compact_task.target_sub_level_id, vnode_partition_count: compact_task.split_weight_by_vnode, diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index 56d27e75c8cb..83fc8fbf07f9 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -29,7 +29,7 @@ use risingwave_hummock_sdk::table_watermark::TableWatermarks; use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{ HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, - ProtoSerializeOwnExt, SstObjectIdRange, + SstObjectIdRange, }; use risingwave_pb::common::{HostAddress, WorkerType}; use risingwave_pb::hummock::compact_task::TaskStatus; @@ -292,7 +292,7 @@ impl HummockMetaClient for MockHummockMetaClient { .unwrap() { let resp = SubscribeCompactionEventResponse { - event: Some(ResponseEvent::CompactTask(task.to_protobuf_own())), + event: Some(ResponseEvent::CompactTask(task.into())), create_at: SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("Clock may have gone backwards") @@ -324,7 +324,7 @@ impl HummockMetaClient for MockHummockMetaClient { TaskStatus::try_from(task_status).unwrap(), sorted_output_ssts .into_iter() - .map(SstableInfo::from_protobuf_own) + .map(SstableInfo::from) .collect_vec(), Some(table_stats_change), ) diff --git a/src/meta/src/hummock/model/ext/hummock.rs b/src/meta/src/hummock/model/ext/hummock.rs index 170c1e9f733e..3faa2bf5c194 100644 --- a/src/meta/src/hummock/model/ext/hummock.rs +++ b/src/meta/src/hummock/model/ext/hummock.rs @@ -14,7 +14,6 @@ use itertools::Itertools; use risingwave_hummock_sdk::version::HummockVersionDelta; -use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta_model_v2::compaction_config::CompactionConfig; use risingwave_meta_model_v2::compaction_status::LevelHandlers; use risingwave_meta_model_v2::compaction_task::CompactionTask; @@ -221,7 +220,7 @@ impl Transactional for HummockVersionDelta { max_committed_epoch: Set(self.max_committed_epoch as _), safe_epoch: Set(self.safe_epoch as _), trivial_move: Set(self.trivial_move), - full_version_delta: Set(FullVersionDelta(self.to_protobuf())), + full_version_delta: Set(FullVersionDelta(self.into())), }; hummock_version_delta::Entity::insert(m) .on_conflict( diff --git a/src/meta/src/hummock/model/version_delta.rs b/src/meta/src/hummock/model/version_delta.rs index c4a36d0ffdf5..cb71a8be88e2 100644 --- a/src/meta/src/hummock/model/version_delta.rs +++ b/src/meta/src/hummock/model/version_delta.rs @@ -29,19 +29,13 @@ impl MetadataModel for HummockVersionDelta { } fn to_protobuf(&self) -> Self::PbType { - use risingwave_hummock_sdk::ProtoSerializeExt; - ::to_protobuf( - self, - ) + self.into() } fn to_protobuf_encoded_vec(&self) -> Vec { use prost::Message; - use risingwave_hummock_sdk::ProtoSerializeExt; - ::to_protobuf( - self, - ) - .encode_to_vec() + let pb = PbHummockVersionDelta::from(self); + pb.encode_to_vec() } fn from_protobuf(prost: Self::PbType) -> Self { diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs index 591988c8942f..8972754ee320 100644 --- a/src/rpc_client/src/meta_client.rs +++ b/src/rpc_client/src/meta_client.rs @@ -39,7 +39,7 @@ use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, - ProtoSerializeExt, SstObjectIdRange, + SstObjectIdRange, }; use risingwave_pb::backup_service::backup_service_client::BackupServiceClient; use risingwave_pb::backup_service::*; @@ -929,7 +929,7 @@ impl MetaClient { version_delta: HummockVersionDelta, ) -> Result<(HummockVersion, Vec)> { let req = ReplayVersionDeltaRequest { - version_delta: Some(version_delta.to_protobuf()), + version_delta: Some(version_delta.into()), }; let resp = self.inner.replay_version_delta(req).await?; Ok(( diff --git a/src/storage/backup/src/meta_snapshot_v1.rs b/src/storage/backup/src/meta_snapshot_v1.rs index 52fec1265603..ac34b4ea6407 100644 --- a/src/storage/backup/src/meta_snapshot_v1.rs +++ b/src/storage/backup/src/meta_snapshot_v1.rs @@ -19,11 +19,10 @@ use bytes::{Buf, BufMut}; use itertools::Itertools; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_pb::catalog::{ Connection, Database, Function, Index, Schema, Sink, Source, Subscription, Table, View, }; -use risingwave_pb::hummock::{CompactionGroup, HummockVersionStats}; +use risingwave_pb::hummock::{CompactionGroup, HummockVersionStats, PbHummockVersion}; use risingwave_pb::meta::{SystemParams, TableFragments}; use risingwave_pb::user::UserInfo; @@ -130,7 +129,7 @@ impl ClusterMetadata { let default_cf_values = self.default_cf.values().collect_vec(); Self::encode_prost_message_list(&default_cf_keys, buf); Self::encode_prost_message_list(&default_cf_values, buf); - Self::encode_prost_message(&self.hummock_version.to_protobuf(), buf); + Self::encode_prost_message(&PbHummockVersion::from(&self.hummock_version), buf); Self::encode_prost_message(&self.version_stats, buf); Self::encode_prost_message_list(&self.compaction_groups.iter().collect_vec(), buf); Self::encode_prost_message_list(&self.table_fragments.iter().collect_vec(), buf); diff --git a/src/storage/backup/src/meta_snapshot_v2.rs b/src/storage/backup/src/meta_snapshot_v2.rs index 9f10ef300de0..0ab2f66fc82f 100644 --- a/src/storage/backup/src/meta_snapshot_v2.rs +++ b/src/storage/backup/src/meta_snapshot_v2.rs @@ -16,8 +16,8 @@ use std::fmt::{Display, Formatter}; use bytes::{Buf, BufMut}; use risingwave_hummock_sdk::version::HummockVersion; -use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta_model_v2 as model_v2; +use risingwave_pb::hummock::PbHummockVersion; use serde::{Deserialize, Serialize}; use crate::meta_snapshot::{MetaSnapshot, Metadata}; @@ -52,7 +52,7 @@ impl Display for MetadataV2 { impl Metadata for MetadataV2 { fn encode_to(&self, buf: &mut Vec) -> BackupResult<()> { put_with_len_prefix(buf, &self.cluster_id)?; - put_with_len_prefix(buf, &self.hummock_version.to_protobuf())?; + put_with_len_prefix(buf, &PbHummockVersion::from(&self.hummock_version))?; put_with_len_prefix(buf, &self.version_stats)?; put_with_len_prefix(buf, &self.compaction_configs)?; // TODO: other metadata diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index bd0ccfaea861..e2c3ab21bdb4 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -33,9 +33,7 @@ use crate::table_watermark::{TableWatermarks, TableWatermarksIndex}; use crate::version::{ HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, }; -use crate::{ - can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId, ProtoSerializeExt, -}; +use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId}; pub struct GroupDeltasSummary { pub delete_sst_levels: Vec, @@ -76,7 +74,7 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary intra_level .inserted_table_infos .iter() - .map(SstableInfo::from_protobuf), + .map(SstableInfo::from), ); } new_vnode_partition_count = intra_level.vnode_partition_count; @@ -431,12 +429,8 @@ impl HummockVersion { if let DeltaType::IntraLevel(delta) = group_delta.get_delta_type().unwrap() { if !delta.inserted_table_infos.is_empty() { info.insert_sst_level = delta.level_idx; - info.insert_sst_infos.extend( - delta - .inserted_table_infos - .iter() - .map(SstableInfo::from_protobuf), - ); + info.insert_sst_infos + .extend(delta.inserted_table_infos.iter().map(SstableInfo::from)); } if !delta.removed_table_ids.is_empty() { for id in &delta.removed_table_ids { @@ -1300,7 +1294,6 @@ mod tests { use crate::version::{ HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, }; - use crate::ProtoSerializeExt; #[test] fn test_get_sst_object_ids() { @@ -1421,7 +1414,7 @@ mod tests { sst_id: 1, ..Default::default() } - .to_protobuf()], + .into()], ..Default::default() })), }], diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index f0d110847a48..853a167b3895 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -359,21 +359,3 @@ impl EpochWithGap { pub trait ProtoSerializeSizeEstimatedExt { fn estimated_encode_len(&self) -> usize; } - -// Define methods for converting Proto struct and rust struct to and from each other -pub trait ProtoSerializeExt { - type PB: Clone + PartialEq + ::prost::Message; - - fn from_protobuf(pb: &Self::PB) -> Self; - - fn to_protobuf(&self) -> Self::PB; -} - -// Similar to `ProtoSerializeExt``, we allow `ProtoSerializeOwnExt to provide a method to obtain ownership. Therefore, copying behavior in memory can be reduced -pub trait ProtoSerializeOwnExt { - type PB: Clone + PartialEq + ::prost::Message; - - fn from_protobuf_own(pb: Self::PB) -> Self; - - fn to_protobuf_own(self) -> Self::PB; -} diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index d1f838671134..8509b059d680 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -29,9 +29,7 @@ use risingwave_pb::hummock::{PbTableWatermarks, PbVnodeWatermark}; use tracing::{debug, warn}; use crate::key::{prefix_slice_with_vnode, vnode, TableKey, TableKeyRange}; -use crate::{ - HummockEpoch, ProtoSerializeExt, ProtoSerializeOwnExt, ProtoSerializeSizeEstimatedExt, -}; +use crate::{HummockEpoch, ProtoSerializeSizeEstimatedExt}; #[derive(Clone)] pub struct ReadTableWatermark { @@ -309,17 +307,17 @@ impl VnodeWatermark { } } -impl ProtoSerializeExt for VnodeWatermark { - type PB = PbVnodeWatermark; - - fn to_protobuf(&self) -> Self::PB { - Self::PB { - watermark: self.watermark.to_vec(), - vnode_bitmap: Some(self.vnode_bitmap.to_protobuf()), +impl From for VnodeWatermark { + fn from(pb: PbVnodeWatermark) -> Self { + Self { + vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), + watermark: Bytes::from(pb.watermark), } } +} - fn from_protobuf(pb: &Self::PB) -> Self { +impl From<&PbVnodeWatermark> for VnodeWatermark { + fn from(pb: &PbVnodeWatermark) -> Self { Self { vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), watermark: Bytes::from(pb.watermark.clone()), @@ -327,20 +325,20 @@ impl ProtoSerializeExt for VnodeWatermark { } } -impl ProtoSerializeOwnExt for VnodeWatermark { - type PB = PbVnodeWatermark; - - fn to_protobuf_own(self) -> Self::PB { - Self::PB { - watermark: self.watermark.into(), - vnode_bitmap: Some(self.vnode_bitmap.to_protobuf()), // TODO: make Bitmao without copy +impl From for PbVnodeWatermark { + fn from(watermark: VnodeWatermark) -> Self { + Self { + watermark: watermark.watermark.into(), + vnode_bitmap: Some(watermark.vnode_bitmap.to_protobuf()), } } +} - fn from_protobuf_own(pb: Self::PB) -> Self { +impl From<&VnodeWatermark> for PbVnodeWatermark { + fn from(watermark: &VnodeWatermark) -> Self { Self { - vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())), - watermark: Bytes::from(pb.watermark), + watermark: watermark.watermark.to_vec(), + vnode_bitmap: Some(watermark.vnode_bitmap.to_protobuf()), } } } @@ -554,27 +552,8 @@ impl ProtoSerializeSizeEstimatedExt for TableWatermarks { } } -impl ProtoSerializeExt for TableWatermarks { - type PB = PbTableWatermarks; - - fn to_protobuf(&self) -> Self::PB { - Self::PB { - epoch_watermarks: self - .watermarks - .iter() - .map(|(epoch, watermarks)| PbEpochNewWatermarks { - watermarks: watermarks.iter().map(VnodeWatermark::to_protobuf).collect(), - epoch: *epoch, - }) - .collect(), - is_ascending: match self.direction { - WatermarkDirection::Ascending => true, - WatermarkDirection::Descending => false, - }, - } - } - - fn from_protobuf(pb: &Self::PB) -> Self { +impl From<&PbTableWatermarks> for TableWatermarks { + fn from(pb: &PbTableWatermarks) -> Self { Self { watermarks: pb .epoch_watermarks @@ -584,7 +563,7 @@ impl ProtoSerializeExt for TableWatermarks { let watermarks = epoch_watermark .watermarks .iter() - .map(VnodeWatermark::from_protobuf) + .map(VnodeWatermark::from) .collect(); (epoch, watermarks) }) @@ -598,30 +577,27 @@ impl ProtoSerializeExt for TableWatermarks { } } -impl ProtoSerializeOwnExt for TableWatermarks { - type PB = PbTableWatermarks; - - fn to_protobuf_own(self) -> Self::PB { - Self::PB { - epoch_watermarks: self +impl From<&TableWatermarks> for PbTableWatermarks { + fn from(table_watermarks: &TableWatermarks) -> Self { + Self { + epoch_watermarks: table_watermarks .watermarks - .into_iter() + .iter() .map(|(epoch, watermarks)| PbEpochNewWatermarks { - watermarks: watermarks - .into_iter() - .map(VnodeWatermark::to_protobuf_own) - .collect(), - epoch, + watermarks: watermarks.iter().map(|wm| wm.into()).collect(), + epoch: *epoch, }) .collect(), - is_ascending: match self.direction { + is_ascending: match table_watermarks.direction { WatermarkDirection::Ascending => true, WatermarkDirection::Descending => false, }, } } +} - fn from_protobuf_own(pb: Self::PB) -> Self { +impl From for TableWatermarks { + fn from(pb: PbTableWatermarks) -> Self { Self { watermarks: pb .epoch_watermarks @@ -631,7 +607,7 @@ impl ProtoSerializeOwnExt for TableWatermarks { let watermarks = epoch_watermark .watermarks .into_iter() - .map(VnodeWatermark::from_protobuf_own) + .map(VnodeWatermark::from) .collect(); (epoch, watermarks) }) @@ -645,6 +621,25 @@ impl ProtoSerializeOwnExt for TableWatermarks { } } +impl From for PbTableWatermarks { + fn from(table_watermarks: TableWatermarks) -> Self { + Self { + epoch_watermarks: table_watermarks + .watermarks + .into_iter() + .map(|(epoch, watermarks)| PbEpochNewWatermarks { + watermarks: watermarks.into_iter().map(VnodeWatermark::into).collect(), + epoch, + }) + .collect(), + is_ascending: match table_watermarks.direction { + WatermarkDirection::Ascending => true, + WatermarkDirection::Descending => false, + }, + } + } +} + #[cfg(test)] mod tests { use std::collections::Bound::Included; diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 2fe687893bdd..ecf25a94468b 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -30,10 +30,7 @@ use serde::Serialize; use crate::key_range::KeyRange; use crate::table_watermark::TableWatermarks; -use crate::{ - CompactionGroupId, HummockSstableObjectId, ProtoSerializeExt, ProtoSerializeOwnExt, - ProtoSerializeSizeEstimatedExt, -}; +use crate::{CompactionGroupId, HummockSstableObjectId, ProtoSerializeSizeEstimatedExt}; #[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct OverlappingLevel { @@ -42,30 +39,58 @@ pub struct OverlappingLevel { pub uncompressed_file_size: u64, } -impl ProtoSerializeExt for OverlappingLevel { - type PB = PbOverlappingLevel; - - fn from_protobuf(pb_overlapping_level: &Self::PB) -> Self { +impl From<&PbOverlappingLevel> for OverlappingLevel { + fn from(pb_overlapping_level: &PbOverlappingLevel) -> Self { Self { sub_levels: pb_overlapping_level .sub_levels .iter() - .map(Level::from_protobuf) + .map(Level::from) .collect_vec(), total_file_size: pb_overlapping_level.total_file_size, uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, } } +} - fn to_protobuf(&self) -> Self::PB { - Self::PB { - sub_levels: self +impl From<&OverlappingLevel> for PbOverlappingLevel { + fn from(overlapping_level: &OverlappingLevel) -> Self { + Self { + sub_levels: overlapping_level .sub_levels .iter() - .map(|pb_level| pb_level.to_protobuf()) + .map(|pb_level| pb_level.into()) .collect_vec(), - total_file_size: self.total_file_size, - uncompressed_file_size: self.uncompressed_file_size, + total_file_size: overlapping_level.total_file_size, + uncompressed_file_size: overlapping_level.uncompressed_file_size, + } + } +} + +impl From for PbOverlappingLevel { + fn from(overlapping_level: OverlappingLevel) -> Self { + Self { + sub_levels: overlapping_level + .sub_levels + .into_iter() + .map(|pb_level| pb_level.into()) + .collect_vec(), + total_file_size: overlapping_level.total_file_size, + uncompressed_file_size: overlapping_level.uncompressed_file_size, + } + } +} + +impl From for OverlappingLevel { + fn from(pb_overlapping_level: PbOverlappingLevel) -> Self { + Self { + sub_levels: pb_overlapping_level + .sub_levels + .into_iter() + .map(Level::from) + .collect_vec(), + total_file_size: pb_overlapping_level.total_file_size, + uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, } } } @@ -98,17 +123,15 @@ pub struct Level { pub vnode_partition_count: u32, } -impl ProtoSerializeExt for Level { - type PB = PbLevel; - - fn from_protobuf(pb_level: &Self::PB) -> Self { +impl From<&PbLevel> for Level { + fn from(pb_level: &PbLevel) -> Self { Self { level_idx: pb_level.level_idx, level_type: LevelType::try_from(pb_level.level_type).unwrap(), table_infos: pb_level .table_infos .iter() - .map(SstableInfo::from_protobuf) + .map(SstableInfo::from) .collect_vec(), total_file_size: pb_level.total_file_size, sub_level_id: pb_level.sub_level_id, @@ -116,20 +139,58 @@ impl ProtoSerializeExt for Level { vnode_partition_count: pb_level.vnode_partition_count, } } +} - fn to_protobuf(&self) -> Self::PB { - Self::PB { - level_idx: self.level_idx, - level_type: self.level_type.into(), - table_infos: self +impl From<&Level> for PbLevel { + fn from(level: &Level) -> Self { + Self { + level_idx: level.level_idx, + level_type: level.level_type.into(), + table_infos: level .table_infos .iter() - .map(|sst| sst.to_protobuf()) + .map(PbSstableInfo::from) + .collect_vec(), + total_file_size: level.total_file_size, + sub_level_id: level.sub_level_id, + uncompressed_file_size: level.uncompressed_file_size, + vnode_partition_count: level.vnode_partition_count, + } + } +} + +impl From for PbLevel { + fn from(level: Level) -> Self { + Self { + level_idx: level.level_idx, + level_type: level.level_type.into(), + table_infos: level + .table_infos + .into_iter() + .map(PbSstableInfo::from) .collect_vec(), - total_file_size: self.total_file_size, - sub_level_id: self.sub_level_id, - uncompressed_file_size: self.uncompressed_file_size, - vnode_partition_count: self.vnode_partition_count, + total_file_size: level.total_file_size, + sub_level_id: level.sub_level_id, + uncompressed_file_size: level.uncompressed_file_size, + vnode_partition_count: level.vnode_partition_count, + } + } +} + +impl From for Level { + fn from(pb_level: PbLevel) -> Self { + Self { + level_idx: pb_level.level_idx, + level_type: LevelType::try_from(pb_level.level_type).unwrap(), + table_infos: pb_level + .table_infos + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + total_file_size: pb_level.total_file_size, + sub_level_id: pb_level.sub_level_id, + uncompressed_file_size: pb_level.uncompressed_file_size, + vnode_partition_count: pb_level.vnode_partition_count, } } } @@ -209,44 +270,66 @@ impl ProtoSerializeSizeEstimatedExt for Levels { } } -impl ProtoSerializeExt for Levels { - type PB = PbLevels; +impl From<&PbLevels> for Levels { + fn from(pb_levels: &PbLevels) -> Self { + Self { + l0: if pb_levels.l0.is_some() { + Some(OverlappingLevel::from(pb_levels.l0.as_ref().unwrap())) + } else { + None + }, + levels: pb_levels.levels.iter().map(Level::from).collect_vec(), + group_id: pb_levels.group_id, + parent_group_id: pb_levels.parent_group_id, + member_table_ids: pb_levels.member_table_ids.clone(), + } + } +} + +impl From<&Levels> for PbLevels { + fn from(levels: &Levels) -> Self { + Self { + l0: if levels.l0.is_some() { + Some(levels.l0.as_ref().unwrap().into()) + } else { + None + }, + levels: levels.levels.iter().map(PbLevel::from).collect_vec(), + group_id: levels.group_id, + parent_group_id: levels.parent_group_id, + member_table_ids: levels.member_table_ids.clone(), + } + } +} - fn from_protobuf(pb_levels: &Self::PB) -> Self { +impl From for Levels { + fn from(pb_levels: PbLevels) -> Self { Self { l0: if pb_levels.l0.is_some() { - Some(OverlappingLevel::from_protobuf( - pb_levels.l0.as_ref().unwrap(), - )) + Some(OverlappingLevel::from(pb_levels.l0.unwrap())) } else { None }, - levels: pb_levels - .levels - .iter() - .map(Level::from_protobuf) - .collect_vec(), + levels: pb_levels.levels.into_iter().map(Level::from).collect_vec(), group_id: pb_levels.group_id, parent_group_id: pb_levels.parent_group_id, member_table_ids: pb_levels.member_table_ids.clone(), } } +} - fn to_protobuf(&self) -> Self::PB { - Self::PB { - l0: if self.l0.is_some() { - Some(self.l0.as_ref().unwrap().to_protobuf()) +impl From for PbLevels { + fn from(levels: Levels) -> Self { + Self { + l0: if levels.l0.is_some() { + Some(levels.l0.unwrap().into()) } else { None }, - levels: self - .levels - .iter() - .map(|level| level.to_protobuf()) - .collect_vec(), - group_id: self.group_id, - parent_group_id: self.parent_group_id, - member_table_ids: self.member_table_ids.clone(), + levels: levels.levels.into_iter().map(PbLevel::from).collect_vec(), + group_id: levels.group_id, + parent_group_id: levels.parent_group_id, + member_table_ids: levels.member_table_ids, } } } @@ -262,7 +345,7 @@ pub struct HummockVersion { impl Default for HummockVersion { fn default() -> Self { - HummockVersion::from_protobuf(&PbHummockVersion::default()) + HummockVersion::from(&PbHummockVersion::default()) } } @@ -270,13 +353,13 @@ impl HummockVersion { /// Convert the `PbHummockVersion` received from rpc to `HummockVersion`. No need to /// maintain backward compatibility. pub fn from_rpc_protobuf(pb_version: &PbHummockVersion) -> Self { - Self::from_protobuf(pb_version) + HummockVersion::from(pb_version) } /// Convert the `PbHummockVersion` deserialized from persisted state to `HummockVersion`. /// We should maintain backward compatibility. pub fn from_persisted_protobuf(pb_version: &PbHummockVersion) -> Self { - Self::from_protobuf(pb_version) + HummockVersion::from(pb_version) } } @@ -297,21 +380,14 @@ impl ProtoSerializeSizeEstimatedExt for HummockVersion { } } -impl ProtoSerializeExt for HummockVersion { - type PB = PbHummockVersion; - - fn from_protobuf(pb_version: &Self::PB) -> Self { +impl From<&PbHummockVersion> for HummockVersion { + fn from(pb_version: &PbHummockVersion) -> Self { Self { id: pb_version.id, levels: pb_version .levels .iter() - .map(|(group_id, levels)| { - ( - *group_id as CompactionGroupId, - Levels::from_protobuf(levels), - ) - }) + .map(|(group_id, levels)| (*group_id as CompactionGroupId, Levels::from(levels))) .collect(), max_committed_epoch: pb_version.max_committed_epoch, safe_epoch: pb_version.safe_epoch, @@ -321,27 +397,49 @@ impl ProtoSerializeExt for HummockVersion { .map(|(table_id, table_watermark)| { ( TableId::new(*table_id), - TableWatermarks::from_protobuf(table_watermark), + TableWatermarks::from(table_watermark), ) }) .collect(), } } +} - fn to_protobuf(&self) -> Self::PB { - Self::PB { - id: self.id, - levels: self +impl From<&HummockVersion> for PbHummockVersion { + fn from(version: &HummockVersion) -> Self { + Self { + id: version.id, + levels: version .levels .iter() - .map(|(group_id, levels)| (*group_id as _, levels.to_protobuf())) + .map(|(group_id, levels)| (*group_id as _, levels.into())) .collect(), - max_committed_epoch: self.max_committed_epoch, - safe_epoch: self.safe_epoch, - table_watermarks: self + max_committed_epoch: version.max_committed_epoch, + safe_epoch: version.safe_epoch, + table_watermarks: version .table_watermarks .iter() - .map(|(table_id, watermark)| (table_id.table_id, watermark.to_protobuf())) + .map(|(table_id, watermark)| (table_id.table_id, watermark.into())) + .collect(), + } + } +} + +impl From for PbHummockVersion { + fn from(version: HummockVersion) -> Self { + Self { + id: version.id, + levels: version + .levels + .into_iter() + .map(|(group_id, levels)| (group_id as _, levels.into())) + .collect(), + max_committed_epoch: version.max_committed_epoch, + safe_epoch: version.safe_epoch, + table_watermarks: version + .table_watermarks + .into_iter() + .map(|(table_id, watermark)| (table_id.table_id, watermark.into())) .collect(), } } @@ -362,7 +460,7 @@ pub struct HummockVersionDelta { impl Default for HummockVersionDelta { fn default() -> Self { - HummockVersionDelta::from_protobuf(&PbHummockVersionDelta::default()) + HummockVersionDelta::from(&PbHummockVersionDelta::default()) } } @@ -370,61 +468,58 @@ impl HummockVersionDelta { /// Convert the `PbHummockVersionDelta` deserialized from persisted state to `HummockVersionDelta`. /// We should maintain backward compatibility. pub fn from_persisted_protobuf(delta: &PbHummockVersionDelta) -> Self { - Self::from_protobuf(delta) + Self::from(delta) } /// Convert the `PbHummockVersionDelta` received from rpc to `HummockVersionDelta`. No need to /// maintain backward compatibility. pub fn from_rpc_protobuf(delta: &PbHummockVersionDelta) -> Self { - Self::from_protobuf(delta) + Self::from(delta) } } -impl ProtoSerializeExt for HummockVersionDelta { - type PB = PbHummockVersionDelta; - - fn from_protobuf(delta: &Self::PB) -> Self { +impl From<&PbHummockVersionDelta> for HummockVersionDelta { + fn from(pb_version_delta: &PbHummockVersionDelta) -> Self { Self { - id: delta.id, - prev_id: delta.prev_id, - group_deltas: delta.group_deltas.clone(), - max_committed_epoch: delta.max_committed_epoch, - safe_epoch: delta.safe_epoch, - trivial_move: delta.trivial_move, - gc_object_ids: delta.gc_object_ids.clone(), - new_table_watermarks: delta + id: pb_version_delta.id, + prev_id: pb_version_delta.prev_id, + group_deltas: pb_version_delta.group_deltas.clone(), + max_committed_epoch: pb_version_delta.max_committed_epoch, + safe_epoch: pb_version_delta.safe_epoch, + trivial_move: pb_version_delta.trivial_move, + gc_object_ids: pb_version_delta.gc_object_ids.clone(), + new_table_watermarks: pb_version_delta .new_table_watermarks .iter() .map(|(table_id, watermarks)| { - ( - TableId::new(*table_id), - TableWatermarks::from_protobuf(watermarks), - ) + (TableId::new(*table_id), TableWatermarks::from(watermarks)) }) .collect(), - removed_table_ids: delta + removed_table_ids: pb_version_delta .removed_table_ids .iter() .map(|table_id| TableId::new(*table_id)) .collect(), } } +} - fn to_protobuf(&self) -> Self::PB { - Self::PB { - id: self.id, - prev_id: self.prev_id, - group_deltas: self.group_deltas.clone(), - max_committed_epoch: self.max_committed_epoch, - safe_epoch: self.safe_epoch, - trivial_move: self.trivial_move, - gc_object_ids: self.gc_object_ids.clone(), - new_table_watermarks: self +impl From<&HummockVersionDelta> for PbHummockVersionDelta { + fn from(version_delta: &HummockVersionDelta) -> Self { + Self { + id: version_delta.id, + prev_id: version_delta.prev_id, + group_deltas: version_delta.group_deltas.clone(), + max_committed_epoch: version_delta.max_committed_epoch, + safe_epoch: version_delta.safe_epoch, + trivial_move: version_delta.trivial_move, + gc_object_ids: version_delta.gc_object_ids.clone(), + new_table_watermarks: version_delta .new_table_watermarks .iter() - .map(|(table_id, watermarks)| (table_id.table_id, watermarks.to_protobuf())) + .map(|(table_id, watermarks)| (table_id.table_id, watermarks.into())) .collect(), - removed_table_ids: self + removed_table_ids: version_delta .removed_table_ids .iter() .map(|table_id| table_id.table_id) @@ -433,6 +528,54 @@ impl ProtoSerializeExt for HummockVersionDelta { } } +impl From for PbHummockVersionDelta { + fn from(version_delta: HummockVersionDelta) -> Self { + Self { + id: version_delta.id, + prev_id: version_delta.prev_id, + group_deltas: version_delta.group_deltas, + max_committed_epoch: version_delta.max_committed_epoch, + safe_epoch: version_delta.safe_epoch, + trivial_move: version_delta.trivial_move, + gc_object_ids: version_delta.gc_object_ids, + new_table_watermarks: version_delta + .new_table_watermarks + .into_iter() + .map(|(table_id, watermarks)| (table_id.table_id, watermarks.into())) + .collect(), + removed_table_ids: version_delta + .removed_table_ids + .into_iter() + .map(|table_id| table_id.table_id) + .collect(), + } + } +} + +impl From for HummockVersionDelta { + fn from(pb_version_delta: PbHummockVersionDelta) -> Self { + Self { + id: pb_version_delta.id, + prev_id: pb_version_delta.prev_id, + group_deltas: pb_version_delta.group_deltas, + max_committed_epoch: pb_version_delta.max_committed_epoch, + safe_epoch: pb_version_delta.safe_epoch, + trivial_move: pb_version_delta.trivial_move, + gc_object_ids: pb_version_delta.gc_object_ids, + new_table_watermarks: pb_version_delta + .new_table_watermarks + .into_iter() + .map(|(table_id, watermarks)| (TableId::new(table_id), watermarks.into())) + .collect(), + removed_table_ids: pb_version_delta + .removed_table_ids + .into_iter() + .map(TableId::new) + .collect(), + } + } +} + #[derive(Debug, PartialEq, Clone, Default, Serialize)] pub struct SstableInfo { pub object_id: u64, @@ -473,18 +616,16 @@ impl ProtoSerializeSizeEstimatedExt for SstableInfo { } } -impl ProtoSerializeExt for SstableInfo { - type PB = PbSstableInfo; - - fn from_protobuf(pb_sstable_info: &Self::PB) -> Self { +impl From for SstableInfo { + fn from(pb_sstable_info: PbSstableInfo) -> Self { Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, key_range: if pb_sstable_info.key_range.is_some() { - let pb_keyrange = pb_sstable_info.key_range.as_ref().unwrap(); + let pb_keyrange = pb_sstable_info.key_range.unwrap(); let key_range = KeyRange { - left: Bytes::from(pb_keyrange.left.clone()), - right: Bytes::from(pb_keyrange.right.clone()), + left: Bytes::from(pb_keyrange.left), + right: Bytes::from(pb_keyrange.right), right_exclusive: pb_keyrange.right_exclusive, }; Some(key_range) @@ -505,49 +646,18 @@ impl ProtoSerializeExt for SstableInfo { .unwrap(), } } - - fn to_protobuf(&self) -> Self::PB { - PbSstableInfo { - object_id: self.object_id, - sst_id: self.sst_id, - key_range: if self.key_range.is_some() { - let keyrange = self.key_range.as_ref().unwrap(); - let pb_key_range = PbKeyRange { - left: keyrange.left.to_vec(), - right: keyrange.right.to_vec(), - right_exclusive: keyrange.right_exclusive, - }; - Some(pb_key_range) - } else { - None - }, - - file_size: self.file_size, - table_ids: self.table_ids.clone(), - meta_offset: self.meta_offset, - stale_key_count: self.stale_key_count, - total_key_count: self.total_key_count, - min_epoch: self.min_epoch, - max_epoch: self.max_epoch, - uncompressed_file_size: self.uncompressed_file_size, - range_tombstone_count: self.range_tombstone_count, - bloom_filter_kind: self.bloom_filter_kind.into(), - } - } } -impl ProtoSerializeOwnExt for SstableInfo { - type PB = PbSstableInfo; - - fn from_protobuf_own(pb_sstable_info: Self::PB) -> Self { +impl From<&PbSstableInfo> for SstableInfo { + fn from(pb_sstable_info: &PbSstableInfo) -> Self { Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, key_range: if pb_sstable_info.key_range.is_some() { - let pb_keyrange = pb_sstable_info.key_range.unwrap(); + let pb_keyrange = pb_sstable_info.key_range.as_ref().unwrap(); let key_range = KeyRange { - left: Bytes::from(pb_keyrange.left), - right: Bytes::from(pb_keyrange.right), + left: Bytes::from(pb_keyrange.left.clone()), + right: Bytes::from(pb_keyrange.right.clone()), right_exclusive: pb_keyrange.right_exclusive, }; Some(key_range) @@ -568,13 +678,15 @@ impl ProtoSerializeOwnExt for SstableInfo { .unwrap(), } } +} - fn to_protobuf_own(self) -> Self::PB { - Self::PB { - object_id: self.object_id, - sst_id: self.sst_id, - key_range: if self.key_range.is_some() { - let keyrange = self.key_range.unwrap(); +impl From for PbSstableInfo { + fn from(sstable_info: SstableInfo) -> Self { + PbSstableInfo { + object_id: sstable_info.object_id, + sst_id: sstable_info.sst_id, + key_range: if sstable_info.key_range.is_some() { + let keyrange = sstable_info.key_range.unwrap(); let pb_key_range = PbKeyRange { left: keyrange.left.into(), right: keyrange.right.into(), @@ -585,16 +697,47 @@ impl ProtoSerializeOwnExt for SstableInfo { None }, - file_size: self.file_size, - table_ids: self.table_ids.clone(), - meta_offset: self.meta_offset, - stale_key_count: self.stale_key_count, - total_key_count: self.total_key_count, - min_epoch: self.min_epoch, - max_epoch: self.max_epoch, - uncompressed_file_size: self.uncompressed_file_size, - range_tombstone_count: self.range_tombstone_count, - bloom_filter_kind: self.bloom_filter_kind.into(), + file_size: sstable_info.file_size, + table_ids: sstable_info.table_ids.clone(), + meta_offset: sstable_info.meta_offset, + stale_key_count: sstable_info.stale_key_count, + total_key_count: sstable_info.total_key_count, + min_epoch: sstable_info.min_epoch, + max_epoch: sstable_info.max_epoch, + uncompressed_file_size: sstable_info.uncompressed_file_size, + range_tombstone_count: sstable_info.range_tombstone_count, + bloom_filter_kind: sstable_info.bloom_filter_kind.into(), + } + } +} + +impl From<&SstableInfo> for PbSstableInfo { + fn from(sstable_info: &SstableInfo) -> Self { + PbSstableInfo { + object_id: sstable_info.object_id, + sst_id: sstable_info.sst_id, + key_range: if sstable_info.key_range.is_some() { + let keyrange = sstable_info.key_range.as_ref().unwrap(); + let pb_key_range = PbKeyRange { + left: keyrange.left.to_vec(), + right: keyrange.right.to_vec(), + right_exclusive: keyrange.right_exclusive, + }; + Some(pb_key_range) + } else { + None + }, + + file_size: sstable_info.file_size, + table_ids: sstable_info.table_ids.clone(), + meta_offset: sstable_info.meta_offset, + stale_key_count: sstable_info.stale_key_count, + total_key_count: sstable_info.total_key_count, + min_epoch: sstable_info.min_epoch, + max_epoch: sstable_info.max_epoch, + uncompressed_file_size: sstable_info.uncompressed_file_size, + range_tombstone_count: sstable_info.range_tombstone_count, + bloom_filter_kind: sstable_info.bloom_filter_kind.into(), } } } @@ -632,57 +775,57 @@ impl ProtoSerializeSizeEstimatedExt for InputLevel { } } -impl ProtoSerializeExt for InputLevel { - type PB = PbInputLevel; - - fn from_protobuf(pb_input_level: &Self::PB) -> Self { +impl From for InputLevel { + fn from(pb_input_level: PbInputLevel) -> Self { Self { level_idx: pb_input_level.level_idx, level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), table_infos: pb_input_level .table_infos - .iter() - .map(SstableInfo::from_protobuf) + .into_iter() + .map(SstableInfo::from) .collect_vec(), } } +} - fn to_protobuf(&self) -> Self::PB { - Self::PB { - level_idx: self.level_idx, - level_type: self.level_type.into(), - table_infos: self +impl From<&PbInputLevel> for InputLevel { + fn from(pb_input_level: &PbInputLevel) -> Self { + Self { + level_idx: pb_input_level.level_idx, + level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), + table_infos: pb_input_level .table_infos .iter() - .map(|sst| sst.to_protobuf()) + .map(SstableInfo::from) .collect_vec(), } } } -impl ProtoSerializeOwnExt for InputLevel { - type PB = PbInputLevel; - - fn from_protobuf_own(pb_input_level: Self::PB) -> Self { +impl From for PbInputLevel { + fn from(input_level: InputLevel) -> Self { Self { - level_idx: pb_input_level.level_idx, - level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), - table_infos: pb_input_level + level_idx: input_level.level_idx, + level_type: input_level.level_type.into(), + table_infos: input_level .table_infos .into_iter() - .map(SstableInfo::from_protobuf_own) + .map(|sst| sst.into()) .collect_vec(), } } +} - fn to_protobuf_own(self) -> Self::PB { - Self::PB { - level_idx: self.level_idx, - level_type: self.level_type.into(), - table_infos: self +impl From<&InputLevel> for PbInputLevel { + fn from(input_level: &InputLevel) -> Self { + Self { + level_idx: input_level.level_idx, + level_type: input_level.level_type.into(), + table_infos: input_level .table_infos - .into_iter() - .map(|sst| sst.to_protobuf_own()) + .iter() + .map(|sst| sst.into()) .collect_vec(), } } @@ -788,30 +931,28 @@ impl ProtoSerializeSizeEstimatedExt for CompactTask { } } -impl ProtoSerializeExt for CompactTask { - type PB = PbCompactTask; - - fn from_protobuf(pb_compact_task: &Self::PB) -> Self { +impl From for CompactTask { + fn from(pb_compact_task: PbCompactTask) -> Self { Self { input_ssts: pb_compact_task .input_ssts - .iter() - .map(InputLevel::from_protobuf) + .into_iter() + .map(InputLevel::from) .collect_vec(), splits: pb_compact_task .splits - .iter() + .into_iter() .map(|pb_keyrange| KeyRange { - left: Bytes::from(pb_keyrange.left.clone()), - right: Bytes::from(pb_keyrange.right.clone()), + left: Bytes::from(pb_keyrange.left), + right: Bytes::from(pb_keyrange.right), right_exclusive: pb_keyrange.right_exclusive, }) .collect_vec(), watermark: pb_compact_task.watermark, sorted_output_ssts: pb_compact_task .sorted_output_ssts - .iter() - .map(SstableInfo::from_protobuf) + .into_iter() + .map(SstableInfo::from) .collect_vec(), task_id: pb_compact_task.task_id, target_level: pb_compact_task.target_level, @@ -833,91 +974,38 @@ impl ProtoSerializeExt for CompactTask { table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), table_watermarks: pb_compact_task .table_watermarks - .iter() + .into_iter() .map(|(table_id, pb_table_watermark)| { - ( - *table_id, - TableWatermarks::from_protobuf(pb_table_watermark), - ) + (table_id, TableWatermarks::from(pb_table_watermark)) }) .collect(), - table_schemas: pb_compact_task.table_schemas.clone(), - } - } - - fn to_protobuf(&self) -> Self::PB { - Self::PB { - input_ssts: self - .input_ssts - .iter() - .map(|input_level| input_level.to_protobuf()) - .collect_vec(), - splits: self - .splits - .iter() - .map(|keyrange| PbKeyRange { - left: keyrange.left.to_vec(), - right: keyrange.right.to_vec(), - right_exclusive: keyrange.right_exclusive, - }) - .collect_vec(), - watermark: self.watermark, - sorted_output_ssts: self - .sorted_output_ssts - .iter() - .map(|sst| sst.to_protobuf()) - .collect_vec(), - task_id: self.task_id, - target_level: self.target_level, - gc_delete_keys: self.gc_delete_keys, - base_level: self.base_level, - task_status: self.task_status.into(), - compaction_group_id: self.compaction_group_id, - existing_table_ids: self.existing_table_ids.clone(), - compression_algorithm: self.compression_algorithm, - target_file_size: self.target_file_size, - compaction_filter_mask: self.compaction_filter_mask, - table_options: self.table_options.clone(), - current_epoch_time: self.current_epoch_time, - target_sub_level_id: self.target_sub_level_id, - task_type: self.task_type.into(), - //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, - split_weight_by_vnode: self.split_weight_by_vnode, - table_vnode_partition: self.table_vnode_partition.clone(), - table_watermarks: self - .table_watermarks - .iter() - .map(|(table_id, table_watermark)| (*table_id, table_watermark.to_protobuf())) - .collect(), - ..Default::default() + table_schemas: pb_compact_task.table_schemas, } } } -impl ProtoSerializeOwnExt for CompactTask { - type PB = PbCompactTask; - - fn from_protobuf_own(pb_compact_task: Self::PB) -> Self { +impl From<&PbCompactTask> for CompactTask { + fn from(pb_compact_task: &PbCompactTask) -> Self { Self { input_ssts: pb_compact_task .input_ssts - .into_iter() - .map(InputLevel::from_protobuf_own) + .iter() + .map(InputLevel::from) .collect_vec(), splits: pb_compact_task .splits - .into_iter() + .iter() .map(|pb_keyrange| KeyRange { - left: Bytes::from(pb_keyrange.left), - right: Bytes::from(pb_keyrange.right), + left: Bytes::from(pb_keyrange.left.clone()), + right: Bytes::from(pb_keyrange.right.clone()), right_exclusive: pb_keyrange.right_exclusive, }) .collect_vec(), watermark: pb_compact_task.watermark, sorted_output_ssts: pb_compact_task .sorted_output_ssts - .into_iter() - .map(SstableInfo::from_protobuf_own) + .iter() + .map(SstableInfo::from) .collect_vec(), task_id: pb_compact_task.task_id, target_level: pb_compact_task.target_level, @@ -939,26 +1027,25 @@ impl ProtoSerializeOwnExt for CompactTask { table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), table_watermarks: pb_compact_task .table_watermarks - .into_iter() + .iter() .map(|(table_id, pb_table_watermark)| { - ( - table_id, - TableWatermarks::from_protobuf_own(pb_table_watermark), - ) + (*table_id, TableWatermarks::from(pb_table_watermark)) }) .collect(), - table_schemas: pb_compact_task.table_schemas, + table_schemas: pb_compact_task.table_schemas.clone(), } } +} - fn to_protobuf_own(self) -> Self::PB { - Self::PB { - input_ssts: self +impl From for PbCompactTask { + fn from(compact_task: CompactTask) -> Self { + Self { + input_ssts: compact_task .input_ssts .into_iter() - .map(|input_level| input_level.to_protobuf_own()) + .map(|input_level| input_level.into()) .collect_vec(), - splits: self + splits: compact_task .splits .into_iter() .map(|keyrange| PbKeyRange { @@ -967,33 +1054,83 @@ impl ProtoSerializeOwnExt for CompactTask { right_exclusive: keyrange.right_exclusive, }) .collect_vec(), - watermark: self.watermark, - sorted_output_ssts: self + watermark: compact_task.watermark, + sorted_output_ssts: compact_task .sorted_output_ssts .into_iter() - .map(|sst| sst.to_protobuf_own()) + .map(|sst| sst.into()) .collect_vec(), - task_id: self.task_id, - target_level: self.target_level, - gc_delete_keys: self.gc_delete_keys, - base_level: self.base_level, - task_status: self.task_status.into(), - compaction_group_id: self.compaction_group_id, - existing_table_ids: self.existing_table_ids.clone(), - compression_algorithm: self.compression_algorithm, - target_file_size: self.target_file_size, - compaction_filter_mask: self.compaction_filter_mask, - table_options: self.table_options.clone(), - current_epoch_time: self.current_epoch_time, - target_sub_level_id: self.target_sub_level_id, - task_type: self.task_type.into(), + task_id: compact_task.task_id, + target_level: compact_task.target_level, + gc_delete_keys: compact_task.gc_delete_keys, + base_level: compact_task.base_level, + task_status: compact_task.task_status.into(), + compaction_group_id: compact_task.compaction_group_id, + existing_table_ids: compact_task.existing_table_ids.clone(), + compression_algorithm: compact_task.compression_algorithm, + target_file_size: compact_task.target_file_size, + compaction_filter_mask: compact_task.compaction_filter_mask, + table_options: compact_task.table_options.clone(), + current_epoch_time: compact_task.current_epoch_time, + target_sub_level_id: compact_task.target_sub_level_id, + task_type: compact_task.task_type.into(), //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, - split_weight_by_vnode: self.split_weight_by_vnode, - table_vnode_partition: self.table_vnode_partition.clone(), - table_watermarks: self + split_weight_by_vnode: compact_task.split_weight_by_vnode, + table_vnode_partition: compact_task.table_vnode_partition.clone(), + table_watermarks: compact_task .table_watermarks .into_iter() - .map(|(table_id, table_watermark)| (table_id, table_watermark.to_protobuf_own())) + .map(|(table_id, table_watermark)| (table_id, table_watermark.into())) + .collect(), + ..Default::default() + } + } +} + +impl From<&CompactTask> for PbCompactTask { + fn from(compact_task: &CompactTask) -> Self { + Self { + input_ssts: compact_task + .input_ssts + .iter() + .map(|input_level| input_level.into()) + .collect_vec(), + splits: compact_task + .splits + .iter() + .map(|keyrange| PbKeyRange { + left: keyrange.left.to_vec(), + right: keyrange.right.to_vec(), + right_exclusive: keyrange.right_exclusive, + }) + .collect_vec(), + watermark: compact_task.watermark, + sorted_output_ssts: compact_task + .sorted_output_ssts + .iter() + .map(|sst| sst.into()) + .collect_vec(), + task_id: compact_task.task_id, + target_level: compact_task.target_level, + gc_delete_keys: compact_task.gc_delete_keys, + base_level: compact_task.base_level, + task_status: compact_task.task_status.into(), + compaction_group_id: compact_task.compaction_group_id, + existing_table_ids: compact_task.existing_table_ids.clone(), + compression_algorithm: compact_task.compression_algorithm, + target_file_size: compact_task.target_file_size, + compaction_filter_mask: compact_task.compaction_filter_mask, + table_options: compact_task.table_options.clone(), + current_epoch_time: compact_task.current_epoch_time, + target_sub_level_id: compact_task.target_sub_level_id, + task_type: compact_task.task_type.into(), + //#[allow(deprecated)] split_by_state_table: self.split_by_state_table, + split_weight_by_vnode: compact_task.split_weight_by_vnode, + table_vnode_partition: compact_task.table_vnode_partition.clone(), + table_watermarks: compact_task + .table_watermarks + .iter() + .map(|(table_id, table_watermark)| (*table_id, table_watermark.into())) .collect(), ..Default::default() } @@ -1029,41 +1166,41 @@ pub struct ValidationTask { pub epoch: u64, } -impl ProtoSerializeSizeEstimatedExt for ValidationTask { - fn estimated_encode_len(&self) -> usize { - self.sst_infos - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) - + size_of::() - } -} - -impl ProtoSerializeOwnExt for ValidationTask { - type PB = PbValidationTask; - - fn from_protobuf_own(pb_validation_task: Self::PB) -> Self { +impl From for ValidationTask { + fn from(pb_validation_task: PbValidationTask) -> Self { Self { sst_infos: pb_validation_task .sst_infos .into_iter() - .map(SstableInfo::from_protobuf_own) + .map(SstableInfo::from) .collect_vec(), sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(), epoch: pb_validation_task.epoch, } } +} - fn to_protobuf_own(self) -> Self::PB { - Self::PB { - sst_infos: self +impl From for PbValidationTask { + fn from(validation_task: ValidationTask) -> Self { + Self { + sst_infos: validation_task .sst_infos .into_iter() - .map(|sst| sst.to_protobuf_own()) + .map(|sst| sst.into()) .collect_vec(), - sst_id_to_worker_id: self.sst_id_to_worker_id.clone(), - epoch: self.epoch, + sst_id_to_worker_id: validation_task.sst_id_to_worker_id.clone(), + epoch: validation_task.epoch, } } } + +impl ProtoSerializeSizeEstimatedExt for ValidationTask { + fn estimated_encode_len(&self) -> usize { + self.sst_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) + + size_of::() + } +} diff --git a/src/storage/hummock_test/src/mock_notification_client.rs b/src/storage/hummock_test/src/mock_notification_client.rs index e9e7095dde56..a99a1cfb2d84 100644 --- a/src/storage/hummock_test/src/mock_notification_client.rs +++ b/src/storage/hummock_test/src/mock_notification_client.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use risingwave_common::util::addr::HostAddr; use risingwave_common_service::observer_manager::{Channel, NotificationClient, ObserverError}; -use risingwave_hummock_sdk::ProtoSerializeExt; use risingwave_meta::hummock::{HummockManager, HummockManagerRef}; use risingwave_meta::manager::{MessageStatus, MetaSrvEnv, NotificationManagerRef, WorkerKey}; use risingwave_pb::backup_service::MetaBackupManifestId; @@ -63,7 +62,7 @@ impl NotificationClient for MockNotificationClient { let hummock_version = self.hummock_manager.get_current_version().await; let meta_snapshot = MetaSnapshot { - hummock_version: Some(hummock_version.to_protobuf()), + hummock_version: Some(hummock_version.into()), version: Some(Default::default()), meta_backup_manifest_id: Some(MetaBackupManifestId { id: 0 }), hummock_write_limits: Some(WriteLimits { diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index 17ffb072b568..31d55711a1b4 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -15,14 +15,13 @@ mod compaction_executor; mod compaction_filter; pub mod compaction_utils; -use itertools::Itertools; use risingwave_hummock_sdk::version::{CompactTask, ValidationTask}; use risingwave_pb::compactor::{dispatch_compaction_task_request, DispatchCompactionTaskRequest}; use risingwave_pb::hummock::report_compaction_task_request::{ Event as ReportCompactionTaskEvent, HeartBeat as SharedHeartBeat, ReportTask as ReportSharedTask, }; -use risingwave_pb::hummock::{ReportFullScanTaskRequest, ReportVacuumTaskRequest}; +use risingwave_pb::hummock::{PbCompactTask, ReportFullScanTaskRequest, ReportVacuumTaskRequest}; use risingwave_rpc_client::GrpcCompactorProxyClient; use thiserror_ext::AsReport; use tokio::sync::mpsc; @@ -50,12 +49,10 @@ pub use compaction_filter::{ pub use context::CompactorContext; use futures::{pin_mut, StreamExt}; pub use iterator::{ConcatSstableIterator, SstableStreamIterator}; +use itertools::Itertools; use more_asserts::assert_ge; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; -use risingwave_hummock_sdk::{ - compact_task_to_string, HummockCompactionTaskId, LocalSstableInfo, ProtoSerializeExt, - ProtoSerializeOwnExt, -}; +use risingwave_hummock_sdk::{compact_task_to_string, HummockCompactionTaskId, LocalSstableInfo}; use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_pb::hummock::subscribe_compaction_event_request::{ Event as RequestEvent, HeartBeat, PullTask, ReportTask, @@ -466,7 +463,7 @@ pub fn start_compactor( executor.spawn(async move { match event { ResponseEvent::CompactTask(compact_task) => { - let compact_task = CompactTask::from_protobuf_own(compact_task); + let compact_task = CompactTask::from(compact_task); let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); @@ -498,7 +495,7 @@ pub fn start_compactor( ReportTask { task_id: compact_task.task_id, task_status: compact_task.task_status.into(), - sorted_output_ssts: compact_task.sorted_output_ssts.iter().map(|sst| sst.to_protobuf()).collect_vec(), + sorted_output_ssts: compact_task.sorted_output_ssts.iter().map(|sst| sst.into()).collect_vec(), table_stats_change:to_prost_table_stats_map(table_stats), } )), @@ -546,7 +543,7 @@ pub fn start_compactor( } } ResponseEvent::ValidationTask(validation_task) => { - let validation_task = ValidationTask::from_protobuf_own(validation_task); + let validation_task = ValidationTask::from(validation_task); validate_ssts( validation_task, context.sstable_store.clone(), @@ -672,7 +669,7 @@ pub fn start_shared_compactor( SharedComapctorObjectIdManager::new(output_object_ids_deque, cloned_grpc_proxy_client.clone(), context.storage_opts.sstable_id_remote_fetch_number); match dispatch_task.unwrap() { dispatch_compaction_task_request::Task::CompactTask(compact_task) => { - let compact_task = CompactTask::from_protobuf(&compact_task); + let compact_task = CompactTask::from(&compact_task); let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); @@ -688,7 +685,7 @@ pub fn start_shared_compactor( shutdown.lock().unwrap().remove(&task_id); let report_compaction_task_request = ReportCompactionTaskRequest { event: Some(ReportCompactionTaskEvent::ReportTask(ReportSharedTask { - compact_task: Some(compact_task.to_protobuf()), + compact_task: Some(PbCompactTask::from(&compact_task)), table_stats_change: to_prost_table_stats_map(table_stats), })), }; @@ -700,11 +697,11 @@ pub fn start_shared_compactor( Ok(_) => { // TODO: remove this method after we have running risingwave cluster with fast compact algorithm stably for a long time. let enable_check_compaction_result = context.storage_opts.check_compaction_result; - let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success; if enable_check_compaction_result && need_check_task { match check_compaction_result(&compact_task, context.clone()).await { Err(e) => { - tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}", compact_task.task_id); + tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}", task_id); }, Ok(true) => (), Ok(false) => { @@ -762,7 +759,7 @@ pub fn start_shared_compactor( } } dispatch_compaction_task_request::Task::ValidationTask(validation_task) => { - let validation_task = ValidationTask::from_protobuf_own(validation_task); + let validation_task = ValidationTask::from(validation_task); validate_ssts(validation_task, context.sstable_store.clone()).await; } dispatch_compaction_task_request::Task::CancelCompactTask(cancel_compact_task) => { diff --git a/src/storage/src/store.rs b/src/storage/src/store.rs index ca766c14c6fa..722465db9bf4 100644 --- a/src/storage/src/store.rs +++ b/src/storage/src/store.rs @@ -31,11 +31,12 @@ use risingwave_hummock_sdk::key::{FullKey, TableKey, TableKeyRange}; use risingwave_hummock_sdk::table_watermark::{ TableWatermarks, VnodeWatermark, WatermarkDirection, }; -use risingwave_hummock_sdk::{HummockReadEpoch, LocalSstableInfo, ProtoSerializeOwnExt}; +use risingwave_hummock_sdk::{HummockReadEpoch, LocalSstableInfo}; use risingwave_hummock_trace::{ TracedInitOptions, TracedNewLocalOptions, TracedOpConsistencyLevel, TracedPrefetchOptions, TracedReadOptions, TracedSealCurrentEpochOptions, TracedWriteOptions, }; +use risingwave_pb::hummock::PbVnodeWatermark; use crate::error::{StorageError, StorageResult}; use crate::hummock::CachePolicy; @@ -588,7 +589,10 @@ impl From for TracedSealCurrentEpochOptions { direction == WatermarkDirection::Ascending, watermarks .into_iter() - .map(|watermark| Message::encode_to_vec(&watermark.to_protobuf_own())) + .map(|watermark| { + let pb_watermark = PbVnodeWatermark::from(watermark); + Message::encode_to_vec(&pb_watermark) + }) .collect(), ) }), @@ -613,7 +617,7 @@ impl From for SealCurrentEpochOptions { .into_iter() .map(|serialized_watermark| { Message::decode(serialized_watermark.as_slice()) - .map(VnodeWatermark::from_protobuf_own) + .map(|pb: PbVnodeWatermark| VnodeWatermark::from(pb)) .expect("should not failed") }) .collect(), From 480fc62161ec727cf3420422a33af571535fa608 Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 9 May 2024 15:56:56 +0800 Subject: [PATCH 09/18] fix(storage): fix compile --- src/meta/service/src/hummock_service.rs | 2 +- src/meta/src/hummock/manager/checkpoint.rs | 5 +- src/meta/src/hummock/manager/compaction.rs | 10 +- .../manager/compaction_group_manager.rs | 4 +- src/meta/src/hummock/manager/mod.rs | 108 ++++++++---------- .../src/hummock/mock_hummock_meta_client.rs | 3 +- src/storage/hummock_sdk/src/version.rs | 88 +++++++------- .../hummock/local_version/pinned_version.rs | 1 - src/storage/src/hummock/sstable_store.rs | 3 - src/stream/src/task/barrier_manager.rs | 2 +- 10 files changed, 100 insertions(+), 126 deletions(-) diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs index 0025b85cc5d9..bfbe49524790 100644 --- a/src/meta/service/src/hummock_service.rs +++ b/src/meta/service/src/hummock_service.rs @@ -17,7 +17,7 @@ use std::time::Duration; use futures::StreamExt; use itertools::Itertools; -use risingwave_common::catalog::{TableId, NON_RESERVED_SYS_CATALOG_ID, SYS_CATALOG_START_ID}; +use risingwave_common::catalog::{TableId, SYS_CATALOG_START_ID}; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::HummockVersionDelta; use risingwave_meta::manager::MetadataManager; diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs index 6b5eeed0c66e..d22ab3c6b922 100644 --- a/src/meta/src/hummock/manager/checkpoint.rs +++ b/src/meta/src/hummock/manager/checkpoint.rs @@ -23,10 +23,9 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ }; use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::HummockVersionId; -use risingwave_pb::hummock::hummock_version_checkpoint::PbStaleObjects; +use risingwave_pb::hummock::hummock_version_checkpoint::{PbStaleObjects, StaleObjects}; use risingwave_pb::hummock::{ - PbHummockVersion, PbHummockVersionArchive, PbHummockVersionArchive, PbHummockVersionCheckpoint, - PbHummockVersionCheckpoint, + PbHummockVersion, PbHummockVersionArchive, PbHummockVersionCheckpoint, }; use thiserror_ext::AsReport; use tracing::warn; diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs index fb507f443c46..e1a941059231 100644 --- a/src/meta/src/hummock/manager/compaction.rs +++ b/src/meta/src/hummock/manager/compaction.rs @@ -28,7 +28,7 @@ use risingwave_pb::hummock::subscribe_compaction_event_response::{ Event as ResponseEvent, PullTaskAck, }; use risingwave_pb::hummock::{ - CompactStatus as PbCompactStatus, CompactTaskAssignment, CompactionConfig, + CompactStatus as PbCompactStatus, CompactionConfig, PbCompactTaskAssignment, }; use thiserror_ext::AsReport; use tokio::sync::mpsc::UnboundedReceiver; @@ -46,7 +46,7 @@ const MAX_REPORT_COUNT: usize = 16; #[derive(Default)] pub struct Compaction { /// Compaction task that is already assigned to a compactor - pub compact_task_assignment: BTreeMap, + pub compact_task_assignment: BTreeMap, /// `CompactStatus` of each compaction group pub compaction_statuses: BTreeMap, @@ -80,7 +80,7 @@ impl HummockManager { #[named] pub async fn list_compaction_status( &self, - ) -> (Vec, Vec) { + ) -> (Vec, Vec) { let compaction = read_lock!(self, compaction).await; ( compaction.compaction_statuses.values().map_into().collect(), @@ -229,7 +229,7 @@ impl HummockManager { } RequestEvent::ReportTask(task) => { - report_events.push(task); + report_events.push(task.into()); } _ => unreachable!(), @@ -247,7 +247,7 @@ impl HummockManager { } RequestEvent::ReportTask(task) => { - report_events.push(task); + report_events.push(task.into()); if report_events.len() >= MAX_REPORT_COUNT { break; } diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs index 18bf9b263e73..ee586ee7987c 100644 --- a/src/meta/src/hummock/manager/compaction_group_manager.rs +++ b/src/meta/src/hummock/manager/compaction_group_manager.rs @@ -24,13 +24,13 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_member_table_ids, try_get_compaction_group_id_by_table_id, TableGroupInfo, }; use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId}; +use risingwave_hummock_sdk::version::ReportTask; use risingwave_hummock_sdk::CompactionGroupId; use risingwave_meta_model_v2::compaction_config; use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_pb::hummock::group_delta::DeltaType; use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config::MutableConfig; -use risingwave_pb::hummock::subscribe_compaction_event_request::ReportTask; use risingwave_pb::hummock::{ compact_task, CompactionConfig, CompactionGroupInfo, CompatibilityVersion, GroupConstruct, GroupDelta, GroupDestroy, GroupMetaChange, GroupTableChange, @@ -660,7 +660,7 @@ impl HummockManager { if need_cancel { canceled_tasks.push(ReportTask { task_id: task.task_id, - task_status: TaskStatus::ManualCanceled as i32, + task_status: TaskStatus::ManualCanceled, table_stats_change: HashMap::default(), sorted_output_ssts: vec![], }); diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs index 8c7edb8c5a2d..ee1b3a8098ed 100644 --- a/src/meta/src/hummock/manager/mod.rs +++ b/src/meta/src/hummock/manager/mod.rs @@ -42,7 +42,7 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_table_compaction_group_id_mapping, BranchedSstInfo, HummockLevelsExt, }; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::version::{CompactTask, HummockVersionDelta, SstableInfo}; +use risingwave_hummock_sdk::version::{CompactTask, HummockVersionDelta, ReportTask, SstableInfo}; use risingwave_hummock_sdk::{ version_archive_dir, version_checkpoint_path, CompactionGroupId, ExtendedSstableInfo, HummockCompactionTaskId, HummockContextId, HummockEpoch, HummockSstableId, @@ -56,13 +56,14 @@ use risingwave_pb::hummock::compact_task::{self, TaskStatus, TaskType}; use risingwave_pb::hummock::group_delta::DeltaType; use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config; use risingwave_pb::hummock::subscribe_compaction_event_request::{ - Event as RequestEvent, HeartBeat, ReportTask, + Event as RequestEvent, HeartBeat, }; use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent; use risingwave_pb::hummock::{ CompactTaskAssignment, GroupDelta, GroupMetaChange, HummockPinnedSnapshot, - HummockPinnedVersion, HummockSnapshot, HummockVersionStats, IntraLevelDelta, PbCompactTask, - PbCompactionGroupInfo, SubscribeCompactionEventRequest, TableOption, TableSchema, + HummockPinnedVersion, HummockSnapshot, HummockVersionStats, IntraLevelDelta, + PbCompactTaskAssignment, PbCompactionGroupInfo, SubscribeCompactionEventRequest, TableOption, + TableSchema, }; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use rw_futures_util::{pending_on_none, select_all}; @@ -492,7 +493,7 @@ impl HummockManager { } compaction_guard.compact_task_assignment = match &meta_store { - MetaStoreImpl::Kv(meta_store) => CompactTaskAssignment::list(meta_store) + MetaStoreImpl::Kv(meta_store) => PbCompactTaskAssignment::list(meta_store) .await? .into_iter() .map(|assigned| (assigned.key().unwrap(), assigned)) @@ -502,7 +503,12 @@ impl HummockManager { .await .map_err(MetadataModelError::from)? .into_iter() - .map(|m| (m.id as HummockCompactionTaskId, m.into())) + .map(|m| { + ( + m.id as HummockCompactionTaskId, + PbCompactTaskAssignment::from(m), + ) + }) .collect(), }; @@ -1001,7 +1007,6 @@ impl HummockManager { _ => 0, }; let vnode_partition_count = compact_task.input.vnode_partition_count; - use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; let mut compact_task = CompactTask { input_ssts: compact_task.input.input_levels, @@ -1121,7 +1126,7 @@ impl HummockManager { compact_task_assignment.insert( compact_task.task_id, CompactTaskAssignment { - compact_task: Some(PbCompactTask::from(&compact_task)), + compact_task: Some(compact_task.clone().into()), context_id: META_NODE_ID, // deprecated }, ); @@ -1270,7 +1275,12 @@ impl HummockManager { assert!(CANCEL_STATUS_SET.contains(&task_status)); let tasks = task_ids .into_iter() - .map(|task_id| (task_id, task_status, vec![], HashMap::default())) + .map(|task_id| ReportTask { + task_id, + task_status, + sorted_output_ssts: vec![], + table_stats_change: HashMap::default(), + }) .collect_vec(); let rets = self.report_compact_tasks(tasks).await?; #[cfg(test)] @@ -1361,25 +1371,6 @@ impl HummockManager { false } - fn is_compact_task_expired_pb( - compact_task: &PbCompactTask, - branched_ssts: &BTreeMap, - ) -> bool { - for input_level in compact_task.get_input_ssts() { - for table_info in input_level.get_table_infos() { - if let Some(mp) = branched_ssts.get(&table_info.object_id) { - if mp - .get(&compact_task.compaction_group_id) - .map_or(true, |sst_id| *sst_id != table_info.sst_id) - { - return true; - } - } - } - } - false - } - pub async fn report_compact_task( &self, task_id: u64, @@ -1388,12 +1379,12 @@ impl HummockManager { table_stats_change: Option, ) -> Result { let rets = self - .report_compact_tasks(vec![( + .report_compact_tasks(vec![ReportTask { task_id, task_status, sorted_output_ssts, - table_stats_change.unwrap_or_default(), - )]) + table_stats_change: table_stats_change.unwrap_or_default(), + }]) .await?; Ok(rets[0]) } @@ -1406,10 +1397,7 @@ impl HummockManager { /// Return Ok(false) indicates either the task is not found, /// or the task is not owned by `context_id` when `context_id` is not None. #[named] - pub async fn report_compact_tasks( - &self, - report_tasks: Vec<(u64, TaskStatus, Vec, PbTableStatsMap)>, - ) -> Result> { + pub async fn report_compact_tasks(&self, report_tasks: Vec) -> Result> { let mut guard = write_lock!(self, compaction).await; let deterministic_mode = self.env.opts.compaction_deterministic_test; let compaction = guard.deref_mut(); @@ -1418,13 +1406,13 @@ impl HummockManager { let mut compact_statuses = create_trx_wrapper!( self.meta_store_ref(), BTreeMapTransactionWrapper, - BTreeMapTransaction::new(&mut compaction.compaction_statuses,) + BTreeMapTransaction::new(&mut compaction.compaction_statuses) ); let mut rets = vec![false; report_tasks.len()]; let mut compact_task_assignment = create_trx_wrapper!( self.meta_store_ref(), BTreeMapTransactionWrapper, - BTreeMapTransaction::new(&mut compaction.compact_task_assignment,) + BTreeMapTransaction::new(&mut compaction.compact_task_assignment) ); // The compaction task is finished. let mut versioning_guard = write_lock!(self, versioning).await; @@ -1443,7 +1431,7 @@ impl HummockManager { let mut hummock_version_deltas = create_trx_wrapper!( self.meta_store_ref(), BTreeMapTransactionWrapper, - BTreeMapTransaction::new(&mut versioning.hummock_version_deltas,) + BTreeMapTransaction::new(&mut versioning.hummock_version_deltas) ); let mut branched_ssts = create_trx_wrapper!( self.meta_store_ref(), @@ -1458,14 +1446,12 @@ impl HummockManager { ); let mut success_count = 0; let last_version_id = current_version.id; - for (idx, (task_id, task_status, sorted_output_ssts, table_stats_change)) in - report_tasks.into_iter().enumerate() - { + for (idx, task) in report_tasks.into_iter().enumerate() { rets[idx] = true; - let mut compact_task = match compact_task_assignment.remove(task_id) { - Some(compact_task) => CompactTask::from(&compact_task.compact_task.unwrap()), + let mut compact_task = match compact_task_assignment.remove(task.task_id) { + Some(compact_task) => CompactTask::from(compact_task.compact_task.unwrap()), None => { - tracing::warn!("{}", format!("compact task {} not found", task_id)); + tracing::warn!("{}", format!("compact task {} not found", task.task_id)); rets[idx] = false; continue; } @@ -1473,8 +1459,8 @@ impl HummockManager { { // apply result - compact_task.set_task_status(task_status); - compact_task.sorted_output_ssts = sorted_output_ssts; + compact_task.set_task_status(task.task_status); + compact_task.sorted_output_ssts = task.sorted_output_ssts; } match compact_statuses.get_mut(compact_task.compaction_group_id) { @@ -1538,12 +1524,12 @@ impl HummockManager { self.metrics.version_stats.reset(); versioning.local_metrics.clear(); } - add_prost_table_stats_map(&mut version_stats.table_stats, &table_stats_change); + add_prost_table_stats_map(&mut version_stats.table_stats, &task.table_stats_change); trigger_local_table_stat( &self.metrics, &mut versioning.local_metrics, &version_stats, - &table_stats_change, + &task.table_stats_change, ); } tasks.push(compact_task); @@ -1786,13 +1772,14 @@ impl HummockManager { tracing::warn!( "table {} in SST {} doesn't belong to any compaction group", table_id, - sst.object_id, + sst.get_object_id(), ); } } } let is_trivial_adjust = group_table_ids.len() == 1 - && group_table_ids.first_key_value().unwrap().1.len() == sst.table_ids.len(); + && group_table_ids.first_key_value().unwrap().1.len() + == sst.get_table_ids().len(); if is_trivial_adjust { *compaction_group_id = *group_table_ids.first_key_value().unwrap().0; // is_sst_belong_to_group_declared = true; @@ -1829,7 +1816,7 @@ impl HummockManager { new_sst_id += 1; } if !branch_groups.is_empty() { - branched_ssts.insert(sst.object_id, branch_groups); + branched_ssts.insert(sst.get_object_id(), branch_groups); } } else { sstables.push(original_sstable); @@ -2111,7 +2098,6 @@ impl HummockManager { table_catalogs: Vec
, compaction_groups: Vec, ) -> Result<()> { - use crate::model::MetadataModel; for table in &table_catalogs { table.insert(self.env.meta_store().as_kv()).await?; } @@ -2326,14 +2312,12 @@ impl HummockManager { sorted_output_ssts: Vec, table_stats_change: Option, ) -> Result<()> { - use risingwave_pb::hummock::PbCompactTask; - if let Some(task) = compact_task { let mut guard = write_lock!(self, compaction).await; guard.compact_task_assignment.insert( task_id, CompactTaskAssignment { - compact_task: Some(PbCompactTask::from(&task)), + compact_task: Some(task.into()), context_id: 0, }, ); @@ -2341,12 +2325,12 @@ impl HummockManager { // In the test, the contents of the compact task may have been modified directly, while the contents of compact_task_assignment were not modified. // So we pass the modified compact_task directly into the `report_compact_task_impl` - self.report_compact_tasks(vec![( + self.report_compact_tasks(vec![ReportTask { task_id, task_status, - sorted_output_ssts.into_iter().map(Into::into).collect(), - table_stats_change.unwrap_or_default(), - )]) + sorted_output_ssts, + table_stats_change: table_stats_change.unwrap_or_default(), + }]) .await?; Ok(()) } @@ -2366,7 +2350,7 @@ impl HummockManager { .last_key_value() .unwrap() .1 - .into()], + .to_protobuf()], }), ); } @@ -2376,7 +2360,7 @@ impl HummockManager { let version_deltas = versioning .hummock_version_deltas .range(start_version_id..) - .map(|(_, delta)| delta.into()) + .map(|(_, delta)| delta.to_protobuf()) .collect_vec(); self.env .notification_manager() @@ -3413,7 +3397,7 @@ fn gen_version_delta<'a>( .table_infos .iter() .map(|sst| { - let object_id = sst.object_id; + let object_id = sst.get_object_id(); let sst_id = sst.get_sst_id(); if !trivial_move { drop_sst( diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index 1c0983e71c44..af7fbd721bd1 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -21,9 +21,10 @@ use async_trait::async_trait; use fail::fail_point; use futures::stream::BoxStream; use futures::{Stream, StreamExt}; +use itertools::Itertools; use risingwave_hummock_sdk::change_log::build_table_change_log_delta; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{ HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, SstObjectIdRange, SyncResult, diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index d11975ad999e..73d6842091c7 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -23,10 +23,11 @@ use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType, TaskStatus, use risingwave_pb::hummock::group_delta::DeltaType; use risingwave_pb::hummock::hummock_version::PbLevels; use risingwave_pb::hummock::hummock_version_delta::{ChangeLogDelta, PbGroupDeltas}; +use risingwave_pb::hummock::subscribe_compaction_event_request::PbReportTask; use risingwave_pb::hummock::{ - BloomFilterType, LevelType, PbCompactTask, PbCompactTaskAssignment, PbHummockVersion, - PbHummockVersionDelta, PbInputLevel, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, - PbSstableInfo, PbValidationTask, TableOption, TableSchema, + BloomFilterType, LevelType, PbCompactTask, PbHummockVersion, PbHummockVersionDelta, + PbInputLevel, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, + PbTableStats, PbValidationTask, TableOption, TableSchema, }; use serde::Serialize; @@ -1302,47 +1303,40 @@ impl ProtoSerializeSizeEstimatedExt for ValidationTask { } } -// #[derive(Clone, PartialEq, Default, Debug)] -// pub struct CompactTaskAssignment { -// pub compact_task: Option, -// pub context_id: u32, -// } - -// impl From for CompactTaskAssignment { -// fn from(pb_compact_task_assignment: PbCompactTaskAssignment) -> Self { -// Self { -// compact_task: if pb_compact_task_assignment.compact_task.is_some() { -// Some(CompactTask::from( -// pb_compact_task_assignment.compact_task.unwrap(), -// )) -// } else { -// None -// }, -// context_id: pb_compact_task_assignment.context_id, -// } -// } -// } - -// impl From<&PbCompactTaskAssignment> for CompactTaskAssignment { -// fn from(pb_compact_task_assignment: &PbCompactTaskAssignment) -> Self { -// Self { -// compact_task: if pb_compact_task_assignment.compact_task.is_some() { -// Some(CompactTask::from( -// pb_compact_task_assignment.compact_task.as_ref().unwrap(), -// )) -// } else { -// None -// }, -// context_id: pb_compact_task_assignment.context_id, -// } -// } -// } - -// impl From for PbCompactTaskAssignment { -// fn from(compact_task_assignment: CompactTaskAssignment) -> Self { -// Self { -// compact_task: compact_task_assignment.compact_task.map(|task| task.into()), -// context_id: compact_task_assignment.context_id, -// } -// } -// } +#[derive(Clone, PartialEq, Default, Debug)] +pub struct ReportTask { + pub table_stats_change: HashMap, + pub task_id: u64, + pub task_status: TaskStatus, + pub sorted_output_ssts: Vec, +} + +impl From for ReportTask { + fn from(value: PbReportTask) -> Self { + Self { + table_stats_change: value.table_stats_change.clone(), + task_id: value.task_id, + task_status: TaskStatus::try_from(value.task_status).unwrap(), + sorted_output_ssts: value + .sorted_output_ssts + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + } + } +} + +impl From for PbCompactTask { + fn from(value: ReportTask) -> Self { + Self { + task_id: value.task_id, + task_status: value.task_status.into(), + sorted_output_ssts: value + .sorted_output_ssts + .into_iter() + .map(SstableInfo::into) + .collect_vec(), + ..Default::default() + } + } +} diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs index 2cf1a0221e55..e90ecd5ba5ad 100644 --- a/src/storage/src/hummock/local_version/pinned_version.rs +++ b/src/storage/src/hummock/local_version/pinned_version.rs @@ -19,7 +19,6 @@ use std::time::{Duration, Instant}; use auto_enums::auto_enum; use risingwave_common::catalog::TableId; -use risingwave_hummock_sdk::table_watermark::TableWatermarksIndex; use risingwave_hummock_sdk::version::{HummockVersion, Level, Levels}; use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId, INVALID_VERSION_ID}; use risingwave_rpc_client::HummockMetaClient; diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs index 2487e97bc0e0..653109b94238 100644 --- a/src/storage/src/hummock/sstable_store.rs +++ b/src/storage/src/hummock/sstable_store.rs @@ -28,9 +28,6 @@ use foyer::memory::{ }; use futures::{future, StreamExt}; use itertools::Itertools; -use risingwave_common::cache::{ - CachePriority, LookupResponse, LruCacheEventListener, LruKey, LruValue, -}; use risingwave_common::config::{EvictionConfig, StorageMemoryConfig}; use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{HummockSstableObjectId, OBJECT_SUFFIX}; diff --git a/src/stream/src/task/barrier_manager.rs b/src/stream/src/task/barrier_manager.rs index 0c4f0c925fd0..3f10fc9e5f67 100644 --- a/src/stream/src/task/barrier_manager.rs +++ b/src/stream/src/task/barrier_manager.rs @@ -584,7 +584,7 @@ impl LocalBarrierWorker { table_stats, }| GroupedSstableInfo { compaction_group_id, - sst: Some(sst_info), + sst: Some(sst_info.into()), table_stats_map: to_prost_table_stats_map(table_stats), }, ) From 5fff8aeb6b6fecec4c619cbd59f9ffc9dfac6687 Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 9 May 2024 18:03:57 +0800 Subject: [PATCH 10/18] fix(storage): fix check --- .../compaction/picker/min_overlap_compaction_picker.rs | 8 ++++---- src/storage/benches/bench_compactor.rs | 1 - src/storage/src/hummock/compactor/compactor_runner.rs | 7 ------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index a99939bc4abe..d37a774a1ee1 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -969,14 +969,14 @@ pub mod tests { let levels = vec![ Level { level_idx: 1, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![generate_table(0, 1, 50, 100, 2)], // 50 total_file_size: 50, ..Default::default() }, Level { level_idx: 2, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(1, 1, 101, 150, 1), // 50 ], @@ -985,7 +985,7 @@ pub mod tests { }, Level { level_idx: 3, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(2, 1, 151, 200, 2), // 50 ], @@ -994,7 +994,7 @@ pub mod tests { }, Level { level_idx: 4, - level_type: LevelType::Nonoverlapping as i32, + level_type: LevelType::Nonoverlapping, table_infos: vec![ generate_table(3, 1, 50, 300, 2), // 250 ], diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index 715aa200ce8a..0fd38096330e 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -28,7 +28,6 @@ use risingwave_common::util::value_encoding::ValueRowSerializer; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::SstableInfo; -use risingwave_hummock_sdk::HummockEpoch; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl}; use risingwave_pb::hummock::compact_task::PbTaskType; diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index 610105419dfd..d69fe35f7cca 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -935,16 +935,9 @@ where #[cfg(test)] pub mod tests { - use bytes::Bytes; - use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::can_concat; - use risingwave_hummock_sdk::key::{TableKey, UserKey}; - use risingwave_hummock_sdk::version::InputLevel; - use super::*; - use crate::filter_key_extractor::FullKeyFilterKeyExtractor; use crate::hummock::compactor::compactor_runner::partition_overlapping_sstable_infos; - use crate::hummock::compactor::StateCleanUpCompactionFilter; use crate::hummock::iterator::test_utils::mock_sstable_store; use crate::hummock::test_utils::{ default_builder_opt_for_test, gen_test_sstable_info, test_key_of, test_value_of, From c5d4b7fcbdef711898d027d4a7cffa2e802c0eaf Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 11 Jul 2024 21:33:05 +0800 Subject: [PATCH 11/18] fix(storage): address comments --- .../rw_catalog/rw_hummock_version.rs | 2 +- .../rw_catalog/rw_hummock_version_deltas.rs | 10 +- src/meta/src/hummock/manager/commit_epoch.rs | 2 +- src/meta/src/hummock/manager/compaction.rs | 46 +-- .../manager/compaction_group_manager.rs | 48 +-- src/meta/src/hummock/manager/transaction.rs | 36 +- src/storage/hummock_sdk/src/change_log.rs | 59 ++- .../compaction_group/hummock_version_ext.rs | 126 +++--- src/storage/hummock_sdk/src/key_range.rs | 3 +- src/storage/hummock_sdk/src/version.rs | 376 ++++++++++++++++-- 10 files changed, 528 insertions(+), 180 deletions(-) diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs index 68874554eb28..5c03bbdd8c5c 100644 --- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs @@ -107,7 +107,7 @@ fn version_to_compaction_group_rows(version: &HummockVersion) -> Vec Result>()) + .into(), }) .collect(); Ok(rows) diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs index 8715f070a074..d05f9af62fae 100644 --- a/src/meta/src/hummock/manager/commit_epoch.rs +++ b/src/meta/src/hummock/manager/commit_epoch.rs @@ -15,6 +15,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use risingwave_common::catalog::TableId; +use risingwave_hummock_sdk::change_log::ChangeLogDelta; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::split_sst; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::table_stats::{ @@ -26,7 +27,6 @@ use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, LocalSstableInfo, }; use risingwave_pb::hummock::compact_task::{self}; -use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta; use risingwave_pb::hummock::HummockSnapshot; use crate::hummock::error::{Error, Result}; diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs index 9b9c0330ecd0..8bc3d108a17a 100644 --- a/src/meta/src/hummock/manager/compaction.rs +++ b/src/meta/src/hummock/manager/compaction.rs @@ -48,14 +48,14 @@ use risingwave_hummock_sdk::table_stats::{ add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap, }; use risingwave_hummock_sdk::version::{ - CompactTask, HummockVersion, InputLevel, Level, Levels, ReportTask, SstableInfo, + CompactTask, GroupDelta, HummockVersion, InputLevel, IntraLevelDelta, Level, Levels, + ReportTask, SstableInfo, }; use risingwave_hummock_sdk::{ compact_task_to_string, statistics_compact_task, CompactionGroupId, HummockCompactionTaskId, HummockVersionId, }; use risingwave_pb::hummock::compact_task::{TaskStatus, TaskType}; -use risingwave_pb::hummock::group_delta::DeltaType; use risingwave_pb::hummock::subscribe_compaction_event_request::{ self, Event as RequestEvent, HeartBeat, PullTask, }; @@ -63,9 +63,9 @@ use risingwave_pb::hummock::subscribe_compaction_event_response::{ Event as ResponseEvent, PullTaskAck, }; use risingwave_pb::hummock::{ - compact_task, CompactStatus as PbCompactStatus, CompactTaskAssignment, CompactionConfig, - GroupDelta, IntraLevelDelta, PbCompactTaskAssignment, StateTableInfoDelta, - SubscribeCompactionEventRequest, TableOption, TableSchema, + compact_task, CompactTaskAssignment, CompactionConfig, PbCompactStatus, + PbCompactTaskAssignment, StateTableInfoDelta, SubscribeCompactionEventRequest, TableOption, + TableSchema, }; use rw_futures_util::pending_on_none; use thiserror_ext::AsReport; @@ -166,29 +166,25 @@ impl<'a> HummockVersionTransaction<'a> { } for (level_idx, removed_table_ids) in removed_table_ids_map { - let group_delta = GroupDelta { - delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { - level_idx, - removed_table_ids, - ..Default::default() - })), - }; + let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new( + level_idx, + 0, // default + removed_table_ids, + vec![], // default + 0, // default + )); + group_deltas.push(group_delta); } - let group_delta = GroupDelta { - delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { - level_idx: compact_task.target_level, - inserted_table_infos: compact_task - .sorted_output_ssts - .iter() - .map(|sst| sst.into()) - .collect(), - l0_sub_level_id: compact_task.target_sub_level_id, - vnode_partition_count: compact_task.split_weight_by_vnode, - ..Default::default() - })), - }; + let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new( + compact_task.target_level, + compact_task.target_sub_level_id, + vec![], // default + compact_task.sorted_output_ssts.clone(), + compact_task.split_weight_by_vnode, + )); + group_deltas.push(group_delta); let new_visible_table_safe_epoch = std::cmp::max( version_delta.latest_version().visible_table_safe_epoch(), diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs index 1cadfd2ba0f5..41b20420be0f 100644 --- a/src/meta/src/hummock/manager/compaction_group_manager.rs +++ b/src/meta/src/hummock/manager/compaction_group_manager.rs @@ -22,17 +22,15 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_compaction_group_ids, TableGroupInfo, }; use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId}; -use risingwave_hummock_sdk::version::ReportTask; +use risingwave_hummock_sdk::version::{GroupDelta, GroupDeltas, ReportTask}; use risingwave_hummock_sdk::CompactionGroupId; use risingwave_meta_model_v2::compaction_config; use risingwave_pb::hummock::compact_task::TaskStatus; -use risingwave_pb::hummock::group_delta::DeltaType; -use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config::MutableConfig; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{ - compact_task, CompactionConfig, CompactionGroupInfo, CompatibilityVersion, GroupConstruct, - GroupDelta, GroupDestroy, StateTableInfoDelta, + compact_task, CompactionConfig, CompactionGroupInfo, CompatibilityVersion, PbGroupConstruct, + PbGroupDestroy, PbStateTableInfoDelta, }; use tokio::sync::OnceCell; @@ -252,20 +250,20 @@ impl HummockManager { } }; - group_deltas.push(GroupDelta { - delta_type: Some(DeltaType::GroupConstruct(GroupConstruct { - group_config: Some(config), - group_id, - ..Default::default() - })), + let group_delta = GroupDelta::GroupConstruct(PbGroupConstruct { + group_config: Some(config), + group_id, + ..Default::default() }); + + group_deltas.push(group_delta); } } assert!(new_version_delta .state_table_info_delta .insert( TableId::new(*table_id), - StateTableInfoDelta { + PbStateTableInfoDelta { committed_epoch: epoch, safe_epoch: epoch, compaction_group_id: *raw_group_id, @@ -343,9 +341,9 @@ impl HummockManager { .entry(*group_id) .or_default() .group_deltas; - group_deltas.push(GroupDelta { - delta_type: Some(DeltaType::GroupDestroy(GroupDestroy {})), - }); + + let group_delta = GroupDelta::GroupDestroy(PbGroupDestroy {}); + group_deltas.push(group_delta); } for (group_id, max_level) in groups_to_remove { @@ -532,16 +530,14 @@ impl HummockManager { new_version_delta.group_deltas.insert( new_compaction_group_id, GroupDeltas { - group_deltas: vec![GroupDelta { - delta_type: Some(DeltaType::GroupConstruct(GroupConstruct { - group_config: Some(config.clone()), - group_id: new_compaction_group_id, - parent_group_id, - new_sst_start_id, - table_ids: vec![], - version: CompatibilityVersion::NoMemberTableIds as i32, - })), - }], + group_deltas: vec![GroupDelta::GroupConstruct(PbGroupConstruct { + group_config: Some(config.clone()), + group_id: new_compaction_group_id, + parent_group_id, + new_sst_start_id, + table_ids: vec![], + version: CompatibilityVersion::NoMemberTableIds as i32, + })], }, ); ((new_compaction_group_id, config), new_compaction_group_id) @@ -561,7 +557,7 @@ impl HummockManager { .state_table_info_delta .insert( table_id, - StateTableInfoDelta { + PbStateTableInfoDelta { committed_epoch: info.committed_epoch, safe_epoch: info.safe_epoch, compaction_group_id: new_compaction_group_id, diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs index 66a2519ae4fc..0d14ccd224d9 100644 --- a/src/meta/src/hummock/manager/transaction.rs +++ b/src/meta/src/hummock/manager/transaction.rs @@ -16,14 +16,14 @@ use std::collections::{BTreeMap, HashMap}; use std::ops::{Deref, DerefMut}; use risingwave_common::catalog::TableId; +use risingwave_hummock_sdk::change_log::ChangeLogDelta; use risingwave_hummock_sdk::table_watermark::TableWatermarks; -use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta, SstableInfo}; -use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, HummockVersionId}; -use risingwave_pb::hummock::group_delta::DeltaType; -use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta; -use risingwave_pb::hummock::{ - GroupDelta, HummockVersionStats, IntraLevelDelta, StateTableInfoDelta, +use risingwave_hummock_sdk::version::{ + GroupDelta, HummockVersion, HummockVersionDelta, IntraLevelDelta, SstableInfo, }; +use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, HummockVersionId}; +use risingwave_pb::hummock::hummock_version_delta::PbChangeLogDelta; +use risingwave_pb::hummock::{HummockVersionStats, StateTableInfoDelta}; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use crate::manager::NotificationManager; @@ -119,7 +119,10 @@ impl<'a> HummockVersionTransaction<'a> { let mut new_version_delta = self.new_delta(); new_version_delta.max_committed_epoch = epoch; new_version_delta.new_table_watermarks = new_table_watermarks; - new_version_delta.change_log_delta = change_log_delta; + new_version_delta.change_log_delta = change_log_delta + .into_iter() + .map(|(table_id, delta)| (table_id, PbChangeLogDelta::from(delta))) + .collect(); // Append SSTs to a new version. for (compaction_group_id, inserted_table_infos) in commit_sstables { @@ -129,17 +132,14 @@ impl<'a> HummockVersionTransaction<'a> { .or_default() .group_deltas; let l0_sub_level_id = epoch; - let group_delta = GroupDelta { - delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { - level_idx: 0, - inserted_table_infos: inserted_table_infos - .iter() - .map(|info| info.into()) - .collect(), - l0_sub_level_id, - ..Default::default() - })), - }; + let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new( + 0, + l0_sub_level_id, + vec![], // default + inserted_table_infos, + 0, // default + )); + group_deltas.push(group_delta); } diff --git a/src/storage/hummock_sdk/src/change_log.rs b/src/storage/hummock_sdk/src/change_log.rs index 94401fb40c0a..ac09033f21b6 100644 --- a/src/storage/hummock_sdk/src/change_log.rs +++ b/src/storage/hummock_sdk/src/change_log.rs @@ -15,8 +15,8 @@ use std::collections::HashMap; use risingwave_common::catalog::TableId; -use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta; -use risingwave_pb::hummock::{PbEpochNewChangeLog, TableChangeLog as PbTableChangeLog}; +use risingwave_pb::hummock::hummock_version_delta::PbChangeLogDelta; +use risingwave_pb::hummock::{PbEpochNewChangeLog, PbTableChangeLog}; use tracing::warn; use crate::version::SstableInfo; @@ -134,7 +134,7 @@ pub fn build_table_change_log_delta<'a>( TableId::new(table_id), ChangeLogDelta { truncate_epoch, - new_log: Some(PbEpochNewChangeLog { + new_log: Some(EpochNewChangeLog { new_value: vec![], old_value: vec![], epochs: epochs.clone(), @@ -147,11 +147,7 @@ pub fn build_table_change_log_delta<'a>( for table_id in &sst.table_ids { match table_change_log.get_mut(&TableId::new(*table_id)) { Some(log) => { - log.new_log - .as_mut() - .unwrap() - .old_value - .push(sst.clone().into()); + log.new_log.as_mut().unwrap().old_value.push(sst.clone()); } None => { warn!(table_id, ?sst, "old value sst contains non-log-store table"); @@ -162,17 +158,54 @@ pub fn build_table_change_log_delta<'a>( for sst in new_value_ssts { for table_id in &sst.table_ids { if let Some(log) = table_change_log.get_mut(&TableId::new(*table_id)) { - log.new_log - .as_mut() - .unwrap() - .new_value - .push(sst.clone().into()); + log.new_log.as_mut().unwrap().new_value.push(sst.clone()); } } } table_change_log } +pub struct ChangeLogDelta { + pub truncate_epoch: u64, + pub new_log: Option, +} + +impl From<&ChangeLogDelta> for PbChangeLogDelta { + fn from(val: &ChangeLogDelta) -> Self { + Self { + truncate_epoch: val.truncate_epoch, + new_log: val.new_log.as_ref().map(|a| a.into()), + } + } +} + +impl From<&PbChangeLogDelta> for ChangeLogDelta { + fn from(val: &PbChangeLogDelta) -> Self { + Self { + truncate_epoch: val.truncate_epoch, + new_log: val.new_log.as_ref().map(|a| a.into()), + } + } +} + +impl From for PbChangeLogDelta { + fn from(val: ChangeLogDelta) -> Self { + Self { + truncate_epoch: val.truncate_epoch, + new_log: val.new_log.map(|a| a.into()), + } + } +} + +impl From for ChangeLogDelta { + fn from(val: PbChangeLogDelta) -> Self { + Self { + truncate_epoch: val.truncate_epoch, + new_log: val.new_log.map(|a| a.into()), + } + } +} + #[cfg(test)] mod tests { use itertools::Itertools; diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 166e0499de6b..f5e1d9350448 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -19,8 +19,6 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_common::catalog::TableId; -use risingwave_pb::hummock::group_delta::DeltaType; -use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; use risingwave_pb::hummock::{ CompactionConfig, CompatibilityVersion, GroupConstruct, GroupDestroy, GroupMetaChange, GroupTableChange, LevelType, PbLevelType, @@ -33,7 +31,8 @@ use crate::compaction_group::StaticCompactionGroupId; use crate::key_range::KeyRangeCommon; use crate::table_watermark::TableWatermarks; use crate::version::{ - HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, + GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, + SstableInfo, }; use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId}; @@ -51,7 +50,7 @@ pub struct GroupDeltasSummary { } pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary { - let mut delete_sst_levels = Vec::with_capacity(group_deltas.group_deltas.len()); + let mut delete_sst_levels = Vec::with_capacity(group_deltas.get_group_deltas().len()); let mut delete_sst_ids_set = HashSet::new(); let mut insert_sst_level_id = u32::MAX; let mut insert_sub_level_id = u64::MAX; @@ -62,37 +61,33 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary let mut group_table_change = None; let mut new_vnode_partition_count = 0; - for group_delta in &group_deltas.group_deltas { - match group_delta.get_delta_type().unwrap() { - DeltaType::IntraLevel(intra_level) => { - if !intra_level.removed_table_ids.is_empty() { - delete_sst_levels.push(intra_level.level_idx); - delete_sst_ids_set.extend(intra_level.removed_table_ids.iter().clone()); + for group_delta in group_deltas.get_group_deltas() { + match group_delta { + GroupDelta::IntraLevel(intra_level) => { + if !intra_level.get_removed_table_ids().is_empty() { + delete_sst_levels.push(intra_level.get_level_idx()); + delete_sst_ids_set.extend(intra_level.get_removed_table_ids().iter().clone()); } - if !intra_level.inserted_table_infos.is_empty() { - insert_sst_level_id = intra_level.level_idx; - insert_sub_level_id = intra_level.l0_sub_level_id; - insert_table_infos.extend( - intra_level - .inserted_table_infos - .iter() - .map(SstableInfo::from), - ); + if !intra_level.get_inserted_table_infos().is_empty() { + insert_sst_level_id = intra_level.get_level_idx(); + insert_sub_level_id = intra_level.get_l0_sub_level_id(); + insert_table_infos + .extend(intra_level.get_inserted_table_infos().iter().cloned()); } - new_vnode_partition_count = intra_level.vnode_partition_count; + new_vnode_partition_count = intra_level.get_vnode_partition_count(); } - DeltaType::GroupConstruct(construct_delta) => { + GroupDelta::GroupConstruct(construct_delta) => { assert!(group_construct.is_none()); group_construct = Some(construct_delta.clone()); } - DeltaType::GroupDestroy(destroy_delta) => { + GroupDelta::GroupDestroy(destroy_delta) => { assert!(group_destroy.is_none()); group_destroy = Some(destroy_delta.clone()); } - DeltaType::GroupMetaChange(meta_delta) => { + GroupDelta::GroupMetaChange(meta_delta) => { group_meta_changes.push(meta_delta.clone()); } - DeltaType::GroupTableChange(meta_delta) => { + GroupDelta::GroupTableChange(meta_delta) => { group_table_change = Some(meta_delta.clone()); } } @@ -402,9 +397,9 @@ impl HummockVersion { // Build only if all deltas are intra level deltas. if !group_deltas - .group_deltas + .get_group_deltas() .iter() - .all(|delta| matches!(delta.get_delta_type().unwrap(), DeltaType::IntraLevel(..))) + .all(|delta| matches!(delta, GroupDelta::IntraLevel(_))) { continue; } @@ -412,22 +407,28 @@ impl HummockVersion { // TODO(MrCroxx): At most one insert delta is allowed here. It's okay for now with the // current `hummock::manager::gen_version_delta` implementation. Better refactor the // struct to reduce conventions. - for group_delta in &group_deltas.group_deltas { - if let DeltaType::IntraLevel(delta) = group_delta.get_delta_type().unwrap() { - if !delta.inserted_table_infos.is_empty() { - info.insert_sst_level = delta.level_idx; - info.insert_sst_infos - .extend(delta.inserted_table_infos.iter().map(SstableInfo::from)); - } - if !delta.removed_table_ids.is_empty() { - for id in &delta.removed_table_ids { - if delta.level_idx == 0 { - removed_l0_ssts.insert(*id); - } else { - removed_ssts.entry(delta.level_idx).or_default().insert(*id); + for group_delta in group_deltas.get_group_deltas() { + match group_delta { + GroupDelta::IntraLevel(intra_level) => { + if !intra_level.get_inserted_table_infos().is_empty() { + info.insert_sst_level = intra_level.get_level_idx(); + info.insert_sst_infos + .extend(intra_level.get_inserted_table_infos().iter().cloned()); + } + if !intra_level.get_removed_table_ids().is_empty() { + for id in intra_level.get_removed_table_ids() { + if intra_level.get_level_idx() == 0 { + removed_l0_ssts.insert(*id); + } else { + removed_ssts + .entry(intra_level.get_level_idx()) + .or_default() + .insert(*id); + } } } } + _ => unreachable!(), } } @@ -1299,14 +1300,15 @@ mod tests { use std::collections::HashMap; use risingwave_pb::hummock::group_delta::DeltaType; - use risingwave_pb::hummock::hummock_version_delta::GroupDeltas; + use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas; use risingwave_pb::hummock::{ - CompactionConfig, GroupConstruct, GroupDelta, GroupDestroy, IntraLevelDelta, LevelType, + CompactionConfig, GroupConstruct, GroupDestroy, LevelType, PbIntraLevelDelta, }; use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels; use crate::version::{ - HummockVersion, HummockVersionDelta, Level, Levels, OverlappingLevel, SstableInfo, + GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, IntraLevelDelta, Level, + Levels, OverlappingLevel, SstableInfo, }; #[test] @@ -1390,40 +1392,36 @@ mod tests { ( 2, GroupDeltas { - group_deltas: vec![GroupDelta { - delta_type: Some(DeltaType::GroupConstruct(GroupConstruct { - group_config: Some(CompactionConfig { - max_level: 6, - ..Default::default() - }), + group_deltas: vec![GroupDelta::GroupConstruct(GroupConstruct { + group_config: Some(CompactionConfig { + max_level: 6, ..Default::default() - })), - }], + }), + ..Default::default() + })], }, ), ( 0, GroupDeltas { - group_deltas: vec![GroupDelta { - delta_type: Some(DeltaType::GroupDestroy(GroupDestroy {})), - }], + group_deltas: vec![GroupDelta::GroupDestroy(GroupDestroy {})], }, ), ( 1, GroupDeltas { - group_deltas: vec![GroupDelta { - delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta { - level_idx: 1, - inserted_table_infos: vec![SstableInfo { - object_id: 1, - sst_id: 1, - ..Default::default() - } - .into()], + group_deltas: vec![GroupDelta::IntraLevel(IntraLevelDelta::new( + 1, + 0, + vec![], + vec![SstableInfo { + object_id: 1, + sst_id: 1, ..Default::default() - })), - }], + } + .into()], + 0, + ))], }, ), ]); diff --git a/src/storage/hummock_sdk/src/key_range.rs b/src/storage/hummock_sdk/src/key_range.rs index 24c5fc386ede..95359e8a5497 100644 --- a/src/storage/hummock_sdk/src/key_range.rs +++ b/src/storage/hummock_sdk/src/key_range.rs @@ -15,12 +15,11 @@ use std::cmp; use bytes::Bytes; -use serde::Serialize; use super::key_cmp::KeyComparator; use crate::key::{FullKey, UserKey}; -#[derive(PartialEq, Eq, Clone, Debug, Default, Serialize)] +#[derive(PartialEq, Eq, Clone, Debug, Default)] pub struct KeyRange { pub left: Bytes, pub right: Bytes, diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index ab80ccd083df..1e1ce5f7c84e 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -23,17 +23,17 @@ use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::INVALID_EPOCH; use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType, TaskStatus, TaskType}; -use risingwave_pb::hummock::group_delta::DeltaType; +use risingwave_pb::hummock::group_delta::PbDeltaType; use risingwave_pb::hummock::hummock_version::PbLevels; use risingwave_pb::hummock::hummock_version_delta::{ChangeLogDelta, PbGroupDeltas}; use risingwave_pb::hummock::subscribe_compaction_event_request::PbReportTask; use risingwave_pb::hummock::{ - BloomFilterType, CompactionConfig, LevelType, PbCompactTask, PbHummockVersion, - PbHummockVersionDelta, PbInputLevel, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, + BloomFilterType, CompactionConfig, LevelType, PbCompactTask, PbGroupConstruct, PbGroupDelta, + PbGroupDestroy, PbGroupMetaChange, PbGroupTableChange, PbHummockVersion, PbHummockVersionDelta, + PbInputLevel, PbIntraLevelDelta, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, PbStateTableInfo, PbTableStats, PbValidationTask, StateTableInfo, StateTableInfoDelta, TableOption, TableSchema, }; -use serde::Serialize; use tracing::warn; use crate::change_log::TableChangeLog; @@ -43,7 +43,7 @@ use crate::key_range::KeyRange; use crate::table_watermark::TableWatermarks; use crate::{CompactionGroupId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID}; -#[derive(Debug, Clone, PartialEq, Default, Serialize)] +#[derive(Debug, Clone, PartialEq, Default)] pub struct OverlappingLevel { pub sub_levels: Vec, pub total_file_size: u64, @@ -123,7 +123,7 @@ impl OverlappingLevel { } } -#[derive(Debug, Clone, PartialEq, Default, Serialize)] +#[derive(Debug, Clone, PartialEq, Default)] pub struct Level { pub level_idx: u32, pub level_type: LevelType, @@ -248,12 +248,14 @@ impl Level { } } -#[derive(Debug, Clone, PartialEq, Default, Serialize)] +#[derive(Debug, Clone, PartialEq, Default)] pub struct Levels { pub levels: Vec, pub l0: Option, pub group_id: u64, pub parent_group_id: u64, + + #[expect(deprecated)] pub member_table_ids: Vec, } @@ -264,6 +266,14 @@ impl Levels { } impl Levels { + pub fn from_protobuf(pb_levels: &PbLevels) -> Self { + Self::from(pb_levels) + } + + pub fn to_protobuf(&self) -> PbLevels { + self.into() + } + pub fn estimated_encode_len(&self) -> usize { let mut basic = self .levels @@ -310,7 +320,7 @@ impl From<&Levels> for PbLevels { group_id: levels.group_id, parent_group_id: levels.parent_group_id, #[expect(deprecated)] - member_table_ids: Default::default(), + member_table_ids: levels.member_table_ids.clone(), } } } @@ -326,6 +336,7 @@ impl From for Levels { levels: pb_levels.levels.into_iter().map(Level::from).collect_vec(), group_id: pb_levels.group_id, parent_group_id: pb_levels.parent_group_id, + #[expect(deprecated)] member_table_ids: pb_levels.member_table_ids.clone(), } } @@ -747,7 +758,7 @@ impl HummockVersion { pub struct HummockVersionDelta { pub id: u64, pub prev_id: u64, - pub group_deltas: HashMap, + pub group_deltas: HashMap, pub max_committed_epoch: u64, safe_epoch: u64, pub trivial_move: bool, @@ -791,17 +802,17 @@ impl HummockVersionDelta { .values() .flat_map(|group_deltas| { group_deltas.group_deltas.iter().flat_map(|group_delta| { - group_delta.delta_type.iter().flat_map(|delta_type| { - static EMPTY_VEC: Vec = Vec::new(); - let sst_slice = match delta_type { - DeltaType::IntraLevel(level_delta) => &level_delta.inserted_table_infos, - DeltaType::GroupConstruct(_) - | DeltaType::GroupDestroy(_) - | DeltaType::GroupMetaChange(_) - | DeltaType::GroupTableChange(_) => &EMPTY_VEC, - }; - sst_slice.iter().map(|sst| sst.object_id) - }) + static EMPTY_VEC: Vec = Vec::new(); + let sst_slice = match group_delta { + GroupDelta::IntraLevel(level_delta) => { + level_delta.get_inserted_table_infos() + } + GroupDelta::GroupConstruct(_) + | GroupDelta::GroupDestroy(_) + | GroupDelta::GroupMetaChange(_) + | GroupDelta::GroupTableChange(_) => &EMPTY_VEC, + }; + sst_slice.iter().map(|sst| sst.object_id) }) }) .chain(self.change_log_delta.values().flat_map(|delta| { @@ -829,7 +840,13 @@ impl From<&PbHummockVersionDelta> for HummockVersionDelta { Self { id: pb_version_delta.id, prev_id: pb_version_delta.prev_id, - group_deltas: pb_version_delta.group_deltas.clone(), + group_deltas: pb_version_delta + .group_deltas + .iter() + .map(|(group_id, deltas)| { + (*group_id as CompactionGroupId, GroupDeltas::from(deltas)) + }) + .collect(), max_committed_epoch: pb_version_delta.max_committed_epoch, safe_epoch: pb_version_delta.safe_epoch, trivial_move: pb_version_delta.trivial_move, @@ -873,7 +890,11 @@ impl From<&HummockVersionDelta> for PbHummockVersionDelta { Self { id: version_delta.id, prev_id: version_delta.prev_id, - group_deltas: version_delta.group_deltas.clone(), + group_deltas: version_delta + .group_deltas + .iter() + .map(|(group_id, deltas)| (*group_id as _, deltas.into())) + .collect(), max_committed_epoch: version_delta.max_committed_epoch, safe_epoch: version_delta.safe_epoch, trivial_move: version_delta.trivial_move, @@ -906,7 +927,11 @@ impl From for PbHummockVersionDelta { Self { id: version_delta.id, prev_id: version_delta.prev_id, - group_deltas: version_delta.group_deltas, + group_deltas: version_delta + .group_deltas + .into_iter() + .map(|(group_id, deltas)| (group_id as _, deltas.into())) + .collect(), max_committed_epoch: version_delta.max_committed_epoch, safe_epoch: version_delta.safe_epoch, trivial_move: version_delta.trivial_move, @@ -939,7 +964,11 @@ impl From for HummockVersionDelta { Self { id: pb_version_delta.id, prev_id: pb_version_delta.prev_id, - group_deltas: pb_version_delta.group_deltas, + group_deltas: pb_version_delta + .group_deltas + .into_iter() + .map(|(group_id, deltas)| (group_id as CompactionGroupId, deltas.into())) + .collect(), max_committed_epoch: pb_version_delta.max_committed_epoch, safe_epoch: pb_version_delta.safe_epoch, trivial_move: pb_version_delta.trivial_move, @@ -975,7 +1004,7 @@ impl From for HummockVersionDelta { } } -#[derive(Debug, PartialEq, Clone, Default, Serialize)] +#[derive(Debug, PartialEq, Clone, Default)] pub struct SstableInfo { pub object_id: u64, pub sst_id: u64, @@ -1163,7 +1192,7 @@ impl SstableInfo { } } -#[derive(Clone, PartialEq, Default, Debug, Serialize)] +#[derive(Clone, PartialEq, Default, Debug)] pub struct InputLevel { pub level_idx: u32, pub level_type: LevelType, @@ -1265,7 +1294,7 @@ pub struct CompactTask { pub sorted_output_ssts: Vec, /// task id assigned by hummock storage service pub task_id: u64, - /// compaction output will be added to `target_level`` of LSM after compaction + /// compaction output will be added to `target_level` of LSM after compaction pub target_level: u32, pub gc_delete_keys: bool, /// Lbase in LSM @@ -1283,7 +1312,7 @@ pub struct CompactTask { pub target_sub_level_id: u64, /// Identifies whether the task is `space_reclaim`, if the `compact_task_type` increases, it will be refactored to enum pub task_type: TaskType, - /// Deprecated. use table_vnode_partition instead; + /// Deprecated. use `table_vnode_partition` instead; pub split_by_state_table: bool, /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed. /// Deprecated. use `table_vnode_partition` instead; @@ -1574,7 +1603,7 @@ impl CompactTask { } } -#[derive(Clone, PartialEq, Default, Serialize)] +#[derive(Clone, PartialEq, Default)] pub struct ValidationTask { pub sst_infos: Vec, pub sst_id_to_worker_id: HashMap, @@ -1657,3 +1686,292 @@ impl From for PbCompactTask { } } } + +// for version delta + +#[derive(Debug, PartialEq, Clone)] +pub struct IntraLevelDelta { + level_idx: u32, + l0_sub_level_id: u64, + removed_table_ids: Vec, + inserted_table_infos: Vec, + vnode_partition_count: u32, +} + +impl IntraLevelDelta { + pub fn estimated_encode_len(&self) -> usize { + size_of::() + + size_of::() + + self.removed_table_ids.len() * size_of::() + + self + .inserted_table_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + size_of::() + } +} + +impl From for IntraLevelDelta { + fn from(pb_intra_level_delta: PbIntraLevelDelta) -> Self { + Self { + level_idx: pb_intra_level_delta.level_idx, + l0_sub_level_id: pb_intra_level_delta.l0_sub_level_id, + removed_table_ids: pb_intra_level_delta.removed_table_ids.clone(), + inserted_table_infos: pb_intra_level_delta + .inserted_table_infos + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + vnode_partition_count: pb_intra_level_delta.vnode_partition_count, + } + } +} + +impl From for PbIntraLevelDelta { + fn from(intra_level_delta: IntraLevelDelta) -> Self { + Self { + level_idx: intra_level_delta.level_idx, + l0_sub_level_id: intra_level_delta.l0_sub_level_id, + removed_table_ids: intra_level_delta.removed_table_ids.clone(), + inserted_table_infos: intra_level_delta + .inserted_table_infos + .into_iter() + .map(|sst| sst.into()) + .collect_vec(), + vnode_partition_count: intra_level_delta.vnode_partition_count, + } + } +} + +impl From<&IntraLevelDelta> for PbIntraLevelDelta { + fn from(intra_level_delta: &IntraLevelDelta) -> Self { + Self { + level_idx: intra_level_delta.level_idx, + l0_sub_level_id: intra_level_delta.l0_sub_level_id, + removed_table_ids: intra_level_delta.removed_table_ids.clone(), + inserted_table_infos: intra_level_delta + .inserted_table_infos + .iter() + .map(|sst| sst.into()) + .collect_vec(), + vnode_partition_count: intra_level_delta.vnode_partition_count, + } + } +} + +impl From<&PbIntraLevelDelta> for IntraLevelDelta { + fn from(pb_intra_level_delta: &PbIntraLevelDelta) -> Self { + Self { + level_idx: pb_intra_level_delta.level_idx, + l0_sub_level_id: pb_intra_level_delta.l0_sub_level_id, + removed_table_ids: pb_intra_level_delta.removed_table_ids.clone(), + inserted_table_infos: pb_intra_level_delta + .inserted_table_infos + .iter() + .map(SstableInfo::from) + .collect_vec(), + vnode_partition_count: pb_intra_level_delta.vnode_partition_count, + } + } +} + +impl IntraLevelDelta { + pub fn get_level_idx(&self) -> u32 { + self.level_idx + } + + pub fn get_l0_sub_level_id(&self) -> u64 { + self.l0_sub_level_id + } + + pub fn get_removed_table_ids(&self) -> &Vec { + &self.removed_table_ids + } + + pub fn get_inserted_table_infos(&self) -> &Vec { + &self.inserted_table_infos + } + + pub fn get_vnode_partition_count(&self) -> u32 { + self.vnode_partition_count + } +} + +impl IntraLevelDelta { + pub fn new( + level_idx: u32, + l0_sub_level_id: u64, + removed_table_ids: Vec, + inserted_table_infos: Vec, + vnode_partition_count: u32, + ) -> Self { + Self { + level_idx, + l0_sub_level_id, + removed_table_ids, + inserted_table_infos, + vnode_partition_count, + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum GroupDelta { + IntraLevel(IntraLevelDelta), + GroupConstruct(PbGroupConstruct), + GroupDestroy(PbGroupDestroy), + GroupMetaChange(PbGroupMetaChange), + + #[allow(dead_code)] + GroupTableChange(PbGroupTableChange), +} + +impl From for GroupDelta { + fn from(pb_group_delta: PbGroupDelta) -> Self { + match pb_group_delta.delta_type { + Some(PbDeltaType::IntraLevel(pb_intra_level_delta)) => { + GroupDelta::IntraLevel(IntraLevelDelta::from(pb_intra_level_delta)) + } + Some(PbDeltaType::GroupConstruct(pb_group_construct)) => { + GroupDelta::GroupConstruct(pb_group_construct) + } + Some(PbDeltaType::GroupDestroy(pb_group_destroy)) => { + GroupDelta::GroupDestroy(pb_group_destroy) + } + Some(PbDeltaType::GroupMetaChange(pb_group_meta_change)) => { + GroupDelta::GroupMetaChange(pb_group_meta_change) + } + Some(PbDeltaType::GroupTableChange(pb_group_table_change)) => { + GroupDelta::GroupTableChange(pb_group_table_change) + } + None => panic!("delta_type is not set"), + } + } +} + +impl From for PbGroupDelta { + fn from(group_delta: GroupDelta) -> Self { + match group_delta { + GroupDelta::IntraLevel(intra_level_delta) => PbGroupDelta { + delta_type: Some(PbDeltaType::IntraLevel(intra_level_delta.into())), + }, + GroupDelta::GroupConstruct(pb_group_construct) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupConstruct(pb_group_construct)), + }, + GroupDelta::GroupDestroy(pb_group_destroy) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupDestroy(pb_group_destroy)), + }, + GroupDelta::GroupMetaChange(pb_group_meta_change) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupMetaChange(pb_group_meta_change)), + }, + GroupDelta::GroupTableChange(pb_group_table_change) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupTableChange(pb_group_table_change)), + }, + } + } +} + +impl From<&GroupDelta> for PbGroupDelta { + fn from(group_delta: &GroupDelta) -> Self { + match group_delta { + GroupDelta::IntraLevel(intra_level_delta) => PbGroupDelta { + delta_type: Some(PbDeltaType::IntraLevel(intra_level_delta.into())), + }, + GroupDelta::GroupConstruct(pb_group_construct) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupConstruct(pb_group_construct.clone())), + }, + GroupDelta::GroupDestroy(pb_group_destroy) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupDestroy(pb_group_destroy.clone())), + }, + GroupDelta::GroupMetaChange(pb_group_meta_change) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupMetaChange(pb_group_meta_change.clone())), + }, + GroupDelta::GroupTableChange(pb_group_table_change) => PbGroupDelta { + delta_type: Some(PbDeltaType::GroupTableChange(pb_group_table_change.clone())), + }, + } + } +} + +impl From<&PbGroupDelta> for GroupDelta { + fn from(pb_group_delta: &PbGroupDelta) -> Self { + match &pb_group_delta.delta_type { + Some(PbDeltaType::IntraLevel(pb_intra_level_delta)) => { + GroupDelta::IntraLevel(IntraLevelDelta::from(pb_intra_level_delta)) + } + Some(PbDeltaType::GroupConstruct(pb_group_construct)) => { + GroupDelta::GroupConstruct(pb_group_construct.clone()) + } + Some(PbDeltaType::GroupDestroy(pb_group_destroy)) => { + GroupDelta::GroupDestroy(pb_group_destroy.clone()) + } + Some(PbDeltaType::GroupMetaChange(pb_group_meta_change)) => { + GroupDelta::GroupMetaChange(pb_group_meta_change.clone()) + } + Some(PbDeltaType::GroupTableChange(pb_group_table_change)) => { + GroupDelta::GroupTableChange(pb_group_table_change.clone()) + } + None => panic!("delta_type is not set"), + } + } +} + +#[derive(Debug, PartialEq, Clone, Default)] +pub struct GroupDeltas { + pub group_deltas: Vec, +} + +impl From for GroupDeltas { + fn from(pb_group_deltas: PbGroupDeltas) -> Self { + Self { + group_deltas: pb_group_deltas + .group_deltas + .into_iter() + .map(GroupDelta::from) + .collect_vec(), + } + } +} + +impl From for PbGroupDeltas { + fn from(group_deltas: GroupDeltas) -> Self { + Self { + group_deltas: group_deltas + .group_deltas + .into_iter() + .map(|group_delta| group_delta.into()) + .collect_vec(), + } + } +} + +impl From<&GroupDeltas> for PbGroupDeltas { + fn from(group_deltas: &GroupDeltas) -> Self { + Self { + group_deltas: group_deltas + .group_deltas + .iter() + .map(|group_delta| group_delta.into()) + .collect_vec(), + } + } +} + +impl From<&PbGroupDeltas> for GroupDeltas { + fn from(pb_group_deltas: &PbGroupDeltas) -> Self { + Self { + group_deltas: pb_group_deltas + .group_deltas + .iter() + .map(GroupDelta::from) + .collect_vec(), + } + } +} + +impl GroupDeltas { + pub fn get_group_deltas(&self) -> &Vec { + &self.group_deltas + } +} From 7a1fbb14e40954da7e3c847d080b4b39c67e8c93 Mon Sep 17 00:00:00 2001 From: Li0k Date: Fri, 12 Jul 2024 13:29:53 +0800 Subject: [PATCH 12/18] fix(storage): address check --- .../src/compaction_group/hummock_version_ext.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index f5e1d9350448..c1b6db4da388 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -1299,11 +1299,7 @@ pub fn split_sst(sst_info: &mut SstableInfo, new_sst_id: &mut u64) -> SstableInf mod tests { use std::collections::HashMap; - use risingwave_pb::hummock::group_delta::DeltaType; - use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas; - use risingwave_pb::hummock::{ - CompactionConfig, GroupConstruct, GroupDestroy, LevelType, PbIntraLevelDelta, - }; + use risingwave_pb::hummock::{CompactionConfig, GroupConstruct, GroupDestroy, LevelType}; use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels; use crate::version::{ @@ -1418,8 +1414,7 @@ mod tests { object_id: 1, sst_id: 1, ..Default::default() - } - .into()], + }], 0, ))], }, From bf1addf10bf0bc48a7bda2a7718cf771be0388eb Mon Sep 17 00:00:00 2001 From: Li0k Date: Tue, 23 Jul 2024 22:21:01 +0800 Subject: [PATCH 13/18] fix(storage): address comments --- Cargo.lock | 1 - src/ctl/src/cmd_impl/hummock/list_version.rs | 10 +- src/ctl/src/cmd_impl/hummock/sst_dump.rs | 6 +- .../src/cmd_impl/hummock/validate_version.rs | 2 +- .../rw_catalog/rw_hummock_version.rs | 4 +- src/meta/service/Cargo.toml | 3 +- src/meta/service/src/hummock_service.rs | 11 +- src/meta/src/barrier/mod.rs | 2 +- src/meta/src/hummock/compaction/mod.rs | 16 +- .../hummock/compaction/overlap_strategy.rs | 22 +- .../picker/base_level_compaction_picker.rs | 31 ++- .../picker/intra_compaction_picker.rs | 42 ++-- .../picker/manual_compaction_picker.rs | 37 ++- .../picker/min_overlap_compaction_picker.rs | 31 ++- .../picker/space_reclaim_compaction_picker.rs | 10 +- .../picker/tier_compaction_picker.rs | 2 +- .../picker/ttl_reclaim_compaction_picker.rs | 30 ++- .../picker/vnode_watermark_picker.rs | 4 +- .../compaction/selector/level_selector.rs | 13 +- .../src/hummock/compaction/selector/mod.rs | 4 +- src/meta/src/hummock/level_handler.rs | 4 +- src/meta/src/hummock/manager/commit_epoch.rs | 4 +- src/meta/src/hummock/manager/compaction.rs | 27 +-- .../manager/compaction_group_manager.rs | 2 +- src/meta/src/hummock/manager/gc.rs | 2 +- src/meta/src/hummock/manager/tests.rs | 54 ++--- src/meta/src/hummock/manager/time_travel.rs | 2 +- src/meta/src/hummock/manager/transaction.rs | 6 +- src/meta/src/hummock/manager/versioning.rs | 2 +- src/meta/src/hummock/metrics_utils.rs | 14 +- .../src/hummock/mock_hummock_meta_client.rs | 2 +- src/meta/src/hummock/test_utils.rs | 8 +- src/meta/src/hummock/vacuum.rs | 2 +- src/storage/hummock_sdk/src/compact.rs | 12 +- .../compaction_group/hummock_version_ext.rs | 133 ++++------- src/storage/hummock_sdk/src/lib.rs | 4 +- .../hummock_sdk/src/table_watermark.rs | 4 +- src/storage/hummock_sdk/src/version.rs | 210 +++++------------- .../hummock_test/src/compactor_tests.rs | 12 +- .../src/hummock_read_version_tests.rs | 6 +- .../hummock_test/src/state_store_tests.rs | 2 +- .../hummock_test/src/sync_point_tests.rs | 6 +- .../src/hummock/compactor/compaction_utils.rs | 10 +- .../src/hummock/compactor/compactor_runner.rs | 16 +- src/storage/src/hummock/compactor/iterator.rs | 2 +- src/storage/src/hummock/compactor/mod.rs | 2 +- .../event_handler/hummock_event_handler.rs | 3 - .../iterator/concat_delete_range_iterator.rs | 26 +-- .../src/hummock/iterator/concat_inner.rs | 7 +- src/storage/src/hummock/sstable/builder.rs | 13 +- src/storage/src/hummock/sstable_store.rs | 2 +- src/storage/src/hummock/store/version.rs | 6 +- src/storage/src/hummock/test_utils.rs | 8 +- src/storage/src/hummock/utils.rs | 26 +-- 54 files changed, 333 insertions(+), 587 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c970f802085..26e2810ab290 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11877,7 +11877,6 @@ version = "1.11.0-alpha" dependencies = [ "anyhow", "async-trait", - "bytes", "either", "futures", "itertools 0.12.1", diff --git a/src/ctl/src/cmd_impl/hummock/list_version.rs b/src/ctl/src/cmd_impl/hummock/list_version.rs index 1c7ea9c40936..282f4edf4d60 100644 --- a/src/ctl/src/cmd_impl/hummock/list_version.rs +++ b/src/ctl/src/cmd_impl/hummock/list_version.rs @@ -34,7 +34,7 @@ pub async fn list_version( let l0 = levels.l0.as_mut().unwrap(); for sub_level in &mut l0.sub_levels { for t in &mut sub_level.table_infos { - t.key_range = None; + t.remove_key_range(); } } } @@ -42,7 +42,7 @@ pub async fn list_version( // l1 ~ lmax for level in &mut levels.levels { for t in &mut level.table_infos { - t.key_range = None; + t.remove_key_range(); } } }); @@ -63,18 +63,18 @@ pub async fn list_version( println!( "sub_level_id {} type {} sst_num {} size {}", sub_level.sub_level_id, - sub_level.level_type().as_str_name(), + sub_level.level_type.as_str_name(), sub_level.table_infos.len(), sub_level.total_file_size ) } } - for level in levels.get_levels() { + for level in &levels.levels { println!( "level_idx {} type {} sst_num {} size {}", level.level_idx, - level.level_type().as_str_name(), + level.level_type.as_str_name(), level.table_infos.len(), level.total_file_size ) diff --git a/src/ctl/src/cmd_impl/hummock/sst_dump.rs b/src/ctl/src/cmd_impl/hummock/sst_dump.rs index f26faaa9c5a5..c024d91ac012 100644 --- a/src/ctl/src/cmd_impl/hummock/sst_dump.rs +++ b/src/ctl/src/cmd_impl/hummock/sst_dump.rs @@ -83,11 +83,11 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result for level in version.get_combined_levels() { for sstable_info in &level.table_infos { if let Some(object_id) = &args.object_id { - if *object_id == sstable_info.get_object_id() { + if *object_id == sstable_info.object_id { print_level(level, sstable_info); sst_dump_via_sstable_store( &sstable_store, - sstable_info.get_object_id(), + sstable_info.object_id, sstable_info.meta_offset, sstable_info.file_size, &table_data, @@ -100,7 +100,7 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result print_level(level, sstable_info); sst_dump_via_sstable_store( &sstable_store, - sstable_info.get_object_id(), + sstable_info.object_id, sstable_info.meta_offset, sstable_info.file_size, &table_data, diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs index de925421866a..db228cfc3dec 100644 --- a/src/ctl/src/cmd_impl/hummock/validate_version.rs +++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs @@ -109,7 +109,7 @@ async fn print_user_key_in_version( .chain(cg.levels.iter()) { for sstable_info in &level.table_infos { - let key_range = sstable_info.key_range.as_ref().unwrap(); + let key_range = &sstable_info.key_range; let left_user_key = FullKey::decode(&key_range.left); let right_user_key = FullKey::decode(&key_range.right); if left_user_key.user_key > *target_key || *target_key > right_user_key.user_key { diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs index 5c03bbdd8c5c..bd54381c2bc9 100644 --- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs +++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs @@ -92,7 +92,7 @@ fn remove_key_range_from_version(mut version: HummockVersion) -> HummockVersion .chain(cg.l0.as_mut().unwrap().sub_levels.iter_mut()) { for sst in &mut level.table_infos { - sst.key_range.take(); + sst.remove_key_range(); } } } @@ -117,7 +117,7 @@ fn version_to_sstable_rows(version: HummockVersion) -> Vec { for cg in version.levels.into_values() { for level in cg.levels.into_iter().chain(cg.l0.unwrap().sub_levels) { for sst in level.table_infos { - let key_range = sst.key_range.unwrap(); + let key_range = sst.key_range; sstables.push(RwHummockSstable { sstable_id: sst.sst_id as _, object_id: sst.object_id as _, diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index cfa9939f00c6..1e3330a2b53a 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -16,11 +16,10 @@ normal = ["workspace-hack"] [dependencies] anyhow = "1" async-trait = "0.1" -bytes = { version = "1" } either = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = { workspace = true } -prost ={ workspace = true } +prost = { workspace = true } rand = { workspace = true } regex = "1" risingwave_common = { workspace = true } diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs index f8e1bbe73579..579c6ffda7de 100644 --- a/src/meta/service/src/hummock_service.rs +++ b/src/meta/service/src/hummock_service.rs @@ -15,6 +15,7 @@ use std::collections::{HashMap, HashSet}; use std::time::Duration; +use compact_task::PbTaskStatus; use futures::StreamExt; use itertools::Itertools; use risingwave_common::catalog::{TableId, SYS_CATALOG_START_ID}; @@ -225,7 +226,6 @@ impl HummockManagerService for HummockServiceImpl { &self, request: Request, ) -> Result, Status> { - use bytes::Bytes; let request = request.into_inner(); let compaction_group_id = request.compaction_group_id; let mut option = ManualCompactionOption { @@ -238,8 +238,8 @@ impl HummockManagerService for HummockServiceImpl { match request.key_range { Some(pb_key_range) => { option.key_range = KeyRange { - left: Bytes::from(pb_key_range.left), - right: Bytes::from(pb_key_range.right), + left: pb_key_range.left.into(), + right: pb_key_range.right.into(), right_exclusive: pb_key_range.right_exclusive, }; } @@ -666,7 +666,10 @@ impl HummockManagerService for HummockServiceImpl { let request = request.into_inner(); let ret = self .hummock_manager - .cancel_compact_task(request.task_id, request.task_status()) + .cancel_compact_task( + request.task_id, + PbTaskStatus::try_from(request.task_status).unwrap(), + ) .await?; let response = Response::new(CancelCompactTaskResponse { ret }); diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs index ba621c562006..51fc54bf3980 100644 --- a/src/meta/src/barrier/mod.rs +++ b/src/meta/src/barrier/mod.rs @@ -1160,7 +1160,7 @@ fn collect_commit_epoch_info( for resp in resps { let ssts_iter = resp.synced_sstables.into_iter().map(|grouped| { let sst_info = grouped.sst.expect("field not None"); - sst_to_worker.insert(sst_info.get_object_id(), resp.worker_id); + sst_to_worker.insert(sst_info.object_id, resp.worker_id); LocalSstableInfo::new( sst_info.into(), from_prost_table_stats_map(grouped.table_stats_map), diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs index f0accf4aafa2..f2feabe77fbb 100644 --- a/src/meta/src/hummock/compaction/mod.rs +++ b/src/meta/src/hummock/compaction/mod.rs @@ -31,7 +31,7 @@ use risingwave_hummock_sdk::table_watermark::TableWatermarks; use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_hummock_sdk::{CompactionGroupId, HummockCompactionTaskId}; use risingwave_pb::hummock::compaction_config::CompactionMode; -use risingwave_pb::hummock::{CompactionConfig, LevelType, PbCompactTask}; +use risingwave_pb::hummock::{CompactionConfig, LevelType}; pub use selector::{CompactionSelector, CompactionSelectorContext}; use self::selector::{EmergencySelector, LocalSelectorStatistic}; @@ -145,12 +145,11 @@ impl CompactStatus { } pub fn is_trivial_move_task(task: &CompactTask) -> bool { - if task.task_type() != TaskType::Dynamic && task.task_type() != TaskType::Emergency { + if task.task_type != TaskType::Dynamic && task.task_type != TaskType::Emergency { return false; } - if task.input_ssts.len() != 2 - || task.input_ssts[0].level_type() != LevelType::Nonoverlapping + if task.input_ssts.len() != 2 || task.input_ssts[0].level_type != LevelType::Nonoverlapping { return false; } @@ -173,7 +172,7 @@ impl CompactStatus { pub fn is_trivial_reclaim(task: &CompactTask) -> bool { // Currently all VnodeWatermark tasks are trivial reclaim. - if task.task_type() == TaskType::VnodeWatermark { + if task.task_type == TaskType::VnodeWatermark { return true; } let exist_table_ids = HashSet::::from_iter(task.existing_table_ids.clone()); @@ -186,13 +185,6 @@ impl CompactStatus { }) } - /// Declares a task as either succeeded, failed or canceled. - pub fn report_compact_task_pb(&mut self, compact_task: &PbCompactTask) { - for level in &compact_task.input_ssts { - self.level_handlers[level.level_idx as usize].remove_task(compact_task.task_id); - } - } - pub fn report_compact_task(&mut self, compact_task: &CompactTask) { for level in &compact_task.input_ssts { self.level_handlers[level.level_idx as usize].remove_task(compact_task.task_id); diff --git a/src/meta/src/hummock/compaction/overlap_strategy.rs b/src/meta/src/hummock/compaction/overlap_strategy.rs index ecd89d8a27eb..3e302d996a35 100644 --- a/src/meta/src/hummock/compaction/overlap_strategy.rs +++ b/src/meta/src/hummock/compaction/overlap_strategy.rs @@ -85,18 +85,14 @@ impl OverlapInfo for RangeOverlapInfo { match self.target_range.as_ref() { Some(key_range) => { let overlap_begin = others.partition_point(|table_status| { - table_status - .key_range - .as_ref() - .unwrap() - .compare_right_with(&key_range.left) + table_status.key_range.compare_right_with(&key_range.left) == cmp::Ordering::Less }); if overlap_begin >= others.len() { return overlap_begin..overlap_begin; } let overlap_end = others.partition_point(|table_status| { - key_range.compare_right_with(&table_status.key_range.as_ref().unwrap().left) + key_range.compare_right_with(&table_status.key_range.left) != cmp::Ordering::Less }); overlap_begin..overlap_end @@ -110,7 +106,7 @@ impl OverlapInfo for RangeOverlapInfo { Some(key_range) => { let overlap_begin = others.partition_point(|table_status| { KeyComparator::compare_encoded_full_key( - &table_status.key_range.as_ref().unwrap().left, + &table_status.key_range.left, &key_range.left, ) == cmp::Ordering::Less }); @@ -119,9 +115,7 @@ impl OverlapInfo for RangeOverlapInfo { } let mut overlap_end = overlap_begin; for table in &others[overlap_begin..] { - if key_range.compare_right_with(&table.key_range.as_ref().unwrap().right) - == cmp::Ordering::Less - { + if key_range.compare_right_with(&table.key_range.right) == cmp::Ordering::Less { break; } overlap_end += 1; @@ -133,7 +127,7 @@ impl OverlapInfo for RangeOverlapInfo { } fn update(&mut self, table: &SstableInfo) { - let other = table.key_range.as_ref().unwrap(); + let other = &table.key_range; if let Some(range) = self.target_range.as_mut() { range.full_key_extend(other); return; @@ -147,8 +141,7 @@ pub struct RangeOverlapStrategy {} impl OverlapStrategy for RangeOverlapStrategy { fn check_overlap(&self, a: &SstableInfo, b: &SstableInfo) -> bool { - let key_range = a.key_range.as_ref().unwrap(); - check_table_overlap(key_range, b) + check_table_overlap(&a.key_range, b) } fn create_overlap_info(&self) -> Box { @@ -157,6 +150,5 @@ impl OverlapStrategy for RangeOverlapStrategy { } fn check_table_overlap(key_range: &KeyRange, table: &SstableInfo) -> bool { - let other = table.key_range.as_ref().unwrap(); - key_range.sstable_overlap(other) + key_range.sstable_overlap(&table.key_range) } diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index ceb660172228..bf9524bb28ae 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_common::config::default::compaction_config; -use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, OverlappingLevel}; use risingwave_pb::hummock::{CompactionConfig, LevelType}; @@ -338,8 +337,8 @@ pub mod tests { .pick_compaction(&levels, &levels_handler, &mut local_stats) .unwrap(); assert_eq!(ret.input_levels[0].table_infos.len(), 1); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 4); - assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 1); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 4); + assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 1); ret.add_pending_task(0, &mut levels_handler); { @@ -350,13 +349,13 @@ pub mod tests { .unwrap(); assert_eq!(ret2.input_levels[0].table_infos.len(), 1); - assert_eq!(ret2.input_levels[0].table_infos[0].get_sst_id(), 6); - assert_eq!(ret2.input_levels[1].table_infos[0].get_sst_id(), 5); + assert_eq!(ret2.input_levels[0].table_infos[0].sst_id, 6); + assert_eq!(ret2.input_levels[1].table_infos[0].sst_id, 5); } levels.l0.as_mut().unwrap().sub_levels[0] .table_infos - .retain(|table| table.get_sst_id() != 4); + .retain(|table| table.sst_id != 4); levels.l0.as_mut().unwrap().total_file_size -= ret.input_levels[0].table_infos[0].file_size; levels_handler[0].remove_task(0); @@ -366,11 +365,11 @@ pub mod tests { .pick_compaction(&levels, &levels_handler, &mut local_stats) .unwrap(); assert_eq!(ret.input_levels.len(), 3); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 6); - assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 5); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6); + assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5); assert_eq!(ret.input_levels[2].table_infos.len(), 2); - assert_eq!(ret.input_levels[2].table_infos[0].get_sst_id(), 3); - assert_eq!(ret.input_levels[2].table_infos[1].get_sst_id(), 2); + assert_eq!(ret.input_levels[2].table_infos[0].sst_id, 3); + assert_eq!(ret.input_levels[2].table_infos[1].sst_id, 2); ret.add_pending_task(1, &mut levels_handler); let mut local_stats = LocalPickerStatistic::default(); @@ -387,8 +386,8 @@ pub mod tests { .pick_compaction(&levels, &levels_handler, &mut local_stats) .unwrap(); assert_eq!(ret.input_levels.len(), 3); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 6); - assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 5); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6); + assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5); assert_eq!(ret.input_levels[2].table_infos.len(), 2); } #[test] @@ -447,7 +446,7 @@ pub mod tests { ret.input_levels[0] .table_infos .iter() - .map(|t| t.get_sst_id()) + .map(|t| t.sst_id) .collect_vec(), vec![1] ); @@ -456,7 +455,7 @@ pub mod tests { ret.input_levels[1] .table_infos .iter() - .map(|t| t.get_sst_id()) + .map(|t| t.sst_id) .collect_vec(), vec![3,] ); @@ -606,7 +605,7 @@ pub mod tests { let ret = picker .pick_compaction(&levels, &levels_handler, &mut local_stats) .unwrap(); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 7); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 7); assert_eq!( 3, ret.input_levels.iter().filter(|l| l.level_idx == 0).count() @@ -634,7 +633,7 @@ pub mod tests { let ret = picker .pick_compaction(&levels, &levels_handler, &mut local_stats) .unwrap(); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 6); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6); assert_eq!( 2, ret.input_levels.iter().filter(|l| l.level_idx == 0).count() diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 96f24579da72..93c505bf48fd 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -142,7 +142,7 @@ impl IntraCompactionPicker { } for (idx, level) in l0.sub_levels.iter().enumerate() { - if level.level_type() != LevelType::Nonoverlapping + if level.level_type != LevelType::Nonoverlapping || level.total_file_size > self.config.sub_level_max_compaction_bytes { continue; @@ -344,7 +344,7 @@ impl WholeLevelCompactionPicker { return None; } for (idx, level) in l0.sub_levels.iter().enumerate() { - if level.level_type() != LevelType::Nonoverlapping + if level.level_type != LevelType::Nonoverlapping || level.vnode_partition_count == partition_count { continue; @@ -539,7 +539,7 @@ pub mod tests { ..Default::default() }; let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)]; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos()); + levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); let config = Arc::new( CompactionConfigBuilder::new() .level0_sub_level_compact_level_count(1) @@ -587,7 +587,7 @@ pub mod tests { ..Default::default() }; let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)]; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos()); + levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); let config = Arc::new( CompactionConfigBuilder::new() .level0_sub_level_compact_level_count(1) @@ -610,9 +610,9 @@ pub mod tests { 3 ); - assert_eq!(4, ret.input_levels[0].table_infos[0].get_sst_id()); - assert_eq!(3, ret.input_levels[1].table_infos[0].get_sst_id()); - assert_eq!(1, ret.input_levels[2].table_infos[0].get_sst_id()); + assert_eq!(4, ret.input_levels[0].table_infos[0].sst_id); + assert_eq!(3, ret.input_levels[1].table_infos[0].sst_id); + assert_eq!(1, ret.input_levels[2].table_infos[0].sst_id); // will pick sst [2, 6, 5] let ret2 = picker @@ -627,9 +627,9 @@ pub mod tests { 3 ); - assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id()); - assert_eq!(6, ret2.input_levels[1].table_infos[0].get_sst_id()); - assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id()); + assert_eq!(5, ret2.input_levels[0].table_infos[0].sst_id); + assert_eq!(6, ret2.input_levels[1].table_infos[0].sst_id); + assert_eq!(2, ret2.input_levels[2].table_infos[0].sst_id); } { @@ -658,7 +658,7 @@ pub mod tests { ..Default::default() }; let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)]; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos()); + levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); let config = Arc::new( CompactionConfigBuilder::new() .level0_sub_level_compact_level_count(1) @@ -681,9 +681,9 @@ pub mod tests { 3 ); - assert_eq!(11, ret.input_levels[0].table_infos[0].get_sst_id()); - assert_eq!(9, ret.input_levels[1].table_infos[0].get_sst_id()); - assert_eq!(7, ret.input_levels[2].table_infos[0].get_sst_id()); + assert_eq!(11, ret.input_levels[0].table_infos[0].sst_id); + assert_eq!(9, ret.input_levels[1].table_infos[0].sst_id); + assert_eq!(7, ret.input_levels[2].table_infos[0].sst_id); let ret2 = picker .pick_compaction(&levels, &levels_handler, &mut local_stats) @@ -697,9 +697,9 @@ pub mod tests { 3 ); - assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id()); - assert_eq!(10, ret2.input_levels[1].table_infos[0].get_sst_id()); - assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id()); + assert_eq!(5, ret2.input_levels[0].table_infos[0].sst_id); + assert_eq!(10, ret2.input_levels[1].table_infos[0].sst_id); + assert_eq!(2, ret2.input_levels[2].table_infos[0].sst_id); } } @@ -715,7 +715,7 @@ pub mod tests { let config = Arc::new( CompactionConfigBuilder::new() .level0_tier_compact_file_number(2) - .target_file_size_base(30) + .tarfile_size_base(30) .level0_sub_level_compact_level_count(20) // reject intra .build(), ); @@ -732,7 +732,7 @@ pub mod tests { levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])], ..Default::default() }; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos()); + levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); let mut local_stats = LocalPickerStatistic::default(); let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats); assert!(ret.is_none()); @@ -860,13 +860,13 @@ pub mod tests { let input = ret.as_ref().unwrap(); assert_eq!(input.input_levels.len(), 2); assert_ne!( - levels.l0.as_ref().unwrap().get_sub_levels()[0] + levels.l0.as_ref().unwrap().sub_levels()[0] .table_infos .len(), input.input_levels[0].table_infos.len() ); assert_ne!( - levels.l0.as_ref().unwrap().get_sub_levels()[1] + levels.l0.as_ref().unwrap().sub_levels()[1] .table_infos .len(), input.input_levels[1].table_infos.len() diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index d8e653cf0cd5..008861b39333 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -16,7 +16,6 @@ use std::collections::HashSet; use std::sync::Arc; use itertools::Itertools; -use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, OverlappingLevel, SstableInfo}; use risingwave_pb::hummock::LevelType; @@ -181,7 +180,7 @@ impl ManualCompactionPicker { let mut hint_sst_ids: HashSet = HashSet::new(); hint_sst_ids.extend(self.option.sst_ids.iter()); let tmp_sst_info = SstableInfo { - key_range: Some(self.option.key_range.clone()), + key_range: self.option.key_range.clone(), ..Default::default() }; if self @@ -233,7 +232,7 @@ impl CompactionPicker for ManualCompactionPicker { hint_sst_ids.extend(self.option.sst_ids.iter()); let mut tmp_sst_info = SstableInfo::default(); let mut range_overlap_info = RangeOverlapInfo::default(); - tmp_sst_info.key_range = Some(self.option.key_range.clone()); + tmp_sst_info.key_range = self.option.key_range.clone(); range_overlap_info.update(&tmp_sst_info); let level = self.option.level; let target_level = self.target_level; @@ -271,15 +270,13 @@ impl CompactionPicker for ManualCompactionPicker { .get_level(level) .table_infos .iter() - .find_position(|p| { - p.get_sst_id() == select_input_ssts.first().unwrap().get_sst_id() - }) + .find_position(|p| p.sst_id == select_input_ssts.first().unwrap().sst_id) .unwrap(); let (right, _) = levels .get_level(level) .table_infos .iter() - .find_position(|p| p.get_sst_id() == select_input_ssts.last().unwrap().get_sst_id()) + .find_position(|p| p.sst_id == select_input_ssts.last().unwrap().sst_id) .unwrap(); select_input_ssts = levels.get_level(level).table_infos[left..=right].to_vec(); vec![] @@ -580,7 +577,7 @@ pub mod tests { for t in &mut l.table_infos { t.table_ids.clear(); if idx == 0 { - t.table_ids.push(((t.get_sst_id() % 2) + 1) as _); + t.table_ids.push(((t.sst_id % 2) + 1) as _); } else { t.table_ids.push(3); } @@ -704,7 +701,7 @@ pub mod tests { result.input_levels[l] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), *e ); @@ -740,7 +737,7 @@ pub mod tests { result.input_levels[l] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), *e ); @@ -792,7 +789,7 @@ pub mod tests { result.input_levels[i] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), *e ); @@ -858,7 +855,7 @@ pub mod tests { .iter() .take(3) .flat_map(|s| s.table_infos.clone()) - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), vec![9, 10, 7, 8, 5, 6] ); @@ -866,7 +863,7 @@ pub mod tests { result.input_levels[3] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), vec![3] ); @@ -900,7 +897,7 @@ pub mod tests { .iter() .take(3) .flat_map(|s| s.table_infos.clone()) - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), vec![9, 10, 7, 8, 5, 6] ); @@ -908,7 +905,7 @@ pub mod tests { result.input_levels[3] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), vec![3] ); @@ -946,7 +943,7 @@ pub mod tests { .iter() .take(1) .flat_map(|s| s.table_infos.clone()) - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), vec![5, 6] ); @@ -954,7 +951,7 @@ pub mod tests { result.input_levels[1] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), vec![3] ); @@ -1049,7 +1046,7 @@ pub mod tests { result.input_levels[l] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), *e ); @@ -1092,7 +1089,7 @@ pub mod tests { result.input_levels[i] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), *e ); @@ -1142,7 +1139,7 @@ pub mod tests { result.input_levels[i] .table_infos .iter() - .map(|s| s.get_sst_id()) + .map(|s| s.sst_id) .collect_vec(), *e ); diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index a5af0d16d739..81f06361f3e5 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use risingwave_hummock_sdk::append_sstable_info_to_string; -use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, SstableInfo}; use risingwave_pb::hummock::LevelType; @@ -252,7 +251,7 @@ impl NonOverlapSubLevelPicker { // Pay attention to the order here: Make sure to select the lowest sub_level to meet the requirements of base compaction. If you break the assumption of this order, you need to redesign it. // TODO: Use binary selection to replace the step algorithm to optimize algorithm complexity 'expand_new_level: for (target_index, target_level) in levels.iter().enumerate().skip(1) { - if target_level.level_type() != LevelType::Nonoverlapping { + if target_level.level_type != LevelType::Nonoverlapping { break; } @@ -309,7 +308,7 @@ impl NonOverlapSubLevelPicker { } basic_overlap_info.update(other); - add_files_size += other.get_file_size(); + add_files_size += other.file_size; add_files_count += 1; } @@ -344,11 +343,7 @@ impl NonOverlapSubLevelPicker { // sort sst per level due to reverse expand ret.sstable_infos.iter_mut().for_each(|level_ssts| { - level_ssts.sort_by(|sst1, sst2| { - let a = sst1.key_range.as_ref().unwrap(); - let b = sst2.key_range.as_ref().unwrap(); - a.cmp(b) - }); + level_ssts.sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range)); }); } else { ret.total_file_count = 1; @@ -590,7 +585,7 @@ pub mod tests { assert_eq!(ret.input_levels[0].level_idx, 1); assert_eq!(ret.target_level, 2); assert_eq!(ret.input_levels[0].table_infos.len(), 1); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 2); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 2); assert_eq!(ret.input_levels[1].table_infos.len(), 0); ret.add_pending_task(0, &mut level_handlers); @@ -600,18 +595,18 @@ pub mod tests { assert_eq!(ret.input_levels[0].level_idx, 1); assert_eq!(ret.target_level, 2); assert_eq!(ret.input_levels[0].table_infos.len(), 1); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 0); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 0); assert_eq!(ret.input_levels[1].table_infos.len(), 1); - assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 4); + assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 4); ret.add_pending_task(1, &mut level_handlers); let ret = picker .pick_compaction(&levels, &level_handlers, &mut local_stats) .unwrap(); assert_eq!(ret.input_levels[0].table_infos.len(), 1); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 1); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 1); assert_eq!(ret.input_levels[1].table_infos.len(), 2); - assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 5); + assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5); } #[test] @@ -663,11 +658,11 @@ pub mod tests { assert_eq!(ret.input_levels[1].level_idx, 2); assert_eq!(ret.input_levels[0].table_infos.len(), 2); - assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 0); - assert_eq!(ret.input_levels[0].table_infos[1].get_sst_id(), 1); + assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 0); + assert_eq!(ret.input_levels[0].table_infos[1].sst_id, 1); assert_eq!(ret.input_levels[1].table_infos.len(), 1); - assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 4); + assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 4); } #[test] @@ -885,7 +880,7 @@ pub mod tests { for plan in ret { let mut sst_id_set = BTreeSet::default(); for sst in &plan.sstable_infos { - sst_id_set.insert(sst[0].get_sst_id()); + sst_id_set.insert(sst[0].sst_id); } assert!(sst_id_set.len() <= max_file_count as usize); } @@ -909,7 +904,7 @@ pub mod tests { for plan in ret { let mut sst_id_set = BTreeSet::default(); for sst in &plan.sstable_infos { - sst_id_set.insert(sst[0].get_sst_id()); + sst_id_set.insert(sst[0].sst_id); } assert!(plan.sstable_infos.len() >= min_depth); } diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 45f00155aedf..7c1adffbf886 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -231,7 +231,7 @@ mod test { { let sst_10 = levels[3].table_infos.get_mut(8).unwrap(); - assert_eq!(10, sst_10.get_sst_id()); + assert_eq!(10, sst_10.sst_id); sst_10.key_range.as_mut().unwrap().right_exclusive = true; } @@ -293,7 +293,7 @@ mod test { let mut start_id = 2; for sst in &task.input.input_levels[0].table_infos { - assert_eq!(start_id, sst.get_sst_id()); + assert_eq!(start_id, sst.sst_id); start_id += 1; } @@ -343,7 +343,7 @@ mod test { )); let mut start_id = 8; for sst in &task.input.input_levels[0].table_infos { - assert_eq!(start_id, sst.get_sst_id()); + assert_eq!(start_id, sst.sst_id); start_id += 1; } @@ -475,7 +475,7 @@ mod test { let select_sst = &task.input.input_levels[0] .table_infos .iter() - .map(|sst| sst.get_sst_id()) + .map(|sst| sst.sst_id) .collect_vec(); assert!(select_sst.is_sorted()); assert_eq!(expect_task_sst_id_range[index], *select_sst); @@ -536,7 +536,7 @@ mod test { let select_sst = &task.input.input_levels[0] .table_infos .iter() - .map(|sst| sst.get_sst_id()) + .map(|sst| sst.sst_id) .collect_vec(); assert!(select_sst.is_sorted()); assert_eq!(expect_task_sst_id_range[index], *select_sst); diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index 6cf48561e06b..b0b71e6c1a84 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -55,7 +55,7 @@ impl TierCompactionPicker { stats: &mut LocalPickerStatistic, ) -> Option { for (idx, level) in l0.sub_levels.iter().enumerate() { - if level.level_type() != LevelType::Overlapping { + if level.level_type != LevelType::Overlapping { continue; } diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index ddaf5cc5a5c3..7f02a140f088 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -137,9 +137,9 @@ impl TtlReclaimCompactionPicker { let last_sst = reclaimed_level.table_infos.last().unwrap(); let key_range_this_round = KeyRange { - left: first_sst.key_range.as_ref().unwrap().left.clone(), - right: last_sst.key_range.as_ref().unwrap().right.clone(), - right_exclusive: last_sst.key_range.as_ref().unwrap().right_exclusive, + left: first_sst.key_range.left.clone(), + right: last_sst.key_range.right.clone(), + right_exclusive: last_sst.key_range.right_exclusive, }; state.init(key_range_this_round); @@ -148,11 +148,7 @@ impl TtlReclaimCompactionPicker { let current_epoch_physical_time = Epoch::now().physical_time(); for sst in &reclaimed_level.table_infos { - let unmatched_sst = sst - .key_range - .as_ref() - .unwrap() - .sstable_overlap(&state.last_select_end_bound); + let unmatched_sst = sst.key_range.sstable_overlap(&state.last_select_end_bound); if unmatched_sst || level_handler.is_pending_compact(&sst.sst_id) @@ -174,8 +170,8 @@ impl TtlReclaimCompactionPicker { let select_last_sst = select_input_ssts.last().unwrap(); state.last_select_end_bound.full_key_extend(&KeyRange { left: Bytes::default(), - right: select_last_sst.key_range.as_ref().unwrap().right.clone(), - right_exclusive: select_last_sst.key_range.as_ref().unwrap().right_exclusive, + right: select_last_sst.key_range.right.clone(), + right_exclusive: select_last_sst.key_range.right_exclusive, }); Some(CompactionInput { @@ -350,7 +346,7 @@ mod test { { let sst_10 = levels[3].table_infos.get_mut(8).unwrap(); - assert_eq!(10, sst_10.get_sst_id()); + assert_eq!(10, sst_10.sst_id); sst_10.key_range.as_mut().unwrap().right_exclusive = true; } @@ -398,7 +394,7 @@ mod test { let mut start_id = 2; for sst in &task.input.input_levels[0].table_infos { - assert_eq!(start_id, sst.get_sst_id()); + assert_eq!(start_id, sst.sst_id); start_id += 1; } @@ -455,7 +451,7 @@ mod test { let mut start_id = 3; for sst in &task.input.input_levels[0].table_infos { - assert_eq!(start_id, sst.get_sst_id()); + assert_eq!(start_id, sst.sst_id); start_id += 1; } @@ -496,7 +492,7 @@ mod test { compact_task::TaskType::Ttl )); for sst in &task.input.input_levels[0].table_infos { - assert_eq!(start_id, sst.get_sst_id()); + assert_eq!(start_id, sst.sst_id); start_id += 1; } } @@ -552,7 +548,7 @@ mod test { // test table_option_filter assert_eq!(task.input.input_levels[0].table_infos.len(), 1); let select_sst = &task.input.input_levels[0].table_infos.first().unwrap(); - assert_eq!(select_sst.get_sst_id(), 5); + assert_eq!(select_sst.sst_id, 5); assert_eq!(task.input.input_levels[1].level_idx, 4); assert_eq!(task.input.input_levels[1].table_infos.len(), 0); @@ -668,7 +664,7 @@ mod test { let select_sst = &task.input.input_levels[0] .table_infos .iter() - .map(|sst| sst.get_sst_id()) + .map(|sst| sst.sst_id) .collect_vec(); assert!(select_sst.is_sorted()); assert_eq!(expect_task_sst_id_range[index], *select_sst); @@ -765,7 +761,7 @@ mod test { let select_sst = &task.input.input_levels[0] .table_infos .iter() - .map(|sst| sst.get_sst_id()) + .map(|sst| sst.sst_id) .collect_vec(); assert!(select_sst.is_sorted()); assert_eq!(expect_task_sst_id_range[index], *select_sst); diff --git a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs index 09143717baa3..c6d04865e6e5 100644 --- a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs +++ b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs @@ -76,8 +76,8 @@ fn should_delete_sst_by_watermark( ) -> bool { // Both table id and vnode must be identical for both the left and right keys in a SST. // As more data is written to the bottommost level, they will eventually become identical. - let left_key = FullKey::decode(&sst_info.key_range.as_ref().unwrap().left); - let right_key = FullKey::decode(&sst_info.key_range.as_ref().unwrap().right); + let left_key = FullKey::decode(&sst_info.key_range.left); + let right_key = FullKey::decode(&sst_info.key_range.right); if left_key.user_key.table_id != right_key.user_key.table_id { return false; } diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index ff7c419cbb8a..3e550d41a0b7 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -18,7 +18,6 @@ // (found in the LICENSE.Apache file in the root directory). use std::sync::Arc; -use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::version::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task::PbTaskType; @@ -228,7 +227,7 @@ impl DynamicLevelSelectorCore { .unwrap() .sub_levels .iter() - .filter(|level| level.level_type() == LevelType::Overlapping) + .filter(|level| level.level_type == LevelType::Overlapping) .map(|level| level.table_infos.len()) .sum::(); if overlapping_file_count > 0 { @@ -257,7 +256,7 @@ impl DynamicLevelSelectorCore { .iter() .filter(|level| { level.vnode_partition_count == self.config.split_weight_by_vnode - && level.level_type() == LevelType::Nonoverlapping + && level.level_type == LevelType::Nonoverlapping }) .map(|level| level.total_file_size) .sum::() @@ -275,7 +274,7 @@ impl DynamicLevelSelectorCore { .unwrap() .sub_levels .iter() - .filter(|level| level.level_type() == LevelType::Nonoverlapping) + .filter(|level| level.level_type == LevelType::Nonoverlapping) .count() as u64; let non_overlapping_level_score = non_overlapping_level_count * SCORE_BASE / std::cmp::max( @@ -376,7 +375,7 @@ impl DynamicLevelSelectorCore { let mut level_bytes; let mut next_level_bytes = 0; for level in &levels.levels[ctx.base_level - 1..levels.levels.len()] { - let level_index = level.get_level_idx() as usize; + let level_index = level.level_idx as usize; if next_level_bytes > 0 { level_bytes = next_level_bytes; @@ -688,7 +687,7 @@ pub mod tests { compaction.input.input_levels[0] .table_infos .iter() - .map(|sst| sst.get_sst_id()) + .map(|sst| sst.sst_id) .collect_vec(), vec![5] ); @@ -696,7 +695,7 @@ pub mod tests { compaction.input.input_levels[1] .table_infos .iter() - .map(|sst| sst.get_sst_id()) + .map(|sst| sst.sst_id) .collect_vec(), vec![10] ); diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index f1466258ea70..2797f39b07b6 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -140,7 +140,7 @@ pub mod tests { level_type: LevelType::Overlapping, total_file_size: sst.file_size, uncompressed_file_size: sst.uncompressed_file_size, - sub_level_id: sst.get_sst_id(), + sub_level_id: sst.sst_id, table_infos: vec![sst], ..Default::default() }); @@ -164,7 +164,7 @@ pub mod tests { .iter() .map(|table| table.uncompressed_file_size) .sum(); - let sub_level_id = table_infos[0].get_sst_id(); + let sub_level_id = table_infos[0].sst_id; levels.l0.as_mut().unwrap().total_file_size += total_file_size; levels.l0.as_mut().unwrap().sub_levels.push(Level { level_idx: 0, diff --git a/src/meta/src/hummock/level_handler.rs b/src/meta/src/hummock/level_handler.rs index 921b80fb87b5..94db30edb8d0 100644 --- a/src/meta/src/hummock/level_handler.rs +++ b/src/meta/src/hummock/level_handler.rs @@ -90,9 +90,9 @@ impl LevelHandler { let mut table_ids = vec![]; let mut total_file_size = 0; for sst in ssts { - self.compacting_files.insert(sst.get_sst_id(), task_id); + self.compacting_files.insert(sst.sst_id, task_id); total_file_size += sst.file_size; - table_ids.push(sst.get_sst_id()); + table_ids.push(sst.sst_id); } self.pending_tasks.push(RunningCompactTask { diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs index 93b7c0c0211b..66d67a23b380 100644 --- a/src/meta/src/hummock/manager/commit_epoch.rs +++ b/src/meta/src/hummock/manager/commit_epoch.rs @@ -349,7 +349,7 @@ impl HummockManager { let mut sst_to_cg_vec = Vec::with_capacity(sstables.len()); for commit_sst in sstables { let mut group_table_ids: BTreeMap> = BTreeMap::new(); - for table_id in commit_sst.sst_info.get_table_ids() { + for table_id in &commit_sst.sst_info.table_ids { match table_compaction_group_mapping.get(&TableId::new(*table_id)) { Some(cg_id_from_meta) => { group_table_ids @@ -361,7 +361,7 @@ impl HummockManager { tracing::warn!( "table {} in SST {} doesn't belong to any compaction group", table_id, - commit_sst.sst_info.get_object_id(), + commit_sst.sst_info.object_id, ); } } diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs index 4b68d8e78fc1..3a296452fa65 100644 --- a/src/meta/src/hummock/manager/compaction.rs +++ b/src/meta/src/hummock/manager/compaction.rs @@ -157,11 +157,8 @@ impl<'a> HummockVersionTransaction<'a> { for level in &compact_task.input_ssts { let level_idx = level.level_idx; - let mut removed_table_ids = level - .table_infos - .iter() - .map(|sst| sst.get_sst_id()) - .collect_vec(); + let mut removed_table_ids = + level.table_infos.iter().map(|sst| sst.sst_id).collect_vec(); removed_table_ids_map .entry(level_idx) @@ -992,7 +989,7 @@ impl HummockManager { "For compaction group {}: pick up {} {} sub_level in level {} to compact to target {}. cost time: {:?} compact_task_statistics {:?}", compaction_group_id, level_count, - compact_task.input_ssts[0].level_type().as_str_name(), + compact_task.input_ssts[0].level_type.as_str_name(), compact_task.input_ssts[0].level_idx, compact_task.target_level, start_time.elapsed(), @@ -1064,7 +1061,7 @@ impl HummockManager { .get_compact_tasks_impl(compaction_groups, max_select_count, selector) .await?; tasks.retain(|task| { - if task.task_status() == TaskStatus::Success { + if task.task_status == TaskStatus::Success { debug_assert!( CompactStatus::is_trivial_reclaim(task) || CompactStatus::is_trivial_move_task(task) @@ -1090,7 +1087,7 @@ impl HummockManager { .get_compact_tasks_impl(vec![compaction_group_id], 1, selector) .await?; for task in normal_tasks { - if task.task_status() != TaskStatus::Success { + if task.task_status != TaskStatus::Success { return Ok(Some(task)); } debug_assert!( @@ -1120,7 +1117,7 @@ impl HummockManager { .levels .get(&compact_task.compaction_group_id) { - for input_level in compact_task.get_input_ssts() { + for input_level in &compact_task.input_ssts { let input_level: &InputLevel = input_level; let mut sst_ids: HashSet<_> = input_level .table_infos @@ -1133,7 +1130,7 @@ impl HummockManager { } } if input_level.level_idx == 0 { - for level in &group.get_level0().sub_levels { + for level in &group.level0().sub_levels { filter_ssts(level, &mut sst_ids); } } else { @@ -1249,7 +1246,7 @@ impl HummockManager { .iter() .map(|level| level.level_idx) .collect(); - let is_success = if let TaskStatus::Success = compact_task.task_status() { + let is_success = if let TaskStatus::Success = compact_task.task_status { // if member_table_ids changes, the data of sstable may stale. let is_expired = Self::is_compact_task_expired(&compact_task, version.latest_version()); @@ -1317,9 +1314,9 @@ impl HummockManager { } let mut success_groups = vec![]; for compact_task in tasks { - let task_status = compact_task.task_status(); + let task_status = compact_task.task_status; let task_status_label = task_status.as_str_name(); - let task_type_label = compact_task.task_type().as_str_name(); + let task_type_label = compact_task.task_type.as_str_name(); self.compactor_manager .remove_task_heartbeat(compact_task.task_id); @@ -1350,8 +1347,8 @@ impl HummockManager { ); if !deterministic_mode - && (matches!(compact_task.task_type(), compact_task::TaskType::Dynamic) - || matches!(compact_task.task_type(), compact_task::TaskType::Emergency)) + && (matches!(compact_task.task_type, compact_task::TaskType::Dynamic) + || matches!(compact_task.task_type, compact_task::TaskType::Emergency)) { // only try send Dynamic compaction self.try_send_compaction_request( diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs index 61a5280d2c00..3b2c4502371a 100644 --- a/src/meta/src/hummock/manager/compaction_group_manager.rs +++ b/src/meta/src/hummock/manager/compaction_group_manager.rs @@ -328,7 +328,7 @@ impl HummockManager { new_version_delta .latest_version() .get_compaction_group_levels(group_id) - .get_levels() + .levels .len(), )); } diff --git a/src/meta/src/hummock/manager/gc.rs b/src/meta/src/hummock/manager/gc.rs index f1e4a4ba426d..97a99945bcf4 100644 --- a/src/meta/src/hummock/manager/gc.rs +++ b/src/meta/src/hummock/manager/gc.rs @@ -430,7 +430,7 @@ mod tests { let committed_object_ids = sst_infos .into_iter() .flatten() - .map(|s| s.get_object_id()) + .map(|s| s.object_id) .sorted() .collect_vec(); assert!(!committed_object_ids.is_empty()); diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs index 2090057d8631..0f161ef677d2 100644 --- a/src/meta/src/hummock/manager/tests.rs +++ b/src/meta/src/hummock/manager/tests.rs @@ -226,14 +226,7 @@ async fn test_hummock_compaction_task() { .await .unwrap() .unwrap(); - assert_eq!( - compact_task - .get_input_ssts() - .first() - .unwrap() - .get_level_idx(), - 0 - ); + assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); assert_eq!(compact_task.get_task_id(), 2); // Cancel the task and succeed. @@ -293,9 +286,9 @@ async fn test_hummock_table() { .iter() .chain(levels.levels.iter()) .flat_map(|level| level.table_infos.iter()) - .map(|info| info.get_object_id()) + .map(|info| info.object_id) .sorted() - .cmp(original_tables.iter().map(|ot| ot.get_object_id()).sorted()) + .cmp(original_tables.iter().map(|ot| ot.object_id).sorted()) ); // Confirm tables got are equal to original tables @@ -762,14 +755,7 @@ async fn test_print_compact_task() { .await .unwrap() .unwrap(); - assert_eq!( - compact_task - .get_input_ssts() - .first() - .unwrap() - .get_level_idx(), - 0 - ); + assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); let s = compact_task_to_string(&compact_task); assert!(s.contains("Compaction task id: 1, group-id: 2, type: Dynamic, target level: 0")); @@ -791,7 +777,7 @@ async fn test_invalid_sst_id() { // reject due to invalid context id let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), WorkerId::MAX)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, WorkerId::MAX)) .collect(); let error = hummock_manager .commit_epoch_for_test(epoch, ssts.clone(), sst_to_worker) @@ -801,7 +787,7 @@ async fn test_invalid_sst_id() { let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id)) .collect(); hummock_manager .commit_epoch_for_test(epoch, ssts, sst_to_worker) @@ -941,14 +927,7 @@ async fn test_hummock_compaction_task_heartbeat() { .unwrap() .unwrap(); - assert_eq!( - compact_task - .get_input_ssts() - .first() - .unwrap() - .get_level_idx(), - 0 - ); + assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); assert_eq!(compact_task.get_task_id(), 2); for i in 0..10 { @@ -1061,14 +1040,7 @@ async fn test_hummock_compaction_task_heartbeat_removal_on_node_removal() { .unwrap() .unwrap(); - assert_eq!( - compact_task - .get_input_ssts() - .first() - .unwrap() - .get_level_idx(), - 0 - ); + assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); assert_eq!(compact_task.get_task_id(), 2); // send heartbeats to the task immediately @@ -1104,8 +1076,8 @@ async fn test_extend_objects_to_delete() { .iter() .map(|ssts| { ssts.iter() - .max_by_key(|s| s.get_object_id()) - .map(|s| s.get_object_id()) + .max_by_key(|s| s.object_id) + .map(|s| s.object_id) .unwrap() }) .max() @@ -1114,7 +1086,7 @@ async fn test_extend_objects_to_delete() { let all_object_ids = sst_infos .iter() .flatten() - .map(|s| s.get_object_id()) + .map(|s| s.object_id) .chain(max_committed_object_id + 1..=max_committed_object_id + orphan_sst_num) .collect_vec(); assert!(hummock_manager.get_objects_to_delete().is_empty()); @@ -1248,7 +1220,7 @@ async fn test_version_stats() { .collect_vec(); let sst_to_worker = ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), worker_node.id)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, worker_node.id)) .collect(); hummock_manager .commit_epoch_for_test(epoch, ssts, sst_to_worker) @@ -1862,7 +1834,7 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { assert_eq!( current_version.get_compaction_group_levels(2).levels[base_level - 1].table_infos[0] .object_id, - sst_1.sst_info.get_object_id() + 1, + sst_1.sst_info.object_id + 1, ); assert_eq!( current_version.get_compaction_group_levels(2).levels[base_level - 1].table_infos[0] diff --git a/src/meta/src/hummock/manager/time_travel.rs b/src/meta/src/hummock/manager/time_travel.rs index 2fc2368a57a8..9d55b3639ce5 100644 --- a/src/meta/src/hummock/manager/time_travel.rs +++ b/src/meta/src/hummock/manager/time_travel.rs @@ -325,7 +325,7 @@ impl HummockManager { .await?; for sst_info in sst_infos { let sst_info: SstableInfo = sst_info.sstable_info.to_protobuf().into(); - sst_id_to_info.insert(sst_info.get_sst_id(), sst_info); + sst_id_to_info.insert(sst_info.sst_id, sst_info); } } if sst_count != sst_id_to_info.len() { diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs index 50d560145d4c..d7559ea1c4be 100644 --- a/src/meta/src/hummock/manager/transaction.rs +++ b/src/meta/src/hummock/manager/transaction.rs @@ -119,11 +119,7 @@ impl<'a> HummockVersionTransaction<'a> { let mut new_version_delta = self.new_delta(); new_version_delta.max_committed_epoch = epoch; new_version_delta.new_table_watermarks = new_table_watermarks; - new_version_delta.change_log_delta = change_log_delta - .into_iter() - .map(|(table_id, delta)| (table_id, delta)) - .collect(); - + new_version_delta.change_log_delta = change_log_delta; // Append SSTs to a new version. for (compaction_group_id, inserted_table_infos) in commit_sstables { let group_deltas = &mut new_version_delta diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs index f257268d9493..6efe5ebcdef8 100644 --- a/src/meta/src/hummock/manager/versioning.rs +++ b/src/meta/src/hummock/manager/versioning.rs @@ -380,7 +380,7 @@ fn estimate_table_stats(sst: &SstableInfo) -> HashMap { let mut changes: HashMap = HashMap::default(); let weighted_value = |value: i64| -> i64 { (value as f64 / sst.table_ids.len() as f64).ceil() as i64 }; - let key_range = sst.key_range.as_ref().unwrap(); + let key_range = &sst.key_range; let estimated_key_size: u64 = (key_range.left.len() + key_range.right.len()) as u64 / 2; let mut estimated_total_key_size = estimated_key_size * sst.total_key_count; if estimated_total_key_size > sst.uncompressed_file_size { diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs index 8e24b71e7ef0..6adcfca18469 100644 --- a/src/meta/src/hummock/metrics_utils.rs +++ b/src/meta/src/hummock/metrics_utils.rs @@ -227,7 +227,7 @@ pub fn trigger_sst_stat( level.l0.as_ref().map(|l0| { l0.sub_levels .iter() - .filter(|sub_level| sub_level.level_type() == LevelType::Overlapping) + .filter(|sub_level| sub_level.level_type == LevelType::Overlapping) .count() }) }) @@ -240,7 +240,7 @@ pub fn trigger_sst_stat( level.l0.as_ref().map(|l0| { l0.sub_levels .iter() - .filter(|sub_level| sub_level.level_type() == LevelType::Nonoverlapping) + .filter(|sub_level| sub_level.level_type == LevelType::Nonoverlapping) .count() }) }) @@ -254,7 +254,7 @@ pub fn trigger_sst_stat( l0.sub_levels .iter() .filter(|sub_level| { - sub_level.level_type() == LevelType::Nonoverlapping + sub_level.level_type == LevelType::Nonoverlapping && sub_level.vnode_partition_count > 0 }) .count() @@ -480,16 +480,16 @@ pub fn trigger_lsm_stat( { // compact_level_compression_ratio let level_compression_ratio = levels - .get_levels() + .levels .iter() .map(|level| { - let ratio = if level.get_uncompressed_file_size() == 0 { + let ratio = if level.uncompressed_file_size == 0 { 0.0 } else { - level.get_total_file_size() as f64 / level.get_uncompressed_file_size() as f64 + level.total_file_size as f64 / level.uncompressed_file_size as f64 }; - (level.get_level_idx(), ratio) + (level.level_idx, ratio) }) .collect_vec(); diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index d496cd882384..bd895b808b74 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -163,7 +163,7 @@ impl HummockMetaClient for MockHummockMetaClient { let sst_to_worker = sync_result .uncommitted_ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), self.context_id)) + .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, self.context_id)) .collect(); let new_table_watermark = sync_result.table_watermarks; let table_change_log = build_table_change_log_delta( diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs index 20f1e635f3fb..bf9ae8cebeea 100644 --- a/src/meta/src/hummock/test_utils.rs +++ b/src/meta/src/hummock/test_utils.rs @@ -168,7 +168,7 @@ pub fn generate_test_sstables_with_table_id( sst_info.push(SstableInfo { object_id: sst_id, sst_id, - key_range: Some(KeyRange { + key_range: KeyRange { left: Bytes::from(key_with_epoch( format!("{:03}\0\0_key_test_{:05}", table_id, i + 1) .as_bytes() @@ -182,7 +182,7 @@ pub fn generate_test_sstables_with_table_id( epoch, )), right_exclusive: false, - }), + }, file_size: 2, table_ids: vec![table_id], uncompressed_file_size: 2, @@ -199,11 +199,11 @@ pub fn generate_test_tables(epoch: u64, sst_ids: Vec) -> sst_info.push(SstableInfo { object_id: sst_id, sst_id, - key_range: Some(KeyRange { + key_range: KeyRange { left: Bytes::from(iterator_test_key_of_epoch(sst_id, i + 1, epoch)), right: Bytes::from(iterator_test_key_of_epoch(sst_id, (i + 1) * 10, epoch)), right_exclusive: false, - }), + }, file_size: 2, table_ids: vec![sst_id as u32, sst_id as u32 * 10000], uncompressed_file_size: 2, diff --git a/src/meta/src/hummock/vacuum.rs b/src/meta/src/hummock/vacuum.rs index 5dffb35dfc6c..08f227dd3184 100644 --- a/src/meta/src/hummock/vacuum.rs +++ b/src/meta/src/hummock/vacuum.rs @@ -272,7 +272,7 @@ mod tests { .first() .unwrap() .iter() - .map(|s| s.get_object_id()) + .map(|s| s.object_id) .collect_vec(), }) .await diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index b71b167a9430..5d827f8adcde 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -27,13 +27,13 @@ pub fn compact_task_output_to_string(compact_task: &CompactTask) -> String { "Compaction task id: {:?}, group-id: {:?}, type: {:?}, target level: {:?}, target sub level: {:?} watermark: {:?}, target_file_size: {:?}, splits: {:?}, status: {:?}", compact_task.task_id, compact_task.compaction_group_id, - compact_task.task_type(), + compact_task.task_type, compact_task.target_level, compact_task.target_sub_level_id, compact_task.watermark, compact_task.target_file_size, compact_task.splits.len(), - compact_task.task_status() + compact_task.task_status ) .unwrap(); s.push_str("Output: \n"); @@ -83,7 +83,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { if table.total_key_count != 0 { format!( "[id: {}, obj_id: {} {}KB stale_ratio {}]", - table.get_sst_id(), + table.sst_id, table.object_id, table.file_size / 1024, (table.stale_key_count * 100 / table.total_key_count), @@ -120,7 +120,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo) { use std::fmt::Write; - let key_range = sstable_info.key_range.as_ref().unwrap(); + let key_range = &sstable_info.key_range; let left_str = if key_range.left.is_empty() { "-inf".to_string() } else { @@ -138,8 +138,8 @@ pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo) writeln!( s, "SstableInfo: object id={}, SST id={}, KeyRange=[{:?},{:?}], table_ids: {:?}, size={}KB, stale_ratio={}%, bloom_filter_kind {:?}", - sstable_info.get_object_id(), - sstable_info.get_sst_id(), + sstable_info.object_id, + sstable_info.sst_id, left_str, right_str, sstable_info.table_ids, diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index b01e3b1ec983..09ed8c9cd791 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -52,7 +52,7 @@ pub struct GroupDeltasSummary { } pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary { - let mut delete_sst_levels = Vec::with_capacity(group_deltas.get_group_deltas().len()); + let mut delete_sst_levels = Vec::with_capacity(group_deltas.group_deltas.len()); let mut delete_sst_ids_set = HashSet::new(); let mut insert_sst_level_id = u32::MAX; let mut insert_sub_level_id = u64::MAX; @@ -63,20 +63,19 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary let mut group_table_change = None; let mut new_vnode_partition_count = 0; - for group_delta in group_deltas.get_group_deltas() { + for group_delta in &group_deltas.group_deltas { match group_delta { GroupDelta::IntraLevel(intra_level) => { - if !intra_level.get_removed_table_ids().is_empty() { - delete_sst_levels.push(intra_level.get_level_idx()); - delete_sst_ids_set.extend(intra_level.get_removed_table_ids().iter().clone()); + if !intra_level.removed_table_ids.is_empty() { + delete_sst_levels.push(intra_level.level_idx); + delete_sst_ids_set.extend(intra_level.removed_table_ids.iter().clone()); } - if !intra_level.get_inserted_table_infos().is_empty() { - insert_sst_level_id = intra_level.get_level_idx(); - insert_sub_level_id = intra_level.get_l0_sub_level_id(); - insert_table_infos - .extend(intra_level.get_inserted_table_infos().iter().cloned()); + if !intra_level.inserted_table_infos.is_empty() { + insert_sst_level_id = intra_level.level_idx; + insert_sub_level_id = intra_level.l0_sub_level_id; + insert_table_infos.extend(intra_level.inserted_table_infos.iter().cloned()); } - new_vnode_partition_count = intra_level.get_vnode_partition_count(); + new_vnode_partition_count = intra_level.vnode_partition_count; } GroupDelta::GroupConstruct(construct_delta) => { assert!(group_construct.is_none()); @@ -315,8 +314,8 @@ pub fn safe_epoch_read_table_watermarks_impl( let vnode_watermarks = &watermarks.watermarks.first().expect("should exist").1; let mut vnode_watermark_map = BTreeMap::new(); for vnode_watermark in vnode_watermarks.iter() { - let watermark = Bytes::copy_from_slice(&vnode_watermark.watermark); - for vnode in vnode_watermark.vnode_bitmap.as_ref().iter_vnodes() { + let watermark = Bytes::copy_from_slice(vnode_watermark.watermark()); + for vnode in vnode_watermark.vnode_bitmap().iter_vnodes() { assert!( vnode_watermark_map .insert(vnode, watermark.clone()) @@ -349,9 +348,9 @@ impl HummockVersion { parent_levels .l0 .iter() - .flat_map(|l0| l0.get_sub_levels()) - .chain(parent_levels.get_levels().iter()) - .flat_map(|level| level.get_table_infos()) + .flat_map(|l0| &l0.sub_levels) + .chain(parent_levels.levels.iter()) + .flat_map(|level| &level.table_infos) .map(|sst_info| { // `sst_info.table_ids` will never be empty. for table_id in &sst_info.table_ids { @@ -426,7 +425,7 @@ impl HummockVersion { insert_new_sub_level( target_l0, sub_level.sub_level_id, - sub_level.level_type(), + sub_level.level_type, insert_table_infos, Some(idx), ); @@ -448,11 +447,9 @@ impl HummockVersion { cur_levels.levels[idx] .table_infos .extend(insert_table_infos); - cur_levels.levels[idx].table_infos.sort_by(|sst1, sst2| { - let a = sst1.key_range.as_ref().unwrap(); - let b = sst2.key_range.as_ref().unwrap(); - a.cmp(b) - }); + cur_levels.levels[idx] + .table_infos + .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range)); assert!(can_concat(&cur_levels.levels[idx].table_infos)); level .table_infos @@ -481,7 +478,7 @@ impl HummockVersion { // Build only if all deltas are intra level deltas. if !group_deltas - .get_group_deltas() + .group_deltas .iter() .all(|delta| matches!(delta, GroupDelta::IntraLevel(_))) { @@ -491,21 +488,21 @@ impl HummockVersion { // TODO(MrCroxx): At most one insert delta is allowed here. It's okay for now with the // current `hummock::manager::gen_version_delta` implementation. Better refactor the // struct to reduce conventions. - for group_delta in group_deltas.get_group_deltas() { + for group_delta in &group_deltas.group_deltas { match group_delta { GroupDelta::IntraLevel(intra_level) => { - if !intra_level.get_inserted_table_infos().is_empty() { - info.insert_sst_level = intra_level.get_level_idx(); + if !intra_level.inserted_table_infos.is_empty() { + info.insert_sst_level = intra_level.level_idx; info.insert_sst_infos - .extend(intra_level.get_inserted_table_infos().iter().cloned()); + .extend(intra_level.inserted_table_infos.iter().cloned()); } - if !intra_level.get_removed_table_ids().is_empty() { - for id in intra_level.get_removed_table_ids() { - if intra_level.get_level_idx() == 0 { + if !intra_level.removed_table_ids.is_empty() { + for id in &intra_level.removed_table_ids { + if intra_level.level_idx == 0 { removed_l0_ssts.insert(*id); } else { removed_ssts - .entry(intra_level.get_level_idx()) + .entry(intra_level.level_idx) .or_default() .insert(*id); } @@ -517,14 +514,14 @@ impl HummockVersion { } let group = self.levels.get(group_id).unwrap(); - for l0_sub_level in &group.get_level0().sub_levels { + for l0_sub_level in &group.level0().sub_levels { for sst_info in &l0_sub_level.table_infos { if removed_l0_ssts.remove(&sst_info.sst_id) { info.delete_sst_object_ids.push(sst_info.object_id); } } } - for level in group.get_levels() { + for level in &group.levels { if let Some(mut removed_level_ssts) = removed_ssts.remove(&level.level_idx) { for sst_info in &level.table_infos { if removed_level_ssts.remove(&sst_info.sst_id) { @@ -829,34 +826,6 @@ impl HummockVersion { #[easy_ext::ext(HummockLevelsExt)] impl Levels { - pub fn get_level0(&self) -> &OverlappingLevel { - self.l0.as_ref().unwrap() - } - - pub fn get_level(&self, level_idx: usize) -> &Level { - &self.levels[level_idx - 1] - } - - pub fn get_level_mut(&mut self, level_idx: usize) -> &mut Level { - &mut self.levels[level_idx - 1] - } - - pub fn is_last_level(&self, level_idx: u32) -> bool { - self.levels - .last() - .as_ref() - .map_or(false, |level| level.level_idx == level_idx) - } - - pub fn count_ssts(&self) -> usize { - self.get_level0() - .get_sub_levels() - .iter() - .chain(self.get_levels().iter()) - .map(|level| level.get_table_infos().len()) - .sum() - } - pub fn apply_compact_ssts( &mut self, summary: GroupDeltasSummary, @@ -1129,11 +1098,7 @@ pub fn add_ssts_to_sub_level( if l0.sub_levels[sub_level_idx].level_type == PbLevelType::Nonoverlapping { l0.sub_levels[sub_level_idx] .table_infos - .sort_by(|sst1, sst2| { - let a = sst1.key_range.as_ref().unwrap(); - let b = sst2.key_range.as_ref().unwrap(); - a.cmp(b) - }); + .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range)); assert!( can_concat(&l0.sub_levels[sub_level_idx].table_infos), "sstable ids: {:?}", @@ -1175,11 +1140,11 @@ pub fn insert_new_sub_level( { if insert_pos > 0 { if let Some(smaller_level) = l0.sub_levels.get(insert_pos - 1) { - debug_assert!(smaller_level.get_sub_level_id() < insert_sub_level_id); + debug_assert!(smaller_level.sub_level_id < insert_sub_level_id); } } if let Some(larger_level) = l0.sub_levels.get(insert_pos) { - debug_assert!(larger_level.get_sub_level_id() > insert_sub_level_id); + debug_assert!(larger_level.sub_level_id > insert_sub_level_id); } } // All files will be committed in one new Overlapping sub-level and become @@ -1224,11 +1189,9 @@ fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec) .map(|sst| sst.uncompressed_file_size) .sum::(); operand.table_infos.extend(insert_table_infos); - operand.table_infos.sort_by(|sst1, sst2| { - let a = sst1.key_range.as_ref().unwrap(); - let b = sst2.key_range.as_ref().unwrap(); - a.cmp(b) - }); + operand + .table_infos + .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range)); if operand.level_type == PbLevelType::Overlapping { operand.level_type = PbLevelType::Nonoverlapping; } @@ -1248,16 +1211,11 @@ pub fn object_size_map(version: &HummockVersion) -> HashMap Vec { if level.table_infos.is_empty() { res.push(format!("{}: empty level", level_identifier)); } - } else if level.level_type() != PbLevelType::Nonoverlapping { + } else if level.level_type != PbLevelType::Nonoverlapping { // Ensure non-L0 level is non-overlapping level res.push(format!( "{}: level type {:?} is not non-overlapping", - level_identifier, - level.level_type() + level_identifier, level.level_type )); } @@ -1325,13 +1282,11 @@ pub fn validate_version(version: &HummockVersion) -> Vec { } // Ensure SSTs in non-overlapping level have non-overlapping key range - if level.level_type() == PbLevelType::Nonoverlapping { + if level.level_type == PbLevelType::Nonoverlapping { if let Some(prev) = prev_table_info.take() { if prev .key_range - .as_ref() - .unwrap() - .compare_right_with(&table_info.key_range.as_ref().unwrap().left) + .compare_right_with(&table_info.key_range.left) != Ordering::Less { res.push(format!( diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index 47ddf7922d61..340f532d6b49 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -210,9 +210,7 @@ pub fn can_concat(ssts: &[SstableInfo]) -> bool { for i in 1..len { if ssts[i - 1] .key_range - .as_ref() - .unwrap() - .compare_right_with(&ssts[i].key_range.as_ref().unwrap().left) + .compare_right_with(&ssts[i].key_range.left) != Ordering::Less { return false; diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index 3ab40b26e560..2de3fd1b6942 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -308,8 +308,8 @@ impl WatermarkDirection { #[derive(Clone, Debug, PartialEq, EstimateSize)] pub struct VnodeWatermark { - pub vnode_bitmap: Arc, - pub watermark: Bytes, + vnode_bitmap: Arc, + watermark: Bytes, } impl VnodeWatermark { diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 3aa8f7acffb3..b41e61502536 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -18,7 +18,6 @@ use std::mem::{replace, size_of}; use std::ops::Deref; use std::sync::{Arc, LazyLock}; -use bytes::Bytes; use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::INVALID_EPOCH; @@ -30,9 +29,9 @@ use risingwave_pb::hummock::subscribe_compaction_event_request::PbReportTask; use risingwave_pb::hummock::{ BloomFilterType, CompactionConfig, LevelType, PbCompactTask, PbGroupConstruct, PbGroupDelta, PbGroupDestroy, PbGroupMetaChange, PbGroupTableChange, PbHummockVersion, PbHummockVersionDelta, - PbInputLevel, PbIntraLevelDelta, PbKeyRange, PbLevel, PbLevelType, PbOverlappingLevel, - PbSstableInfo, PbStateTableInfo, PbTableStats, PbValidationTask, StateTableInfo, - StateTableInfoDelta, TableOption, TableSchema, + PbInputLevel, PbIntraLevelDelta, PbKeyRange, PbLevel, PbOverlappingLevel, PbSstableInfo, + PbStateTableInfo, PbTableStats, PbValidationTask, StateTableInfo, StateTableInfoDelta, + TableOption, TableSchema, }; use tracing::warn; @@ -117,12 +116,6 @@ impl OverlappingLevel { } } -impl OverlappingLevel { - pub fn get_sub_levels(&self) -> &Vec { - &self.sub_levels - } -} - #[derive(Debug, Clone, PartialEq, Default)] pub struct Level { pub level_idx: u32, @@ -222,32 +215,6 @@ impl Level { } } -impl Level { - pub fn get_table_infos(&self) -> &Vec { - &self.table_infos - } - - pub fn level_type(&self) -> LevelType { - self.level_type - } - - pub fn get_sub_level_id(&self) -> u64 { - self.sub_level_id - } - - pub fn get_level_idx(&self) -> u32 { - self.level_idx - } - - pub fn get_total_file_size(&self) -> u64 { - self.total_file_size - } - - pub fn get_uncompressed_file_size(&self) -> u64 { - self.uncompressed_file_size - } -} - #[derive(Debug, Clone, PartialEq, Default)] pub struct Levels { pub levels: Vec, @@ -260,8 +227,32 @@ pub struct Levels { } impl Levels { - pub fn get_levels(&self) -> &Vec { - &self.levels + pub fn level0(&self) -> &OverlappingLevel { + self.l0.as_ref().unwrap() + } + + pub fn get_level(&self, level_idx: usize) -> &Level { + &self.levels[level_idx - 1] + } + + pub fn get_level_mut(&mut self, level_idx: usize) -> &mut Level { + &mut self.levels[level_idx - 1] + } + + pub fn is_last_level(&self, level_idx: u32) -> bool { + self.levels + .last() + .as_ref() + .map_or(false, |level| level.level_idx == level_idx) + } + + pub fn count_ssts(&self) -> usize { + self.level0() + .sub_levels + .iter() + .chain(self.levels.iter()) + .map(|level| level.table_infos.len()) + .sum() } } @@ -805,9 +796,7 @@ impl HummockVersionDelta { group_deltas.group_deltas.iter().flat_map(|group_delta| { static EMPTY_VEC: Vec = Vec::new(); let sst_slice = match group_delta { - GroupDelta::IntraLevel(level_delta) => { - level_delta.get_inserted_table_infos() - } + GroupDelta::IntraLevel(level_delta) => &level_delta.inserted_table_infos, GroupDelta::GroupConstruct(_) | GroupDelta::GroupDestroy(_) | GroupDelta::GroupMetaChange(_) @@ -1068,7 +1057,7 @@ impl From for HummockVersionDelta { pub struct SstableInfo { pub object_id: u64, pub sst_id: u64, - pub key_range: Option, + pub key_range: KeyRange, pub file_size: u64, pub table_ids: Vec, pub meta_offset: u64, @@ -1095,10 +1084,7 @@ impl SstableInfo { + size_of::() // uncompressed_file_size + size_of::() // range_tombstone_count + size_of::(); // bloom_filter_kind - - if let Some(key_range) = &self.key_range { - basic += key_range.left.len() + key_range.right.len() + size_of::(); - } + basic += self.key_range.left.len() + self.key_range.right.len() + size_of::(); basic } @@ -1113,18 +1099,14 @@ impl From for SstableInfo { Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, - key_range: if pb_sstable_info.key_range.is_some() { + key_range: { let pb_keyrange = pb_sstable_info.key_range.unwrap(); - let key_range = KeyRange { - left: Bytes::from(pb_keyrange.left), - right: Bytes::from(pb_keyrange.right), + KeyRange { + left: pb_keyrange.left.into(), + right: pb_keyrange.right.into(), right_exclusive: pb_keyrange.right_exclusive, - }; - Some(key_range) - } else { - None + } }, - file_size: pb_sstable_info.file_size, table_ids: pb_sstable_info.table_ids.clone(), meta_offset: pb_sstable_info.meta_offset, @@ -1145,18 +1127,14 @@ impl From<&PbSstableInfo> for SstableInfo { Self { object_id: pb_sstable_info.object_id, sst_id: pb_sstable_info.sst_id, - key_range: if pb_sstable_info.key_range.is_some() { + key_range: { let pb_keyrange = pb_sstable_info.key_range.as_ref().unwrap(); - let key_range = KeyRange { - left: Bytes::from(pb_keyrange.left.clone()), - right: Bytes::from(pb_keyrange.right.clone()), + KeyRange { + left: pb_keyrange.left.clone().into(), + right: pb_keyrange.right.clone().into(), right_exclusive: pb_keyrange.right_exclusive, - }; - Some(key_range) - } else { - None + } }, - file_size: pb_sstable_info.file_size, table_ids: pb_sstable_info.table_ids.clone(), meta_offset: pb_sstable_info.meta_offset, @@ -1177,16 +1155,14 @@ impl From for PbSstableInfo { PbSstableInfo { object_id: sstable_info.object_id, sst_id: sstable_info.sst_id, - key_range: if sstable_info.key_range.is_some() { - let keyrange = sstable_info.key_range.unwrap(); + key_range: { + let keyrange = sstable_info.key_range; let pb_key_range = PbKeyRange { left: keyrange.left.into(), right: keyrange.right.into(), right_exclusive: keyrange.right_exclusive, }; Some(pb_key_range) - } else { - None }, file_size: sstable_info.file_size, @@ -1208,16 +1184,14 @@ impl From<&SstableInfo> for PbSstableInfo { PbSstableInfo { object_id: sstable_info.object_id, sst_id: sstable_info.sst_id, - key_range: if sstable_info.key_range.is_some() { - let keyrange = sstable_info.key_range.as_ref().unwrap(); + key_range: { + let keyrange = &sstable_info.key_range; let pb_key_range = PbKeyRange { left: keyrange.left.to_vec(), right: keyrange.right.to_vec(), right_exclusive: keyrange.right_exclusive, }; Some(pb_key_range) - } else { - None }, file_size: sstable_info.file_size, @@ -1235,24 +1209,8 @@ impl From<&SstableInfo> for PbSstableInfo { } impl SstableInfo { - pub fn get_sst_id(&self) -> u64 { - self.sst_id - } - - pub fn get_object_id(&self) -> u64 { - self.sst_id - } - - pub fn get_file_size(&self) -> u64 { - self.file_size - } - - pub fn get_table_ids(&self) -> &Vec { - &self.table_ids - } - - pub fn get_bloom_filter_kind(&self) -> BloomFilterType { - self.bloom_filter_kind + pub fn remove_key_range(&mut self) { + self.key_range = KeyRange::default(); } } @@ -1331,20 +1289,6 @@ impl From<&InputLevel> for PbInputLevel { } } -impl InputLevel { - pub fn get_table_infos(&self) -> &Vec { - &self.table_infos - } - - pub fn level_type(&self) -> PbLevelType { - self.level_type - } - - pub fn get_level_idx(&self) -> u32 { - self.level_idx - } -} - #[derive(Clone, PartialEq, Default, Debug)] pub struct CompactTask { /// SSTs to be compacted, which will be removed from LSM after compaction @@ -1363,7 +1307,7 @@ pub struct CompactTask { pub gc_delete_keys: bool, /// Lbase in LSM pub base_level: u32, - pub task_status: TaskStatus, + pub task_status: PbTaskStatus, /// compaction group the task belongs to pub compaction_group_id: u64, /// `existing_table_ids` for compaction drop key @@ -1375,7 +1319,7 @@ pub struct CompactTask { pub current_epoch_time: u64, pub target_sub_level_id: u64, /// Identifies whether the task is `space_reclaim`, if the `compact_task_type` increases, it will be refactored to enum - pub task_type: TaskType, + pub task_type: PbTaskType, /// Deprecated. use `table_vnode_partition` instead; pub split_by_state_table: bool, /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed. @@ -1446,8 +1390,8 @@ impl From for CompactTask { .splits .into_iter() .map(|pb_keyrange| KeyRange { - left: Bytes::from(pb_keyrange.left), - right: Bytes::from(pb_keyrange.right), + left: pb_keyrange.left.into(), + right: pb_keyrange.right.into(), right_exclusive: pb_keyrange.right_exclusive, }) .collect_vec(), @@ -1500,8 +1444,8 @@ impl From<&PbCompactTask> for CompactTask { .splits .iter() .map(|pb_keyrange| KeyRange { - left: Bytes::from(pb_keyrange.left.clone()), - right: Bytes::from(pb_keyrange.right.clone()), + left: pb_keyrange.left.clone().into(), + right: pb_keyrange.right.clone().into(), right_exclusive: pb_keyrange.right_exclusive, }) .collect_vec(), @@ -1646,29 +1590,9 @@ impl From<&CompactTask> for PbCompactTask { } impl CompactTask { - pub fn task_type(&self) -> PbTaskType { - PbTaskType::try_from(self.task_type).unwrap() - } - - pub fn task_status(&self) -> PbTaskStatus { - PbTaskStatus::try_from(self.task_status).unwrap() - } - pub fn set_task_status(&mut self, s: PbTaskStatus) { self.task_status = s; } - - pub fn get_input_ssts(&self) -> &Vec { - &self.input_ssts - } - - pub fn get_task_id(&self) -> u64 { - self.task_id - } - - pub fn get_max_sub_compaction(&self) -> u32 { - self.max_sub_compaction - } } #[derive(Clone, PartialEq, Default)] @@ -1844,28 +1768,6 @@ impl From<&PbIntraLevelDelta> for IntraLevelDelta { } } -impl IntraLevelDelta { - pub fn get_level_idx(&self) -> u32 { - self.level_idx - } - - pub fn get_l0_sub_level_id(&self) -> u64 { - self.l0_sub_level_id - } - - pub fn get_removed_table_ids(&self) -> &Vec { - &self.removed_table_ids - } - - pub fn get_inserted_table_infos(&self) -> &Vec { - &self.inserted_table_infos - } - - pub fn get_vnode_partition_count(&self) -> u32 { - self.vnode_partition_count - } -} - impl IntraLevelDelta { pub fn new( level_idx: u32, @@ -2039,10 +1941,6 @@ impl From<&PbGroupDeltas> for GroupDeltas { } impl GroupDeltas { - pub fn get_group_deltas(&self) -> &Vec { - &self.group_deltas - } - pub fn to_protobuf(&self) -> PbGroupDeltas { self.into() } diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index 1e9b2b5e71f9..e689c08808b2 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -315,7 +315,7 @@ pub(crate) mod tests { .report_compact_task_for_test( result_task.task_id, Some(compact_task), - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -470,7 +470,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -804,7 +804,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1003,7 +1003,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1195,7 +1195,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1366,7 +1366,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index c99ca382080d..ad1e667c17f2 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -235,8 +235,8 @@ async fn test_read_version_basic() { let staging_ssts = staging_sst_iter.cloned().collect_vec(); assert_eq!(2, staging_ssts.len()); - assert_eq!(1, staging_ssts[0].get_object_id()); - assert_eq!(2, staging_ssts[1].get_object_id()); + assert_eq!(1, staging_ssts[0].object_id); + assert_eq!(2, staging_ssts[1].object_id); } { @@ -259,7 +259,7 @@ async fn test_read_version_basic() { let staging_ssts = staging_sst_iter.cloned().collect_vec(); assert_eq!(1, staging_ssts.len()); - assert_eq!(2, staging_ssts[0].get_object_id()); + assert_eq!(2, staging_ssts[0].object_id); } } diff --git a/src/storage/hummock_test/src/state_store_tests.rs b/src/storage/hummock_test/src/state_store_tests.rs index 5c13d73f07ec..8df417072207 100644 --- a/src/storage/hummock_test/src/state_store_tests.rs +++ b/src/storage/hummock_test/src/state_store_tests.rs @@ -1302,7 +1302,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() { sync_result .uncommitted_ssts .iter() - .map(|LocalSstableInfo { sst_info, .. }| sst_info.get_object_id()) + .map(|LocalSstableInfo { sst_info, .. }| sst_info.object_id) .min() .unwrap() }; diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs index 3074bba6dd75..5cdd6b5f5257 100644 --- a/src/storage/hummock_test/src/sync_point_tests.rs +++ b/src/storage/hummock_test/src/sync_point_tests.rs @@ -213,7 +213,7 @@ pub async fn compact_once( hummock_manager_ref .report_compact_task( result_task.task_id, - result_task.task_status(), + result_tasktask_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -425,8 +425,8 @@ async fn test_syncpoints_get_in_delete_range_boundary() { .right_exclusive ); assert_eq!( - user_key(&base_level.table_infos[0].key_range.as_ref().unwrap().right), - user_key(&base_level.table_infos[1].key_range.as_ref().unwrap().left), + user_key(&base_level.table_infos[0].key_range.right), + user_key(&base_level.table_infos[1].key_range.left), ); storage.wait_version(version).await; let read_options = ReadOptions { diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index a383a4c2165e..a6c9d692e2ec 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -189,7 +189,7 @@ fn generate_splits_fast( ); let mut indexes = vec![]; for sst in sstable_infos { - let key_range = sst.key_range.as_ref().unwrap(); + let key_range = &sst.key_range; indexes.push( FullKey { user_key: FullKey::decode(&key_range.left).user_key, @@ -512,7 +512,7 @@ pub fn optimize_by_copy_block(compact_task: &CompactTask, context: &CompactorCon let all_ssts_are_blocked_filter = sstable_infos .iter() - .all(|table_info| table_info.get_bloom_filter_kind() == BloomFilterType::Blocked); + .all(|table_info| table_info.bloom_filter_kind == BloomFilterType::Blocked); let delete_key_count = sstable_infos .iter() @@ -552,7 +552,7 @@ pub fn optimize_by_copy_block(compact_task: &CompactTask, context: &CompactorCon && compaction_size < context.storage_opts.compactor_fast_max_compact_task_size && delete_key_count * 100 < context.storage_opts.compactor_fast_max_compact_delete_ratio as u64 * total_key_count - && compact_task.task_type() == PbTaskType::Dynamic + && compact_task.task_type == PbTaskType::Dynamic } pub async fn generate_splits_for_task( @@ -582,7 +582,7 @@ pub async fn generate_splits_for_task( &sstable_infos, compaction_size, context, - compact_task.get_max_sub_compaction(), + compact_task.max_sub_compaction, ) .await?; if !splits.is_empty() { @@ -666,7 +666,7 @@ pub fn calculate_task_parallelism(compact_task: &CompactTask, context: &Compacto context.compaction_executor.worker_num(), parallel_compact_size, compaction_size, - compact_task.get_max_sub_compaction(), + compact_task.max_sub_compaction, ) } diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index da953fe6f8f3..cae809b97993 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -179,9 +179,7 @@ impl CompactorRunner { .iter() .any(|table_id| self.compact_task.existing_table_ids.contains(table_id)); - self.key_range - .full_key_overlap(table_info.key_range.as_ref().unwrap()) - && exist_table + self.key_range.full_key_overlap(&table_info.key_range) && exist_table }) .cloned() .collect_vec(); @@ -267,8 +265,8 @@ pub fn partition_overlapping_sstable_infos( } let mut groups: BinaryHeap = BinaryHeap::default(); origin_infos.sort_by(|a, b| { - let x = a.key_range.as_ref().unwrap(); - let y = b.key_range.as_ref().unwrap(); + let x = &a.key_range; + let y = &b.key_range; KeyComparator::compare_encoded_full_key(&x.left, &y.left) }); for sst in origin_infos { @@ -276,17 +274,15 @@ pub fn partition_overlapping_sstable_infos( if let Some(mut prev_group) = groups.peek_mut() { if KeyComparator::encoded_full_key_less_than( &prev_group.max_right_bound, - &sst.key_range.as_ref().unwrap().left, + &sst.key_range.left, ) { - prev_group - .max_right_bound - .clone_from(&sst.key_range.as_ref().unwrap().right); + prev_group.max_right_bound.clone_from(&sst.key_range.right); prev_group.ssts.push(sst); continue; } } groups.push(SstableGroup { - max_right_bound: sst.key_range.as_ref().unwrap().right.clone(), + max_right_bound: sst.key_range.right.clone(), ssts: vec![sst], }); } diff --git a/src/storage/src/hummock/compactor/iterator.rs b/src/storage/src/hummock/compactor/iterator.rs index 1b3b66738a92..cc2cb4a588c3 100644 --- a/src/storage/src/hummock/compactor/iterator.rs +++ b/src/storage/src/hummock/compactor/iterator.rs @@ -478,7 +478,7 @@ impl HummockIterator for ConcatSstableIterator { // Note that we need to use `<` instead of `<=` to ensure that all keys in an SST // (including its max. key) produce the same search result. - let max_sst_key = &table.key_range.as_ref().unwrap().right; + let max_sst_key = &table.key_range.right; FullKey::decode(max_sst_key).cmp(&seek_key) == Ordering::Less }); diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index e451c99a80f3..bcc9663f4a6a 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -554,7 +554,7 @@ pub fn start_compactor( let enable_check_compaction_result = context.storage_opts.check_compaction_result; - let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success; if enable_check_compaction_result && need_check_task { match check_compaction_result(&compact_task, context.clone()) diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index addb2d08d5fc..cd177e4cf34f 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -53,7 +53,6 @@ use crate::hummock::local_version::pinned_version::PinnedVersion; use crate::hummock::store::version::{ HummockReadVersion, StagingData, StagingSstableInfo, VersionUpdate, }; -use crate::hummock::utils::validate_table_key_range; use crate::hummock::{ HummockResult, MemoryLimiter, SstableObjectIdManager, SstableStoreRef, TrackerId, }; @@ -612,8 +611,6 @@ impl HummockEventHandler { HummockVersionUpdate::PinnedVersion(version) => *version, }; - validate_table_key_range(&newly_pinned_version); - pinned_version.new_pin_version(newly_pinned_version) } diff --git a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs index bbc278b8739d..cf4a38e85d9b 100644 --- a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs +++ b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs @@ -51,32 +51,26 @@ impl ConcatDeleteRangeIterator { && iter .next_extended_user_key() .left_user_key - .eq(&FullKey::decode( - &self.sstables[self.idx].key_range.as_ref().unwrap().right, - ) - .user_key) + .eq(&FullKey::decode(&self.sstables[self.idx].key_range.right).user_key) { // When the last range of the current sstable is equal to the first range of the // next sstable, the `next` method would return two same `PointRange`. So we // must skip one. let exclusive_range_start = iter.next_extended_user_key().is_exclude_left_key; - let last_key_in_sst_start = - iter.next_extended_user_key() - .left_user_key - .eq(&FullKey::decode( - &self.sstables[self.idx + 1].key_range.as_ref().unwrap().left, - ) - .user_key); + let last_key_in_sst_start = iter + .next_extended_user_key() + .left_user_key + .eq(&FullKey::decode(&self.sstables[self.idx + 1].key_range.left).user_key); iter.next().await?; if !iter.is_valid() && last_key_in_sst_start { self.seek_idx(self.idx + 1, None).await?; let next_range = self.next_extended_user_key(); debug_assert!(self.is_valid()); if next_range.is_exclude_left_key == exclusive_range_start - && next_range.left_user_key.eq(&FullKey::decode( - &self.sstables[self.idx].key_range.as_ref().unwrap().left, - ) - .user_key) + && next_range + .left_user_key + .eq(&FullKey::decode(&self.sstables[self.idx].key_range.left) + .user_key) { self.current.as_mut().unwrap().next().await?; } @@ -158,7 +152,7 @@ impl DeleteRangeIterator for ConcatDeleteRangeIterator { let mut idx = self .sstables .partition_point(|sst| { - FullKey::decode(&sst.key_range.as_ref().unwrap().left) + FullKey::decode(&sst.key_range.left) .user_key .le(&target_user_key) }) diff --git a/src/storage/src/hummock/iterator/concat_inner.rs b/src/storage/src/hummock/iterator/concat_inner.rs index f0250b34cf8f..ad4e115af927 100644 --- a/src/storage/src/hummock/iterator/concat_inner.rs +++ b/src/storage/src/hummock/iterator/concat_inner.rs @@ -27,11 +27,11 @@ use crate::hummock::{HummockResult, SstableIteratorType, SstableStoreRef}; use crate::monitor::StoreLocalStatistic; fn smallest_key(sstable_info: &SstableInfo) -> &[u8] { - &sstable_info.key_range.as_ref().unwrap().left + &sstable_info.key_range.left } fn largest_key(sstable_info: &SstableInfo) -> &[u8] { - &sstable_info.key_range.as_ref().unwrap().right + &sstable_info.key_range.right } /// Served as the concrete implementation of `ConcatIterator` and `BackwardConcatIterator`. @@ -160,8 +160,7 @@ impl HummockIterator for ConcatIteratorInner { } DirectionEnum::Backward => { let ord = FullKey::decode(largest_key(table)).cmp(&key); - ord == Greater - || (ord == Equal && !table.key_range.as_ref().unwrap().right_exclusive) + ord == Greater || (ord == Equal && !table.key_range.right_exclusive) } }) .saturating_sub(1); // considering the boundary of 0 diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index fc49b3d74515..7d957803305f 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -501,11 +501,11 @@ impl SstableBuilder { object_id: self.sstable_id, sst_id: self.sstable_id, bloom_filter_kind, - key_range: Some(KeyRange { + key_range: KeyRange { left: Bytes::from(meta.smallest_key.clone()), right: Bytes::from(meta.largest_key.clone()), right_exclusive, - }), + }, file_size: meta.estimated_size as u64, table_ids: self.table_ids.into_iter().collect(), meta_offset: meta.meta_offset, @@ -527,7 +527,7 @@ impl SstableBuilder { self.epoch_set.len() ); let bloom_filter_size = meta.bloom_filter.len(); - let sstable_file_size = sst_info.get_file_size() as usize; + let sstable_file_size = sst_info.file_size as usize; let writer_output = self.writer.finish(meta).await?; Ok(SstableBuilderOutput:: { @@ -717,13 +717,10 @@ pub(super) mod tests { let output = b.finish().await.unwrap(); let info = output.sst_info.sst_info; - assert_bytes_eq!( - test_key_of(0).encode(), - info.key_range.as_ref().unwrap().left - ); + assert_bytes_eq!(test_key_of(0).encode(), info.key_range.left); assert_bytes_eq!( test_key_of(TEST_KEYS_COUNT - 1).encode(), - info.key_range.as_ref().unwrap().right + info.key_range.right ); let (data, meta) = output.writer_output; assert_eq!(info.file_size, meta.estimated_size as u64); diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs index bf22bcc0ff03..5c24d0143ea6 100644 --- a/src/storage/src/hummock/sstable_store.rs +++ b/src/storage/src/hummock/sstable_store.rs @@ -572,7 +572,7 @@ impl SstableStore { sst: &SstableInfo, stats: &mut StoreLocalStatistic, ) -> impl Future> + Send + 'static { - let object_id = sst.get_object_id(); + let object_id = sst.object_id; let entry = self.meta_cache.fetch(object_id, || { let store = self.store.clone(); diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs index 56e601df7d9c..f0bf53b16fa3 100644 --- a/src/storage/src/hummock/store/version.rs +++ b/src/storage/src/hummock/store/version.rs @@ -614,7 +614,7 @@ impl HummockVersionReader { continue; } - match level.level_type() { + match level.level_type { LevelType::Overlapping | LevelType::Unspecified => { let sstable_infos = prune_overlapping_ssts( &level.table_infos, @@ -650,8 +650,6 @@ impl HummockVersionReader { table_info_idx = table_info_idx.saturating_sub(1); let ord = level.table_infos[table_info_idx] .key_range - .as_ref() - .unwrap() .compare_right_with_user_key(full_key.user_key.as_ref()); // the case that the key falls into the gap between two ssts if ord == Ordering::Less { @@ -929,7 +927,7 @@ impl HummockVersionReader { .sstable_store .sstable(sstable_info, local_stats) .await?; - assert_eq!(sstable_info.get_object_id(), sstable.id); + assert_eq!(sstable_info.object_id, sstable.id); if let Some(dist_hash) = bloom_filter_prefix_hash.as_ref() { if !hit_sstable_bloom_filter( &sstable, diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs index 6a4d47af1222..74b9141e83b6 100644 --- a/src/storage/src/hummock/test_utils.rs +++ b/src/storage/src/hummock/test_utils.rs @@ -113,11 +113,11 @@ pub fn gen_dummy_sst_info( SstableInfo { object_id: id, sst_id: id, - key_range: Some(KeyRange { + key_range: KeyRange { left: Bytes::from(FullKey::for_test(table_id, min_table_key, epoch).encode()), right: Bytes::from(FullKey::for_test(table_id, max_table_key, epoch).encode()), right_exclusive: false, - }), + }, file_size, table_ids: vec![table_id.table_id], uncompressed_file_size: file_size, @@ -190,11 +190,11 @@ pub async fn put_sst( let sst = SstableInfo { object_id: sst_object_id, sst_id: sst_object_id, - key_range: Some(KeyRange { + key_range: KeyRange { left: Bytes::from(meta.smallest_key.clone()), right: Bytes::from(meta.largest_key.clone()), right_exclusive: false, - }), + }, file_size: meta.estimated_size as u64, meta_offset: meta.meta_offset, uncompressed_file_size: meta.estimated_size as u64, diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs index 2202609dc750..327b023fd040 100644 --- a/src/storage/src/hummock/utils.rs +++ b/src/storage/src/hummock/utils.rs @@ -87,32 +87,12 @@ pub fn validate_safe_epoch( Ok(()) } -pub fn validate_table_key_range(version: &HummockVersion) { - for l in version.levels.values().flat_map(|levels| { - levels - .l0 - .as_ref() - .unwrap() - .sub_levels - .iter() - .chain(levels.levels.iter()) - }) { - for t in &l.table_infos { - assert!( - t.key_range.is_some(), - "key_range in table [{}] is none", - t.object_id - ); - } - } -} - pub fn filter_single_sst(info: &SstableInfo, table_id: TableId, table_key_range: &R) -> bool where R: RangeBounds>, B: AsRef<[u8]> + EmptySliceRef, { - let table_range = info.key_range.as_ref().unwrap(); + let table_range = &info.key_range; let table_start = FullKey::decode(table_range.left.as_ref()).user_key; let table_end = FullKey::decode(table_range.right.as_ref()).user_key; let (left, right) = bound_table_key_range(table_id, table_key_range); @@ -132,9 +112,7 @@ where /// Search the SST containing the specified key within a level, using binary search. pub(crate) fn search_sst_idx(ssts: &[SstableInfo], key: UserKey<&[u8]>) -> usize { ssts.partition_point(|table| { - let ord = FullKey::decode(&table.key_range.as_ref().unwrap().left) - .user_key - .cmp(&key); + let ord = FullKey::decode(&table.key_range.left).user_key.cmp(&key); ord == Ordering::Less || ord == Ordering::Equal }) } From a22660b6cacfcca3cee725b1edc1c8932d79823d Mon Sep 17 00:00:00 2001 From: Li0k Date: Wed, 24 Jul 2024 00:00:18 +0800 Subject: [PATCH 14/18] fix(storage): fix compile --- .../picker/intra_compaction_picker.rs | 20 ++--- .../picker/manual_compaction_picker.rs | 1 - .../picker/min_overlap_compaction_picker.rs | 1 - .../picker/space_reclaim_compaction_picker.rs | 2 +- .../picker/ttl_reclaim_compaction_picker.rs | 2 +- .../picker/vnode_watermark_picker.rs | 63 ++++++++++----- .../src/hummock/compaction/selector/mod.rs | 8 +- src/meta/src/hummock/manager/tests.rs | 76 +++++++++---------- src/meta/src/hummock/manager/versioning.rs | 8 +- .../hummock_test/src/compactor_tests.rs | 24 +++--- .../src/hummock_read_version_tests.rs | 21 ++--- .../hummock_test/src/sync_point_tests.rs | 2 +- .../event_handler/uploader/test_utils.rs | 13 ++-- 13 files changed, 129 insertions(+), 112 deletions(-) diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index 93c505bf48fd..bfb7e198d2fd 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -417,7 +417,7 @@ impl WholeLevelCompactionPicker { #[cfg(test)] pub mod tests { - use risingwave_hummock_sdk::version::{Level, Levels}; + use risingwave_hummock_sdk::version::Level; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; @@ -539,7 +539,7 @@ pub mod tests { ..Default::default() }; let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)]; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); + levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos); let config = Arc::new( CompactionConfigBuilder::new() .level0_sub_level_compact_level_count(1) @@ -587,7 +587,7 @@ pub mod tests { ..Default::default() }; let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)]; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); + levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos); let config = Arc::new( CompactionConfigBuilder::new() .level0_sub_level_compact_level_count(1) @@ -658,7 +658,7 @@ pub mod tests { ..Default::default() }; let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)]; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); + levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos); let config = Arc::new( CompactionConfigBuilder::new() .level0_sub_level_compact_level_count(1) @@ -715,7 +715,7 @@ pub mod tests { let config = Arc::new( CompactionConfigBuilder::new() .level0_tier_compact_file_number(2) - .tarfile_size_base(30) + .target_file_size_base(30) .level0_sub_level_compact_level_count(20) // reject intra .build(), ); @@ -732,7 +732,7 @@ pub mod tests { levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])], ..Default::default() }; - levels_handler[1].add_pending_task(100, 1, levels.levels[0].table_infos()); + levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos); let mut local_stats = LocalPickerStatistic::default(); let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats); assert!(ret.is_none()); @@ -860,15 +860,11 @@ pub mod tests { let input = ret.as_ref().unwrap(); assert_eq!(input.input_levels.len(), 2); assert_ne!( - levels.l0.as_ref().unwrap().sub_levels()[0] - .table_infos - .len(), + levels.l0.as_ref().unwrap().sub_levels[0].table_infos.len(), input.input_levels[0].table_infos.len() ); assert_ne!( - levels.l0.as_ref().unwrap().sub_levels()[1] - .table_infos - .len(), + levels.l0.as_ref().unwrap().sub_levels[1].table_infos.len(), input.input_levels[1].table_infos.len() ); } diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index 008861b39333..ee712737725b 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -329,7 +329,6 @@ pub mod tests { use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_pb::hummock::compact_task; - pub use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 81f06361f3e5..17d723c2d621 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -529,7 +529,6 @@ pub mod tests { use std::collections::BTreeSet; use risingwave_common::config::default::compaction_config; - pub use risingwave_pb::hummock::LevelType; use super::*; use crate::hummock::compaction::overlap_strategy::RangeOverlapStrategy; diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 7c1adffbf886..35a99cd3c1db 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -232,7 +232,7 @@ mod test { { let sst_10 = levels[3].table_infos.get_mut(8).unwrap(); assert_eq!(10, sst_10.sst_id); - sst_10.key_range.as_mut().unwrap().right_exclusive = true; + sst_10.key_range.right_exclusive = true; } assert_eq!(levels.len(), 4); diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index 7f02a140f088..af1b3a62f7a3 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -347,7 +347,7 @@ mod test { { let sst_10 = levels[3].table_infos.get_mut(8).unwrap(); assert_eq!(10, sst_10.sst_id); - sst_10.key_range.as_mut().unwrap().right_exclusive = true; + sst_10.key_range.right_exclusive = true; } assert_eq!(levels.len(), 4); diff --git a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs index c6d04865e6e5..ed42727cac4a 100644 --- a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs +++ b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs @@ -107,8 +107,9 @@ mod tests { use bytes::{BufMut, Bytes, BytesMut}; use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::key::{FullKey, TableKey}; + use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, WatermarkDirection}; - use risingwave_pb::hummock::{KeyRange, SstableInfo}; + use risingwave_hummock_sdk::version::SstableInfo; use crate::hummock::compaction::picker::vnode_watermark_picker::should_delete_sst_by_watermark; @@ -133,11 +134,15 @@ mod tests { let sst_info = SstableInfo { object_id: 1, sst_id: 1, - key_range: Some(KeyRange { - left: FullKey::new(2.into(), table_key(16, "some_watermark_key_1"), 0).encode(), - right: FullKey::new(2.into(), table_key(16, "some_watermark_key_2"), 0).encode(), + key_range: KeyRange { + left: FullKey::new(2.into(), table_key(16, "some_watermark_key_1"), 0) + .encode() + .into(), + right: FullKey::new(2.into(), table_key(16, "some_watermark_key_2"), 0) + .encode() + .into(), right_exclusive: true, - }), + }, table_ids: vec![2], ..Default::default() }; @@ -149,11 +154,15 @@ mod tests { let sst_info = SstableInfo { object_id: 1, sst_id: 1, - key_range: Some(KeyRange { - left: FullKey::new(1.into(), table_key(13, "some_watermark_key_1"), 0).encode(), - right: FullKey::new(1.into(), table_key(14, "some_watermark_key_2"), 0).encode(), + key_range: KeyRange { + left: FullKey::new(1.into(), table_key(13, "some_watermark_key_1"), 0) + .encode() + .into(), + right: FullKey::new(1.into(), table_key(14, "some_watermark_key_2"), 0) + .encode() + .into(), right_exclusive: true, - }), + }, table_ids: vec![1], ..Default::default() }; @@ -165,11 +174,15 @@ mod tests { let sst_info = SstableInfo { object_id: 1, sst_id: 1, - key_range: Some(KeyRange { - left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0).encode(), - right: FullKey::new(1.into(), table_key(17, "some_watermark_key_2"), 0).encode(), + key_range: KeyRange { + left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0) + .encode() + .into(), + right: FullKey::new(1.into(), table_key(17, "some_watermark_key_2"), 0) + .encode() + .into(), right_exclusive: true, - }), + }, table_ids: vec![1], ..Default::default() }; @@ -181,11 +194,15 @@ mod tests { let sst_info = SstableInfo { object_id: 1, sst_id: 1, - key_range: Some(KeyRange { - left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0).encode(), - right: FullKey::new(1.into(), table_key(16, "some_watermark_key_9"), 0).encode(), + key_range: KeyRange { + left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0) + .encode() + .into(), + right: FullKey::new(1.into(), table_key(16, "some_watermark_key_9"), 0) + .encode() + .into(), right_exclusive: true, - }), + }, table_ids: vec![1], ..Default::default() }; @@ -197,11 +214,15 @@ mod tests { let sst_info = SstableInfo { object_id: 1, sst_id: 1, - key_range: Some(KeyRange { - left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0).encode(), - right: FullKey::new(1.into(), table_key(16, "some_watermark_key_2"), 0).encode(), + key_range: KeyRange { + left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0) + .encode() + .into(), + right: FullKey::new(1.into(), table_key(16, "some_watermark_key_2"), 0) + .encode() + .into(), right_exclusive: true, - }), + }, table_ids: vec![1], ..Default::default() }; diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index 2797f39b07b6..7a77c736efc4 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -187,11 +187,11 @@ pub mod tests { SstableInfo { object_id: id, sst_id: id, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(table_prefix, left, epoch).into(), right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(), right_exclusive: false, - }), + }, file_size: (right - left + 1) as u64, table_ids: vec![table_prefix as u32], uncompressed_file_size: (right - left + 1) as u64, @@ -214,11 +214,11 @@ pub mod tests { SstableInfo { object_id: id, sst_id: id, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(table_prefix, left, epoch).into(), right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(), right_exclusive: false, - }), + }, file_size: (right - left + 1) as u64, table_ids, uncompressed_file_size: (right - left + 1) as u64, diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs index 0f161ef677d2..b59f48343e12 100644 --- a/src/meta/src/hummock/manager/tests.rs +++ b/src/meta/src/hummock/manager/tests.rs @@ -64,11 +64,11 @@ fn pin_snapshots_epoch(pin_snapshots: &[HummockPinnedSnapshot]) -> Vec { fn gen_sstable_info(sst_id: u64, idx: usize, table_ids: Vec) -> SstableInfo { SstableInfo { sst_id, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, idx, 1).into(), right: iterator_test_key_of_epoch(1, idx, 1).into(), right_exclusive: false, - }), + }, table_ids, object_id: sst_id, min_epoch: 20, @@ -227,7 +227,7 @@ async fn test_hummock_compaction_task() { .unwrap() .unwrap(); assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); - assert_eq!(compact_task.get_task_id(), 2); + assert_eq!(compact_task.task_id, 2); // Cancel the task and succeed. assert!(hummock_manager @@ -244,7 +244,7 @@ async fn test_hummock_compaction_task() { .await .unwrap() .unwrap(); - assert_eq!(compact_task.get_task_id(), 3); + assert_eq!(compact_task.task_id, 3); // Finish the task and succeed. assert!(hummock_manager @@ -928,7 +928,7 @@ async fn test_hummock_compaction_task_heartbeat() { .unwrap(); assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); - assert_eq!(compact_task.get_task_id(), 2); + assert_eq!(compact_task.task_id, 2); for i in 0..10 { // send heartbeats to the task over 2.5 seconds @@ -962,7 +962,7 @@ async fn test_hummock_compaction_task_heartbeat() { .unwrap() .unwrap(); - assert_eq!(compact_task.get_task_id(), 3); + assert_eq!(compact_task.task_id, 3); // Cancel the task after heartbeat has triggered and fail. @@ -1041,7 +1041,7 @@ async fn test_hummock_compaction_task_heartbeat_removal_on_node_removal() { .unwrap(); assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0); - assert_eq!(compact_task.get_task_id(), 2); + assert_eq!(compact_task.task_id, 2); // send heartbeats to the task immediately let req = CompactTaskProgress { @@ -1203,11 +1203,11 @@ async fn test_version_stats() { sst_info: SstableInfo { object_id: sst_ids[idx], sst_id: sst_ids[idx], - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, 1, 1).into(), right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, - }), + }, file_size: 1024 * 1024 * 1024, table_ids: table_ids.clone(), ..Default::default() @@ -1312,7 +1312,7 @@ async fn test_split_compaction_group_on_commit() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: None, + key_range: KeyRange::default(), table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, @@ -1400,11 +1400,11 @@ async fn test_split_compaction_group_on_demand_basic() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(100, 1, 20).into(), right: iterator_test_key_of_epoch(100, 100, 20).into(), right_exclusive: false, - }), + }, table_ids: vec![100], min_epoch: 20, max_epoch: 20, @@ -1416,11 +1416,11 @@ async fn test_split_compaction_group_on_demand_basic() { sst_info: SstableInfo { object_id: 11, sst_id: 11, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(100, 101, 20).into(), right: iterator_test_key_of_epoch(101, 100, 20).into(), right_exclusive: false, - }), + }, table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, @@ -1498,7 +1498,7 @@ async fn test_split_compaction_group_on_demand_non_trivial() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: None, + key_range: KeyRange::default(), table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, @@ -1583,11 +1583,11 @@ async fn test_split_compaction_group_trivial_expired() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(100, 1, 20).into(), right: iterator_test_key_of_epoch(100, 100, 20).into(), right_exclusive: false, - }), + }, table_ids: vec![100], min_epoch: 20, max_epoch: 20, @@ -1602,11 +1602,11 @@ async fn test_split_compaction_group_trivial_expired() { table_ids: vec![101], min_epoch: 20, max_epoch: 20, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(101, 1, 20).into(), right: iterator_test_key_of_epoch(101, 100, 20).into(), right_exclusive: false, - }), + }, ..Default::default() }, table_stats: Default::default(), @@ -1694,7 +1694,7 @@ async fn test_split_compaction_group_trivial_expired() { vec![SstableInfo { object_id: 12, sst_id: 12, - key_range: None, + key_range: KeyRange::default(), table_ids: vec![100], min_epoch: 20, max_epoch: 20, @@ -1749,11 +1749,11 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, 1, 1).into(), right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, - }), + }, table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, @@ -1780,22 +1780,22 @@ async fn test_split_compaction_group_on_demand_bottom_levels() { object_id: 11, sst_id: 11, table_ids: vec![100, 101], - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, 1, 1).into(), right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, - }), + }, ..Default::default() }, SstableInfo { object_id: 12, sst_id: 12, table_ids: vec![100], - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, 2, 2).into(), right: iterator_test_key_of_epoch(1, 2, 2).into(), right_exclusive: false, - }), + }, ..Default::default() }, ], @@ -1884,11 +1884,11 @@ async fn test_compaction_task_expiration_due_to_split_group() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, 1, 1).into(), right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, - }), + }, table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, @@ -1900,11 +1900,11 @@ async fn test_compaction_task_expiration_due_to_split_group() { sst_info: SstableInfo { object_id: 11, sst_id: 11, - key_range: Some(KeyRange { + key_range: KeyRange { left: iterator_test_key_of_epoch(1, 1, 1).into(), right: iterator_test_key_of_epoch(1, 1, 1).into(), right_exclusive: false, - }), + }, table_ids: vec![101], min_epoch: 20, max_epoch: 20, @@ -2263,11 +2263,11 @@ async fn test_unregister_moved_table() { sst_info: SstableInfo { object_id: 10, sst_id: 10, - key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(100, 1, 20), - right: iterator_test_key_of_epoch(100, 100, 20), + key_range: KeyRange { + left: iterator_test_key_of_epoch(100, 1, 20).into(), + right: iterator_test_key_of_epoch(100, 100, 20).into(), right_exclusive: false, - }), + }, table_ids: vec![100], min_epoch: 20, max_epoch: 20, @@ -2279,11 +2279,11 @@ async fn test_unregister_moved_table() { sst_info: SstableInfo { object_id: 11, sst_id: 11, - key_range: Some(KeyRange { - left: iterator_test_key_of_epoch(100, 101, 20), - right: iterator_test_key_of_epoch(101, 100, 20), + key_range: KeyRange { + left: iterator_test_key_of_epoch(100, 101, 20).into(), + right: iterator_test_key_of_epoch(101, 100, 20).into(), right_exclusive: false, - }), + }, table_ids: vec![100, 101], min_epoch: 20, max_epoch: 20, diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs index 6efe5ebcdef8..c6c1ea1070ed 100644 --- a/src/meta/src/hummock/manager/versioning.rs +++ b/src/meta/src/hummock/manager/versioning.rs @@ -539,11 +539,11 @@ mod tests { #[test] fn test_estimate_table_stats() { let sst = SstableInfo { - key_range: Some(KeyRange { + key_range: KeyRange { left: vec![1; 10].into(), right: vec![1; 20].into(), ..Default::default() - }), + }, table_ids: vec![1, 2, 3], total_key_count: 6000, uncompressed_file_size: 6_000_000, @@ -601,11 +601,11 @@ mod tests { #[test] fn test_estimate_table_stats_large_key_range() { let sst = SstableInfo { - key_range: Some(KeyRange { + key_range: KeyRange { left: vec![1; 1000].into(), right: vec![1; 2000].into(), ..Default::default() - }), + }, table_ids: vec![1, 2, 3], total_key_count: 6000, uncompressed_file_size: 60_000, diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index e689c08808b2..1acc04d67aa0 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -315,7 +315,7 @@ pub(crate) mod tests { .report_compact_task_for_test( result_task.task_id, Some(compact_task), - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -470,7 +470,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -804,7 +804,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1003,7 +1003,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1195,7 +1195,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1366,7 +1366,7 @@ pub(crate) mod tests { hummock_manager_ref .report_compact_task( result_task.task_id, - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) @@ -1895,13 +1895,11 @@ pub(crate) mod tests { table_watermarks.insert( 1, TableWatermarks { - epoch_watermarks: vec![PbEpochNewWatermarks { - watermarks: vec![ - VnodeWatermark::new(bitmap.clone(), watermark_key.clone()).to_protobuf() - ], - epoch: test_epoch(500), - }], - is_ascending: true, + watermarks: vec![( + test_epoch(500), + vec![VnodeWatermark::new(bitmap.clone(), watermark_key.clone())].into(), + )], + direction: WatermarkDirection::Ascending, }, ); diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index ad1e667c17f2..6e6a66543772 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -28,7 +28,6 @@ use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::LocalSstableInfo; use risingwave_meta::hummock::test_utils::setup_compute_env; -use risingwave_pb::hummock::{KeyRange, SstableInfo}; use risingwave_storage::hummock::event_handler::TEST_LOCAL_INSTANCE_ID; use risingwave_storage::hummock::iterator::test_utils::{ iterator_test_table_key_of, iterator_test_user_key_of, @@ -155,11 +154,13 @@ async fn test_read_version_basic() { LocalSstableInfo::for_test(SstableInfo { object_id: 1, sst_id: 1, - key_range: Some(KeyRange { - left: key_with_epoch(iterator_test_user_key_of(1).encode(), test_epoch(1)), - right: key_with_epoch(iterator_test_user_key_of(2).encode(), test_epoch(2)), + key_range: KeyRange { + left: key_with_epoch(iterator_test_user_key_of(1).encode(), test_epoch(1)) + .into(), + right: key_with_epoch(iterator_test_user_key_of(2).encode(), test_epoch(2)) + .into(), right_exclusive: false, - }), + }, file_size: 1, table_ids: vec![0], meta_offset: 1, @@ -171,11 +172,13 @@ async fn test_read_version_basic() { LocalSstableInfo::for_test(SstableInfo { object_id: 2, sst_id: 2, - key_range: Some(KeyRange { - left: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3)), - right: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3)), + key_range: KeyRange { + left: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3)) + .into(), + right: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3)) + .into(), right_exclusive: false, - }), + }, file_size: 1, table_ids: vec![0], meta_offset: 1, diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs index 5cdd6b5f5257..f50747339c44 100644 --- a/src/storage/hummock_test/src/sync_point_tests.rs +++ b/src/storage/hummock_test/src/sync_point_tests.rs @@ -213,7 +213,7 @@ pub async fn compact_once( hummock_manager_ref .report_compact_task( result_task.task_id, - result_tasktask_status, + result_task.task_status, result_task.sorted_output_ssts, Some(to_prost_table_stats_map(task_stats)), ) diff --git a/src/storage/src/hummock/event_handler/uploader/test_utils.rs b/src/storage/src/hummock/event_handler/uploader/test_utils.rs index 2fa574c72fc2..a73acb6bdb5e 100644 --- a/src/storage/src/hummock/event_handler/uploader/test_utils.rs +++ b/src/storage/src/hummock/event_handler/uploader/test_utils.rs @@ -31,9 +31,10 @@ use risingwave_common::must_match; use risingwave_common::util::epoch::{test_epoch, EpochExt}; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::{FullKey, TableKey}; -use risingwave_hummock_sdk::version::HummockVersion; +use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; -use risingwave_pb::hummock::{KeyRange, SstableInfo, StateTableInfoDelta}; +use risingwave_pb::hummock::StateTableInfoDelta; use spin::Mutex; use tokio::spawn; use tokio::sync::mpsc::unbounded_channel; @@ -159,11 +160,11 @@ pub(super) fn gen_sstable_info( vec![LocalSstableInfo::for_test(SstableInfo { object_id: gen_sst_object_id, sst_id: gen_sst_object_id, - key_range: Some(KeyRange { - left: start_full_key.encode(), - right: end_full_key.encode(), + key_range: KeyRange { + left: start_full_key.encode().into(), + right: end_full_key.encode().into(), right_exclusive: true, - }), + }, table_ids: vec![TEST_TABLE_ID.table_id], ..Default::default() })] From 75d72d4ac30b40f5698089c873c9127966fcecf1 Mon Sep 17 00:00:00 2001 From: Li0k Date: Wed, 24 Jul 2024 00:09:03 +0800 Subject: [PATCH 15/18] fix(storage): fix sync_point and bench code --- src/storage/benches/bench_multi_builder.rs | 2 +- src/storage/hummock_test/src/sync_point_tests.rs | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs index 08d744189a1d..d9b745ce892b 100644 --- a/src/storage/benches/bench_multi_builder.rs +++ b/src/storage/benches/bench_multi_builder.rs @@ -25,10 +25,10 @@ use rand::random; use risingwave_common::catalog::TableId; use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_hummock_sdk::key::{FullKey, UserKey}; +use risingwave_hummock_sdk::version::SstableInfo; use risingwave_object_store::object::{ InMemObjectStore, ObjectStore, ObjectStoreImpl, S3ObjectStore, }; -use risingwave_pb::hummock::SstableInfo; use risingwave_storage::hummock::iterator::{ConcatIterator, ConcatIteratorInner, HummockIterator}; use risingwave_storage::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory}; use risingwave_storage::hummock::value::HummockValue; diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs index f50747339c44..f5ee41783813 100644 --- a/src/storage/hummock_test/src/sync_point_tests.rs +++ b/src/storage/hummock_test/src/sync_point_tests.rs @@ -417,13 +417,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() { .get_compaction_group_levels(StaticCompactionGroupId::StateDefault.into()) .levels[4]; assert_eq!(base_level.table_infos.len(), 3); - assert!( - base_level.table_infos[0] - .key_range - .as_ref() - .unwrap() - .right_exclusive - ); + assert!(base_level.table_infos[0].key_range.right_exclusive); assert_eq!( user_key(&base_level.table_infos[0].key_range.right), user_key(&base_level.table_infos[1].key_range.left), From 2de6a48d845afb49fd3bb7fbb817f7a2086a0cb1 Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 25 Jul 2024 14:59:47 +0800 Subject: [PATCH 16/18] fix(storage): reorg and address comments --- src/ctl/src/cmd_impl/hummock/sst_dump.rs | 3 +- .../src/cmd_impl/hummock/validate_version.rs | 3 +- src/meta/model_v2/src/hummock_sstable_info.rs | 1 - src/meta/src/hummock/compaction/mod.rs | 3 +- .../hummock/compaction/overlap_strategy.rs | 2 +- .../picker/base_level_compaction_picker.rs | 2 +- .../picker/emergency_compaction_picker.rs | 2 +- .../picker/intra_compaction_picker.rs | 4 +- .../picker/manual_compaction_picker.rs | 3 +- .../picker/min_overlap_compaction_picker.rs | 6 +- src/meta/src/hummock/compaction/picker/mod.rs | 2 +- .../picker/space_reclaim_compaction_picker.rs | 3 +- .../picker/tier_compaction_picker.rs | 2 +- .../tombstone_reclaim_compaction_picker.rs | 4 +- .../picker/trivial_move_compaction_picker.rs | 3 +- .../picker/ttl_reclaim_compaction_picker.rs | 3 +- .../picker/vnode_watermark_picker.rs | 5 +- .../compaction/selector/level_selector.rs | 2 +- .../src/hummock/compaction/selector/mod.rs | 3 +- src/meta/src/hummock/compactor_manager.rs | 2 +- src/meta/src/hummock/level_handler.rs | 3 +- src/meta/src/hummock/manager/commit_epoch.rs | 3 +- src/meta/src/hummock/manager/compaction.rs | 20 +- .../manager/compaction_group_manager.rs | 3 +- src/meta/src/hummock/manager/time_travel.rs | 3 +- src/meta/src/hummock/manager/transaction.rs | 3 +- src/meta/src/hummock/manager/versioning.rs | 3 +- src/meta/src/hummock/metrics_utils.rs | 3 +- .../src/hummock/mock_hummock_meta_client.rs | 4 +- src/meta/src/hummock/test_utils.rs | 6 +- src/meta/src/manager/diagnose.rs | 2 +- src/storage/benches/bench_compactor.rs | 2 +- src/storage/benches/bench_multi_builder.rs | 2 +- src/storage/hummock_sdk/src/change_log.rs | 2 +- src/storage/hummock_sdk/src/compact.rs | 3 +- src/storage/hummock_sdk/src/compact_task.rs | 412 ++++++++ .../compaction_group/hummock_version_ext.rs | 41 +- src/storage/hummock_sdk/src/level.rs | 406 ++++++++ src/storage/hummock_sdk/src/lib.rs | 5 +- src/storage/hummock_sdk/src/sstable_info.rs | 180 ++++ src/storage/hummock_sdk/src/time_travel.rs | 4 +- src/storage/hummock_sdk/src/version.rs | 952 +----------------- .../src/hummock_read_version_tests.rs | 2 +- .../src/hummock/compactor/compaction_utils.rs | 3 +- .../src/hummock/compactor/compactor_runner.rs | 3 +- .../compactor/fast_compactor_runner.rs | 3 +- src/storage/src/hummock/compactor/iterator.rs | 2 +- src/storage/src/hummock/compactor/mod.rs | 6 +- .../iterator/concat_delete_range_iterator.rs | 2 +- .../src/hummock/iterator/concat_inner.rs | 2 +- .../hummock/iterator/delete_range_iterator.rs | 2 +- src/storage/src/hummock/iterator/mod.rs | 2 +- .../src/hummock/iterator/test_utils.rs | 2 +- .../hummock/local_version/pinned_version.rs | 3 +- src/storage/src/hummock/mod.rs | 2 +- src/storage/src/hummock/sstable/builder.rs | 2 +- src/storage/src/hummock/sstable_store.rs | 4 +- .../src/hummock/store/hummock_storage.rs | 3 +- .../hummock/store/local_hummock_storage.rs | 2 +- src/storage/src/hummock/store/version.rs | 2 +- src/storage/src/hummock/test_utils.rs | 2 +- src/storage/src/hummock/utils.rs | 3 +- src/storage/src/hummock/validator.rs | 2 +- 63 files changed, 1130 insertions(+), 1044 deletions(-) create mode 100644 src/storage/hummock_sdk/src/compact_task.rs create mode 100644 src/storage/hummock_sdk/src/level.rs create mode 100644 src/storage/hummock_sdk/src/sstable_info.rs diff --git a/src/ctl/src/cmd_impl/hummock/sst_dump.rs b/src/ctl/src/cmd_impl/hummock/sst_dump.rs index c024d91ac012..03a7b35a8519 100644 --- a/src/ctl/src/cmd_impl/hummock/sst_dump.rs +++ b/src/ctl/src/cmd_impl/hummock/sst_dump.rs @@ -28,7 +28,8 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAw use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, ValueRowDeserializer}; use risingwave_frontend::TableCatalog; use risingwave_hummock_sdk::key::FullKey; -use risingwave_hummock_sdk::version::{Level, SstableInfo}; +use risingwave_hummock_sdk::level::Level; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::HummockSstableObjectId; use risingwave_object_store::object::{ObjectMetadata, ObjectStoreImpl}; use risingwave_rpc_client::MetaClient; diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs index db228cfc3dec..7d0af4da9fee 100644 --- a/src/ctl/src/cmd_impl/hummock/validate_version.rs +++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs @@ -20,7 +20,8 @@ use itertools::Itertools; use risingwave_common::util::epoch::Epoch; use risingwave_hummock_sdk::compaction_group::hummock_version_ext; use risingwave_hummock_sdk::key::{FullKey, UserKey}; -use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta, SstableInfo}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::{version_archive_dir, HummockSstableObjectId, HummockVersionId}; use risingwave_object_store::object::ObjectStoreRef; use risingwave_pb::hummock::group_delta::DeltaType; diff --git a/src/meta/model_v2/src/hummock_sstable_info.rs b/src/meta/model_v2/src/hummock_sstable_info.rs index f3495186b618..672d1f0e05a8 100644 --- a/src/meta/model_v2/src/hummock_sstable_info.rs +++ b/src/meta/model_v2/src/hummock_sstable_info.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// use risingwave_hummock_sdk::version::SstableInfo; use risingwave_pb::hummock::PbSstableInfo; use sea_orm::entity::prelude::*; use sea_orm::{DeriveEntityModel, DeriveRelation, EnumIter}; diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs index f2feabe77fbb..2ebbab619a79 100644 --- a/src/meta/src/hummock/compaction/mod.rs +++ b/src/meta/src/hummock/compaction/mod.rs @@ -17,7 +17,8 @@ pub mod compaction_config; mod overlap_strategy; use risingwave_common::catalog::{TableId, TableOption}; -use risingwave_hummock_sdk::version::{CompactTask, Levels}; +use risingwave_hummock_sdk::compact_task::CompactTask; +use risingwave_hummock_sdk::level::Levels; use risingwave_pb::hummock::compact_task::{self, TaskType}; mod picker; diff --git a/src/meta/src/hummock/compaction/overlap_strategy.rs b/src/meta/src/hummock/compaction/overlap_strategy.rs index 3e302d996a35..9a8617d0e927 100644 --- a/src/meta/src/hummock/compaction/overlap_strategy.rs +++ b/src/meta/src/hummock/compaction/overlap_strategy.rs @@ -18,7 +18,7 @@ use std::ops::Range; use itertools::Itertools; use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::KeyComparator; pub trait OverlapInfo: Debug { diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs index bf9524bb28ae..8b2831d28e1d 100644 --- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_common::config::default::compaction_config; -use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, OverlappingLevel}; +use risingwave_hummock_sdk::level::{InputLevel, Level, Levels, OverlappingLevel}; use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker; diff --git a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs index 8598ba837109..f0fc3f237659 100644 --- a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs @@ -14,7 +14,7 @@ use std::sync::Arc; -use risingwave_hummock_sdk::version::Levels; +use risingwave_hummock_sdk::level::Levels; use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::{ diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs index bfb7e198d2fd..dcd41073f05b 100644 --- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use risingwave_common::config::default::compaction_config; -use risingwave_hummock_sdk::version::{InputLevel, Levels, OverlappingLevel}; +use risingwave_hummock_sdk::level::{InputLevel, Levels, OverlappingLevel}; use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker; @@ -417,7 +417,7 @@ impl WholeLevelCompactionPicker { #[cfg(test)] pub mod tests { - use risingwave_hummock_sdk::version::Level; + use risingwave_hummock_sdk::level::Level; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs index ee712737725b..251a0a694b1a 100644 --- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs @@ -16,7 +16,8 @@ use std::collections::HashSet; use std::sync::Arc; use itertools::Itertools; -use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, OverlappingLevel, SstableInfo}; +use risingwave_hummock_sdk::level::{InputLevel, Level, Levels, OverlappingLevel}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_pb::hummock::LevelType; use super::{CompactionInput, CompactionPicker, LocalPickerStatistic}; diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index 17d723c2d621..aa6777acf2b4 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -15,7 +15,8 @@ use std::sync::Arc; use risingwave_hummock_sdk::append_sstable_info_to_string; -use risingwave_hummock_sdk::version::{InputLevel, Level, Levels, SstableInfo}; +use risingwave_hummock_sdk::level::{InputLevel, Level, Levels}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_pb::hummock::LevelType; use super::{CompactionInput, CompactionPicker, LocalPickerStatistic}; @@ -529,6 +530,7 @@ pub mod tests { use std::collections::BTreeSet; use risingwave_common::config::default::compaction_config; + use risingwave_hummock_sdk::level::{Level, Levels}; use super::*; use crate::hummock::compaction::overlap_strategy::RangeOverlapStrategy; @@ -750,7 +752,7 @@ pub mod tests { true, compaction_config::max_l0_compact_level_count() as usize, ); - let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); + let ret = picker.pick_l0_multi_non_overlap_level(&Levels, &levels_handlers[0]); assert_eq!(6, ret.len()); } diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs index 770b5575bfa5..3bed84fc57b9 100644 --- a/src/meta/src/hummock/compaction/picker/mod.rs +++ b/src/meta/src/hummock/compaction/picker/mod.rs @@ -32,7 +32,7 @@ pub use emergency_compaction_picker::EmergencyCompactionPicker; pub use intra_compaction_picker::IntraCompactionPicker; pub use manual_compaction_picker::ManualCompactionPicker; pub use min_overlap_compaction_picker::MinOverlappingPicker; -use risingwave_hummock_sdk::version::{InputLevel, Levels}; +use risingwave_hummock_sdk::level::{InputLevel, Levels}; pub use space_reclaim_compaction_picker::{SpaceReclaimCompactionPicker, SpaceReclaimPickerState}; pub use tier_compaction_picker::TierCompactionPicker; pub use tombstone_reclaim_compaction_picker::{ diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 35a99cd3c1db..6e06629b9ebd 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -14,7 +14,8 @@ use std::collections::HashSet; -use risingwave_hummock_sdk::version::{InputLevel, Levels, SstableInfo}; +use risingwave_hummock_sdk::level::{InputLevel, Levels}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use super::CompactionInput; use crate::hummock::level_handler::LevelHandler; diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index b0b71e6c1a84..f78b8ebadb56 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -14,7 +14,7 @@ use std::sync::Arc; -use risingwave_hummock_sdk::version::{InputLevel, Levels, OverlappingLevel}; +use risingwave_hummock_sdk::level::{InputLevel, Levels, OverlappingLevel}; use risingwave_pb::hummock::{CompactionConfig, LevelType}; use super::{ diff --git a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs index f717c13a7ade..0b3aad7ef8b2 100644 --- a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs @@ -14,7 +14,7 @@ use std::sync::Arc; -use risingwave_hummock_sdk::version::{InputLevel, Levels}; +use risingwave_hummock_sdk::level::{InputLevel, Levels}; use crate::hummock::compaction::overlap_strategy::OverlapStrategy; use crate::hummock::compaction::picker::CompactionInput; @@ -132,7 +132,7 @@ impl TombstoneReclaimCompactionPicker { #[cfg(test)] pub mod tests { - use risingwave_hummock_sdk::version::OverlappingLevel; + use risingwave_hummock_sdk::level::OverlappingLevel; use super::*; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs index 8478602ff62d..96fcc69175bd 100644 --- a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs @@ -14,7 +14,8 @@ use std::sync::Arc; -use risingwave_hummock_sdk::version::{InputLevel, SstableInfo}; +use risingwave_hummock_sdk::level::InputLevel; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_pb::hummock::LevelType; use super::{CompactionInput, LocalPickerStatistic}; diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index af1b3a62f7a3..f5fe6dd86116 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -19,7 +19,8 @@ use risingwave_common::catalog::TableOption; use risingwave_common::util::epoch::Epoch; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; -use risingwave_hummock_sdk::version::{InputLevel, Levels, SstableInfo}; +use risingwave_hummock_sdk::level::{InputLevel, Levels}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use super::CompactionInput; use crate::hummock::level_handler::LevelHandler; diff --git a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs index ed42727cac4a..50a33c7d42e4 100644 --- a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs +++ b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs @@ -16,8 +16,9 @@ use std::collections::BTreeMap; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::{FullKey, TableKey}; +use risingwave_hummock_sdk::level::{InputLevel, Levels}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_watermark::ReadTableWatermark; -use risingwave_hummock_sdk::version::{InputLevel, Levels, SstableInfo}; use crate::hummock::compaction::picker::CompactionInput; use crate::hummock::level_handler::LevelHandler; @@ -108,8 +109,8 @@ mod tests { use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::key::{FullKey, TableKey}; use risingwave_hummock_sdk::key_range::KeyRange; + use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, WatermarkDirection}; - use risingwave_hummock_sdk::version::SstableInfo; use crate::hummock::compaction::picker::vnode_watermark_picker::should_delete_sst_by_watermark; diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index 3e550d41a0b7..5597cdd3ea17 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -18,7 +18,7 @@ // (found in the LICENSE.Apache file in the root directory). use std::sync::Arc; -use risingwave_hummock_sdk::version::Levels; +use risingwave_hummock_sdk::level::Levels; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task::PbTaskType; use risingwave_pb::hummock::{CompactionConfig, LevelType}; diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index 7a77c736efc4..9018707d9e9e 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -32,8 +32,9 @@ pub use emergency_selector::EmergencySelector; pub use level_selector::{DynamicLevelSelector, DynamicLevelSelectorCore}; pub use manual_selector::{ManualCompactionOption, ManualCompactionSelector}; use risingwave_common::catalog::{TableId, TableOption}; +use risingwave_hummock_sdk::level::Levels; use risingwave_hummock_sdk::table_watermark::TableWatermarks; -use risingwave_hummock_sdk::version::{HummockVersionStateTableInfo, Levels}; +use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_hummock_sdk::HummockCompactionTaskId; use risingwave_pb::hummock::compact_task; pub use space_reclaim_selector::SpaceReclaimCompactionSelector; diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs index 095a920b038e..252f92c40401 100644 --- a/src/meta/src/hummock/compactor_manager.rs +++ b/src/meta/src/hummock/compactor_manager.rs @@ -19,7 +19,7 @@ use std::time::{Duration, Instant, SystemTime}; use fail::fail_point; use parking_lot::RwLock; use risingwave_hummock_sdk::compact::statistics_compact_task; -use risingwave_hummock_sdk::version::CompactTask; +use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId}; use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent; use risingwave_pb::hummock::{ diff --git a/src/meta/src/hummock/level_handler.rs b/src/meta/src/hummock/level_handler.rs index 94db30edb8d0..2f729b7123d5 100644 --- a/src/meta/src/hummock/level_handler.rs +++ b/src/meta/src/hummock/level_handler.rs @@ -15,7 +15,8 @@ use std::collections::HashMap; use itertools::Itertools; -use risingwave_hummock_sdk::version::{Level, SstableInfo}; +use risingwave_hummock_sdk::level::Level; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockSstableId}; use risingwave_pb::hummock::level_handler::RunningCompactTask; diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs index 66d67a23b380..2e8535e97de7 100644 --- a/src/meta/src/hummock/manager/commit_epoch.rs +++ b/src/meta/src/hummock/manager/commit_epoch.rs @@ -18,11 +18,12 @@ use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::change_log::ChangeLogDelta; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::split_sst; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::{ add_prost_table_stats_map, purge_prost_table_stats, to_prost_table_stats_map, PbTableStatsMap, }; use risingwave_hummock_sdk::table_watermark::TableWatermarks; -use risingwave_hummock_sdk::version::{HummockVersionStateTableInfo, SstableInfo}; +use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, LocalSstableInfo, }; diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs index 3a296452fa65..fbe0f1fe8a66 100644 --- a/src/meta/src/hummock/manager/compaction.rs +++ b/src/meta/src/hummock/manager/compaction.rs @@ -41,16 +41,16 @@ use parking_lot::Mutex; use rand::seq::SliceRandom; use rand::thread_rng; use risingwave_common::util::epoch::Epoch; +use risingwave_hummock_sdk::compact_task::{CompactTask, ReportTask}; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::level::{InputLevel, Level, Levels}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::{ add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap, }; -use risingwave_hummock_sdk::version::{ - CompactTask, GroupDelta, HummockVersion, InputLevel, IntraLevelDelta, Level, Levels, - ReportTask, SstableInfo, -}; +use risingwave_hummock_sdk::version::{GroupDelta, HummockVersion, IntraLevelDelta}; use risingwave_hummock_sdk::{ compact_task_to_string, statistics_compact_task, CompactionGroupId, HummockCompactionTaskId, HummockVersionId, @@ -837,7 +837,7 @@ impl HummockManager { compact_task.input_ssts, start_time.elapsed() ); - compact_task.set_task_status(TaskStatus::Success); + compact_task.task_status = TaskStatus::Success; compact_status.report_compact_task(&compact_task); if !is_trivial_reclaim { compact_task @@ -951,7 +951,7 @@ impl HummockManager { .initiate_task_heartbeat(compact_task.clone()); // this task has been finished. - compact_task.set_task_status(TaskStatus::Pending); + compact_task.task_status = TaskStatus::Pending; let compact_task_statistics = statistics_compact_task(compact_task); let level_type_label = build_compact_task_level_type_metrics_label( @@ -1223,7 +1223,7 @@ impl HummockManager { { // apply result - compact_task.set_task_status(task.task_status); + compact_task.task_status = task.task_status; compact_task.sorted_output_ssts = task.sorted_output_ssts; } @@ -1232,7 +1232,7 @@ impl HummockManager { compact_status.report_compact_task(&compact_task); } None => { - compact_task.set_task_status(TaskStatus::InvalidGroupCanceled); + compact_task.task_status = TaskStatus::InvalidGroupCanceled; } } @@ -1251,7 +1251,7 @@ impl HummockManager { let is_expired = Self::is_compact_task_expired(&compact_task, version.latest_version()); if is_expired { - compact_task.set_task_status(TaskStatus::InputOutdatedCanceled); + compact_task.task_status = TaskStatus::InputOutdatedCanceled; false } else { let group = version @@ -1262,7 +1262,7 @@ impl HummockManager { let input_exist = group.check_deleted_sst_exist(&input_level_ids, input_sst_ids); if !input_exist { - compact_task.set_task_status(TaskStatus::InvalidGroupCanceled); + compact_task.task_status = TaskStatus::InvalidGroupCanceled; warn!( "The task may be expired because of group split, task:\n {:?}", compact_task_to_string(&compact_task) diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs index 3b2c4502371a..61191722f800 100644 --- a/src/meta/src/hummock/manager/compaction_group_manager.rs +++ b/src/meta/src/hummock/manager/compaction_group_manager.rs @@ -18,11 +18,12 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_common::catalog::TableId; +use risingwave_hummock_sdk::compact_task::ReportTask; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_compaction_group_ids, TableGroupInfo, }; use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId}; -use risingwave_hummock_sdk::version::{GroupDelta, GroupDeltas, ReportTask}; +use risingwave_hummock_sdk::version::{GroupDelta, GroupDeltas}; use risingwave_hummock_sdk::CompactionGroupId; use risingwave_meta_model_v2::compaction_config; use risingwave_pb::hummock::compact_task::TaskStatus; diff --git a/src/meta/src/hummock/manager/time_travel.rs b/src/meta/src/hummock/manager/time_travel.rs index 9d55b3639ce5..6ac687ad39c0 100644 --- a/src/meta/src/hummock/manager/time_travel.rs +++ b/src/meta/src/hummock/manager/time_travel.rs @@ -18,10 +18,11 @@ use anyhow::anyhow; use itertools::Itertools; use risingwave_common::util::epoch::Epoch; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::time_travel::{ refill_version, IncompleteHummockVersion, IncompleteHummockVersionDelta, }; -use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta, SstableInfo}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockEpoch, HummockSstableId, HummockSstableObjectId, }; diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs index d7559ea1c4be..b6a41e8ffa8a 100644 --- a/src/meta/src/hummock/manager/transaction.rs +++ b/src/meta/src/hummock/manager/transaction.rs @@ -17,9 +17,10 @@ use std::ops::{Deref, DerefMut}; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::change_log::ChangeLogDelta; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_watermark::TableWatermarks; use risingwave_hummock_sdk::version::{ - GroupDelta, HummockVersion, HummockVersionDelta, IntraLevelDelta, SstableInfo, + GroupDelta, HummockVersion, HummockVersionDelta, IntraLevelDelta, }; use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, HummockVersionId}; use risingwave_pb::hummock::{HummockVersionStats, StateTableInfoDelta}; diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs index c6c1ea1070ed..7b004d0e13a6 100644 --- a/src/meta/src/hummock/manager/versioning.rs +++ b/src/meta/src/hummock/manager/versioning.rs @@ -21,8 +21,9 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{ get_compaction_group_ids, get_table_compaction_group_id_mapping, BranchedSstInfo, }; use risingwave_hummock_sdk::compaction_group::StateTableId; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::add_prost_table_stats_map; -use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta, SstableInfo}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableId, HummockSstableObjectId, HummockVersionId, diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs index 6adcfca18469..992d5b114690 100644 --- a/src/meta/src/hummock/metrics_utils.rs +++ b/src/meta/src/hummock/metrics_utils.rs @@ -21,8 +21,9 @@ use itertools::{enumerate, Itertools}; use prometheus::core::{AtomicU64, GenericCounter}; use prometheus::IntGauge; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::object_size_map; +use risingwave_hummock_sdk::level::Levels; use risingwave_hummock_sdk::table_stats::PbTableStatsMap; -use risingwave_hummock_sdk::version::{HummockVersion, Levels}; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{CompactionGroupId, HummockContextId, HummockEpoch, HummockVersionId}; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{ diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs index bd895b808b74..8499605e04fb 100644 --- a/src/meta/src/hummock/mock_hummock_meta_client.rs +++ b/src/meta/src/hummock/mock_hummock_meta_client.rs @@ -24,8 +24,10 @@ use futures::stream::BoxStream; use futures::{Stream, StreamExt}; use itertools::Itertools; use risingwave_hummock_sdk::change_log::build_table_change_log_delta; +use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; -use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{ HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, SstObjectIdRange, SyncResult, diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs index bf9ae8cebeea..84c19cfd4d93 100644 --- a/src/meta/src/hummock/test_utils.rs +++ b/src/meta/src/hummock/test_utils.rs @@ -25,10 +25,10 @@ use risingwave_common::util::epoch::test_epoch; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::key_with_epoch; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::level::Levels; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_watermark::TableWatermarks; -use risingwave_hummock_sdk::version::{ - HummockVersion, HummockVersionStateTableInfo, Levels, SstableInfo, -}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionStateTableInfo}; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, LocalSstableInfo, }; diff --git a/src/meta/src/manager/diagnose.rs b/src/meta/src/manager/diagnose.rs index dbba917a73eb..1d6fcdd7f8fe 100644 --- a/src/meta/src/manager/diagnose.rs +++ b/src/meta/src/manager/diagnose.rs @@ -21,7 +21,7 @@ use itertools::Itertools; use prometheus_http_query::response::Data::Vector; use risingwave_common::types::Timestamptz; use risingwave_common::util::StackTraceResponseExt; -use risingwave_hummock_sdk::version::Level; +use risingwave_hummock_sdk::level::Level; use risingwave_pb::common::WorkerType; use risingwave_pb::meta::event_log::Event; use risingwave_pb::meta::EventLog; diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index 0e9a52649fed..e5f929a312a8 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -27,7 +27,7 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAw use risingwave_common::util::value_encoding::ValueRowSerializer; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl}; use risingwave_pb::hummock::compact_task::PbTaskType; diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs index d9b745ce892b..730534a81831 100644 --- a/src/storage/benches/bench_multi_builder.rs +++ b/src/storage/benches/bench_multi_builder.rs @@ -25,7 +25,7 @@ use rand::random; use risingwave_common::catalog::TableId; use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_hummock_sdk::key::{FullKey, UserKey}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_object_store::object::{ InMemObjectStore, ObjectStore, ObjectStoreImpl, S3ObjectStore, }; diff --git a/src/storage/hummock_sdk/src/change_log.rs b/src/storage/hummock_sdk/src/change_log.rs index c7102658f55d..885880933ce1 100644 --- a/src/storage/hummock_sdk/src/change_log.rs +++ b/src/storage/hummock_sdk/src/change_log.rs @@ -19,7 +19,7 @@ use risingwave_pb::hummock::hummock_version_delta::PbChangeLogDelta; use risingwave_pb::hummock::{PbEpochNewChangeLog, PbTableChangeLog}; use tracing::warn; -use crate::version::SstableInfo; +use crate::sstable_info::SstableInfo; #[derive(Debug, Clone, PartialEq)] pub struct TableChangeLog(pub Vec); diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index 5d827f8adcde..ff1241447791 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -16,7 +16,8 @@ use std::collections::HashSet; use risingwave_pb::hummock::LevelType; -use crate::version::{CompactTask, SstableInfo}; +use crate::compact_task::CompactTask; +use crate::sstable_info::SstableInfo; pub fn compact_task_output_to_string(compact_task: &CompactTask) -> String { use std::fmt::Write; diff --git a/src/storage/hummock_sdk/src/compact_task.rs b/src/storage/hummock_sdk/src/compact_task.rs new file mode 100644 index 000000000000..61f96e1ee3b5 --- /dev/null +++ b/src/storage/hummock_sdk/src/compact_task.rs @@ -0,0 +1,412 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{BTreeMap, HashMap}; +use std::mem::size_of; + +use itertools::Itertools; +use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType, TaskStatus}; +use risingwave_pb::hummock::subscribe_compaction_event_request::PbReportTask; +use risingwave_pb::hummock::{ + PbCompactTask, PbKeyRange, PbTableOption, PbTableSchema, PbTableStats, PbValidationTask, +}; + +use crate::key_range::KeyRange; +use crate::level::InputLevel; +use crate::sstable_info::SstableInfo; +use crate::table_watermark::TableWatermarks; + +#[derive(Clone, PartialEq, Default, Debug)] +pub struct CompactTask { + /// SSTs to be compacted, which will be removed from LSM after compaction + pub input_ssts: Vec, + /// In ideal case, the compaction will generate `splits.len()` tables which have key range + /// corresponding to that in `splits`, respectively + pub splits: Vec, + /// low watermark in 'ts-aware compaction' + pub watermark: u64, + /// compaction output, which will be added to `target_level` of LSM after compaction + pub sorted_output_ssts: Vec, + /// task id assigned by hummock storage service + pub task_id: u64, + /// compaction output will be added to `target_level` of LSM after compaction + pub target_level: u32, + pub gc_delete_keys: bool, + /// Lbase in LSM + pub base_level: u32, + pub task_status: PbTaskStatus, + /// compaction group the task belongs to + pub compaction_group_id: u64, + /// `existing_table_ids` for compaction drop key + pub existing_table_ids: Vec, + pub compression_algorithm: u32, + pub target_file_size: u64, + pub compaction_filter_mask: u32, + pub table_options: BTreeMap, + pub current_epoch_time: u64, + pub target_sub_level_id: u64, + /// Identifies whether the task is `space_reclaim`, if the `compact_task_type` increases, it will be refactored to enum + pub task_type: PbTaskType, + /// Deprecated. use `table_vnode_partition` instead; + pub split_by_state_table: bool, + /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed. + /// Deprecated. use `table_vnode_partition` instead; + pub split_weight_by_vnode: u32, + pub table_vnode_partition: BTreeMap, + /// The table watermark of any table id. In compaction we only use the table watermarks on safe epoch, + /// so we only need to include the table watermarks on safe epoch to reduce the size of metadata. + pub table_watermarks: BTreeMap, + + pub table_schemas: BTreeMap, + + pub max_sub_compaction: u32, +} + +impl CompactTask { + pub fn estimated_encode_len(&self) -> usize { + self.input_ssts + .iter() + .map(|input_level| input_level.estimated_encode_len()) + .sum::() + + self + .splits + .iter() + .map(|split| split.left.len() + split.right.len() + size_of::()) + .sum::() + + size_of::() + + self + .sorted_output_ssts + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.existing_table_ids.len() * size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.table_options.len() * size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + + self.table_vnode_partition.len() * size_of::() + + self + .table_watermarks + .values() + .map(|table_watermark| size_of::() + table_watermark.estimated_encode_len()) + .sum::() + } +} + +impl From for CompactTask { + #[expect(deprecated)] + fn from(pb_compact_task: PbCompactTask) -> Self { + Self { + input_ssts: pb_compact_task + .input_ssts + .into_iter() + .map(InputLevel::from) + .collect_vec(), + splits: pb_compact_task + .splits + .into_iter() + .map(|pb_keyrange| KeyRange { + left: pb_keyrange.left.into(), + right: pb_keyrange.right.into(), + right_exclusive: pb_keyrange.right_exclusive, + }) + .collect_vec(), + watermark: pb_compact_task.watermark, + sorted_output_ssts: pb_compact_task + .sorted_output_ssts + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + task_id: pb_compact_task.task_id, + target_level: pb_compact_task.target_level, + gc_delete_keys: pb_compact_task.gc_delete_keys, + base_level: pb_compact_task.base_level, + task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(), + compaction_group_id: pb_compact_task.compaction_group_id, + existing_table_ids: pb_compact_task.existing_table_ids.clone(), + compression_algorithm: pb_compact_task.compression_algorithm, + target_file_size: pb_compact_task.target_file_size, + compaction_filter_mask: pb_compact_task.compaction_filter_mask, + table_options: pb_compact_task.table_options.clone(), + current_epoch_time: pb_compact_task.current_epoch_time, + target_sub_level_id: pb_compact_task.target_sub_level_id, + task_type: PbTaskType::try_from(pb_compact_task.task_type).unwrap(), + split_by_state_table: pb_compact_task.split_by_state_table, + split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, + table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), + table_watermarks: pb_compact_task + .table_watermarks + .into_iter() + .map(|(table_id, pb_table_watermark)| { + (table_id, TableWatermarks::from(pb_table_watermark)) + }) + .collect(), + table_schemas: pb_compact_task.table_schemas, + max_sub_compaction: pb_compact_task.max_sub_compaction, + } + } +} + +impl From<&PbCompactTask> for CompactTask { + #[expect(deprecated)] + fn from(pb_compact_task: &PbCompactTask) -> Self { + Self { + input_ssts: pb_compact_task + .input_ssts + .iter() + .map(InputLevel::from) + .collect_vec(), + splits: pb_compact_task + .splits + .iter() + .map(|pb_keyrange| KeyRange { + left: pb_keyrange.left.clone().into(), + right: pb_keyrange.right.clone().into(), + right_exclusive: pb_keyrange.right_exclusive, + }) + .collect_vec(), + watermark: pb_compact_task.watermark, + sorted_output_ssts: pb_compact_task + .sorted_output_ssts + .iter() + .map(SstableInfo::from) + .collect_vec(), + task_id: pb_compact_task.task_id, + target_level: pb_compact_task.target_level, + gc_delete_keys: pb_compact_task.gc_delete_keys, + base_level: pb_compact_task.base_level, + task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(), + compaction_group_id: pb_compact_task.compaction_group_id, + existing_table_ids: pb_compact_task.existing_table_ids.clone(), + compression_algorithm: pb_compact_task.compression_algorithm, + target_file_size: pb_compact_task.target_file_size, + compaction_filter_mask: pb_compact_task.compaction_filter_mask, + table_options: pb_compact_task.table_options.clone(), + current_epoch_time: pb_compact_task.current_epoch_time, + target_sub_level_id: pb_compact_task.target_sub_level_id, + task_type: PbTaskType::try_from(pb_compact_task.task_type).unwrap(), + split_by_state_table: pb_compact_task.split_by_state_table, + split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, + table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), + table_watermarks: pb_compact_task + .table_watermarks + .iter() + .map(|(table_id, pb_table_watermark)| { + (*table_id, TableWatermarks::from(pb_table_watermark)) + }) + .collect(), + table_schemas: pb_compact_task.table_schemas.clone(), + max_sub_compaction: pb_compact_task.max_sub_compaction, + } + } +} + +impl From for PbCompactTask { + #[expect(deprecated)] + fn from(compact_task: CompactTask) -> Self { + Self { + input_ssts: compact_task + .input_ssts + .into_iter() + .map(|input_level| input_level.into()) + .collect_vec(), + splits: compact_task + .splits + .into_iter() + .map(|keyrange| PbKeyRange { + left: keyrange.left.into(), + right: keyrange.right.into(), + right_exclusive: keyrange.right_exclusive, + }) + .collect_vec(), + watermark: compact_task.watermark, + sorted_output_ssts: compact_task + .sorted_output_ssts + .into_iter() + .map(|sst| sst.into()) + .collect_vec(), + task_id: compact_task.task_id, + target_level: compact_task.target_level, + gc_delete_keys: compact_task.gc_delete_keys, + base_level: compact_task.base_level, + task_status: compact_task.task_status.into(), + compaction_group_id: compact_task.compaction_group_id, + existing_table_ids: compact_task.existing_table_ids.clone(), + compression_algorithm: compact_task.compression_algorithm, + target_file_size: compact_task.target_file_size, + compaction_filter_mask: compact_task.compaction_filter_mask, + table_options: compact_task.table_options.clone(), + current_epoch_time: compact_task.current_epoch_time, + target_sub_level_id: compact_task.target_sub_level_id, + task_type: compact_task.task_type.into(), + split_weight_by_vnode: compact_task.split_weight_by_vnode, + table_vnode_partition: compact_task.table_vnode_partition.clone(), + table_watermarks: compact_task + .table_watermarks + .into_iter() + .map(|(table_id, table_watermark)| (table_id, table_watermark.into())) + .collect(), + split_by_state_table: compact_task.split_by_state_table, + table_schemas: compact_task.table_schemas.clone(), + max_sub_compaction: compact_task.max_sub_compaction, + } + } +} + +impl From<&CompactTask> for PbCompactTask { + #[expect(deprecated)] + fn from(compact_task: &CompactTask) -> Self { + Self { + input_ssts: compact_task + .input_ssts + .iter() + .map(|input_level| input_level.into()) + .collect_vec(), + splits: compact_task + .splits + .iter() + .map(|keyrange| PbKeyRange { + left: keyrange.left.to_vec(), + right: keyrange.right.to_vec(), + right_exclusive: keyrange.right_exclusive, + }) + .collect_vec(), + watermark: compact_task.watermark, + sorted_output_ssts: compact_task + .sorted_output_ssts + .iter() + .map(|sst| sst.into()) + .collect_vec(), + task_id: compact_task.task_id, + target_level: compact_task.target_level, + gc_delete_keys: compact_task.gc_delete_keys, + base_level: compact_task.base_level, + task_status: compact_task.task_status.into(), + compaction_group_id: compact_task.compaction_group_id, + existing_table_ids: compact_task.existing_table_ids.clone(), + compression_algorithm: compact_task.compression_algorithm, + target_file_size: compact_task.target_file_size, + compaction_filter_mask: compact_task.compaction_filter_mask, + table_options: compact_task.table_options.clone(), + current_epoch_time: compact_task.current_epoch_time, + target_sub_level_id: compact_task.target_sub_level_id, + task_type: compact_task.task_type.into(), + split_weight_by_vnode: compact_task.split_weight_by_vnode, + table_vnode_partition: compact_task.table_vnode_partition.clone(), + table_watermarks: compact_task + .table_watermarks + .iter() + .map(|(table_id, table_watermark)| (*table_id, table_watermark.into())) + .collect(), + split_by_state_table: compact_task.split_by_state_table, + table_schemas: compact_task.table_schemas.clone(), + max_sub_compaction: compact_task.max_sub_compaction, + } + } +} + +#[derive(Clone, PartialEq, Default)] +pub struct ValidationTask { + pub sst_infos: Vec, + pub sst_id_to_worker_id: HashMap, + pub epoch: u64, +} + +impl From for ValidationTask { + fn from(pb_validation_task: PbValidationTask) -> Self { + Self { + sst_infos: pb_validation_task + .sst_infos + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(), + epoch: pb_validation_task.epoch, + } + } +} + +impl From for PbValidationTask { + fn from(validation_task: ValidationTask) -> Self { + Self { + sst_infos: validation_task + .sst_infos + .into_iter() + .map(|sst| sst.into()) + .collect_vec(), + sst_id_to_worker_id: validation_task.sst_id_to_worker_id.clone(), + epoch: validation_task.epoch, + } + } +} + +impl ValidationTask { + pub fn estimated_encode_len(&self) -> usize { + self.sst_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) + + size_of::() + } +} + +#[derive(Clone, PartialEq, Default, Debug)] +pub struct ReportTask { + pub table_stats_change: HashMap, + pub task_id: u64, + pub task_status: TaskStatus, + pub sorted_output_ssts: Vec, +} + +impl From for ReportTask { + fn from(value: PbReportTask) -> Self { + Self { + table_stats_change: value.table_stats_change.clone(), + task_id: value.task_id, + task_status: PbTaskStatus::try_from(value.task_status).unwrap(), + sorted_output_ssts: value + .sorted_output_ssts + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + } + } +} + +impl From for PbReportTask { + fn from(value: ReportTask) -> Self { + Self { + table_stats_change: value.table_stats_change.clone(), + task_id: value.task_id, + task_status: value.task_status.into(), + sorted_output_ssts: value + .sorted_output_ssts + .into_iter() + .map(|sst| sst.into()) + .collect_vec(), + } + } +} diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 09ed8c9cd791..b98e2d1abd78 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -31,10 +31,11 @@ use super::StateTableId; use crate::change_log::TableChangeLog; use crate::compaction_group::StaticCompactionGroupId; use crate::key_range::KeyRangeCommon; +use crate::level::{Level, Levels, OverlappingLevel}; +use crate::sstable_info::SstableInfo; use crate::table_watermark::{ReadTableWatermark, TableWatermarks}; use crate::version::{ GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, HummockVersionStateTableInfo, - Level, Levels, OverlappingLevel, SstableInfo, }; use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId}; @@ -489,27 +490,24 @@ impl HummockVersion { // current `hummock::manager::gen_version_delta` implementation. Better refactor the // struct to reduce conventions. for group_delta in &group_deltas.group_deltas { - match group_delta { - GroupDelta::IntraLevel(intra_level) => { - if !intra_level.inserted_table_infos.is_empty() { - info.insert_sst_level = intra_level.level_idx; - info.insert_sst_infos - .extend(intra_level.inserted_table_infos.iter().cloned()); - } - if !intra_level.removed_table_ids.is_empty() { - for id in &intra_level.removed_table_ids { - if intra_level.level_idx == 0 { - removed_l0_ssts.insert(*id); - } else { - removed_ssts - .entry(intra_level.level_idx) - .or_default() - .insert(*id); - } + if let GroupDelta::IntraLevel(intra_level) = group_delta { + if !intra_level.inserted_table_infos.is_empty() { + info.insert_sst_level = intra_level.level_idx; + info.insert_sst_infos + .extend(intra_level.inserted_table_infos.iter().cloned()); + } + if !intra_level.removed_table_ids.is_empty() { + for id in &intra_level.removed_table_ids { + if intra_level.level_idx == 0 { + removed_l0_ssts.insert(*id); + } else { + removed_ssts + .entry(intra_level.level_idx) + .or_default() + .insert(*id); } } } - _ => unreachable!(), } } @@ -1341,9 +1339,10 @@ mod tests { use risingwave_pb::hummock::{CompactionConfig, GroupConstruct, GroupDestroy, LevelType}; use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels; + use crate::level::{Level, Levels, OverlappingLevel}; + use crate::sstable_info::SstableInfo; use crate::version::{ - GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, IntraLevelDelta, Level, - Levels, OverlappingLevel, SstableInfo, + GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, IntraLevelDelta, }; #[test] diff --git a/src/storage/hummock_sdk/src/level.rs b/src/storage/hummock_sdk/src/level.rs new file mode 100644 index 000000000000..d81e153ebd91 --- /dev/null +++ b/src/storage/hummock_sdk/src/level.rs @@ -0,0 +1,406 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::mem::size_of; + +use itertools::Itertools; +use risingwave_pb::hummock::hummock_version::PbLevels; +use risingwave_pb::hummock::{ + PbInputLevel, PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo, +}; + +use crate::sstable_info::SstableInfo; + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct OverlappingLevel { + pub sub_levels: Vec, + pub total_file_size: u64, + pub uncompressed_file_size: u64, +} + +impl From<&PbOverlappingLevel> for OverlappingLevel { + fn from(pb_overlapping_level: &PbOverlappingLevel) -> Self { + Self { + sub_levels: pb_overlapping_level + .sub_levels + .iter() + .map(Level::from) + .collect_vec(), + total_file_size: pb_overlapping_level.total_file_size, + uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, + } + } +} + +impl From<&OverlappingLevel> for PbOverlappingLevel { + fn from(overlapping_level: &OverlappingLevel) -> Self { + Self { + sub_levels: overlapping_level + .sub_levels + .iter() + .map(|pb_level| pb_level.into()) + .collect_vec(), + total_file_size: overlapping_level.total_file_size, + uncompressed_file_size: overlapping_level.uncompressed_file_size, + } + } +} + +impl From for PbOverlappingLevel { + fn from(overlapping_level: OverlappingLevel) -> Self { + Self { + sub_levels: overlapping_level + .sub_levels + .into_iter() + .map(|pb_level| pb_level.into()) + .collect_vec(), + total_file_size: overlapping_level.total_file_size, + uncompressed_file_size: overlapping_level.uncompressed_file_size, + } + } +} + +impl From for OverlappingLevel { + fn from(pb_overlapping_level: PbOverlappingLevel) -> Self { + Self { + sub_levels: pb_overlapping_level + .sub_levels + .into_iter() + .map(Level::from) + .collect_vec(), + total_file_size: pb_overlapping_level.total_file_size, + uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, + } + } +} + +impl OverlappingLevel { + pub fn estimated_encode_len(&self) -> usize { + self.sub_levels + .iter() + .map(|level| level.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + } +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Level { + pub level_idx: u32, + pub level_type: PbLevelType, + pub table_infos: Vec, + pub total_file_size: u64, + pub sub_level_id: u64, + pub uncompressed_file_size: u64, + pub vnode_partition_count: u32, +} + +impl From<&PbLevel> for Level { + fn from(pb_level: &PbLevel) -> Self { + Self { + level_idx: pb_level.level_idx, + level_type: PbLevelType::try_from(pb_level.level_type).unwrap(), + table_infos: pb_level + .table_infos + .iter() + .map(SstableInfo::from) + .collect_vec(), + total_file_size: pb_level.total_file_size, + sub_level_id: pb_level.sub_level_id, + uncompressed_file_size: pb_level.uncompressed_file_size, + vnode_partition_count: pb_level.vnode_partition_count, + } + } +} + +impl From<&Level> for PbLevel { + fn from(level: &Level) -> Self { + Self { + level_idx: level.level_idx, + level_type: level.level_type.into(), + table_infos: level + .table_infos + .iter() + .map(PbSstableInfo::from) + .collect_vec(), + total_file_size: level.total_file_size, + sub_level_id: level.sub_level_id, + uncompressed_file_size: level.uncompressed_file_size, + vnode_partition_count: level.vnode_partition_count, + } + } +} + +impl From for PbLevel { + fn from(level: Level) -> Self { + Self { + level_idx: level.level_idx, + level_type: level.level_type.into(), + table_infos: level + .table_infos + .into_iter() + .map(PbSstableInfo::from) + .collect_vec(), + total_file_size: level.total_file_size, + sub_level_id: level.sub_level_id, + uncompressed_file_size: level.uncompressed_file_size, + vnode_partition_count: level.vnode_partition_count, + } + } +} + +impl From for Level { + fn from(pb_level: PbLevel) -> Self { + Self { + level_idx: pb_level.level_idx, + level_type: PbLevelType::try_from(pb_level.level_type).unwrap(), + table_infos: pb_level + .table_infos + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + total_file_size: pb_level.total_file_size, + sub_level_id: pb_level.sub_level_id, + uncompressed_file_size: pb_level.uncompressed_file_size, + vnode_partition_count: pb_level.vnode_partition_count, + } + } +} + +impl Level { + pub fn estimated_encode_len(&self) -> usize { + size_of::() + + size_of::() + + self + .table_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + } +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Levels { + pub levels: Vec, + pub l0: Option, + pub group_id: u64, + pub parent_group_id: u64, + + #[deprecated] + pub member_table_ids: Vec, +} + +impl Levels { + pub fn level0(&self) -> &OverlappingLevel { + self.l0.as_ref().unwrap() + } + + pub fn get_level(&self, level_idx: usize) -> &Level { + &self.levels[level_idx - 1] + } + + pub fn get_level_mut(&mut self, level_idx: usize) -> &mut Level { + &mut self.levels[level_idx - 1] + } + + pub fn is_last_level(&self, level_idx: u32) -> bool { + self.levels + .last() + .as_ref() + .map_or(false, |level| level.level_idx == level_idx) + } + + pub fn count_ssts(&self) -> usize { + self.level0() + .sub_levels + .iter() + .chain(self.levels.iter()) + .map(|level| level.table_infos.len()) + .sum() + } +} + +impl Levels { + pub fn from_protobuf(pb_levels: &PbLevels) -> Self { + Self::from(pb_levels) + } + + pub fn to_protobuf(&self) -> PbLevels { + self.into() + } + + pub fn estimated_encode_len(&self) -> usize { + let mut basic = self + .levels + .iter() + .map(|level| level.estimated_encode_len()) + .sum::() + + size_of::() + + size_of::() + + size_of::(); + if let Some(l0) = self.l0.as_ref() { + basic += l0.estimated_encode_len(); + } + + basic + } +} + +impl From<&PbLevels> for Levels { + #[expect(deprecated)] + fn from(pb_levels: &PbLevels) -> Self { + Self { + l0: if pb_levels.l0.is_some() { + Some(OverlappingLevel::from(pb_levels.l0.as_ref().unwrap())) + } else { + None + }, + levels: pb_levels.levels.iter().map(Level::from).collect_vec(), + group_id: pb_levels.group_id, + parent_group_id: pb_levels.parent_group_id, + member_table_ids: pb_levels.member_table_ids.clone(), + } + } +} + +impl From<&Levels> for PbLevels { + #[expect(deprecated)] + fn from(levels: &Levels) -> Self { + Self { + l0: if levels.l0.is_some() { + Some(levels.l0.as_ref().unwrap().into()) + } else { + None + }, + levels: levels.levels.iter().map(PbLevel::from).collect_vec(), + group_id: levels.group_id, + parent_group_id: levels.parent_group_id, + member_table_ids: levels.member_table_ids.clone(), + } + } +} + +impl From for Levels { + #[expect(deprecated)] + fn from(pb_levels: PbLevels) -> Self { + Self { + l0: if pb_levels.l0.is_some() { + Some(OverlappingLevel::from(pb_levels.l0.unwrap())) + } else { + None + }, + levels: pb_levels.levels.into_iter().map(Level::from).collect_vec(), + group_id: pb_levels.group_id, + parent_group_id: pb_levels.parent_group_id, + member_table_ids: pb_levels.member_table_ids.clone(), + } + } +} + +impl From for PbLevels { + fn from(levels: Levels) -> Self { + #[expect(deprecated)] + Self { + l0: if levels.l0.is_some() { + Some(levels.l0.unwrap().into()) + } else { + None + }, + levels: levels.levels.into_iter().map(PbLevel::from).collect_vec(), + group_id: levels.group_id, + parent_group_id: levels.parent_group_id, + member_table_ids: levels.member_table_ids.clone(), + } + } +} + +#[derive(Clone, PartialEq, Default, Debug)] +pub struct InputLevel { + pub level_idx: u32, + pub level_type: PbLevelType, + pub table_infos: Vec, +} + +impl InputLevel { + pub fn estimated_encode_len(&self) -> usize { + size_of::() + + size_of::() + + self + .table_infos + .iter() + .map(|sst| sst.estimated_encode_len()) + .sum::() + } +} + +impl From for InputLevel { + fn from(pb_input_level: PbInputLevel) -> Self { + Self { + level_idx: pb_input_level.level_idx, + level_type: PbLevelType::try_from(pb_input_level.level_type).unwrap(), + table_infos: pb_input_level + .table_infos + .into_iter() + .map(SstableInfo::from) + .collect_vec(), + } + } +} + +impl From<&PbInputLevel> for InputLevel { + fn from(pb_input_level: &PbInputLevel) -> Self { + Self { + level_idx: pb_input_level.level_idx, + level_type: PbLevelType::try_from(pb_input_level.level_type).unwrap(), + table_infos: pb_input_level + .table_infos + .iter() + .map(SstableInfo::from) + .collect_vec(), + } + } +} + +impl From for PbInputLevel { + fn from(input_level: InputLevel) -> Self { + Self { + level_idx: input_level.level_idx, + level_type: input_level.level_type.into(), + table_infos: input_level + .table_infos + .into_iter() + .map(|sst| sst.into()) + .collect_vec(), + } + } +} + +impl From<&InputLevel> for PbInputLevel { + fn from(input_level: &InputLevel) -> Self { + Self { + level_idx: input_level.level_idx, + level_type: input_level.level_type.into(), + table_infos: input_level + .table_infos + .iter() + .map(|sst| sst.into()) + .collect_vec(), + } + } +} diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index 340f532d6b49..2ad827d8cae5 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -31,17 +31,20 @@ use std::collections::HashMap; pub use key_cmp::*; use risingwave_common::util::epoch::EPOCH_SPILL_TIME_MASK; use risingwave_pb::common::{batch_query_epoch, BatchQueryEpoch}; -use version::SstableInfo; +use sstable_info::SstableInfo; use crate::key_range::KeyRangeCommon; use crate::table_stats::TableStatsMap; pub mod change_log; pub mod compact; +pub mod compact_task; pub mod compaction_group; pub mod key; pub mod key_range; +pub mod level; pub mod prost_key_range; +pub mod sstable_info; pub mod table_stats; pub mod table_watermark; pub mod time_travel; diff --git a/src/storage/hummock_sdk/src/sstable_info.rs b/src/storage/hummock_sdk/src/sstable_info.rs new file mode 100644 index 000000000000..3fd7ff49686e --- /dev/null +++ b/src/storage/hummock_sdk/src/sstable_info.rs @@ -0,0 +1,180 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::mem::size_of; + +use risingwave_pb::hummock::{PbBloomFilterType, PbKeyRange, PbSstableInfo}; + +use crate::key_range::KeyRange; + +#[derive(Debug, PartialEq, Clone, Default)] +pub struct SstableInfo { + pub object_id: u64, + pub sst_id: u64, + pub key_range: KeyRange, + pub file_size: u64, + pub table_ids: Vec, + pub meta_offset: u64, + pub stale_key_count: u64, + pub total_key_count: u64, + pub min_epoch: u64, + pub max_epoch: u64, + pub uncompressed_file_size: u64, + pub range_tombstone_count: u64, + pub bloom_filter_kind: PbBloomFilterType, +} + +impl SstableInfo { + pub fn estimated_encode_len(&self) -> usize { + let mut basic = size_of::() // object_id + + size_of::() // sstable_id + + size_of::() // file_size + + self.table_ids.len() * size_of::() // table_ids + + size_of::() // meta_offset + + size_of::() // stale_key_count + + size_of::() // total_key_count + + size_of::() // min_epoch + + size_of::() // max_epoch + + size_of::() // uncompressed_file_size + + size_of::() // range_tombstone_count + + size_of::(); // bloom_filter_kind + basic += self.key_range.left.len() + self.key_range.right.len() + size_of::(); + + basic + } + + pub fn to_protobuf(&self) -> PbSstableInfo { + self.into() + } +} + +impl From for SstableInfo { + fn from(pb_sstable_info: PbSstableInfo) -> Self { + Self { + object_id: pb_sstable_info.object_id, + sst_id: pb_sstable_info.sst_id, + key_range: { + let pb_keyrange = pb_sstable_info.key_range.unwrap(); + KeyRange { + left: pb_keyrange.left.into(), + right: pb_keyrange.right.into(), + right_exclusive: pb_keyrange.right_exclusive, + } + }, + file_size: pb_sstable_info.file_size, + table_ids: pb_sstable_info.table_ids.clone(), + meta_offset: pb_sstable_info.meta_offset, + stale_key_count: pb_sstable_info.stale_key_count, + total_key_count: pb_sstable_info.total_key_count, + min_epoch: pb_sstable_info.min_epoch, + max_epoch: pb_sstable_info.max_epoch, + uncompressed_file_size: pb_sstable_info.uncompressed_file_size, + range_tombstone_count: pb_sstable_info.range_tombstone_count, + bloom_filter_kind: PbBloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) + .unwrap(), + } + } +} + +impl From<&PbSstableInfo> for SstableInfo { + fn from(pb_sstable_info: &PbSstableInfo) -> Self { + Self { + object_id: pb_sstable_info.object_id, + sst_id: pb_sstable_info.sst_id, + key_range: { + let pb_keyrange = pb_sstable_info.key_range.as_ref().unwrap(); + KeyRange { + left: pb_keyrange.left.clone().into(), + right: pb_keyrange.right.clone().into(), + right_exclusive: pb_keyrange.right_exclusive, + } + }, + file_size: pb_sstable_info.file_size, + table_ids: pb_sstable_info.table_ids.clone(), + meta_offset: pb_sstable_info.meta_offset, + stale_key_count: pb_sstable_info.stale_key_count, + total_key_count: pb_sstable_info.total_key_count, + min_epoch: pb_sstable_info.min_epoch, + max_epoch: pb_sstable_info.max_epoch, + uncompressed_file_size: pb_sstable_info.uncompressed_file_size, + range_tombstone_count: pb_sstable_info.range_tombstone_count, + bloom_filter_kind: PbBloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) + .unwrap(), + } + } +} + +impl From for PbSstableInfo { + fn from(sstable_info: SstableInfo) -> Self { + PbSstableInfo { + object_id: sstable_info.object_id, + sst_id: sstable_info.sst_id, + key_range: { + let keyrange = sstable_info.key_range; + let pb_key_range = PbKeyRange { + left: keyrange.left.into(), + right: keyrange.right.into(), + right_exclusive: keyrange.right_exclusive, + }; + Some(pb_key_range) + }, + + file_size: sstable_info.file_size, + table_ids: sstable_info.table_ids.clone(), + meta_offset: sstable_info.meta_offset, + stale_key_count: sstable_info.stale_key_count, + total_key_count: sstable_info.total_key_count, + min_epoch: sstable_info.min_epoch, + max_epoch: sstable_info.max_epoch, + uncompressed_file_size: sstable_info.uncompressed_file_size, + range_tombstone_count: sstable_info.range_tombstone_count, + bloom_filter_kind: sstable_info.bloom_filter_kind.into(), + } + } +} + +impl From<&SstableInfo> for PbSstableInfo { + fn from(sstable_info: &SstableInfo) -> Self { + PbSstableInfo { + object_id: sstable_info.object_id, + sst_id: sstable_info.sst_id, + key_range: { + let keyrange = &sstable_info.key_range; + let pb_key_range = PbKeyRange { + left: keyrange.left.to_vec(), + right: keyrange.right.to_vec(), + right_exclusive: keyrange.right_exclusive, + }; + Some(pb_key_range) + }, + + file_size: sstable_info.file_size, + table_ids: sstable_info.table_ids.clone(), + meta_offset: sstable_info.meta_offset, + stale_key_count: sstable_info.stale_key_count, + total_key_count: sstable_info.total_key_count, + min_epoch: sstable_info.min_epoch, + max_epoch: sstable_info.max_epoch, + uncompressed_file_size: sstable_info.uncompressed_file_size, + range_tombstone_count: sstable_info.range_tombstone_count, + bloom_filter_kind: sstable_info.bloom_filter_kind.into(), + } + } +} + +impl SstableInfo { + pub fn remove_key_range(&mut self) { + self.key_range = KeyRange::default(); + } +} diff --git a/src/storage/hummock_sdk/src/time_travel.rs b/src/storage/hummock_sdk/src/time_travel.rs index 12df25dc4136..7da9197e54dd 100644 --- a/src/storage/hummock_sdk/src/time_travel.rs +++ b/src/storage/hummock_sdk/src/time_travel.rs @@ -20,10 +20,12 @@ use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas; use risingwave_pb::hummock::{PbHummockVersion, PbHummockVersionDelta, PbStateTableInfoDelta}; use crate::change_log::{ChangeLogDelta, EpochNewChangeLog, TableChangeLog}; +use crate::level::{Level, Levels, OverlappingLevel}; +use crate::sstable_info::SstableInfo; use crate::table_watermark::TableWatermarks; use crate::version::{ GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, HummockVersionStateTableInfo, - IntraLevelDelta, Level, Levels, OverlappingLevel, SstableInfo, + IntraLevelDelta, }; use crate::{CompactionGroupId, HummockSstableId}; diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index b41e61502536..d38376a7fda0 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::collections::hash_map::Entry; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::mem::{replace, size_of}; use std::ops::Deref; use std::sync::{Arc, LazyLock}; @@ -21,335 +21,23 @@ use std::sync::{Arc, LazyLock}; use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::INVALID_EPOCH; -use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType, TaskStatus, TaskType}; use risingwave_pb::hummock::group_delta::PbDeltaType; -use risingwave_pb::hummock::hummock_version::PbLevels; use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas; -use risingwave_pb::hummock::subscribe_compaction_event_request::PbReportTask; use risingwave_pb::hummock::{ - BloomFilterType, CompactionConfig, LevelType, PbCompactTask, PbGroupConstruct, PbGroupDelta, - PbGroupDestroy, PbGroupMetaChange, PbGroupTableChange, PbHummockVersion, PbHummockVersionDelta, - PbInputLevel, PbIntraLevelDelta, PbKeyRange, PbLevel, PbOverlappingLevel, PbSstableInfo, - PbStateTableInfo, PbTableStats, PbValidationTask, StateTableInfo, StateTableInfoDelta, - TableOption, TableSchema, + CompactionConfig, PbGroupConstruct, PbGroupDelta, PbGroupDestroy, PbGroupMetaChange, + PbGroupTableChange, PbHummockVersion, PbHummockVersionDelta, PbIntraLevelDelta, + PbStateTableInfo, StateTableInfo, StateTableInfoDelta, }; use tracing::warn; use crate::change_log::{ChangeLogDelta, TableChangeLog}; use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels; use crate::compaction_group::StaticCompactionGroupId; -use crate::key_range::KeyRange; +use crate::level::Levels; +use crate::sstable_info::SstableInfo; use crate::table_watermark::TableWatermarks; use crate::{CompactionGroupId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID}; -#[derive(Debug, Clone, PartialEq, Default)] -pub struct OverlappingLevel { - pub sub_levels: Vec, - pub total_file_size: u64, - pub uncompressed_file_size: u64, -} - -impl From<&PbOverlappingLevel> for OverlappingLevel { - fn from(pb_overlapping_level: &PbOverlappingLevel) -> Self { - Self { - sub_levels: pb_overlapping_level - .sub_levels - .iter() - .map(Level::from) - .collect_vec(), - total_file_size: pb_overlapping_level.total_file_size, - uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, - } - } -} - -impl From<&OverlappingLevel> for PbOverlappingLevel { - fn from(overlapping_level: &OverlappingLevel) -> Self { - Self { - sub_levels: overlapping_level - .sub_levels - .iter() - .map(|pb_level| pb_level.into()) - .collect_vec(), - total_file_size: overlapping_level.total_file_size, - uncompressed_file_size: overlapping_level.uncompressed_file_size, - } - } -} - -impl From for PbOverlappingLevel { - fn from(overlapping_level: OverlappingLevel) -> Self { - Self { - sub_levels: overlapping_level - .sub_levels - .into_iter() - .map(|pb_level| pb_level.into()) - .collect_vec(), - total_file_size: overlapping_level.total_file_size, - uncompressed_file_size: overlapping_level.uncompressed_file_size, - } - } -} - -impl From for OverlappingLevel { - fn from(pb_overlapping_level: PbOverlappingLevel) -> Self { - Self { - sub_levels: pb_overlapping_level - .sub_levels - .into_iter() - .map(Level::from) - .collect_vec(), - total_file_size: pb_overlapping_level.total_file_size, - uncompressed_file_size: pb_overlapping_level.uncompressed_file_size, - } - } -} - -impl OverlappingLevel { - pub fn estimated_encode_len(&self) -> usize { - self.sub_levels - .iter() - .map(|level| level.estimated_encode_len()) - .sum::() - + size_of::() - + size_of::() - } -} - -#[derive(Debug, Clone, PartialEq, Default)] -pub struct Level { - pub level_idx: u32, - pub level_type: LevelType, - pub table_infos: Vec, - pub total_file_size: u64, - pub sub_level_id: u64, - pub uncompressed_file_size: u64, - pub vnode_partition_count: u32, -} - -impl From<&PbLevel> for Level { - fn from(pb_level: &PbLevel) -> Self { - Self { - level_idx: pb_level.level_idx, - level_type: LevelType::try_from(pb_level.level_type).unwrap(), - table_infos: pb_level - .table_infos - .iter() - .map(SstableInfo::from) - .collect_vec(), - total_file_size: pb_level.total_file_size, - sub_level_id: pb_level.sub_level_id, - uncompressed_file_size: pb_level.uncompressed_file_size, - vnode_partition_count: pb_level.vnode_partition_count, - } - } -} - -impl From<&Level> for PbLevel { - fn from(level: &Level) -> Self { - Self { - level_idx: level.level_idx, - level_type: level.level_type.into(), - table_infos: level - .table_infos - .iter() - .map(PbSstableInfo::from) - .collect_vec(), - total_file_size: level.total_file_size, - sub_level_id: level.sub_level_id, - uncompressed_file_size: level.uncompressed_file_size, - vnode_partition_count: level.vnode_partition_count, - } - } -} - -impl From for PbLevel { - fn from(level: Level) -> Self { - Self { - level_idx: level.level_idx, - level_type: level.level_type.into(), - table_infos: level - .table_infos - .into_iter() - .map(PbSstableInfo::from) - .collect_vec(), - total_file_size: level.total_file_size, - sub_level_id: level.sub_level_id, - uncompressed_file_size: level.uncompressed_file_size, - vnode_partition_count: level.vnode_partition_count, - } - } -} - -impl From for Level { - fn from(pb_level: PbLevel) -> Self { - Self { - level_idx: pb_level.level_idx, - level_type: LevelType::try_from(pb_level.level_type).unwrap(), - table_infos: pb_level - .table_infos - .into_iter() - .map(SstableInfo::from) - .collect_vec(), - total_file_size: pb_level.total_file_size, - sub_level_id: pb_level.sub_level_id, - uncompressed_file_size: pb_level.uncompressed_file_size, - vnode_partition_count: pb_level.vnode_partition_count, - } - } -} - -impl Level { - pub fn estimated_encode_len(&self) -> usize { - size_of::() - + size_of::() - + self - .table_infos - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - } -} - -#[derive(Debug, Clone, PartialEq, Default)] -pub struct Levels { - pub levels: Vec, - pub l0: Option, - pub group_id: u64, - pub parent_group_id: u64, - - #[deprecated] - pub member_table_ids: Vec, -} - -impl Levels { - pub fn level0(&self) -> &OverlappingLevel { - self.l0.as_ref().unwrap() - } - - pub fn get_level(&self, level_idx: usize) -> &Level { - &self.levels[level_idx - 1] - } - - pub fn get_level_mut(&mut self, level_idx: usize) -> &mut Level { - &mut self.levels[level_idx - 1] - } - - pub fn is_last_level(&self, level_idx: u32) -> bool { - self.levels - .last() - .as_ref() - .map_or(false, |level| level.level_idx == level_idx) - } - - pub fn count_ssts(&self) -> usize { - self.level0() - .sub_levels - .iter() - .chain(self.levels.iter()) - .map(|level| level.table_infos.len()) - .sum() - } -} - -impl Levels { - pub fn from_protobuf(pb_levels: &PbLevels) -> Self { - Self::from(pb_levels) - } - - pub fn to_protobuf(&self) -> PbLevels { - self.into() - } - - pub fn estimated_encode_len(&self) -> usize { - let mut basic = self - .levels - .iter() - .map(|level| level.estimated_encode_len()) - .sum::() - + size_of::() - + size_of::() - + size_of::(); - if let Some(l0) = self.l0.as_ref() { - basic += l0.estimated_encode_len(); - } - - basic - } -} - -impl From<&PbLevels> for Levels { - #[expect(deprecated)] - fn from(pb_levels: &PbLevels) -> Self { - Self { - l0: if pb_levels.l0.is_some() { - Some(OverlappingLevel::from(pb_levels.l0.as_ref().unwrap())) - } else { - None - }, - levels: pb_levels.levels.iter().map(Level::from).collect_vec(), - group_id: pb_levels.group_id, - parent_group_id: pb_levels.parent_group_id, - member_table_ids: pb_levels.member_table_ids.clone(), - } - } -} - -impl From<&Levels> for PbLevels { - #[expect(deprecated)] - fn from(levels: &Levels) -> Self { - Self { - l0: if levels.l0.is_some() { - Some(levels.l0.as_ref().unwrap().into()) - } else { - None - }, - levels: levels.levels.iter().map(PbLevel::from).collect_vec(), - group_id: levels.group_id, - parent_group_id: levels.parent_group_id, - member_table_ids: levels.member_table_ids.clone(), - } - } -} - -impl From for Levels { - #[expect(deprecated)] - fn from(pb_levels: PbLevels) -> Self { - Self { - l0: if pb_levels.l0.is_some() { - Some(OverlappingLevel::from(pb_levels.l0.unwrap())) - } else { - None - }, - levels: pb_levels.levels.into_iter().map(Level::from).collect_vec(), - group_id: pb_levels.group_id, - parent_group_id: pb_levels.parent_group_id, - member_table_ids: pb_levels.member_table_ids.clone(), - } - } -} - -impl From for PbLevels { - fn from(levels: Levels) -> Self { - #[expect(deprecated)] - Self { - l0: if levels.l0.is_some() { - Some(levels.l0.unwrap().into()) - } else { - None - }, - levels: levels.levels.into_iter().map(PbLevel::from).collect_vec(), - group_id: levels.group_id, - parent_group_id: levels.parent_group_id, - member_table_ids: levels.member_table_ids.clone(), - } - } -} - #[derive(Debug, Clone, PartialEq)] pub struct HummockVersionStateTableInfo { state_table_info: HashMap, @@ -1053,634 +741,6 @@ impl From for HummockVersionDelta { } } -#[derive(Debug, PartialEq, Clone, Default)] -pub struct SstableInfo { - pub object_id: u64, - pub sst_id: u64, - pub key_range: KeyRange, - pub file_size: u64, - pub table_ids: Vec, - pub meta_offset: u64, - pub stale_key_count: u64, - pub total_key_count: u64, - pub min_epoch: u64, - pub max_epoch: u64, - pub uncompressed_file_size: u64, - pub range_tombstone_count: u64, - pub bloom_filter_kind: BloomFilterType, -} - -impl SstableInfo { - pub fn estimated_encode_len(&self) -> usize { - let mut basic = size_of::() // object_id - + size_of::() // sstable_id - + size_of::() // file_size - + self.table_ids.len() * size_of::() // table_ids - + size_of::() // meta_offset - + size_of::() // stale_key_count - + size_of::() // total_key_count - + size_of::() // min_epoch - + size_of::() // max_epoch - + size_of::() // uncompressed_file_size - + size_of::() // range_tombstone_count - + size_of::(); // bloom_filter_kind - basic += self.key_range.left.len() + self.key_range.right.len() + size_of::(); - - basic - } - - pub fn to_protobuf(&self) -> PbSstableInfo { - self.into() - } -} - -impl From for SstableInfo { - fn from(pb_sstable_info: PbSstableInfo) -> Self { - Self { - object_id: pb_sstable_info.object_id, - sst_id: pb_sstable_info.sst_id, - key_range: { - let pb_keyrange = pb_sstable_info.key_range.unwrap(); - KeyRange { - left: pb_keyrange.left.into(), - right: pb_keyrange.right.into(), - right_exclusive: pb_keyrange.right_exclusive, - } - }, - file_size: pb_sstable_info.file_size, - table_ids: pb_sstable_info.table_ids.clone(), - meta_offset: pb_sstable_info.meta_offset, - stale_key_count: pb_sstable_info.stale_key_count, - total_key_count: pb_sstable_info.total_key_count, - min_epoch: pb_sstable_info.min_epoch, - max_epoch: pb_sstable_info.max_epoch, - uncompressed_file_size: pb_sstable_info.uncompressed_file_size, - range_tombstone_count: pb_sstable_info.range_tombstone_count, - bloom_filter_kind: BloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) - .unwrap(), - } - } -} - -impl From<&PbSstableInfo> for SstableInfo { - fn from(pb_sstable_info: &PbSstableInfo) -> Self { - Self { - object_id: pb_sstable_info.object_id, - sst_id: pb_sstable_info.sst_id, - key_range: { - let pb_keyrange = pb_sstable_info.key_range.as_ref().unwrap(); - KeyRange { - left: pb_keyrange.left.clone().into(), - right: pb_keyrange.right.clone().into(), - right_exclusive: pb_keyrange.right_exclusive, - } - }, - file_size: pb_sstable_info.file_size, - table_ids: pb_sstable_info.table_ids.clone(), - meta_offset: pb_sstable_info.meta_offset, - stale_key_count: pb_sstable_info.stale_key_count, - total_key_count: pb_sstable_info.total_key_count, - min_epoch: pb_sstable_info.min_epoch, - max_epoch: pb_sstable_info.max_epoch, - uncompressed_file_size: pb_sstable_info.uncompressed_file_size, - range_tombstone_count: pb_sstable_info.range_tombstone_count, - bloom_filter_kind: BloomFilterType::try_from(pb_sstable_info.bloom_filter_kind) - .unwrap(), - } - } -} - -impl From for PbSstableInfo { - fn from(sstable_info: SstableInfo) -> Self { - PbSstableInfo { - object_id: sstable_info.object_id, - sst_id: sstable_info.sst_id, - key_range: { - let keyrange = sstable_info.key_range; - let pb_key_range = PbKeyRange { - left: keyrange.left.into(), - right: keyrange.right.into(), - right_exclusive: keyrange.right_exclusive, - }; - Some(pb_key_range) - }, - - file_size: sstable_info.file_size, - table_ids: sstable_info.table_ids.clone(), - meta_offset: sstable_info.meta_offset, - stale_key_count: sstable_info.stale_key_count, - total_key_count: sstable_info.total_key_count, - min_epoch: sstable_info.min_epoch, - max_epoch: sstable_info.max_epoch, - uncompressed_file_size: sstable_info.uncompressed_file_size, - range_tombstone_count: sstable_info.range_tombstone_count, - bloom_filter_kind: sstable_info.bloom_filter_kind.into(), - } - } -} - -impl From<&SstableInfo> for PbSstableInfo { - fn from(sstable_info: &SstableInfo) -> Self { - PbSstableInfo { - object_id: sstable_info.object_id, - sst_id: sstable_info.sst_id, - key_range: { - let keyrange = &sstable_info.key_range; - let pb_key_range = PbKeyRange { - left: keyrange.left.to_vec(), - right: keyrange.right.to_vec(), - right_exclusive: keyrange.right_exclusive, - }; - Some(pb_key_range) - }, - - file_size: sstable_info.file_size, - table_ids: sstable_info.table_ids.clone(), - meta_offset: sstable_info.meta_offset, - stale_key_count: sstable_info.stale_key_count, - total_key_count: sstable_info.total_key_count, - min_epoch: sstable_info.min_epoch, - max_epoch: sstable_info.max_epoch, - uncompressed_file_size: sstable_info.uncompressed_file_size, - range_tombstone_count: sstable_info.range_tombstone_count, - bloom_filter_kind: sstable_info.bloom_filter_kind.into(), - } - } -} - -impl SstableInfo { - pub fn remove_key_range(&mut self) { - self.key_range = KeyRange::default(); - } -} - -#[derive(Clone, PartialEq, Default, Debug)] -pub struct InputLevel { - pub level_idx: u32, - pub level_type: LevelType, - pub table_infos: Vec, -} - -impl InputLevel { - pub fn estimated_encode_len(&self) -> usize { - size_of::() - + size_of::() - + self - .table_infos - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - } -} - -impl From for InputLevel { - fn from(pb_input_level: PbInputLevel) -> Self { - Self { - level_idx: pb_input_level.level_idx, - level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), - table_infos: pb_input_level - .table_infos - .into_iter() - .map(SstableInfo::from) - .collect_vec(), - } - } -} - -impl From<&PbInputLevel> for InputLevel { - fn from(pb_input_level: &PbInputLevel) -> Self { - Self { - level_idx: pb_input_level.level_idx, - level_type: LevelType::try_from(pb_input_level.level_type).unwrap(), - table_infos: pb_input_level - .table_infos - .iter() - .map(SstableInfo::from) - .collect_vec(), - } - } -} - -impl From for PbInputLevel { - fn from(input_level: InputLevel) -> Self { - Self { - level_idx: input_level.level_idx, - level_type: input_level.level_type.into(), - table_infos: input_level - .table_infos - .into_iter() - .map(|sst| sst.into()) - .collect_vec(), - } - } -} - -impl From<&InputLevel> for PbInputLevel { - fn from(input_level: &InputLevel) -> Self { - Self { - level_idx: input_level.level_idx, - level_type: input_level.level_type.into(), - table_infos: input_level - .table_infos - .iter() - .map(|sst| sst.into()) - .collect_vec(), - } - } -} - -#[derive(Clone, PartialEq, Default, Debug)] -pub struct CompactTask { - /// SSTs to be compacted, which will be removed from LSM after compaction - pub input_ssts: Vec, - /// In ideal case, the compaction will generate `splits.len()` tables which have key range - /// corresponding to that in `splits`, respectively - pub splits: Vec, - /// low watermark in 'ts-aware compaction' - pub watermark: u64, - /// compaction output, which will be added to `target_level` of LSM after compaction - pub sorted_output_ssts: Vec, - /// task id assigned by hummock storage service - pub task_id: u64, - /// compaction output will be added to `target_level` of LSM after compaction - pub target_level: u32, - pub gc_delete_keys: bool, - /// Lbase in LSM - pub base_level: u32, - pub task_status: PbTaskStatus, - /// compaction group the task belongs to - pub compaction_group_id: u64, - /// `existing_table_ids` for compaction drop key - pub existing_table_ids: Vec, - pub compression_algorithm: u32, - pub target_file_size: u64, - pub compaction_filter_mask: u32, - pub table_options: BTreeMap, - pub current_epoch_time: u64, - pub target_sub_level_id: u64, - /// Identifies whether the task is `space_reclaim`, if the `compact_task_type` increases, it will be refactored to enum - pub task_type: PbTaskType, - /// Deprecated. use `table_vnode_partition` instead; - pub split_by_state_table: bool, - /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed. - /// Deprecated. use `table_vnode_partition` instead; - pub split_weight_by_vnode: u32, - pub table_vnode_partition: BTreeMap, - /// The table watermark of any table id. In compaction we only use the table watermarks on safe epoch, - /// so we only need to include the table watermarks on safe epoch to reduce the size of metadata. - pub table_watermarks: BTreeMap, - - pub table_schemas: BTreeMap, - - pub max_sub_compaction: u32, -} - -impl CompactTask { - pub fn estimated_encode_len(&self) -> usize { - self.input_ssts - .iter() - .map(|input_level| input_level.estimated_encode_len()) - .sum::() - + self - .splits - .iter() - .map(|split| split.left.len() + split.right.len() + size_of::()) - .sum::() - + size_of::() - + self - .sorted_output_ssts - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + self.existing_table_ids.len() * size_of::() - + size_of::() - + size_of::() - + size_of::() - + self.table_options.len() * size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - + self.table_vnode_partition.len() * size_of::() - + self - .table_watermarks - .values() - .map(|table_watermark| size_of::() + table_watermark.estimated_encode_len()) - .sum::() - } -} - -impl From for CompactTask { - #[expect(deprecated)] - fn from(pb_compact_task: PbCompactTask) -> Self { - Self { - input_ssts: pb_compact_task - .input_ssts - .into_iter() - .map(InputLevel::from) - .collect_vec(), - splits: pb_compact_task - .splits - .into_iter() - .map(|pb_keyrange| KeyRange { - left: pb_keyrange.left.into(), - right: pb_keyrange.right.into(), - right_exclusive: pb_keyrange.right_exclusive, - }) - .collect_vec(), - watermark: pb_compact_task.watermark, - sorted_output_ssts: pb_compact_task - .sorted_output_ssts - .into_iter() - .map(SstableInfo::from) - .collect_vec(), - task_id: pb_compact_task.task_id, - target_level: pb_compact_task.target_level, - gc_delete_keys: pb_compact_task.gc_delete_keys, - base_level: pb_compact_task.base_level, - task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(), - compaction_group_id: pb_compact_task.compaction_group_id, - existing_table_ids: pb_compact_task.existing_table_ids.clone(), - compression_algorithm: pb_compact_task.compression_algorithm, - target_file_size: pb_compact_task.target_file_size, - compaction_filter_mask: pb_compact_task.compaction_filter_mask, - table_options: pb_compact_task.table_options.clone(), - current_epoch_time: pb_compact_task.current_epoch_time, - target_sub_level_id: pb_compact_task.target_sub_level_id, - task_type: TaskType::try_from(pb_compact_task.task_type).unwrap(), - split_by_state_table: pb_compact_task.split_by_state_table, - split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, - table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), - table_watermarks: pb_compact_task - .table_watermarks - .into_iter() - .map(|(table_id, pb_table_watermark)| { - (table_id, TableWatermarks::from(pb_table_watermark)) - }) - .collect(), - table_schemas: pb_compact_task.table_schemas, - max_sub_compaction: pb_compact_task.max_sub_compaction, - } - } -} - -impl From<&PbCompactTask> for CompactTask { - #[expect(deprecated)] - fn from(pb_compact_task: &PbCompactTask) -> Self { - Self { - input_ssts: pb_compact_task - .input_ssts - .iter() - .map(InputLevel::from) - .collect_vec(), - splits: pb_compact_task - .splits - .iter() - .map(|pb_keyrange| KeyRange { - left: pb_keyrange.left.clone().into(), - right: pb_keyrange.right.clone().into(), - right_exclusive: pb_keyrange.right_exclusive, - }) - .collect_vec(), - watermark: pb_compact_task.watermark, - sorted_output_ssts: pb_compact_task - .sorted_output_ssts - .iter() - .map(SstableInfo::from) - .collect_vec(), - task_id: pb_compact_task.task_id, - target_level: pb_compact_task.target_level, - gc_delete_keys: pb_compact_task.gc_delete_keys, - base_level: pb_compact_task.base_level, - task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(), - compaction_group_id: pb_compact_task.compaction_group_id, - existing_table_ids: pb_compact_task.existing_table_ids.clone(), - compression_algorithm: pb_compact_task.compression_algorithm, - target_file_size: pb_compact_task.target_file_size, - compaction_filter_mask: pb_compact_task.compaction_filter_mask, - table_options: pb_compact_task.table_options.clone(), - current_epoch_time: pb_compact_task.current_epoch_time, - target_sub_level_id: pb_compact_task.target_sub_level_id, - task_type: TaskType::try_from(pb_compact_task.task_type).unwrap(), - split_by_state_table: pb_compact_task.split_by_state_table, - split_weight_by_vnode: pb_compact_task.split_weight_by_vnode, - table_vnode_partition: pb_compact_task.table_vnode_partition.clone(), - table_watermarks: pb_compact_task - .table_watermarks - .iter() - .map(|(table_id, pb_table_watermark)| { - (*table_id, TableWatermarks::from(pb_table_watermark)) - }) - .collect(), - table_schemas: pb_compact_task.table_schemas.clone(), - max_sub_compaction: pb_compact_task.max_sub_compaction, - } - } -} - -impl From for PbCompactTask { - #[expect(deprecated)] - fn from(compact_task: CompactTask) -> Self { - Self { - input_ssts: compact_task - .input_ssts - .into_iter() - .map(|input_level| input_level.into()) - .collect_vec(), - splits: compact_task - .splits - .into_iter() - .map(|keyrange| PbKeyRange { - left: keyrange.left.into(), - right: keyrange.right.into(), - right_exclusive: keyrange.right_exclusive, - }) - .collect_vec(), - watermark: compact_task.watermark, - sorted_output_ssts: compact_task - .sorted_output_ssts - .into_iter() - .map(|sst| sst.into()) - .collect_vec(), - task_id: compact_task.task_id, - target_level: compact_task.target_level, - gc_delete_keys: compact_task.gc_delete_keys, - base_level: compact_task.base_level, - task_status: compact_task.task_status.into(), - compaction_group_id: compact_task.compaction_group_id, - existing_table_ids: compact_task.existing_table_ids.clone(), - compression_algorithm: compact_task.compression_algorithm, - target_file_size: compact_task.target_file_size, - compaction_filter_mask: compact_task.compaction_filter_mask, - table_options: compact_task.table_options.clone(), - current_epoch_time: compact_task.current_epoch_time, - target_sub_level_id: compact_task.target_sub_level_id, - task_type: compact_task.task_type.into(), - split_weight_by_vnode: compact_task.split_weight_by_vnode, - table_vnode_partition: compact_task.table_vnode_partition.clone(), - table_watermarks: compact_task - .table_watermarks - .into_iter() - .map(|(table_id, table_watermark)| (table_id, table_watermark.into())) - .collect(), - split_by_state_table: compact_task.split_by_state_table, - table_schemas: compact_task.table_schemas.clone(), - max_sub_compaction: compact_task.max_sub_compaction, - } - } -} - -impl From<&CompactTask> for PbCompactTask { - #[expect(deprecated)] - fn from(compact_task: &CompactTask) -> Self { - Self { - input_ssts: compact_task - .input_ssts - .iter() - .map(|input_level| input_level.into()) - .collect_vec(), - splits: compact_task - .splits - .iter() - .map(|keyrange| PbKeyRange { - left: keyrange.left.to_vec(), - right: keyrange.right.to_vec(), - right_exclusive: keyrange.right_exclusive, - }) - .collect_vec(), - watermark: compact_task.watermark, - sorted_output_ssts: compact_task - .sorted_output_ssts - .iter() - .map(|sst| sst.into()) - .collect_vec(), - task_id: compact_task.task_id, - target_level: compact_task.target_level, - gc_delete_keys: compact_task.gc_delete_keys, - base_level: compact_task.base_level, - task_status: compact_task.task_status.into(), - compaction_group_id: compact_task.compaction_group_id, - existing_table_ids: compact_task.existing_table_ids.clone(), - compression_algorithm: compact_task.compression_algorithm, - target_file_size: compact_task.target_file_size, - compaction_filter_mask: compact_task.compaction_filter_mask, - table_options: compact_task.table_options.clone(), - current_epoch_time: compact_task.current_epoch_time, - target_sub_level_id: compact_task.target_sub_level_id, - task_type: compact_task.task_type.into(), - split_weight_by_vnode: compact_task.split_weight_by_vnode, - table_vnode_partition: compact_task.table_vnode_partition.clone(), - table_watermarks: compact_task - .table_watermarks - .iter() - .map(|(table_id, table_watermark)| (*table_id, table_watermark.into())) - .collect(), - split_by_state_table: compact_task.split_by_state_table, - table_schemas: compact_task.table_schemas.clone(), - max_sub_compaction: compact_task.max_sub_compaction, - } - } -} - -impl CompactTask { - pub fn set_task_status(&mut self, s: PbTaskStatus) { - self.task_status = s; - } -} - -#[derive(Clone, PartialEq, Default)] -pub struct ValidationTask { - pub sst_infos: Vec, - pub sst_id_to_worker_id: HashMap, - pub epoch: u64, -} - -impl From for ValidationTask { - fn from(pb_validation_task: PbValidationTask) -> Self { - Self { - sst_infos: pb_validation_task - .sst_infos - .into_iter() - .map(SstableInfo::from) - .collect_vec(), - sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(), - epoch: pb_validation_task.epoch, - } - } -} - -impl From for PbValidationTask { - fn from(validation_task: ValidationTask) -> Self { - Self { - sst_infos: validation_task - .sst_infos - .into_iter() - .map(|sst| sst.into()) - .collect_vec(), - sst_id_to_worker_id: validation_task.sst_id_to_worker_id.clone(), - epoch: validation_task.epoch, - } - } -} - -impl ValidationTask { - pub fn estimated_encode_len(&self) -> usize { - self.sst_infos - .iter() - .map(|sst| sst.estimated_encode_len()) - .sum::() - + self.sst_id_to_worker_id.len() * (size_of::() + size_of::()) - + size_of::() - } -} - -#[derive(Clone, PartialEq, Default, Debug)] -pub struct ReportTask { - pub table_stats_change: HashMap, - pub task_id: u64, - pub task_status: TaskStatus, - pub sorted_output_ssts: Vec, -} - -impl From for ReportTask { - fn from(value: PbReportTask) -> Self { - Self { - table_stats_change: value.table_stats_change.clone(), - task_id: value.task_id, - task_status: TaskStatus::try_from(value.task_status).unwrap(), - sorted_output_ssts: value - .sorted_output_ssts - .into_iter() - .map(SstableInfo::from) - .collect_vec(), - } - } -} - -impl From for PbReportTask { - fn from(value: ReportTask) -> Self { - Self { - table_stats_change: value.table_stats_change.clone(), - task_id: value.task_id, - task_status: value.task_status.into(), - sorted_output_ssts: value - .sorted_output_ssts - .into_iter() - .map(|sst| sst.into()) - .collect_vec(), - } - } -} - -// for version delta - #[derive(Debug, PartialEq, Clone)] pub struct IntraLevelDelta { pub level_idx: u32, diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index 6e6a66543772..67c3aa8b059b 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -25,7 +25,7 @@ use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::{test_epoch, EpochExt}; use risingwave_hummock_sdk::key::{key_with_epoch, map_table_key_range}; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::LocalSstableInfo; use risingwave_meta::hummock::test_utils::setup_compute_env; use risingwave_storage::hummock::event_handler::TEST_LOCAL_INSTANCE_ID; diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index a6c9d692e2ec..6eb49d627b34 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -21,11 +21,12 @@ use std::sync::Arc; use bytes::Bytes; use itertools::Itertools; use risingwave_common::constants::hummock::CompactionFilterFlag; +use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::TableStatsMap; -use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, EpochWithGap, KeyComparator}; use risingwave_pb::hummock::compact_task::PbTaskType; use risingwave_pb::hummock::{BloomFilterType, PbLevelType, PbTableSchema}; diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index cae809b97993..773b2d565550 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -23,10 +23,11 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::try_drop use risingwave_hummock_sdk::compact::{ compact_task_to_string, estimate_memory_for_compact_task, statistics_compact_task, }; +use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker}; use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, TableStatsMap}; -use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{ can_concat, compact_task_output_to_string, HummockSstableObjectId, KeyComparator, }; diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs index 264ae74907e1..227a50cb5813 100644 --- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs +++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs @@ -22,10 +22,11 @@ use std::time::Instant; use await_tree::InstrumentAwait; use bytes::Bytes; use itertools::Itertools; +use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::TableStats; -use risingwave_hummock_sdk::version::{CompactTask, SstableInfo}; use risingwave_hummock_sdk::{can_concat, compact_task_to_string, EpochWithGap, LocalSstableInfo}; use crate::filter_key_extractor::FilterKeyExtractorImpl; diff --git a/src/storage/src/hummock/compactor/iterator.rs b/src/storage/src/hummock/compactor/iterator.rs index cc2cb4a588c3..e96164cb6be0 100644 --- a/src/storage/src/hummock/compactor/iterator.rs +++ b/src/storage/src/hummock/compactor/iterator.rs @@ -23,7 +23,7 @@ use fail::fail_point; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::KeyComparator; use crate::hummock::block_stream::BlockDataStream; diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index bcc9663f4a6a..12e009bb60cb 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -15,7 +15,7 @@ mod compaction_executor; mod compaction_filter; pub mod compaction_utils; -use risingwave_hummock_sdk::version::{CompactTask, ValidationTask}; +use risingwave_hummock_sdk::compact_task::{CompactTask, ValidationTask}; use risingwave_pb::compactor::{dispatch_compaction_task_request, DispatchCompactionTaskRequest}; use risingwave_pb::hummock::report_compaction_task_request::{ Event as ReportCompactionTaskEvent, HeartBeat as SharedHeartBeat, @@ -537,9 +537,7 @@ pub fn start_compactor( Err(err) => { tracing::warn!(error = %err.as_report(), "Failed to track pending SST object id"); let mut compact_task = compact_task; - compact_task.set_task_status( - TaskStatus::TrackSstObjectIdFailed, - ); + compact_task.task_status = TaskStatus::TrackSstObjectIdFailed; ((compact_task, HashMap::default()), None) } }; diff --git a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs index cf4a38e85d9b..a7c5215439bf 100644 --- a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs +++ b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs @@ -15,7 +15,7 @@ use std::future::Future; use risingwave_hummock_sdk::key::{FullKey, PointRange, UserKey}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; use crate::hummock::iterator::DeleteRangeIterator; diff --git a/src/storage/src/hummock/iterator/concat_inner.rs b/src/storage/src/hummock/iterator/concat_inner.rs index ad4e115af927..a71a9d940186 100644 --- a/src/storage/src/hummock/iterator/concat_inner.rs +++ b/src/storage/src/hummock/iterator/concat_inner.rs @@ -16,7 +16,7 @@ use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::Arc; use risingwave_hummock_sdk::key::FullKey; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use crate::hummock::iterator::{ DirectionEnum, HummockIterator, HummockIteratorDirection, ValueMeta, diff --git a/src/storage/src/hummock/iterator/delete_range_iterator.rs b/src/storage/src/hummock/iterator/delete_range_iterator.rs index d504ad4bdf0d..bcc2f3e3ea26 100644 --- a/src/storage/src/hummock/iterator/delete_range_iterator.rs +++ b/src/storage/src/hummock/iterator/delete_range_iterator.rs @@ -17,7 +17,7 @@ use std::future::Future; use risingwave_common::util::epoch::is_max_epoch; use risingwave_hummock_sdk::key::{PointRange, UserKey}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::HummockEpoch; use crate::hummock::iterator::concat_delete_range_iterator::ConcatDeleteRangeIterator; diff --git a/src/storage/src/hummock/iterator/mod.rs b/src/storage/src/hummock/iterator/mod.rs index 29fd7126a45b..fdfcd26a3a59 100644 --- a/src/storage/src/hummock/iterator/mod.rs +++ b/src/storage/src/hummock/iterator/mod.rs @@ -18,7 +18,7 @@ use std::ops::{Deref, DerefMut}; use std::sync::Arc; use more_asserts::{assert_gt, assert_lt}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use super::{ HummockResult, HummockValue, SstableIteratorReadOptions, SstableIteratorType, SstableStoreRef, diff --git a/src/storage/src/hummock/iterator/test_utils.rs b/src/storage/src/hummock/iterator/test_utils.rs index a09fccf28d9c..971e3b7f97c7 100644 --- a/src/storage/src/hummock/iterator/test_utils.rs +++ b/src/storage/src/hummock/iterator/test_utils.rs @@ -22,7 +22,7 @@ use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::test_epoch; use risingwave_hummock_sdk::key::{prefix_slice_with_vnode, FullKey, TableKey, UserKey}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId}; use risingwave_object_store::object::{ InMemObjectStore, ObjectStore, ObjectStoreImpl, ObjectStoreRef, diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs index ec8dc13105a8..8819d0162a5e 100644 --- a/src/storage/src/hummock/local_version/pinned_version.rs +++ b/src/storage/src/hummock/local_version/pinned_version.rs @@ -19,7 +19,8 @@ use std::time::{Duration, Instant}; use auto_enums::auto_enum; use risingwave_common::catalog::TableId; -use risingwave_hummock_sdk::version::{HummockVersion, Level, Levels}; +use risingwave_hummock_sdk::level::{Level, Levels}; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId, INVALID_VERSION_ID}; use risingwave_rpc_client::HummockMetaClient; use thiserror_ext::AsReport; diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs index 21f1bbb247a2..14ac9532c8cb 100644 --- a/src/storage/src/hummock/mod.rs +++ b/src/storage/src/hummock/mod.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use bytes::Bytes; use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKeyRangeRef}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::{HummockEpoch, *}; pub mod block_cache; diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index 7d957803305f..77399eb3e3d0 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -20,8 +20,8 @@ use bytes::{Bytes, BytesMut}; use risingwave_common::util::epoch::is_max_epoch; use risingwave_hummock_sdk::key::{user_key, FullKey, MAX_KEY_LEN}; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::{TableStats, TableStatsMap}; -use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; use risingwave_pb::hummock::BloomFilterType; diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs index 5c24d0143ea6..cea2c42529ce 100644 --- a/src/storage/src/hummock/sstable_store.rs +++ b/src/storage/src/hummock/sstable_store.rs @@ -26,7 +26,7 @@ use foyer::{ use futures::{future, StreamExt}; use itertools::Itertools; use risingwave_common::config::StorageMemoryConfig; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::{HummockSstableObjectId, OBJECT_SUFFIX}; use risingwave_hummock_trace::TracedCachePolicy; use risingwave_object_store::object::{ @@ -1131,7 +1131,7 @@ mod tests { use std::ops::Range; use std::sync::Arc; - use risingwave_hummock_sdk::version::SstableInfo; + use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::HummockSstableObjectId; use super::{SstableStoreRef, SstableWriterOptions}; diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs index e2de603518e4..f98ee38458b6 100644 --- a/src/storage/src/hummock/store/hummock_storage.rs +++ b/src/storage/src/hummock/store/hummock_storage.rs @@ -29,8 +29,9 @@ use risingwave_common_service::{NotificationClient, ObserverManager}; use risingwave_hummock_sdk::key::{ is_empty_key_range, vnode, vnode_range, TableKey, TableKeyRange, }; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_watermark::TableWatermarksIndex; -use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::HummockReadEpoch; use risingwave_rpc_client::HummockMetaClient; use thiserror_ext::AsReport; diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs index b6f0b1c9aa06..ae0d775219c5 100644 --- a/src/storage/src/hummock/store/local_hummock_storage.rs +++ b/src/storage/src/hummock/store/local_hummock_storage.rs @@ -24,7 +24,7 @@ use risingwave_common::catalog::{TableId, TableOption}; use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::MAX_SPILL_TIMES; use risingwave_hummock_sdk::key::{is_empty_key_range, vnode_range, TableKey, TableKeyRange}; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch}; use tracing::{warn, Instrument}; diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs index f0bf53b16fa3..4733b961a37d 100644 --- a/src/storage/src/hummock/store/version.rs +++ b/src/storage/src/hummock/store/version.rs @@ -32,10 +32,10 @@ use risingwave_hummock_sdk::key::{ bound_table_key_range, FullKey, TableKey, TableKeyRange, UserKey, }; use risingwave_hummock_sdk::key_range::KeyRangeCommon; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_watermark::{ TableWatermarksIndex, VnodeWatermark, WatermarkDirection, }; -use risingwave_hummock_sdk::version::SstableInfo; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, LocalSstableInfo}; use risingwave_pb::hummock::LevelType; use sync_point::sync_point; diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs index 74b9141e83b6..53b58d895c80 100644 --- a/src/storage/src/hummock/test_utils.rs +++ b/src/storage/src/hummock/test_utils.rs @@ -28,7 +28,7 @@ use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::test_epoch; use risingwave_hummock_sdk::key::{FullKey, PointRange, TableKey, UserKey}; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::version::SstableInfo; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId}; use super::iterator::test_utils::iterator_test_table_key_of; diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs index 327b023fd040..1e1b23096402 100644 --- a/src/storage/src/hummock/utils.rs +++ b/src/storage/src/hummock/utils.rs @@ -28,7 +28,8 @@ use risingwave_common::catalog::{TableId, TableOption}; use risingwave_hummock_sdk::key::{ bound_table_key_range, EmptySliceRef, FullKey, TableKey, UserKey, }; -use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{can_concat, HummockEpoch}; use tokio::sync::oneshot::{channel, Receiver, Sender}; diff --git a/src/storage/src/hummock/validator.rs b/src/storage/src/hummock/validator.rs index 03994c7beae2..cc95b7089b66 100644 --- a/src/storage/src/hummock/validator.rs +++ b/src/storage/src/hummock/validator.rs @@ -17,8 +17,8 @@ use std::cmp; use std::collections::HashMap; use std::sync::Arc; +use risingwave_hummock_sdk::compact_task::ValidationTask; use risingwave_hummock_sdk::key::FullKey; -use risingwave_hummock_sdk::version::ValidationTask; use crate::hummock::iterator::HummockIterator; use crate::hummock::sstable::SstableIteratorReadOptions; From 6018f587b53bf3309edc23d82c703fb1583d5272 Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 25 Jul 2024 15:28:04 +0800 Subject: [PATCH 17/18] fix(storage): fix compile --- src/storage/src/hummock/event_handler/uploader/test_utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/src/hummock/event_handler/uploader/test_utils.rs b/src/storage/src/hummock/event_handler/uploader/test_utils.rs index a73acb6bdb5e..f866ef18f9e2 100644 --- a/src/storage/src/hummock/event_handler/uploader/test_utils.rs +++ b/src/storage/src/hummock/event_handler/uploader/test_utils.rs @@ -32,7 +32,8 @@ use risingwave_common::util::epoch::{test_epoch, EpochExt}; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::{FullKey, TableKey}; use risingwave_hummock_sdk::key_range::KeyRange; -use risingwave_hummock_sdk::version::{HummockVersion, SstableInfo}; +use risingwave_hummock_sdk::sstable_info::SstableInfo; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; use risingwave_pb::hummock::StateTableInfoDelta; use spin::Mutex; From ffc5ac6dc898c1dfad6f95d66c6c45077aba0701 Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 25 Jul 2024 15:57:08 +0800 Subject: [PATCH 18/18] fix(storage): fix test compile --- .../compaction/picker/min_overlap_compaction_picker.rs | 3 +-- .../compaction/picker/space_reclaim_compaction_picker.rs | 3 ++- .../src/hummock/compaction/picker/tier_compaction_picker.rs | 2 +- .../compaction/picker/ttl_reclaim_compaction_picker.rs | 3 ++- src/meta/src/hummock/compaction/selector/level_selector.rs | 3 ++- src/meta/src/hummock/compaction/selector/mod.rs | 3 ++- src/meta/src/hummock/manager/tests.rs | 4 +++- src/meta/src/hummock/manager/versioning.rs | 6 +++--- src/storage/hummock_test/src/compactor_tests.rs | 5 ++++- 9 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs index aa6777acf2b4..c444f1f647eb 100644 --- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs @@ -530,7 +530,6 @@ pub mod tests { use std::collections::BTreeSet; use risingwave_common::config::default::compaction_config; - use risingwave_hummock_sdk::level::{Level, Levels}; use super::*; use crate::hummock::compaction::overlap_strategy::RangeOverlapStrategy; @@ -752,7 +751,7 @@ pub mod tests { true, compaction_config::max_l0_compact_level_count() as usize, ); - let ret = picker.pick_l0_multi_non_overlap_level(&Levels, &levels_handlers[0]); + let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]); assert_eq!(6, ret.len()); } diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs index 6e06629b9ebd..8d69e85bcc48 100644 --- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs @@ -173,7 +173,8 @@ mod test { use itertools::Itertools; use risingwave_common::catalog::TableId; - use risingwave_hummock_sdk::version::{HummockVersionStateTableInfo, Level}; + use risingwave_hummock_sdk::level::Level; + use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_pb::hummock::compact_task; pub use risingwave_pb::hummock::LevelType; diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index f78b8ebadb56..776a83019dc6 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -165,7 +165,7 @@ pub mod tests { use std::sync::Arc; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::new_sub_level; - use risingwave_hummock_sdk::version::{Levels, OverlappingLevel}; + use risingwave_hummock_sdk::level::{Levels, OverlappingLevel}; use risingwave_pb::hummock::LevelType; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs index f5fe6dd86116..51a1d56f34dc 100644 --- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs @@ -202,7 +202,8 @@ mod test { use std::sync::Arc; use itertools::Itertools; - use risingwave_hummock_sdk::version::{HummockVersionStateTableInfo, Level}; + use risingwave_hummock_sdk::level::Level; + use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_pb::hummock::compact_task; pub use risingwave_pb::hummock::LevelType; diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs index 5597cdd3ea17..918361bc4d44 100644 --- a/src/meta/src/hummock/compaction/selector/level_selector.rs +++ b/src/meta/src/hummock/compaction/selector/level_selector.rs @@ -484,7 +484,8 @@ pub mod tests { use itertools::Itertools; use risingwave_common::constants::hummock::CompactionFilterFlag; - use risingwave_hummock_sdk::version::{HummockVersionStateTableInfo, Levels}; + use risingwave_hummock_sdk::level::Levels; + use risingwave_hummock_sdk::version::HummockVersionStateTableInfo; use risingwave_pb::hummock::compaction_config::CompactionMode; use crate::hummock::compaction::compaction_config::CompactionConfigBuilder; diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs index 9018707d9e9e..86571bf1f65b 100644 --- a/src/meta/src/hummock/compaction/selector/mod.rs +++ b/src/meta/src/hummock/compaction/selector/mod.rs @@ -128,7 +128,8 @@ pub mod tests { use itertools::Itertools; use risingwave_hummock_sdk::key_range::KeyRange; - use risingwave_hummock_sdk::version::{Level, OverlappingLevel, SstableInfo}; + use risingwave_hummock_sdk::level::{Level, OverlappingLevel}; + use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_pb::hummock::LevelType; use super::*; diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs index b59f48343e12..71f0d117acb0 100644 --- a/src/meta/src/hummock/manager/tests.rs +++ b/src/meta/src/hummock/manager/tests.rs @@ -24,11 +24,13 @@ use prometheus::Registry; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::{test_epoch, EpochExt, INVALID_EPOCH}; use risingwave_hummock_sdk::compact::compact_task_to_string; +use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::get_compaction_group_ssts; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key_range::KeyRange; +use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::{to_prost_table_stats_map, TableStats, TableStatsMap}; -use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, SstableInfo}; +use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{ CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo, FIRST_VERSION_ID, diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs index 7b004d0e13a6..4777a203bb3c 100644 --- a/src/meta/src/hummock/manager/versioning.rs +++ b/src/meta/src/hummock/manager/versioning.rs @@ -404,9 +404,9 @@ mod tests { use std::sync::Arc; use risingwave_hummock_sdk::key_range::KeyRange; - use risingwave_hummock_sdk::version::{ - HummockVersion, Level, Levels, OverlappingLevel, SstableInfo, - }; + use risingwave_hummock_sdk::level::{Level, Levels, OverlappingLevel}; + use risingwave_hummock_sdk::sstable_info::SstableInfo; + use risingwave_hummock_sdk::version::HummockVersion; use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId}; use risingwave_pb::hummock::write_limits::WriteLimit; use risingwave_pb::hummock::{HummockPinnedVersion, HummockVersionStats}; diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index 1acc04d67aa0..78a49788b6c5 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -30,17 +30,20 @@ pub(crate) mod tests { use risingwave_common::util::epoch::{test_epoch, Epoch, EpochExt}; use risingwave_common_service::NotificationClient; use risingwave_hummock_sdk::can_concat; + use risingwave_hummock_sdk::compact_task::CompactTask; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::{ next_key, prefix_slice_with_vnode, prefixed_range_with_vnode, FullKey, TableKey, TABLE_PREFIX_LEN, }; use risingwave_hummock_sdk::key_range::KeyRange; + use risingwave_hummock_sdk::level::InputLevel; + use risingwave_hummock_sdk::sstable_info::SstableInfo; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; use risingwave_hummock_sdk::table_watermark::{ ReadTableWatermark, TableWatermarks, VnodeWatermark, WatermarkDirection, }; - use risingwave_hummock_sdk::version::{CompactTask, HummockVersion, InputLevel, SstableInfo}; + use risingwave_hummock_sdk::version::HummockVersion; use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder; use risingwave_meta::hummock::compaction::selector::{ default_compaction_selector, ManualCompactionOption,