diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs index f6d45d8de02ef..3c3e41e590d59 100644 --- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs +++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs @@ -525,6 +525,9 @@ fn generate_splits( } } } + + // Meta node will calculate size of each state-table in one task in `risingwave_meta::hummock::manager::compaction::calculate_vnode_partition`. + // To make the calculate result more accurately we shall split the large state-table from other small ones. for table_id in existing_table_ids { if let Some(table_size) = table_size_infos.get(table_id) && *table_size > min_sstable_size @@ -709,13 +712,13 @@ mod tests { ..Default::default() }; let payload = vec![imm1, imm2, imm3, imm4, imm5]; - let (splits, _sstable_capacity, vnode) = + let (splits, _sstable_capacity, vnodes) = generate_splits(&payload, &HashSet::from_iter([1, 2]), &storage_opts); assert_eq!( splits.len(), storage_opts.share_buffers_sync_parallelism as usize ); - assert_eq!(vnode, 0); + assert!(vnodes.is_empty()); for i in 1..splits.len() { assert_eq!(splits[i].left, splits[i - 1].right); assert!(splits[i].left > splits[i - 1].left); diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index 25667b1c818e8..f3f6c2345c70c 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -45,7 +45,6 @@ pub const MIN_BLOCK_SIZE: usize = 8 * 1024; pub struct SstableBuilderOptions { /// Approximate sstable capacity. pub capacity: usize, - pub min_sstable_size: usize, /// Approximate block capacity. pub block_capacity: usize, /// Restart point interval. @@ -60,10 +59,8 @@ pub struct SstableBuilderOptions { impl From<&StorageOpts> for SstableBuilderOptions { fn from(options: &StorageOpts) -> SstableBuilderOptions { let capacity: usize = (options.sstable_size_mb as usize) * (1 << 20); - let min_sstable_size: usize = (options.min_sstable_size_mb as usize) * (1 << 20); SstableBuilderOptions { capacity, - min_sstable_size, block_capacity: (options.block_size_kb as usize) * (1 << 10), restart_interval: DEFAULT_RESTART_INTERVAL, bloom_false_positive: options.bloom_false_positive, @@ -76,7 +73,6 @@ impl From<&StorageOpts> for SstableBuilderOptions { impl Default for SstableBuilderOptions { fn default() -> Self { Self { - min_sstable_size: DEFAULT_SSTABLE_SIZE, capacity: DEFAULT_SSTABLE_SIZE, block_capacity: DEFAULT_BLOCK_SIZE, restart_interval: DEFAULT_RESTART_INTERVAL, diff --git a/src/storage/src/hummock/sstable/multi_builder.rs b/src/storage/src/hummock/sstable/multi_builder.rs index 9a330f40e4a6d..4049811f78c9e 100644 --- a/src/storage/src/hummock/sstable/multi_builder.rs +++ b/src/storage/src/hummock/sstable/multi_builder.rs @@ -35,7 +35,6 @@ use crate::hummock::{ use crate::monitor::CompactorMetrics; pub type UploadJoinHandle = JoinHandle>; -const MIN_SST_SIZE: usize = 4 * 1024 * 1024; // 4MB #[async_trait::async_trait] pub trait TableBuilderFactory {