From bfbdc142b2bed5bf0301fdfb062d8d2ec7b82a42 Mon Sep 17 00:00:00 2001 From: Li0k Date: Wed, 13 Nov 2024 21:37:49 +0800 Subject: [PATCH] feat(compaction): cherry-pick #19218 (#19375) --- src/common/src/config.rs | 11 +++++++++++ src/config/docs.md | 1 + src/config/example.toml | 1 + src/storage/src/hummock/compactor/compaction_utils.rs | 3 +-- src/storage/src/opts.rs | 4 ++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/common/src/config.rs b/src/common/src/config.rs index 9766f5b601a3f..06b37179f31d3 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -814,6 +814,12 @@ pub struct StorageConfig { #[serde(default = "default::storage::compactor_max_overlap_sst_count")] pub compactor_max_overlap_sst_count: usize, + /// The maximum number of meta files that can be preloaded. + /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`. + /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient. + #[serde(default = "default::storage::compactor_max_preload_meta_file_count")] + pub compactor_max_preload_meta_file_count: usize, + /// Object storage configuration /// 1. General configuration /// 2. Some special configuration of Backend @@ -1712,6 +1718,11 @@ pub mod default { 64 } + pub fn compactor_max_preload_meta_file_count() -> usize { + 32 + } + + // deprecated pub fn table_info_statistic_history_times() -> usize { 240 } diff --git a/src/config/docs.md b/src/config/docs.md index 99cd5f2b4dd2e..a349bf98cc21a 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -122,6 +122,7 @@ This page is automatically generated by `./risedev generate-example-config` | compactor_fast_max_compact_task_size | | 2147483648 | | compactor_iter_max_io_retry_times | | 8 | | compactor_max_overlap_sst_count | | 64 | +| compactor_max_preload_meta_file_count | The maximum number of meta files that can be preloaded. If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`. This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient. | 32 | | compactor_max_sst_key_count | | 2097152 | | compactor_max_sst_size | | 536870912 | | compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on `worker_num` and `compactor_max_task_multiplier`. `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier` | 3.0 | diff --git a/src/config/example.toml b/src/config/example.toml index f097df23fc528..6aec9f8d70f83 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -170,6 +170,7 @@ compactor_iter_max_io_retry_times = 8 table_info_statistic_history_times = 240 mem_table_spill_threshold = 4194304 compactor_max_overlap_sst_count = 64 +compactor_max_preload_meta_file_count = 32 time_travel_version_cache_capacity = 32 [storage.cache.block_cache_eviction] diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index 3b032123f426a..b6a5ebc918593 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -232,10 +232,9 @@ pub async fn generate_splits( context: &CompactorContext, max_sub_compaction: u32, ) -> HummockResult> { - const MAX_FILE_COUNT: usize = 32; let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20; if compaction_size > parallel_compact_size { - if sstable_infos.len() > MAX_FILE_COUNT { + if sstable_infos.len() > context.storage_opts.compactor_max_preload_meta_file_count { return Ok(generate_splits_fast( sstable_infos, compaction_size, diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs index 14500f7c5113e..e41e07dafc44a 100644 --- a/src/storage/src/opts.rs +++ b/src/storage/src/opts.rs @@ -140,6 +140,9 @@ pub struct StorageOpts { pub compactor_max_overlap_sst_count: usize, + /// The maximum number of meta files that can be preloaded. + pub compactor_max_preload_meta_file_count: usize, + pub object_store_config: ObjectStoreConfig, pub time_travel_version_cache_capacity: u64, } @@ -243,6 +246,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt .compactor_concurrent_uploading_sst_count, time_travel_version_cache_capacity: c.storage.time_travel_version_cache_capacity, compactor_max_overlap_sst_count: c.storage.compactor_max_overlap_sst_count, + compactor_max_preload_meta_file_count: c.storage.compactor_max_preload_meta_file_count, } } }