diff --git a/src/common/src/config.rs b/src/common/src/config.rs index c3fc78919a3e..64d7675903ec 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -871,6 +871,12 @@ pub struct StorageConfig { #[serde(default = "default::storage::compactor_max_overlap_sst_count")] pub compactor_max_overlap_sst_count: usize, + /// The maximum number of meta files that can be preloaded. + /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`. + /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient. + #[serde(default = "default::storage::compactor_max_preload_meta_file_count")] + pub compactor_max_preload_meta_file_count: usize, + /// Object storage configuration /// 1. General configuration /// 2. Some special configuration of Backend @@ -1795,6 +1801,10 @@ pub mod default { 64 } + pub fn compactor_max_preload_meta_file_count() -> usize { + 32 + } + // deprecated pub fn table_info_statistic_history_times() -> usize { 240 diff --git a/src/config/docs.md b/src/config/docs.md index 4a25867ed63c..bfe6a2fb5429 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -125,6 +125,7 @@ This page is automatically generated by `./risedev generate-example-config` | compactor_fast_max_compact_task_size | | 2147483648 | | compactor_iter_max_io_retry_times | | 8 | | compactor_max_overlap_sst_count | | 64 | +| compactor_max_preload_meta_file_count | The maximum number of meta files that can be preloaded. If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`. This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient. | 32 | | compactor_max_sst_key_count | | 2097152 | | compactor_max_sst_size | | 536870912 | | compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on `worker_num` and `compactor_max_task_multiplier`. `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier` | 3.0 | diff --git a/src/config/example.toml b/src/config/example.toml index 1d540b429a87..7a01ff5254e7 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -176,6 +176,7 @@ compactor_iter_max_io_retry_times = 8 table_info_statistic_history_times = 240 mem_table_spill_threshold = 4194304 compactor_max_overlap_sst_count = 64 +compactor_max_preload_meta_file_count = 32 time_travel_version_cache_capacity = 32 [storage.cache.block_cache_eviction] diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index 5e22d3e45701..73b178d73e80 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -232,10 +232,9 @@ pub async fn generate_splits( context: &CompactorContext, max_sub_compaction: u32, ) -> HummockResult> { - const MAX_FILE_COUNT: usize = 32; let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20; if compaction_size > parallel_compact_size { - if sstable_infos.len() > MAX_FILE_COUNT { + if sstable_infos.len() > context.storage_opts.compactor_max_preload_meta_file_count { return Ok(generate_splits_fast( sstable_infos, compaction_size, diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs index 14500f7c5113..e41e07dafc44 100644 --- a/src/storage/src/opts.rs +++ b/src/storage/src/opts.rs @@ -140,6 +140,9 @@ pub struct StorageOpts { pub compactor_max_overlap_sst_count: usize, + /// The maximum number of meta files that can be preloaded. + pub compactor_max_preload_meta_file_count: usize, + pub object_store_config: ObjectStoreConfig, pub time_travel_version_cache_capacity: u64, } @@ -243,6 +246,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt .compactor_concurrent_uploading_sst_count, time_travel_version_cache_capacity: c.storage.time_travel_version_cache_capacity, compactor_max_overlap_sst_count: c.storage.compactor_max_overlap_sst_count, + compactor_max_preload_meta_file_count: c.storage.compactor_max_preload_meta_file_count, } } }