feat(compaction): Introducing configuration for compactor preload SstableMeta

src/common/src/config.rs

@@ -871,6 +871,12 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::compactor_max_overlap_sst_count")]
     pub compactor_max_overlap_sst_count: usize,
 
+    /// The maximum number of meta files that can be preloaded.
+    /// If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`.
+    /// This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient.
+    #[serde(default = "default::storage::compactor_max_preload_meta_file_count")]
+    pub compactor_max_preload_meta_file_count: usize,
+
     /// Object storage configuration
     /// 1. General configuration
     /// 2. Some special configuration of Backend
@@ -1795,6 +1801,10 @@ pub mod default {
             64
         }
 
+        pub fn compactor_max_preload_meta_file_count() -> usize {
+            32
+        }
+
         // deprecated
         pub fn table_info_statistic_history_times() -> usize {
             240

src/config/docs.md

@@ -125,6 +125,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | compactor_fast_max_compact_task_size |  | 2147483648 |
 | compactor_iter_max_io_retry_times |  | 8 |
 | compactor_max_overlap_sst_count |  | 64 |
+| compactor_max_preload_meta_file_count | The maximum number of meta files that can be preloaded. If the number of meta files exceeds this value, the compactor will try to compute parallelism only through `SstableInfo`, no longer preloading `SstableMeta`. This is to prevent the compactor from consuming too much memory, but it may cause the compactor to be less efficient. | 32 |
 | compactor_max_sst_key_count |  | 2097152 |
 | compactor_max_sst_size |  | 536870912 |
 | compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on `worker_num` and `compactor_max_task_multiplier`. `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier` | 3.0 |

src/config/example.toml

@@ -176,6 +176,7 @@ compactor_iter_max_io_retry_times = 8
 table_info_statistic_history_times = 240
 mem_table_spill_threshold = 4194304
 compactor_max_overlap_sst_count = 64
+compactor_max_preload_meta_file_count = 32
 time_travel_version_cache_capacity = 32
 
 [storage.cache.block_cache_eviction]

src/storage/src/hummock/compactor/compaction_utils.rs

@@ -232,10 +232,9 @@ pub async fn generate_splits(
     context: &CompactorContext,
     max_sub_compaction: u32,
 ) -> HummockResult<Vec<KeyRange>> {
-    const MAX_FILE_COUNT: usize = 32;
     let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
     if compaction_size > parallel_compact_size {
-        if sstable_infos.len() > MAX_FILE_COUNT {
+        if sstable_infos.len() > context.storage_opts.compactor_max_preload_meta_file_count {
             return Ok(generate_splits_fast(
                 sstable_infos,
                 compaction_size,

src/storage/src/opts.rs

@@ -140,6 +140,9 @@ pub struct StorageOpts {
 
     pub compactor_max_overlap_sst_count: usize,
 
+    /// The maximum number of meta files that can be preloaded.
+    pub compactor_max_preload_meta_file_count: usize,
+
     pub object_store_config: ObjectStoreConfig,
     pub time_travel_version_cache_capacity: u64,
 }
@@ -243,6 +246,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
                 .compactor_concurrent_uploading_sst_count,
             time_travel_version_cache_capacity: c.storage.time_travel_version_cache_capacity,
             compactor_max_overlap_sst_count: c.storage.compactor_max_overlap_sst_count,
+            compactor_max_preload_meta_file_count: c.storage.compactor_max_preload_meta_file_count,
         }
     }
 }