From 52f4c630ba60a4a2e2c7915e776744d3cbcfb901 Mon Sep 17 00:00:00 2001 From: Croxx Date: Wed, 18 Oct 2023 03:10:10 -0500 Subject: [PATCH] fix: don't count file cache buffer pool size if disabled (#12941) Signed-off-by: MrCroxx --- src/compute/src/memory_management/mod.rs | 51 ++++++++++++++++-------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/src/compute/src/memory_management/mod.rs b/src/compute/src/memory_management/mod.rs index 3ae47729b336a..0323cb2ddc093 100644 --- a/src/compute/src/memory_management/mod.rs +++ b/src/compute/src/memory_management/mod.rs @@ -163,22 +163,30 @@ pub fn storage_memory_config( // // Adding more buffers can prevent allocators from waiting for buffers to be freed by flushers. - let data_file_cache_buffer_pool_capacity_mb = storage_config - .data_file_cache - .buffer_pool_size_mb - .unwrap_or( - storage_config.data_file_cache.file_capacity_mb - * (storage_config.data_file_cache.flushers - + 2 * (1 << storage_config.data_file_cache.allocation_bits)), - ); - let meta_file_cache_buffer_pool_capacity_mb = storage_config - .meta_file_cache - .buffer_pool_size_mb - .unwrap_or( - storage_config.meta_file_cache.file_capacity_mb - * (storage_config.meta_file_cache.flushers - + 2 * (1 << storage_config.meta_file_cache.allocation_bits)), - ); + let data_file_cache_buffer_pool_capacity_mb = if storage_config.data_file_cache.dir.is_empty() { + 0 + } else { + storage_config + .data_file_cache + .buffer_pool_size_mb + .unwrap_or( + storage_config.data_file_cache.file_capacity_mb + * (storage_config.data_file_cache.flushers + + 2 * (1 << storage_config.data_file_cache.allocation_bits)), + ) + }; + let meta_file_cache_buffer_pool_capacity_mb = if storage_config.meta_file_cache.dir.is_empty() { + 0 + } else { + storage_config + .meta_file_cache + .buffer_pool_size_mb + .unwrap_or( + storage_config.meta_file_cache.file_capacity_mb + * (storage_config.meta_file_cache.flushers + + 2 * (1 << storage_config.meta_file_cache.allocation_bits)), + ) + }; let compactor_memory_limit_mb = storage_config.compactor_memory_limit_mb.unwrap_or( ((non_reserved_memory_bytes as f64 * compactor_memory_proportion).ceil() as usize) >> 20, @@ -238,6 +246,17 @@ mod tests { let mut storage_config = StorageConfig::default(); let total_non_reserved_memory_bytes = 8 << 30; + let memory_config = + storage_memory_config(total_non_reserved_memory_bytes, true, &storage_config); + assert_eq!(memory_config.block_cache_capacity_mb, 737); + assert_eq!(memory_config.meta_cache_capacity_mb, 860); + assert_eq!(memory_config.shared_buffer_capacity_mb, 737); + assert_eq!(memory_config.data_file_cache_buffer_pool_capacity_mb, 0); + assert_eq!(memory_config.meta_file_cache_buffer_pool_capacity_mb, 0); + assert_eq!(memory_config.compactor_memory_limit_mb, 819); + + storage_config.data_file_cache.dir = "data".to_string(); + storage_config.meta_file_cache.dir = "meta".to_string(); let memory_config = storage_memory_config(total_non_reserved_memory_bytes, true, &storage_config); assert_eq!(memory_config.block_cache_capacity_mb, 737);