diff --git a/src/common/src/cache.rs b/src/common/src/cache.rs index 111fb823d6f46..99a373d6a94a8 100644 --- a/src/common/src/cache.rs +++ b/src/common/src/cache.rs @@ -1049,7 +1049,7 @@ mod tests { #[test] fn test_cache_shard() { - let cache = Arc::new(LruCache::<(u64, u64), Block>::new(2, 256, 0)); + let cache = Arc::new(LruCache::<(u64, u64), Block>::new(4, 256, 0)); assert_eq!(cache.shard(0), 0); assert_eq!(cache.shard(1), 1); assert_eq!(cache.shard(10), 2); @@ -1349,7 +1349,7 @@ mod tests { #[test] fn test_write_request_pending() { - let cache = Arc::new(LruCache::new(0, 5, 0)); + let cache = Arc::new(LruCache::new(1, 5, 0)); { let mut shard = cache.shards[0].lock(); insert(&mut shard, "a", "v1"); @@ -1394,7 +1394,7 @@ mod tests { #[test] fn test_event_listener() { let listener = Arc::new(TestLruCacheEventListener::default()); - let cache = Arc::new(LruCache::with_event_listener(0, 2, 0, listener.clone())); + let cache = Arc::new(LruCache::with_event_listener(1, 2, 0, listener.clone())); // full-fill cache let h = cache.insert( @@ -1489,7 +1489,7 @@ mod tests { #[tokio::test] async fn test_future_cancel() { - let cache: Arc> = Arc::new(LruCache::new(0, 5, 0)); + let cache: Arc> = Arc::new(LruCache::new(1, 5, 0)); // do not need sender because this receiver will be cancelled. let (_, recv) = channel::<()>(); let polled = Arc::new(AtomicBool::new(false)); diff --git a/src/compute/src/memory/config.rs b/src/compute/src/memory/config.rs index f37869e53d57d..a6eee7c2416d3 100644 --- a/src/compute/src/memory/config.rs +++ b/src/compute/src/memory/config.rs @@ -33,7 +33,7 @@ const COMPACTOR_MEMORY_PROPORTION: f64 = 0.1; const STORAGE_BLOCK_CACHE_MEMORY_PROPORTION: f64 = 0.3; const STORAGE_META_CACHE_MAX_MEMORY_MB: usize = 4096; -const STORAGE_SHARED_BUFFER_MAX_MEMORY_MB: usize = 1024; +const STORAGE_SHARED_BUFFER_MAX_MEMORY_MB: usize = 4096; const STORAGE_META_CACHE_MEMORY_PROPORTION: f64 = 0.35; const STORAGE_SHARED_BUFFER_MEMORY_PROPORTION: f64 = 0.3; const STORAGE_DEFAULT_HIGH_PRIORITY_BLOCK_CACHE_RATIO: usize = 50; @@ -95,7 +95,8 @@ pub fn storage_memory_config( if meta_cache_capacity_mb != default_meta_cache_capacity_mb { default_block_cache_capacity_mb += default_meta_cache_capacity_mb; - default_block_cache_capacity_mb -= meta_cache_capacity_mb; + default_block_cache_capacity_mb = + default_block_cache_capacity_mb.saturating_sub(meta_cache_capacity_mb); } let default_shared_buffer_capacity_mb = ((non_reserved_memory_bytes as f64 @@ -112,7 +113,8 @@ pub fn storage_memory_config( )); if shared_buffer_capacity_mb != default_shared_buffer_capacity_mb { default_block_cache_capacity_mb += default_shared_buffer_capacity_mb; - default_block_cache_capacity_mb -= shared_buffer_capacity_mb; + default_block_cache_capacity_mb = + default_block_cache_capacity_mb.saturating_sub(shared_buffer_capacity_mb); } let block_cache_capacity_mb = storage_config .block_cache_capacity_mb @@ -170,15 +172,15 @@ pub fn storage_memory_config( StorageMemoryConfig { block_cache_capacity_mb, + block_shard_num, meta_cache_capacity_mb, + meta_shard_num, shared_buffer_capacity_mb, data_file_cache_ring_buffer_capacity_mb, meta_file_cache_ring_buffer_capacity_mb, compactor_memory_limit_mb, prefetch_buffer_capacity_mb, high_priority_ratio_in_percent, - block_shard_num, - meta_shard_num, } } diff --git a/src/config/docs.md b/src/config/docs.md index 452caa81e83fc..3730b4e828f6d 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -94,6 +94,7 @@ This page is automatically generated by `./risedev generate-example-config` | Config | Description | Default | |--------|-------------|---------| | block_cache_capacity_mb | Capacity of sstable block cache. | | +| block_shard_num | | | | cache_refill | | | | check_compaction_result | | false | | compact_iter_recreate_timeout_ms | | 600000 | @@ -118,6 +119,7 @@ This page is automatically generated by `./risedev generate-example-config` | mem_table_spill_threshold | The spill threshold for mem table. | 4194304 | | meta_cache_capacity_mb | Capacity of sstable meta cache. | | | meta_file_cache | | | +| meta_shard_num | | | | min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 | | object_store | | | | prefetch_buffer_capacity_mb | max memory usage for large query | | diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index edab0d067778d..9d960572b8d0f 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -65,6 +65,8 @@ pub fn mock_sstable_store() -> SstableStoreRef { block_cache_capacity: 64 << 20, meta_cache_capacity: 128 << 20, high_priority_ratio: 0, + meta_shard_num: 2, + block_shard_num: 2, prefetch_buffer_capacity: 64 << 20, max_prefetch_block_number: 16, data_file_cache: FileCache::none(), diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs index bdc288b4d6925..590fb1a6f497e 100644 --- a/src/storage/benches/bench_multi_builder.rs +++ b/src/storage/benches/bench_multi_builder.rs @@ -147,6 +147,8 @@ fn bench_builder( block_cache_capacity: 64 << 20, meta_cache_capacity: 128 << 20, high_priority_ratio: 0, + meta_shard_num: 2, + block_shard_num: 2, prefetch_buffer_capacity: 64 << 20, max_prefetch_block_number: 16, data_file_cache: FileCache::none(),