From c94358d119940b91acd3d015f142f34da9fb6b58 Mon Sep 17 00:00:00 2001 From: congyi <15605187270@163.com> Date: Mon, 26 Feb 2024 11:35:05 +0800 Subject: [PATCH 1/5] enable mem table spill --- src/common/src/config.rs | 2 +- src/config/example.toml | 2 +- src/storage/src/hummock/store/local_hummock_storage.rs | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/common/src/config.rs b/src/common/src/config.rs index b1415a00b1362..fb1c3d6304aed 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -1253,7 +1253,7 @@ pub mod default { 3 } pub fn mem_table_spill_threshold() -> usize { - 0 // disable + 4 << 20 } pub fn compactor_fast_max_compact_delete_ratio() -> u32 { diff --git a/src/config/example.toml b/src/config/example.toml index 59c68aff3c7c0..5a50b758b7496 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -124,7 +124,7 @@ check_compaction_result = false max_preload_io_retry_times = 3 compactor_fast_max_compact_delete_ratio = 40 compactor_fast_max_compact_task_size = 2147483648 -mem_table_spill_threshold = 0 +mem_table_spill_threshold = 4194304 [storage.data_file_cache] dir = "" diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs index fa35f71975063..096425b64fc4e 100644 --- a/src/storage/src/hummock/store/local_hummock_storage.rs +++ b/src/storage/src/hummock/store/local_hummock_storage.rs @@ -350,9 +350,7 @@ impl LocalStateStore for LocalHummockStorage { } async fn try_flush(&mut self) -> StorageResult<()> { - if self.mem_table_spill_threshold != 0 - && self.mem_table.kv_size.size() > self.mem_table_spill_threshold - { + if self.mem_table.kv_size.size() > self.mem_table_spill_threshold { if self.spill_offset < MAX_SPILL_TIMES { let table_id_label = self.table_id.table_id().to_string(); self.flush().await?; From 84f13c2bfaf1b3e013542c0e03bcb981f0415c7d Mon Sep 17 00:00:00 2001 From: congyi <15605187270@163.com> Date: Mon, 26 Feb 2024 13:49:54 +0800 Subject: [PATCH 2/5] config generated --- src/config/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/docs.md b/src/config/docs.md index 36fd40ce2d13a..fdf732c4d8cf2 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -110,7 +110,7 @@ This page is automatically generated by `./risedev generate-example-config` | max_preload_wait_time_mill | | 0 | | max_sub_compaction | Max sub compaction task numbers | 4 | | max_version_pinning_duration_sec | | 10800 | -| mem_table_spill_threshold | The spill threshold for mem table. | 0 | +| mem_table_spill_threshold | The spill threshold for mem table. | 4194304 | | meta_cache_capacity_mb | Capacity of sstable meta cache. | | | meta_file_cache | | | | min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 | From c5ea8d346e3b53e18f5d0660b21ccbf1b988f4f0 Mon Sep 17 00:00:00 2001 From: congyi <15605187270@163.com> Date: Mon, 26 Feb 2024 14:06:22 +0800 Subject: [PATCH 3/5] keep 0 as disable judgement --- src/storage/src/hummock/store/local_hummock_storage.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs index 096425b64fc4e..fa35f71975063 100644 --- a/src/storage/src/hummock/store/local_hummock_storage.rs +++ b/src/storage/src/hummock/store/local_hummock_storage.rs @@ -350,7 +350,9 @@ impl LocalStateStore for LocalHummockStorage { } async fn try_flush(&mut self) -> StorageResult<()> { - if self.mem_table.kv_size.size() > self.mem_table_spill_threshold { + if self.mem_table_spill_threshold != 0 + && self.mem_table.kv_size.size() > self.mem_table_spill_threshold + { if self.spill_offset < MAX_SPILL_TIMES { let table_id_label = self.table_id.table_id().to_string(); self.flush().await?; From 4b1a5588d59ce568a8dd7c94ff5462ca1be4a562 Mon Sep 17 00:00:00 2001 From: congyi <15605187270@163.com> Date: Mon, 26 Feb 2024 18:36:53 +0800 Subject: [PATCH 4/5] update doc --- src/config/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/docs.md b/src/config/docs.md index fdf732c4d8cf2..3ebdf17cfffad 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -110,7 +110,7 @@ This page is automatically generated by `./risedev generate-example-config` | max_preload_wait_time_mill | | 0 | | max_sub_compaction | Max sub compaction task numbers | 4 | | max_version_pinning_duration_sec | | 10800 | -| mem_table_spill_threshold | The spill threshold for mem table. | 4194304 | +| mem_table_spill_threshold | The spill threshold measured in bytes for mem table | 4194304 | | meta_cache_capacity_mb | Capacity of sstable meta cache. | | | meta_file_cache | | | | min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 | From 2d2527ee3a3535385239b97417b5f2f219a410c8 Mon Sep 17 00:00:00 2001 From: congyi <15605187270@163.com> Date: Mon, 26 Feb 2024 21:51:39 +0800 Subject: [PATCH 5/5] generate-example-config --- src/config/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/docs.md b/src/config/docs.md index 3ebdf17cfffad..fdf732c4d8cf2 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -110,7 +110,7 @@ This page is automatically generated by `./risedev generate-example-config` | max_preload_wait_time_mill | | 0 | | max_sub_compaction | Max sub compaction task numbers | 4 | | max_version_pinning_duration_sec | | 10800 | -| mem_table_spill_threshold | The spill threshold measured in bytes for mem table | 4194304 | +| mem_table_spill_threshold | The spill threshold for mem table. | 4194304 | | meta_cache_capacity_mb | Capacity of sstable meta cache. | | | meta_file_cache | | | | min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 |