From 33f6d9262e8bc572980f27e64769b8391da76305 Mon Sep 17 00:00:00 2001 From: William Wen Date: Wed, 11 Sep 2024 15:25:44 +0800 Subject: [PATCH] make configurable and enable time travel by default --- src/common/src/config.rs | 3 +++ src/common/src/system_param/mod.rs | 2 +- src/config/docs.md | 3 ++- src/config/example.toml | 2 +- .../src/hummock/event_handler/hummock_event_handler.rs | 2 +- src/storage/src/opts.rs | 6 ++++++ 6 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/common/src/config.rs b/src/common/src/config.rs index d78fdbe51fa9b..e2b4dd7b0f97c 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -699,6 +699,9 @@ pub struct StorageConfig { #[serde(default)] pub prefetch_buffer_capacity_mb: Option, + #[serde(default)] + pub max_cached_recent_versions_number: Option, + /// max prefetch block number #[serde(default = "default::storage::max_prefetch_block_number")] pub max_prefetch_block_number: usize, diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs index afe7adaacff5a..095875551fc51 100644 --- a/src/common/src/system_param/mod.rs +++ b/src/common/src/system_param/mod.rs @@ -91,7 +91,7 @@ macro_rules! for_all_params { { enable_tracing, bool, Some(false), true, "Whether to enable distributed tracing.", }, { use_new_object_prefix_strategy, bool, None, false, "Whether to split object prefix.", }, { license_key, risingwave_license::LicenseKey, Some(Default::default()), true, "The license key to activate enterprise features.", }, - { time_travel_retention_ms, u64, Some(0_u64), true, "The data retention period for time travel, where 0 indicates that it's disabled.", }, + { time_travel_retention_ms, u64, Some(600000_u64), true, "The data retention period for time travel, where 0 indicates that it's disabled.", }, } }; } diff --git a/src/config/docs.md b/src/config/docs.md index 47905d71e5e0c..65a0a010dc086 100644 --- a/src/config/docs.md +++ b/src/config/docs.md @@ -119,6 +119,7 @@ This page is automatically generated by `./risedev generate-example-config` | enable_fast_compaction | | true | | high_priority_ratio_in_percent | DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead. | | | imm_merge_threshold | The threshold for the number of immutable memtables to merge to a new imm. | 0 | +| max_cached_recent_versions_number | | | | max_concurrent_compaction_task_number | | 16 | | max_prefetch_block_number | max prefetch block number | 16 | | max_preload_io_retry_times | | 3 | @@ -169,5 +170,5 @@ This page is automatically generated by `./risedev generate-example-config` | pause_on_next_bootstrap | Whether to pause all data sources on next bootstrap. | false | | sstable_size_mb | Target size of the Sstable. | 256 | | state_store | URL for the state store | | -| time_travel_retention_ms | The data retention period for time travel, where 0 indicates that it's disabled. | 0 | +| time_travel_retention_ms | The data retention period for time travel, where 0 indicates that it's disabled. | 600000 | | use_new_object_prefix_strategy | Whether to split object prefix. | | diff --git a/src/config/example.toml b/src/config/example.toml index f3c127cdc7825..3268827804080 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -247,4 +247,4 @@ max_concurrent_creating_streaming_jobs = 1 pause_on_next_bootstrap = false enable_tracing = false license_key = "" -time_travel_retention_ms = 0 +time_travel_retention_ms = 600000 diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index c1621f71a3f4a..8bc13a88c5feb 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -358,7 +358,7 @@ impl HummockEventHandler { version_update_notifier_tx, recent_versions: Arc::new(ArcSwap::from_pointee(RecentVersions::new( pinned_version, - 60, + storage_opts.max_cached_recent_versions_number, ))), write_conflict_detector, read_version_mapping, diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs index f6d6f31fb3a4f..a3a787f55c97d 100644 --- a/src/storage/src/opts.rs +++ b/src/storage/src/opts.rs @@ -63,6 +63,8 @@ pub struct StorageOpts { /// max memory usage for large query. pub prefetch_buffer_capacity_mb: usize, + pub max_cached_recent_versions_number: usize, + pub max_prefetch_block_number: usize, pub disable_remote_compactor: bool, @@ -170,6 +172,10 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt meta_cache_shard_num: s.meta_cache_shard_num, meta_cache_eviction_config: s.meta_cache_eviction_config.clone(), prefetch_buffer_capacity_mb: s.prefetch_buffer_capacity_mb, + max_cached_recent_versions_number: c + .storage + .max_cached_recent_versions_number + .unwrap_or(60), max_prefetch_block_number: c.storage.max_prefetch_block_number, disable_remote_compactor: c.storage.disable_remote_compactor, share_buffer_upload_concurrency: c.storage.share_buffer_upload_concurrency,