Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(compaction): partition all levels by vnode #11903

Closed
wants to merge 19 commits into from
Closed

feat(compaction): partition all levels by vnode #11903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
49e4bcb
tmp commit
Little-Wallace Aug 22, 2023
e19ee6d
initial commit
Little-Wallace Aug 26, 2023
3fd6541
add ut
Little-Wallace Aug 28, 2023
71a0b73
adjust configure
Little-Wallace Aug 31, 2023
a7d8451
add test
Little-Wallace Aug 31, 2023
9d87c8b
fix hot vnode
Little-Wallace Aug 31, 2023
bd6842e
fix base level size
Little-Wallace Sep 1, 2023
7b0a6e4
fix size base level score
Little-Wallace Sep 4, 2023
6e317b1
small base level
Little-Wallace Sep 5, 2023
a9ffb6c
tmp commit
Little-Wallace Sep 11, 2023
d8ba92f
Merge branch 'main' into wallace/partition-group
Little-Wallace Sep 11, 2023
5f0cbdc
fix conflict
Little-Wallace Sep 11, 2023
de82a7f
fix test
Little-Wallace Sep 11, 2023
3525d6b
fix overflow
Little-Wallace Sep 12, 2023
2325058
address comment
Little-Wallace Sep 12, 2023
2173b0d
Merge branch 'main' into wallace/partition-group
Little-Wallace Sep 21, 2023
cef7255
fix conflict
Little-Wallace Sep 21, 2023
1c8ab1c
add more metrics and fix bug
Little-Wallace Sep 22, 2023
f99bb37
add info log and fix trivial move change vnode partition
Little-Wallace Sep 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions proto/hummock.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ message Level {
uint64 total_file_size = 4;
uint64 sub_level_id = 5;
uint64 uncompressed_file_size = 6;
uint32 vnode_partition_count = 7;
}

message InputLevel {
Expand All @@ -62,6 +63,7 @@ message IntraLevelDelta {
uint64 l0_sub_level_id = 2;
repeated uint64 removed_table_ids = 3;
repeated SstableInfo inserted_table_infos = 4;
uint32 vnode_partition_count = 5;
}

enum CompatibilityVersion {
Expand Down Expand Up @@ -116,6 +118,7 @@ message HummockVersion {
uint64 group_id = 3;
uint64 parent_group_id = 4;
repeated uint32 member_table_ids = 5;
uint32 vnode_partition_count = 6;
}
uint64 id = 1;
// Levels of each compaction group
Expand Down
10 changes: 5 additions & 5 deletions src/common/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,8 @@ pub struct MetaConfig {
#[serde(default)]
pub do_not_config_object_storage_lifecycle: bool,

#[serde(default = "default::meta::partition_vnode_count")]
pub partition_vnode_count: u32,
#[serde(default = "default::meta::vnode_partition_count")]
pub vnode_partition_count: u32,

#[serde(default = "default::meta::table_write_throughput_threshold")]
pub table_write_throughput_threshold: u64,
Expand Down Expand Up @@ -891,8 +891,8 @@ pub mod default {
64 * 1024 * 1024 * 1024 // 64GB
}

pub fn partition_vnode_count() -> u32 {
64
pub fn vnode_partition_count() -> u32 {
32
}

pub fn table_write_throughput_threshold() -> u64 {
Expand Down Expand Up @@ -1210,7 +1210,7 @@ pub mod default {
const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * 1024 * 1024; // 512MB;
const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 300;
const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 96;
const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3;
const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 6;
const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 6;
const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
const DEFAULT_EMERGENCY_PICKER: bool = true;
Expand Down
5 changes: 2 additions & 3 deletions src/config/example.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ periodic_split_compact_group_interval_sec = 180
move_table_size_limit = 10737418240
split_group_size_limit = 68719476736
do_not_config_object_storage_lifecycle = false
partition_vnode_count = 64
partition_vnode_count = 32
table_write_throughput_threshold = 16777216
min_table_split_write_throughput = 4194304
compaction_task_max_heartbeat_interval_secs = 60
Expand All @@ -50,7 +50,7 @@ target_file_size_base = 33554432
compaction_filter_mask = 6
max_sub_compaction = 4
level0_stop_write_threshold_sub_level_number = 300
level0_sub_level_compact_level_count = 3
level0_sub_level_compact_level_count = 6
level0_overlapping_sub_level_compact_level_count = 6
max_space_reclaim_bytes = 536870912
level0_max_compact_file_number = 96
Expand Down Expand Up @@ -110,7 +110,6 @@ compact_iter_recreate_timeout_ms = 600000
compactor_max_sst_size = 536870912
enable_fast_compaction = true


[storage.data_file_cache]
dir = ""
capacity_mb = 1024
Expand Down
Loading