Skip to content

Commit

Permalink
add batching toggle to GC settings
Browse files Browse the repository at this point in the history
  • Loading branch information
@teh-cmc
teh-cmc committed Nov 30, 2023
1 parent d25e7c2 commit 3189266
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 37 deletions.
2 changes: 2 additions & 0 deletions crates/re_arrow_store/benches/data_store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,7 @@ fn gc(c: &mut Criterion) {
protect_latest: 0,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
});
stats_diff
});
Expand All @@ -315,6 +316,7 @@ fn gc(c: &mut Criterion) {
protect_latest: 0,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
});
stats_diff
});
Expand Down
100 changes: 63 additions & 37 deletions crates/re_arrow_store/benches/gc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,17 @@ mod constants {

use constants::{NUM_ENTITY_PATHS, NUM_ROWS_PER_ENTITY_PATH};

fn gc_batching() -> &'static [bool] {
#[cfg(feature = "core_benchmarks_only")]
{
&[false]
}
#[cfg(not(feature = "core_benchmarks_only"))]
{
&[false, true]
}
}

fn num_rows_per_bucket() -> &'static [u64] {
#[cfg(feature = "core_benchmarks_only")]
{
Expand Down Expand Up @@ -63,6 +74,7 @@ fn plotting_dashboard(c: &mut Criterion) {
protect_latest: 1,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
};

let mut timegen = |i| {
Expand Down Expand Up @@ -98,26 +110,33 @@ fn plotting_dashboard(c: &mut Criterion) {

// Emulate more or less bucket
for &num_rows_per_bucket in num_rows_per_bucket() {
group.bench_function(format!("bucketsz={num_rows_per_bucket}"), |b| {
let store = build_store(
DataStoreConfig {
indexed_bucket_num_rows: num_rows_per_bucket,
..Default::default()
for &gc_batching in gc_batching() {
group.bench_function(
format!("bucketsz={num_rows_per_bucket}/gc_batching={gc_batching}"),
|b| {
let store = build_store(
DataStoreConfig {
indexed_bucket_num_rows: num_rows_per_bucket,
..Default::default()
},
InstanceKey::name(),
false,
&mut timegen,
&mut datagen,
);
let mut gc_settings = gc_settings.clone();
gc_settings.enable_batching = gc_batching;
b.iter_batched(
|| store.clone(),
|mut store| {
let (_, stats_diff) = store.gc(&gc_settings);
stats_diff
},
BatchSize::LargeInput,
);
},
InstanceKey::name(),
false,
&mut timegen,
&mut datagen,
);
b.iter_batched(
|| store.clone(),
|mut store| {
let (_, stats_diff) = store.gc(&gc_settings);
stats_diff
},
BatchSize::LargeInput,
);
});
}
}
}

Expand All @@ -138,6 +157,7 @@ fn timeless_logs(c: &mut Criterion) {
protect_latest: 1,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
};

let mut timegen = |_| TimePoint::timeless();
Expand Down Expand Up @@ -165,28 +185,34 @@ fn timeless_logs(c: &mut Criterion) {
);
});

// Emulate more or less bucket
for &num_rows_per_bucket in num_rows_per_bucket() {
group.bench_function(format!("bucketsz={num_rows_per_bucket}"), |b| {
let store = build_store(
DataStoreConfig {
indexed_bucket_num_rows: num_rows_per_bucket,
..Default::default()
for &gc_batching in gc_batching() {
group.bench_function(
format!("bucketsz={num_rows_per_bucket}/gc_batching={gc_batching}"),
|b| {
let store = build_store(
DataStoreConfig {
indexed_bucket_num_rows: num_rows_per_bucket,
..Default::default()
},
InstanceKey::name(),
false,
&mut timegen,
&mut datagen,
);
let mut gc_settings = gc_settings.clone();
gc_settings.enable_batching = gc_batching;
b.iter_batched(
|| store.clone(),
|mut store| {
let (_, stats_diff) = store.gc(&gc_settings);
stats_diff
},
BatchSize::LargeInput,
);
},
InstanceKey::name(),
false,
&mut timegen,
&mut datagen,
);
b.iter_batched(
|| store.clone(),
|mut store| {
let (_, stats_diff) = store.gc(&gc_settings);
stats_diff
},
BatchSize::LargeInput,
);
});
}
}
}

Expand Down
2 changes: 2 additions & 0 deletions crates/re_arrow_store/tests/correctness.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,13 +588,15 @@ fn gc_metadata_size() -> anyhow::Result<()> {
protect_latest: 1,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
});
_ = store.gc(&GarbageCollectionOptions {
target: re_arrow_store::GarbageCollectionTarget::DropAtLeastFraction(1.0),
gc_timeless: false,
protect_latest: 1,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
});
}

Expand Down
4 changes: 4 additions & 0 deletions crates/re_arrow_store/tests/data_store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -932,6 +932,7 @@ fn gc_impl(store: &mut DataStore) {
protect_latest: 0,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
});
for event in store_events {
assert!(store.get_msg_metadata(&event.row_id).is_none());
Expand Down Expand Up @@ -1011,6 +1012,7 @@ fn protected_gc_impl(store: &mut DataStore) {
protect_latest: 1,
purge_empty_tables: true,
dont_protect: Default::default(),
enable_batching: false,
});

let mut assert_latest_components = |frame_nr: TimeInt, rows: &[(ComponentName, &DataRow)]| {
Expand Down Expand Up @@ -1107,6 +1109,7 @@ fn protected_gc_clear_impl(store: &mut DataStore) {
protect_latest: 1,
purge_empty_tables: true,
dont_protect: Default::default(),
enable_batching: false,
});

let mut assert_latest_components = |frame_nr: TimeInt, rows: &[(ComponentName, &DataRow)]| {
Expand Down Expand Up @@ -1149,6 +1152,7 @@ fn protected_gc_clear_impl(store: &mut DataStore) {
protect_latest: 1,
purge_empty_tables: true,
dont_protect: Default::default(),
enable_batching: false,
});

// No rows should remain because the table should have been purged
Expand Down
2 changes: 2 additions & 0 deletions crates/re_data_store/src/store_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ impl StoreDb {
]
.into_iter()
.collect(),
enable_batching: false,
});
}

Expand All @@ -449,6 +450,7 @@ impl StoreDb {
protect_latest: 1,
purge_empty_tables: false,
dont_protect: Default::default(),
enable_batching: false,
});
}

Expand Down

0 comments on commit 3189266

Please sign in to comment.