From 66b5bb08935d3671e8f76b46ab0392ab615ffa68 Mon Sep 17 00:00:00 2001 From: Hiram Chirino Date: Fri, 26 Apr 2024 09:44:33 -0400 Subject: [PATCH] Adding benchmarks for the async rate limiters. Signed-off-by: Hiram Chirino --- Cargo.lock | 2 + limitador/Cargo.toml | 2 +- limitador/benches/bench.rs | 205 ++++++++++++++++++++++++++++++++++--- 3 files changed, 195 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 301842c1..40a22640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -739,6 +739,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", + "futures", "is-terminal", "itertools 0.10.5", "num-traits", @@ -751,6 +752,7 @@ dependencies = [ "serde_derive", "serde_json", "tinytemplate", + "tokio", "walkdir", ] diff --git a/limitador/Cargo.toml b/limitador/Cargo.toml index 2d6646c6..47881469 100644 --- a/limitador/Cargo.toml +++ b/limitador/Cargo.toml @@ -53,7 +53,7 @@ base64 = { version = "0.22", optional = true } [dev-dependencies] serial_test = "3.0" -criterion = { version = "0.5.1", features = ["html_reports"] } +criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } redis-test = { version = "0.4.0", features = ["aio"] } paste = "1" rand = "0.8" diff --git a/limitador/benches/bench.rs b/limitador/benches/bench.rs index 1926efcf..21f03a52 100644 --- a/limitador/benches/bench.rs +++ b/limitador/benches/bench.rs @@ -5,11 +5,14 @@ use limitador::limit::Limit; #[cfg(feature = "disk_storage")] use limitador::storage::disk::{DiskStorage, OptimizeFor}; use limitador::storage::in_memory::InMemoryStorage; -use limitador::storage::CounterStorage; -use limitador::RateLimiter; +use limitador::storage::redis::CachedRedisStorageBuilder; +use limitador::storage::{AsyncCounterStorage, CounterStorage}; +use limitador::{AsyncRateLimiter, RateLimiter}; use rand::SeedableRng; use std::collections::HashMap; use std::fmt::{Display, Formatter}; +use std::future::Future; +use std::time::Instant; const SEED: u64 = 42; @@ -18,9 +21,15 @@ criterion_group!(benches, bench_in_mem); #[cfg(all(feature = "disk_storage", not(feature = "redis_storage")))] criterion_group!(benches, bench_in_mem, bench_disk); #[cfg(all(not(feature = "disk_storage"), feature = "redis_storage"))] -criterion_group!(benches, bench_in_mem, bench_redis); +criterion_group!(benches, bench_in_mem, bench_redis, bench_cached_redis); #[cfg(all(feature = "disk_storage", feature = "redis_storage"))] -criterion_group!(benches, bench_in_mem, bench_disk, bench_redis); +criterion_group!( + benches, + bench_in_mem, + bench_disk, + bench_redis, + bench_cached_redis +); criterion_main!(benches); @@ -138,6 +147,55 @@ fn bench_disk(c: &mut Criterion) { group.finish(); } +#[cfg(feature = "redis_storage")] +fn bench_cached_redis(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + async fn create_storage() -> Box { + let storage_builder = CachedRedisStorageBuilder::new("redis://127.0.0.1:6379"); + let storage = storage_builder + .build() + .await + .expect("We need a Redis running locally"); + storage.clear().await.unwrap(); + return Box::new(storage); + } + + let mut group = c.benchmark_group("CachedRedis"); + for scenario in TEST_SCENARIOS { + group.bench_with_input( + BenchmarkId::new("is_rate_limited", scenario), + scenario, + |b: &mut Bencher, test_scenario: &&TestScenario| { + async_bench_is_rate_limited(&runtime, b, test_scenario, create_storage); + }, + ); + group.bench_with_input( + BenchmarkId::new("update_counters", scenario), + scenario, + |b: &mut Bencher, test_scenario: &&TestScenario| { + async_bench_update_counters(&runtime, b, test_scenario, create_storage); + }, + ); + group.bench_with_input( + BenchmarkId::new("check_rate_limited_and_update", scenario), + scenario, + |b: &mut Bencher, test_scenario: &&TestScenario| { + async_bench_check_rate_limited_and_update( + &runtime, + b, + test_scenario, + create_storage, + ); + }, + ); + } + group.finish(); +} + #[cfg(feature = "redis_storage")] fn bench_redis(c: &mut Criterion) { let mut group = c.benchmark_group("Redis"); @@ -195,6 +253,37 @@ fn bench_is_rate_limited( }) } +fn async_bench_is_rate_limited( + runtime: &tokio::runtime::Runtime, + b: &mut Bencher, + test_scenario: &TestScenario, + storage: fn() -> F, +) where + F: Future>, +{ + b.to_async(runtime).iter_custom(|iters| async move { + let storage = storage().await; + let (rate_limiter, call_params) = generate_async_test_data(test_scenario, storage); + let rng = &mut rand::rngs::StdRng::seed_from_u64(SEED); + + let start = Instant::now(); + for _i in 0..iters { + black_box({ + let params = call_params.choose(rng).unwrap(); + rate_limiter + .is_rate_limited( + ¶ms.namespace.to_owned().into(), + ¶ms.values, + params.delta, + ) + .await + .unwrap() + }); + } + start.elapsed() + }) +} + fn bench_update_counters( b: &mut Bencher, test_scenario: &TestScenario, @@ -214,7 +303,38 @@ fn bench_update_counters( params.delta, ) .unwrap(); - black_box(()) + }) +} + +fn async_bench_update_counters( + runtime: &tokio::runtime::Runtime, + b: &mut Bencher, + test_scenario: &TestScenario, + storage: fn() -> F, +) where + F: Future>, +{ + b.to_async(runtime).iter_custom(|iters| async move { + let storage = storage().await; + let (rate_limiter, call_params) = generate_async_test_data(test_scenario, storage); + let rng = &mut rand::rngs::StdRng::seed_from_u64(SEED); + + let start = Instant::now(); + for _i in 0..iters { + black_box({ + let params = call_params.choose(rng).unwrap(); + + rate_limiter + .update_counters( + ¶ms.namespace.to_owned().into(), + ¶ms.values, + params.delta, + ) + .await + .unwrap(); + }); + } + start.elapsed() }) } @@ -243,6 +363,39 @@ fn bench_check_rate_limited_and_update( }) } +fn async_bench_check_rate_limited_and_update( + runtime: &tokio::runtime::Runtime, + b: &mut Bencher, + test_scenario: &TestScenario, + storage: fn() -> F, +) where + F: Future>, +{ + b.to_async(runtime).iter_custom(|iters| async move { + let storage = storage().await; + let (rate_limiter, call_params) = generate_async_test_data(test_scenario, storage); + let rng = &mut rand::rngs::StdRng::seed_from_u64(SEED); + + let start = Instant::now(); + for _i in 0..iters { + black_box({ + let params = call_params.choose(rng).unwrap(); + + rate_limiter + .check_rate_limited_and_update( + ¶ms.namespace.to_owned().into(), + ¶ms.values, + params.delta, + false, + ) + .await + .unwrap() + }); + } + start.elapsed() + }) +} + // Notice that this function creates all the limits with the same conditions and // variables. Also, all the conditions have the same format: "cond_x == 1". // That's to simplify things, those are not the aspects that should have the @@ -255,6 +408,39 @@ fn generate_test_data( scenario: &TestScenario, storage: Box, ) -> (RateLimiter, Vec) { + let rate_limiter = RateLimiter::new_with_storage(storage); + + let (test_limits, call_params) = generate_test_limits(scenario); + for limit in test_limits { + rate_limiter.add_limit(limit); + } + + (rate_limiter, call_params) +} + +// Notice that this function creates all the limits with the same conditions and +// variables. Also, all the conditions have the same format: "cond_x == 1". +// That's to simplify things, those are not the aspects that should have the +// greatest impact on performance. +// The limits generated are big enough to avoid being rate-limited during the +// benchmark. +// Note that with this test data each request only increases one counter, we can +// that as another variable in the future. +fn generate_async_test_data( + scenario: &TestScenario, + storage: Box, +) -> (AsyncRateLimiter, Vec) { + let rate_limiter = AsyncRateLimiter::new_with_storage(storage); + + let (test_limits, call_params) = generate_test_limits(scenario); + for limit in test_limits { + rate_limiter.add_limit(limit); + } + + (rate_limiter, call_params) +} + +fn generate_test_limits(scenario: &TestScenario) -> (Vec, Vec) { let mut test_values: HashMap = HashMap::new(); let mut conditions = vec![]; @@ -293,12 +479,5 @@ fn generate_test_data( delta: 1, }); } - - let rate_limiter = RateLimiter::new_with_storage(storage); - - for limit in test_limits { - rate_limiter.add_limit(limit); - } - - (rate_limiter, call_params) + (test_limits, call_params) }