diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ff6eac5fa..0890a4baf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - - run: cargo llvm-cov nextest --all-features --lcov --output-path lcov.info + - run: cargo build -r --bin katana && cargo llvm-cov nextest --all-features --lcov --output-path lcov.info - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/crates/benches/README.md b/crates/benches/README.md index ef39c7bd49..dafea2439d 100644 --- a/crates/benches/README.md +++ b/crates/benches/README.md @@ -1,19 +1,38 @@ # This crate is dedicated for benchmarking purposes -## Quick start +## Prerequisites + +- `cargo` - for test case generation and runtime +- `katana` - as a local RPC server +- `sozo` - for contract compilation and deployment + + +## Katana Benchmarks + +TMP: Due to the number of accounts overfowing `u8` at the moment, katana binary is run from `target` directory. +After the change to `u16` is released it should take the system installed version. Because of this additional first step is required. + +```bash +cargo build -r --bin katana +``` + +And to run the benchamarks + +```bash +cargo test --manifest-path crates/benches/Cargo.toml -- --nocapture +``` + +## Gas usage Benchmarks + +### Quick start ```bash katana bash scripts/cargo_bench.sh ``` -## Prerequisites - -- `cargo` - for test case generation and runtime -- `katana` - as a local RPC server -- `sozo` - for contract compilation and deployment -## Requirements for running +### Requirements for running While benchmarks are running a Katana instance has to be online either remotely or locally... @@ -42,7 +61,7 @@ cargo test bench -- --ignored Benchmarks are ignored by default because they need a while to complete and need a running Katana. Their names should start with bench. -## Running with compiled `sozo` +### Running with compiled `sozo` While during testing release version of the tool worked better, Sozo can be run from source with diff --git a/crates/benches/bench_results.txt b/crates/benches/bench_results.txt index 6e9dd59ea5..6d9c15b5d4 100644 --- a/crates/benches/bench_results.txt +++ b/crates/benches/bench_results.txt @@ -7,3 +7,7 @@ {"name":"Benchmark: 2000 accounts, is_prime transactions, 1 calls","sending_time":13649,"responses_span":783,"longest_confirmation_difference":9,"stats":{"estimated_tps":4.386705649857541,"estimated_sps":3866350.1863253224,"relevant_blocks":[[538,125111],[1462,330812]],"relevant_time":455923},"block_times":[0,3000,2999,3000,2998,3001,3000,3000,2999,3000,3000,3000,2998,3001,2998,3000,3001,2997,3000,2999,2999,2999,3000,2999,3000,2999,3000,2999,3000,3000,2999,3000,125111,330812,2993],"block_sizes":[0,0,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,538,1462,0],"steps":1762757976} {"name":"Benchmark: 2000 accounts, is_prime transactions, 1 calls","sending_time":13638,"responses_span":765,"longest_confirmation_difference":6,"stats":{"estimated_tps":4.378465829358052,"estimated_sps":3859087.78167218,"relevant_blocks":[[685,158378],[1315,298403]],"relevant_time":456781},"block_times":[0,2999,2999,3001,2999,2999,3000,3000,2999,2999,3000,3000,3007,3002,2999,3000,3000,2998,3003,2996,2999,3000,3000,2999,2999,3000,2998,3001,2999,3000,2999,2999,158378,298403,2993],"block_sizes":[0,0,1,1,1,0,1,1,1,1,1,1,2,0,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,685,1315,0],"steps":1762757976} {"name":"Benchmark: 2000 accounts, is_prime transactions, 23 calls","sending_time":78297,"responses_span":5477,"longest_confirmation_difference":10,"stats":{"estimated_tps":4.1648532201603885,"estimated_sps":4441395.259147579,"relevant_blocks":[[85,21448],[1915,458761]],"relevant_time":480209},"block_times":[0,2999,3000,2999,2999,3000,2999,3000,3000,2999,2999,3000,2998,3000,3000,2999,3000,2999,3001,3000,2998,3000,3001,2998,3000,3000,2999,3000,3000,3000,2999,3001,2999,2999,3000,2999,3000,3000,2999,3000,2999,2999,2999,3000,2999,3000,2999,3001,2999,2999,3000,2999,21448,458761,2982],"block_sizes":[0,0,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,85,1915,0],"steps":2132797976} +{"name":"Benchmark: 1 accounts, is_prime transactions, 1 calls","sending_time":14,"responses_span":0,"longest_confirmation_difference":0,"stats":{"estimated_tps":0.33288948069241014,"estimated_sps":293403.1291611185,"relevant_blocks":[[1,3004]],"relevant_time":3004},"block_times":[0,3000,2999,3000,3000,3000,2999,2999,3000,2999,3000,2999,3000,2999,2999,3000,3000,2999,3000,3004,2994,3000],"block_sizes":[2,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,0,0],"steps":881383} +{"name":"Benchmark: 100 accounts, is_prime transactions, 1 calls","sending_time":670,"responses_span":8,"longest_confirmation_difference":1,"stats":{"estimated_tps":4.053341980462892,"estimated_sps":3572530.1771310447,"relevant_blocks":[[100,24671]],"relevant_time":24671},"block_times":[0,2998,3000,3001,2998,2999,3000,2999,3000,2999,3000,3000,2999,2999,3000,2999,3000,2999,3000,24671,3000,2999,2999,2999,2999,2999,3000,2999,3000,2999,2999,3000,2999,3000,3000,3000,2999,3003,2997,3000,2999],"block_sizes":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"steps":88137892} +{"name":"Benchmark: 1 accounts, is_prime transactions, 1 calls","sending_time":14,"responses_span":0,"longest_confirmation_difference":0,"stats":{"estimated_tps":0.3333333333333333,"estimated_sps":293794.3333333333,"relevant_blocks":[[1,3000]],"relevant_time":3000},"block_times":[0,3000,2999,2999,3000,3000,3000,2999,2999,3000,3004,2995,2999,3000,3001,2997,3000,2999,3000,3000,2999,2999],"block_sizes":[2,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,0,0],"steps":881383} +{"name":"Benchmark: 100 accounts, is_prime transactions, 1 calls","sending_time":673,"responses_span":9,"longest_confirmation_difference":1,"stats":{"estimated_tps":4.014290875516839,"estimated_sps":3538111.3564288868,"relevant_blocks":[[100,24911]],"relevant_time":24911},"block_times":[0,3000,2999,3000,3000,3000,2999,3000,2999,2999,2999,2999,3000,3004,2994,3000,2999,2999,3000,24911,2999,3000,3000,2999,3000,3000,3000,2999,3000,2999,2999,2999,2999,3000,2999,2999,2999,2999,2999,2999,2999],"block_sizes":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"steps":88137892} diff --git a/crates/benches/src/lib.rs b/crates/benches/src/lib.rs index a60d15304f..42218e5b4b 100644 --- a/crates/benches/src/lib.rs +++ b/crates/benches/src/lib.rs @@ -74,8 +74,6 @@ mod tests { // does not need proptest, as it doesn't use any input #[katana_runner::katana_test] async fn bench_default_spawn() { - runner.deploy("contracts/Scarb.toml", "contracts/scripts/auth.sh").await.unwrap(); - let fee = estimate_gas(&runner.account(0), BenchCall("spawn", vec![]), &contract_address) .unwrap(); diff --git a/crates/benches/src/spammer.rs b/crates/benches/src/spammer.rs index 2ad9910c85..9c7f3b8690 100644 --- a/crates/benches/src/spammer.rs +++ b/crates/benches/src/spammer.rs @@ -52,7 +52,7 @@ pub async fn spam_katana( // generating all needed accounts let accounts = runner.accounts(); - let wait_time = Duration::from_millis(accounts.len() as u64 * 30 + 3000 + additional_sleep); + let wait_time = Duration::from_millis(accounts.len() as u64 * 35 + 3000 + additional_sleep); let name = format!( "Benchmark: {} accounts, {} transactions, {} calls", accounts.len(), diff --git a/crates/benches/tests/heavy.rs b/crates/benches/tests/heavy.rs index acaafd8619..816ba44e89 100644 --- a/crates/benches/tests/heavy.rs +++ b/crates/benches/tests/heavy.rs @@ -31,7 +31,7 @@ async fn katana_heavy_prime_1000_a() { let arg = FieldElement::from_dec_str(&(2111u64 * 2111u64).to_string()).unwrap(); let result = - spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 180000, true) + spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 220000, true) .await; assert!(result.steps > 800000000); @@ -43,7 +43,7 @@ async fn katana_heavy_prime_1000_b() { let arg = FieldElement::from_dec_str(&(2111u64 * 2111u64).to_string()).unwrap(); let result = - spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 180000, true) + spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 220000, true) .await; assert!(result.steps > 800000000); @@ -55,7 +55,7 @@ async fn katana_heavy_prime_1000_c() { let arg = FieldElement::from_dec_str(&(2111u64 * 2111u64).to_string()).unwrap(); let result = - spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 180000, true) + spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 220000, true) .await; assert!(result.steps > 800000000); @@ -67,7 +67,7 @@ async fn katana_heavy_prime_2000_a() { let arg = FieldElement::from_dec_str(&(2111u64 * 2111u64).to_string()).unwrap(); let result = - spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 250000, true) + spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 300000, true) .await; assert!(result.steps > 1600000000); @@ -79,7 +79,7 @@ async fn katana_heavy_prime_2000_b() { let arg = FieldElement::from_dec_str(&(2111u64 * 2111u64).to_string()).unwrap(); let result = - spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 250000, true) + spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 300000, true) .await; assert!(result.steps > 1600000000); @@ -91,7 +91,7 @@ async fn katana_heavy_prime_2000_c() { let arg = FieldElement::from_dec_str(&(2111u64 * 2111u64).to_string()).unwrap(); let result = - spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 250000, true) + spam_katana(runner, contract_address, vec![BenchCall("is_prime", vec![arg])], 300000, true) .await; assert!(result.steps > 1600000000);