From a8a234c0d1a6e2b45f5e2852d0846a97c6a00133 Mon Sep 17 00:00:00 2001 From: theochap <80177219+theochap@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:57:52 +0100 Subject: [PATCH] Refactor and augment demo rollup benches (#1282) * Fix ed25519 version * Comment * Add sov-data-generators dependency + add cycle trackers for more finer performance tracking * Update Cargo.toml Co-authored-by: Nikolai Golub * Lints * Add benchmarks with prepopulated accounts + Add env variables to prover_benchmark * Update default values for prover data generation * Fixing lints --------- Co-authored-by: Nikolai Golub --- .github/workflows/rust.yml | 20 +- Cargo.lock | 2 + Cargo.toml | 2 +- examples/demo-rollup/Cargo.toml | 1 + examples/demo-rollup/benches/prover/Makefile | 22 ++ examples/demo-rollup/benches/prover/README.md | 193 ++++++++++++++---- .../demo-rollup/benches/prover/datagen.rs | 89 ++++++-- .../benches/prover/prover_bench.rs | 27 ++- .../provers/risc0/guest-celestia/Cargo.lock | 1 + .../provers/risc0/guest-mock/Cargo.lock | 1 + examples/demo-rollup/stf/Cargo.toml | 2 +- .../test-data/genesis/benchmark/accounts.json | 1 + .../test-data/genesis/benchmark/bank.json | 17 ++ .../genesis/benchmark/chain_state.json | 7 + examples/test-data/genesis/benchmark/evm.json | 25 +++ examples/test-data/genesis/benchmark/nft.json | 1 + .../genesis/benchmark/sequencer_registry.json | 9 + .../genesis/benchmark/value_setter.json | 3 + .../integration-tests/Cargo.toml | 2 +- .../sov-chain-state/Cargo.toml | 2 +- .../sov-nft-module/Cargo.toml | 2 +- module-system/sov-data-generators/Cargo.toml | 25 +++ .../sov-data-generators/src/bank_data.rs | 87 +++++++- .../sov-data-generators/src/lib.rs | 10 + .../src/value_setter_data.rs | 0 .../sov-modules-stf-blueprint/Cargo.toml | 1 + .../src/stf_blueprint.rs | 116 +++++++---- .../src/tx_verifier.rs | 12 +- module-system/sov-state/src/zk_storage.rs | 95 +++++---- .../utils/sov-data-generators/Cargo.toml | 26 --- 30 files changed, 615 insertions(+), 186 deletions(-) create mode 100644 examples/demo-rollup/benches/prover/Makefile create mode 100644 examples/test-data/genesis/benchmark/accounts.json create mode 100644 examples/test-data/genesis/benchmark/bank.json create mode 100644 examples/test-data/genesis/benchmark/chain_state.json create mode 100644 examples/test-data/genesis/benchmark/evm.json create mode 100644 examples/test-data/genesis/benchmark/nft.json create mode 100644 examples/test-data/genesis/benchmark/sequencer_registry.json create mode 100644 examples/test-data/genesis/benchmark/value_setter.json create mode 100644 module-system/sov-data-generators/Cargo.toml rename module-system/{utils => }/sov-data-generators/src/bank_data.rs (82%) rename module-system/{utils => }/sov-data-generators/src/lib.rs (93%) rename module-system/{utils => }/sov-data-generators/src/value_setter_data.rs (100%) delete mode 100644 module-system/utils/sov-data-generators/Cargo.toml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 649838e56..c9c8bccd1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -83,11 +83,11 @@ jobs: - check-demo-rollup-bash-commands - validate-packages-to-publish-yml steps: - - name: Compute whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - allowed-skips: deploy-github-pages - jobs: ${{ toJSON(needs) }} + - name: Compute whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + allowed-skips: deploy-github-pages + jobs: ${{ toJSON(needs) }} check: name: check @@ -184,6 +184,10 @@ jobs: . fuzz - name: cargo prover bench check + env: + BLOCKS: 1 + TXNS_PER_BLOCK: 10 + NUM_PUB_KEYS: 100 run: cargo bench --bench prover_bench --features bench bench_check: @@ -218,6 +222,10 @@ jobs: . fuzz - name: cargo bench check + env: + BLOCKS: 1 + TXNS_PER_BLOCK: 10 + NUM_PUB_KEYS: 100 run: cargo bench # Check that every combination of features is working properly. @@ -492,7 +500,7 @@ jobs: run: cargo run --bin bashtestmd -- --input examples/demo-rollup/README.md --output demo-rollup-readme.sh --tag test-ci - run: cat demo-rollup-readme.sh - run: chmod +x demo-rollup-readme.sh && ./demo-rollup-readme.sh - + validate-packages-to-publish-yml: runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index 1898ada31..423ca8ad9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8765,6 +8765,7 @@ dependencies = [ "sov-bank", "sov-celestia-adapter", "sov-cli", + "sov-data-generators", "sov-db", "sov-ethereum", "sov-evm", @@ -9037,6 +9038,7 @@ dependencies = [ "sov-blob-storage", "sov-chain-state", "sov-modules-api", + "sov-modules-core", "sov-rollup-interface", "sov-state", "sov-zk-cycle-macros", diff --git a/Cargo.toml b/Cargo.toml index c395f1c64..30ed41c0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ members = [ "module-system/sov-state", "module-system/sov-modules-api", "module-system/module-schemas", - "module-system/utils/sov-data-generators", + "module-system/sov-data-generators", "module-system/module-implementations/sov-accounts", "module-system/module-implementations/sov-bank", "module-system/module-implementations/sov-nft-module", diff --git a/examples/demo-rollup/Cargo.toml b/examples/demo-rollup/Cargo.toml index f516c0047..793d28b07 100644 --- a/examples/demo-rollup/Cargo.toml +++ b/examples/demo-rollup/Cargo.toml @@ -57,6 +57,7 @@ sov-evm = { path = "../../module-system/module-implementations/sov-evm", feature sov-bank = { path = "../../module-system/module-implementations/sov-bank", features = ["native"] } sov-nft-module = { path = "../../module-system/module-implementations/sov-nft-module", features = ["native"] } sov-zk-cycle-macros = { path = "../../utils/zk-cycle-macros" } +sov-data-generators = { path = "../../module-system/sov-data-generators" } humantime = "2.1" borsh = { workspace = true } diff --git a/examples/demo-rollup/benches/prover/Makefile b/examples/demo-rollup/benches/prover/Makefile new file mode 100644 index 000000000..529aa623f --- /dev/null +++ b/examples/demo-rollup/benches/prover/Makefile @@ -0,0 +1,22 @@ +# Default values for num blocks and transactions per block +BLOCKS ?= 10 +TXNS_PER_BLOCK ?= 100 +NUM_PUB_KEYS ?= 10000 +GENESIS_CONFIG_DIR ?= ../test-data/genesis/benchmark + +export BLOCKS +export TXNS_PER_BLOCK +export NUM_PUB_KEYS +export GENESIS_CONFIG_DIR + +small_bench: + @echo "Small benchmark configuration:" + @BLOCKS=2 TXNS_PER_BLOCK=10 NUM_PUB_KEYS=10 make standard_bench + +realistic_bench: + @echo "Realistic benchmark configuration:" + @BLOCKS=100 TXNS_PER_BLOCK=1000 NUM_PUB_KEYS=1000000 make standard_bench + +standard_bench: + @echo "Running benchmark with $(BLOCKS) transaction blocks, $(TXNS_PER_BLOCK) transactions per block, and $(NUM_PUB_KEYS) public keys" + @cd ../.. && cargo bench --features="bench" --bench=prover_bench \ No newline at end of file diff --git a/examples/demo-rollup/benches/prover/README.md b/examples/demo-rollup/benches/prover/README.md index cc51f25cd..2870c6b1e 100644 --- a/examples/demo-rollup/benches/prover/README.md +++ b/examples/demo-rollup/benches/prover/README.md @@ -1,17 +1,22 @@ # Prover Benchmarks -* For benchmarking the prover, we measure the number of risc0 vm cycles for each of the major functions. -* The reason for using the cycles is the assumption that proving works off a cycles/second (KHz, MHz) based on the hardware used + +- For benchmarking the prover, we measure the number of risc0 vm cycles for each of the major functions. +- The reason for using the cycles is the assumption that proving works off a cycles/second (KHz, MHz) based on the hardware used ## Running the bench -* From sovereign-sdk + +- From sovereign-sdk + ``` $ cd examples/demo-rollup/benches/prover $ cargo bench --features bench --bench prover_bench ``` ## Methodology -* We have `cycle_tracker` macro defined which can be used to annotate a function in zk that we want to measure the cycles for -* The `cycle_tracker` macro is defined at `sovereign-sdk/zk-cycle-util` + +- We have `cycle_tracker` macro defined which can be used to annotate a function in zk that we want to measure the cycles for +- The `cycle_tracker` macro is defined at `sovereign-sdk/zk-cycle-util` + ```rust #[cfg_attr(all(target_os = "zkvm", feature="bench"), cycle_tracker)] fn begin_slot(&mut self, witness: Self::Witness) { @@ -21,7 +26,9 @@ $ cargo bench --features bench --bench prover_bench )); } ``` -* The method we use to track metrics is by registering the `io_callback` syscall when creating the risc0 host. + +- The method we use to track metrics is by registering the `io_callback` syscall when creating the risc0 host. + ``` pub fn get_syscall_name_handler() -> (SyscallName, fn(&[u8]) -> Vec) { let cycle_string = "cycle_metrics\0"; @@ -48,35 +55,41 @@ pub fn get_syscall_name_handler() -> (SyscallName, fn(&[u8]) -> Vec) { default_env.io_callback(metrics_syscall_name, metrics_callback); } ``` -* The above allows us to use `risc0_zkvm::guest::env::send_recv_slice` which lets the guest pass a slice of raw bytes to host and get back a vector of bytes -* We use it to pass cycle metrics to the host -* Cycles are tracked by the macro which gets a cycle count at the beginning and end of the function + +- The above allows us to use `risc0_zkvm::guest::env::send_recv_slice` which lets the guest pass a slice of raw bytes to host and get back a vector of bytes +- We use it to pass cycle metrics to the host +- Cycles are tracked by the macro which gets a cycle count at the beginning and end of the function + ```rust let before = risc0_zkvm::guest::env::get_cycle_count(); let result = (|| #block)(); let after = risc0_zkvm::guest::env::get_cycle_count(); ``` -* We feature gate the application of the macro `cycle_tracker` with both the target_os set to `zkvm` and the feature flag `bench` -* The reason for using both is that we need conditional compilation to work in all cases -* For the purpose of this profiling we run the prover without generating the proof + +- We feature gate the application of the macro `cycle_tracker` with both the target_os set to `zkvm` and the feature flag `bench` +- The reason for using both is that we need conditional compilation to work in all cases +- For the purpose of this profiling we run the prover without generating the proof ## Input set -* Unlike demo-prover it's harder to generate fake data since all the proofs and checks need to succeed. -* This means the DA samples, hashes, signatures etc need to succeed -* To make this easier we use a static input set consisting of 3 blocks - * we avoid using empty blocks because they skew average metrics - * we have 3 blocks - * block 1 -> 1 blob containing 1 create token transaction - * block 2 -> 1 blob containing 1 transfer transaction - * block 3 -> 1 blob containing 2 transfer transactions -* This dataset is stored at `demo-prover/benches/blocks.hex` -* The dataset can be substituted with another valid dataset as well from Celestia (TBD: automate parametrized generation of blocks.hex) -* We can run this on different kinds of workloads to gauge the efficiency of different parts of the code + +- Unlike demo-prover it's harder to generate fake data since all the proofs and checks need to succeed. +- This means the DA samples, hashes, signatures etc need to succeed +- To make this easier we use a static input set consisting of 3 blocks + - we avoid using empty blocks because they skew average metrics + - we have 3 blocks + - block 1 -> 1 blob containing 1 create token transaction + - block 2 -> 1 blob containing 1 transfer transaction + - block 3 -> 1 blob containing 2 transfer transactions +- This dataset is stored at `demo-prover/benches/blocks.hex` +- The dataset can be substituted with another valid dataset as well from Celestia (TBD: automate parametrized generation of blocks.hex) +- We can run this on different kinds of workloads to gauge the efficiency of different parts of the code ## Result -* Standard hash function patched with risc0/rust_crypto -* Signature verification currently NOT patched (TBD) -* Signature verification takes about 60% of the total cycles + +- Standard hash function patched with risc0/rust_crypto +- Signature verification currently NOT patched (TBD) +- Signature verification takes about 60% of the total cycles + ``` Block stats @@ -114,20 +127,25 @@ Cycle Metrics ``` ## Custom annotations -* We can also get finer grained information by annotating low level functions, but the process for this isn't straightforward. -* For code that we control, it's as simple as adding the `cycle_tracker` annotation to our function and then feature gating it (not feature gating it causes compilation errors) -* For external dependencies, we need to fork and include a path dependency locally after annotating -* We did this for the `jmt` jellyfish merkle tree library to measure cycle gains when we use the risc0 accelerated sha function vs without -* We apply the risc0 patch in the following way in demo-prover/methods/guest/Cargo.toml + +- We can also get finer grained information by annotating low level functions, but the process for this isn't straightforward. +- For code that we control, it's as simple as adding the `cycle_tracker` annotation to our function and then feature gating it (not feature gating it causes compilation errors) +- For external dependencies, we need to fork and include a path dependency locally after annotating +- We did this for the `jmt` jellyfish merkle tree library to measure cycle gains when we use the risc0 accelerated sha function vs without +- We apply the risc0 patch in the following way in demo-prover/methods/guest/Cargo.toml + ```yaml [patch.crates-io] sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6-risc0" } ``` -* Note that the specific tag needs to be pointed to, since master and other branches don't contain acceleration + +- Note that the specific tag needs to be pointed to, since master and other branches don't contain acceleration ## Accelerated vs Non-accelerated libs -* Accelerated and risc0 optimized crypto libraries give a significant (nearly 10x) cycle gain -* With sha2 acceleration + +- Accelerated and risc0 optimized crypto libraries give a significant (nearly 10x) cycle gain +- With sha2 acceleration + ``` =====> hash: 1781 =====> hash: 1781 @@ -135,7 +153,9 @@ sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6 =====> hash: 1781 =====> hash: 1781 ``` -* Without sha2 acceleration + +- Without sha2 acceleration + ``` =====> hash: 13901 =====> hash: 13901 @@ -143,8 +163,10 @@ sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6 =====> hash: 13901 =====> hash: 13901 ``` -* Overall performance difference when using sha acceleration vs without for the same dataset (3 blocks, 4 transactions) as described above -* With sha acceleration + +- Overall performance difference when using sha acceleration vs without for the same dataset (3 blocks, 4 transactions) as described above +- With sha acceleration + ``` +-------------------------+----------------+-----------+ | Function | Average Cycles | Num Calls | @@ -157,7 +179,9 @@ sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6 +-------------------------+----------------+-----------+ Total cycles consumed for test: 20834815 ``` -* Without sha acceleration + +- Without sha acceleration + ``` +-------------------------+----------------+-----------+ | Function | Average Cycles | Num Calls | @@ -170,10 +194,12 @@ Total cycles consumed for test: 20834815 +-------------------------+----------------+-----------+ Total cycles consumed for test: 26152702 ``` -* There's an overall efficiency of 6 million cycles in total for 3 blocks. -* Keep in mind that the above table shows average number of cycles per call, so they give an efficiency per call, but the "Total cycles consumed for test" metric at the bottom shows total for 3 blocks -* With ed25519 acceleration +- There's an overall efficiency of 6 million cycles in total for 3 blocks. +- Keep in mind that the above table shows average number of cycles per call, so they give an efficiency per call, but the "Total cycles consumed for test" metric at the bottom shows total for 3 blocks + +- With ed25519 acceleration + ``` +----------------------+---------------------+----------------------+----------+-----------+ | Function | Avg Cycles w/o Accel | Avg Cycles w/ Accel | % Change | Num Calls | @@ -196,10 +222,91 @@ Total cycles consumed for test: 26152702 +----------------------+----------------------+---------------------+----------+-----------+ ``` -* We can see a ~4x speedup for the `verify` function when using risc0 accelerated ed25519-dalek patch + +- We can see a ~4x speedup for the `verify` function when using risc0 accelerated ed25519-dalek patch + ``` [patch.crates-io] sha2 = { git = "https://github.com/risc0/RustCrypto-hashes", tag = "sha2/v0.10.6-risc0" } ed25519-dalek = { git = "https://github.com/risc0/curve25519-dalek", tag = "curve25519-4.1.0-risczero.1" } crypto-bigint = {git = "https://github.com/risc0/RustCrypto-crypto-bigint", tag = "v0.5.2-risc0"} -``` \ No newline at end of file +``` + +## Augmented input set + +- In order to increase the accuracy of the benchmarks, and get estimates closer to real use-cases, we have integrated the data-generation module `sov-data-generators`, to be able to generate transaction data more easily. We have added cycle-tracking methods to have a finer understanding of the system's performances. + +For our benchmark, we have used two block types: + +- block 1 -> 1 blob containing 1 create token transaction +- block 2 -> 1 blob containing 100 transfer transaction to random addresses, repeated 10 times + +Here are the results (including ed25519 acceleration): + +### Block Stats + +| Description | Value | +| ---------------------------------------- | ----- | +| Total blocks | 11 | +| Blocks with transactions | 11 | +| Number of blobs | 11 | +| Total number of transactions | 1001 | +| Average number of transactions per block | 91 | + +### Cycle Metrics + +| Function | Average Cycles | Num Calls | +| -------------------- | -------------- | --------- | +| Cycles per block | 78,058,312 | 11 | +| apply_blob | 74,186,372 | 11 | +| pre_process_batch | 71,891,297 | 11 | +| verify_txs_stateless | 71,555,628 | 11 | +| apply_txs | 2,258,064 | 11 | +| end_slot | 2,008,051 | 11 | +| jmt_verify_update | 1,086,936 | 11 | +| jmt_verify_existence | 792,805 | 11 | +| verify | 734,681 | 1001 | +| decode_txs | 238,998 | 11 | +| begin_slot | 98,566 | 11 | +| deserialize_batch | 88,472 | 11 | +| deserialize | 23,515 | 1001 | +| hash | 5,556 | 1001 | +| commit | 7 | 11 | + +**Total cycles consumed for test: 858,641,427** + +## Benchmarks with prepopulated accounts + +Now we compare these results by prepopulating the accounts module with 1M accounts. + +### Block Stats + +| Description | Value | +| ---------------------------------------- | ----- | +| Total blocks | 11 | +| Blocks with transactions | 11 | +| Number of blobs | 11 | +| Total number of transactions | 1001 | +| Average number of transactions per block | 91 | + +### Cycle Metrics + +| Function | Average Cycles | Num Calls | +| -------------------- | -------------- | --------- | +| Cycles per block | 82,501,342 | 11 | +| apply_blob | 73,774,539 | 11 | +| pre_process_batch | 71,614,640 | 11 | +| verify_txs_stateless | 71,203,340 | 11 | +| end_slot | 5,277,919 | 11 | +| jmt_verify_update | 3,007,153 | 11 | +| jmt_verify_existence | 2,143,099 | 11 | +| apply_txs | 2,120,704 | 11 | +| verify | 731,327 | 1001 | +| decode_txs | 308,557 | 11 | +| begin_slot | 184,097 | 11 | +| deserialize_batch | 82,908 | 11 | +| deserialize | 24,004 | 1001 | +| hash | 5,852 | 1001 | +| commit | 7 | 11 | + +**Total cycles consumed for test: 907,514,763** diff --git a/examples/demo-rollup/benches/prover/datagen.rs b/examples/demo-rollup/benches/prover/datagen.rs index 77da4bed2..6171cad95 100644 --- a/examples/demo-rollup/benches/prover/datagen.rs +++ b/examples/demo-rollup/benches/prover/datagen.rs @@ -1,25 +1,86 @@ +use std::env; +use std::fs::File; +use std::io::BufWriter; +use std::path::Path; + +use serde::Serialize; +use sov_data_generators::bank_data::BankMessageGenerator; +use sov_data_generators::MessageGenerator; +use sov_demo_rollup::MockDemoRollup; use sov_mock_da::{MockAddress, MockBlock, MockDaService}; -use sov_rng_da_service::{generate_create_token_payload, generate_transfers}; +use sov_modules_api::default_signature::private_key::DefaultPrivateKey; +use sov_modules_api::default_signature::DefaultPublicKey; +use sov_modules_api::PrivateKey; use sov_rollup_interface::services::da::DaService; -pub async fn get_bench_blocks() -> Vec { - let da_service = MockDaService::new(MockAddress::default()); +#[derive(Serialize)] +struct AccountsData { + pub_keys: Vec, +} + +const DEFAULT_BLOCKS: u64 = 10; +const DEFAULT_TXNS_PER_BLOCK: u64 = 100; +const DEFAULT_NUM_PUB_KEYS: u64 = 1000; + +pub fn generate_genesis_config(config_dir: &str) -> anyhow::Result<()> { + let num_pub_keys = match env::var("NUM_PUB_KEYS") { + Ok(num_pub_keys_str) => num_pub_keys_str.parse::()?, + Err(_) => { + println!("NUM_PUB_KEYS not set, using default"); + DEFAULT_NUM_PUB_KEYS + } + }; + + let file = File::create(Path::join(Path::new(config_dir), "accounts.json")).unwrap(); + let accounts_pub_keys: Vec<_> = (0..num_pub_keys) + .map(|_| { + let pkey = DefaultPrivateKey::generate(); + pkey.pub_key() + }) + .collect(); + + let data = AccountsData { + pub_keys: accounts_pub_keys, + }; + + let data_buf = BufWriter::new(file); + Ok(serde_json::ser::to_writer(data_buf, &data)?) +} + +pub async fn get_bench_blocks() -> anyhow::Result> { + let txns_per_block = match env::var("TXNS_PER_BLOCK") { + Ok(txns_per_block) => txns_per_block.parse::()?, + Err(_) => { + println!("TXNS_PER_BLOCK not set, using default"); + DEFAULT_TXNS_PER_BLOCK + } + }; + let block_cnt = match env::var("BLOCKS") { + Ok(block_cnt_str) => block_cnt_str.parse::()?, + Err(_) => { + println!("BLOCKS not set, using default"); + DEFAULT_BLOCKS + } + }; + + let da_service = MockDaService::new(MockAddress::default()); let mut blocks = vec![]; - let blob = generate_create_token_payload(0); + + let create_token_message_gen = BankMessageGenerator::default_generate_create_token(); + let blob = create_token_message_gen.create_blobs::<::NativeRuntime>(); da_service.send_transaction(&blob).await.unwrap(); let block1 = da_service.get_block_at(1).await.unwrap(); blocks.push(block1); - let blob = generate_transfers(3, 1); - da_service.send_transaction(&blob).await.unwrap(); - let block2 = da_service.get_block_at(2).await.unwrap(); - blocks.push(block2); - - let blob = generate_transfers(10, 4); - da_service.send_transaction(&blob).await.unwrap(); - let block2 = da_service.get_block_at(3).await.unwrap(); - blocks.push(block2); + let create_transfer_message_gen = + BankMessageGenerator::default_generate_random_transfers(txns_per_block); + for i in 0..block_cnt { + let blob = create_transfer_message_gen.create_blobs::<::NativeRuntime>(); + da_service.send_transaction(&blob).await.unwrap(); + let blocki = da_service.get_block_at(2 + i).await.unwrap(); + blocks.push(blocki); + } - blocks + Ok(blocks) } diff --git a/examples/demo-rollup/benches/prover/prover_bench.rs b/examples/demo-rollup/benches/prover/prover_bench.rs index 5d7755760..7bbaa812c 100644 --- a/examples/demo-rollup/benches/prover/prover_bench.rs +++ b/examples/demo-rollup/benches/prover/prover_bench.rs @@ -36,7 +36,7 @@ use sov_state::DefaultStorageSpec; use sov_stf_runner::{from_toml_path, read_json_file, RollupConfig}; use tempfile::TempDir; -use crate::datagen::get_bench_blocks; +use crate::datagen::{generate_genesis_config, get_bench_blocks}; #[derive(Debug)] struct RegexAppender { @@ -44,6 +44,8 @@ struct RegexAppender { file: Arc>, } +const DEFAULT_GENESIS_CONFIG_DIR: &str = "../test-data/genesis/benchmark"; + impl RegexAppender { fn new(pattern: &str, file_path: &str) -> Self { if Path::new(file_path).exists() { @@ -151,6 +153,14 @@ async fn main() -> Result<(), anyhow::Error> { } } + let genesis_conf_dir = match env::var("GENESIS_CONFIG_DIR") { + Ok(dir) => dir, + Err(_) => { + println!("GENESIS_CONFIG_DIR not set, using default"); + String::from(DEFAULT_GENESIS_CONFIG_DIR) + } + }; + let rollup_config_path = "benches/prover/rollup_config.toml".to_string(); let mut rollup_config: RollupConfig = from_toml_path(rollup_config_path) .context("Failed to read rollup configuration") @@ -173,13 +183,16 @@ async fn main() -> Result<(), anyhow::Error> { .expect("ProverStorageManager initialization has failed"); let stf = BenchSTF::new(); + generate_genesis_config(genesis_conf_dir.as_str())?; + let genesis_config = { - let integ_test_conf_dir: &Path = "../test-data/genesis/integration-tests".as_ref(); - let rt_params = - get_genesis_config::(&GenesisPaths::from_dir(integ_test_conf_dir)) - .unwrap(); + let rt_params = get_genesis_config::(&GenesisPaths::from_dir( + genesis_conf_dir.as_str(), + )) + .unwrap(); - let chain_state = read_json_file(integ_test_conf_dir.join("chain_state.json")).unwrap(); + let chain_state = + read_json_file(Path::new(genesis_conf_dir.as_str()).join("chain_state.json")).unwrap(); let kernel_params = BasicKernelGenesisConfig { chain_state }; GenesisParams { runtime: rt_params, @@ -202,7 +215,7 @@ async fn main() -> Result<(), anyhow::Error> { storage_manager.finalize(&genesis_block.header).unwrap(); // TODO: Fix this with genesis logic. - let blocks = get_bench_blocks().await; + let blocks = get_bench_blocks().await?; for filtered_block in &blocks { num_blocks += 1; diff --git a/examples/demo-rollup/provers/risc0/guest-celestia/Cargo.lock b/examples/demo-rollup/provers/risc0/guest-celestia/Cargo.lock index 824fc0e8a..400bea68e 100644 --- a/examples/demo-rollup/provers/risc0/guest-celestia/Cargo.lock +++ b/examples/demo-rollup/provers/risc0/guest-celestia/Cargo.lock @@ -2212,6 +2212,7 @@ dependencies = [ "sov-blob-storage", "sov-chain-state", "sov-modules-api", + "sov-modules-core", "sov-rollup-interface", "sov-state", "thiserror", diff --git a/examples/demo-rollup/provers/risc0/guest-mock/Cargo.lock b/examples/demo-rollup/provers/risc0/guest-mock/Cargo.lock index a0155317f..47200d87a 100644 --- a/examples/demo-rollup/provers/risc0/guest-mock/Cargo.lock +++ b/examples/demo-rollup/provers/risc0/guest-mock/Cargo.lock @@ -1157,6 +1157,7 @@ dependencies = [ "sov-blob-storage", "sov-chain-state", "sov-modules-api", + "sov-modules-core", "sov-rollup-interface", "sov-state", "sov-zk-cycle-macros", diff --git a/examples/demo-rollup/stf/Cargo.toml b/examples/demo-rollup/stf/Cargo.toml index bfe36358d..47934fb96 100644 --- a/examples/demo-rollup/stf/Cargo.toml +++ b/examples/demo-rollup/stf/Cargo.toml @@ -48,7 +48,7 @@ sov-evm = { path = "../../../module-system/module-implementations/sov-evm", opti demo-stf = { path = ".", features = ["native"] } tempfile = { workspace = true } rand = { workspace = true } -sov-data-generators = { path = "../../../module-system/utils/sov-data-generators" } +sov-data-generators = { path = "../../../module-system/sov-data-generators" } sov-mock-zkvm = { path = "../../../adapters/mock-zkvm" } sov-prover-storage-manager = { path = "../../../full-node/sov-prover-storage-manager", features = ["test-utils"] } diff --git a/examples/test-data/genesis/benchmark/accounts.json b/examples/test-data/genesis/benchmark/accounts.json new file mode 100644 index 000000000..33120351b --- /dev/null +++ b/examples/test-data/genesis/benchmark/accounts.json @@ -0,0 +1 @@ +{"pub_keys":["ef9a976bd15a3c03ce3724cd1dc157a68cd5ac4bc7f0a5ddd93cb85a1afe6f31","5ce85246565dffeae49501857798fe51b45a05990f6e9abfce7cd657d81a9e23","9724083b1cd97d6c7a4965c173fd04102dbb179917febf4cfcba9c750587a88f","01be91dec3da20185894100a5c938ab5ec937e2c5ac57a2345ef3381e7234aac","58686a2f9a60426ab03635ae214564b9500e3f2027fbb36d339ecaeeb31a444a","456747c9917bf6ab3773c17ba5b04af4d9adfc02fe0c901bd7bc86e44c19672c","0b991f025b7a9fe95483921c236d8d06ab82f5327ab9c3a7103142242df0e276","29bff255ed5eb9a36af2137c1b45858611197d37ab094299d2755baa69fed2ae","923c23104a6c153783dd549fb0e0810acded2da5c8cac7c5539cd93a72c502c4","bb81ae26ae53083613134604db8163297b94439e61a7e5be3a583b09e8d60624","32e2dea36a505669040b884c95c5f26fb11cff56d3daaef7de2f635f821784b0","0a9978f06537ab89d02b0435fff1008864c713a3fea497a1186c3b79dd727f95","a0a912ce8a40b7e6aae360b70d28ad72ef29806addcd153df59e39e79070d813","0f7a72092231c98438506a95a7535b589dc59ac431ba97dbe86381da679cd579","66984f0fa0db06061ec256840cb029dfac7971c2d8f0eb5f9bddff2368a420da","bdf03072d32b08e3586e74856df757be549f2caacffa2d3487e8924c7572f819","93f71d183b262624b6a675829dfee37ceec4cf5dba25ab426cf4094744052df2","cfd9ea4adf2097a485b28e431bb396bbc384ab07d3465203fa205bd7d1df21ee","35634c2e64f439d854ae5536279b79f173c2866441fb10478590f3a25bed211b","ec72ca4c2bce81a94da44c842e72978a7c490bf7dfac533f7c4c4672cf9fb12a","f93871e6ff1d3a5ca9b2c2c02e49af228e524f8a55c9a7987434fd2a8b3da16d","3541d02623afcc16d1754b85d718898020ff3fd6507d9c8b81c577c6cd60ae54","a823caef073513ac27b0ebf241b570d3ccbcf7d9d8730f7da44803c95e0916c2","91ae5754eefd6300778628087738b21388f8633849e135b66d3256e460ca366b","411696003f4c4b6b7d188c72b282302f5fa1914702eb260f32b81a38d8983d7b","1d285bd04de6a02801ebe26caa0dc49040ee212b403357b14bc58c907baa4173","b2f5545641e93757a873dcfad5a062f7af196295a3cba298cdcd9d6fa64fd103","9772eadd16e12845d553e8064dab04eedec80388562ebf1103a63514d7c555a3","36ece635f223a1fb69ee199ec309acd6786ecc2a99095ca4139f9b1c9c78c2bf","7150ae7012967088389b64c3bbdb2cb56e305b5aaaa0ce4e14d3bf2580f21eac","85eabac16d575f2417929302041c511ccdcd48449abfc21d5d15ef661a4ed2e2","60fb354db91b0b03141f0f3049070c2b275b97e38f2dc828aedd4edaeefaaf2d","d10f44d41deffd3ab58176d4175f28177e00165449cf3f889168cb1a581efdd9","1ffb54cb9d0675a7bee4971b9f4b306988cf2629e34d7aca67be4c4c6e3f4102","e435a7c8ddfb5a23b11db362b301cccd67894c2def5c6a5ad924c4de1bff4780","0a8b49f9c921402375a0cb474de564320e47df1f5599541116d966eeef15331c","dc1b7fbb68d2ec8424ffa9b4ceb6acbbc2012d3de1602e84cf3f9325fc90328c","6af0557577e913c9e1aed08de7b13d20377e7e1ade2e837ba38fa9afcd878a49","acee040cda210e7144aabe5e8fbad840882fd63e892148bf314435621302cfe9","0c525007766925c1baad98abcfde9bf71c3b3585513c00ecc964aeb76fe4b3ce","f87fa3899179b3ebf447596b676c96a2266fa832d148e7e690fbdeccc4b1f73a","1f59e1618f1d983998aa7bed6391538941ef8d94b01265efe754044781508004","2038ed260f3667d53324cefca77ba491c0c97f14b81d153fde10839f3b4f4fe3","fdc9415f2cddc6c95a969d2e79a6361ad5750de5efda444bcb53bf71ad559c36","76f73bcad97c740fdfa5ccb12647801937c96575177406ae7d64d3a9a54d8099","b640535ac6baeb975e9ff0e956b5a94b7d00cc5c59f94fd2f954051465a445b4","dadca2a91ff0d89b22c6696bf2a8253b1d3b285a1bd60da4384573c4bf61b455","0f11bdc5bf545a552114709f023326779f3a1dff0a522f8c8d7f1316512957cf","4032ddd455801c32c43888848e901b7eb032567a832350fea15ac616e135f62a","a8c62d079688e8aa3b46e14939502c245951138c859b09deefaf1aa722e968ab","84e9de6ea30017e99dc991b617a9932e88fc67769ac62b6c5655b803f5288513","0e178b50a0fe9073dca11696e1f6266c4bf256a5b32d56d8170c9d3c84ee59aa","594250af9e58ef7c942c286fcd3c1e1abd1d26e175093c5b5b3795842240d4b3","322bf49c6947aba4c1968523f3926eba64065cf6166995de4c7a28be1b5e1f5a","0c1ca0b1cf7cdfe721178d86d10eefbf0f5f9b808958f778caefecd679851329","b9cdd0dd208a88017c1ddcf6e0d9de7a72039c559409dbc59cf5574565981972","fc574d21e5679b119d48c1c331973e3ba4a06a27da8cd2cdbb3c164771e803d7","5befb5e155d008c47c1f16e598d3cb6993585d9e45c1b561a4e7996e2186cc32","34dcc09d557799eb6f3e69ebfbe2bef386221f6e7c6e5d438ffb496eb132e6e1","3730486135ca59f82b528f28106fddaa85a8c6fc17a5eb21280acf32f51c64dd","2f87680bde60406a46aa08599972e6fe0f6f899b67c0af572acef0dfd85209dc","4ff269e2f0c4bafe87894f7434cdb4874f64f2ce1d0304b286f9554939d13a69","9c276fe32b67427ed915468d32615163c59bb50ef38305b2d8789301b13fd81d","51e1542ebeef79a5e85b7cfbccbc125e0f530aab9bcfe933a94c4a4815c4a6e0","671c01ec06cce8f6e3cdd56970535bb599c383f03a63a870315e2b445ff9558a","e2031163f857691568e45c49a1a28fa17d2a4d1c7b4149aa9ee7c1166d83aa4f","4d70a9b83791720988fc08059b302873aae86dfbb41b5ef0ae5e2c953580dcba","d53bb423e79e05e75e3485ce869cd0333a176c72c8b6ab9ed924e8d3c00e1f93","7c5f8623df0d6a90f701dff19fec7137472d2b2fdab122e2d274c78a075a49e7","df7cfc2f42d085b6cdf326fd6ac47fc0c1bdc2051c6710b60f0c9be17dbc83c6","97eca276973c8f70e12a18ca35be27eb66e747c51aedf7c161c5db39b73e3693","47a9c26f4b836b21f4a5f8b5ebb92ba86d7acb604e5d13504082c3e9be165eec","4ba34f3f118e3f71adfe354107e896a795782f2c95a3381d0315f5f24c6dafab","26614786da9351028312d535d9245af4a24aea7fc452ca4553a1976c558876da","ff90cb39c600db33b06c417fadb25dbeee248e1062791ef518a5d55bca0d5493","f972307f956cc3d1b47e6510fcc960cf12e424293f6e0f83bb2b856b6655a765","00e18cc51ffa72f7e2843e05815464b09b0f224867c1d590931ea7dfb29bb4fb","f2df33363bd7d10c2787e42c830d4a2d5d3873e0cc4ddc8d5c659c0b7ea42470","75d2a52cfb95f982032e1faf0bcc9531fbb810f8e263334b0fb4c59c84665ec3","832deb165ed1af5ac142fd4475d4f262b918c7a49072b175759edfce2c47a288","639cab1ff0db19645820dd154ce0f40c22ce70f744792c167a7d01444f8e618a","371c33b3991801f8ce6a304dfa10b4e55805152fe68f300438c4b318ff8bb248","80bd2c9e6f271bf7cd65dab50f33e0eb0e9f0d50bfcc4f0ad63eb99c0de162b6","5263b3878c28a46d16238647808f00f12ab398f308ea3136be3cb7ea0a8f306c","af39755d55fd32554e513b1fe673d03d1b07209d17a16c2cf64caaef2e9fcc92","f6bc6bbe4f3b9a90465dabbb7d03687fe72a0e102be16cd6f11cea45ee5dc662","507cff2fec32364929e066484b2fdb8c90d8e978bee37e2fd8f1a50483330347","0936591a6c113982f6cdaa0bf0aa6b1e9c7d7017c60d59e50c27161537d34fbc","4bdf17a4a16522bda11cfefeaf9bc58041302fc26d760f032ae414e05634fa40","dca3af087624844c419113b9ac1a5f86b38813204082858be8bd9ad169c4185a","ac1a21eacbd9b4a9ca0f876cb2c5d8d1b2baad7c0e85d6c1c50a248726b2c686","126fd6a00474ea5f09ef850fb3bcf1aab7754f3a2c7a0388d167debb2d440da6","25172c51dd51c7261967fd9e76fcf232887774a0e50f2e4fbb05d7b62e966123","6d19a6a703d3d9c9c57e11bbc99904def4af3a01fbf7f203a2debd44dbc2a381","6d0dae9e583678fb044eb026a1d977de1f4abe62b990a08827ea597dca322c92","843ba282ada316256eca702e1122ec629b27f0cad6e4f3203ea1ed89cb26d7da","c21a0f99f03aa5c58f8e973e82d0d15d4d5b3d0140e6f453cc6127859be5c367","33d9eda534e38de28c602285bbcd065f61ce44a2d96184741bdd6d4d4b9a301d","b482f47802a0f6b7a7702f57715adcdd5fcb5fc61972741aea1b667702f48f8c","b34d81ba04046c938c543345dd78c90c8393f6b6fe22007c3edd9e510ca27a6d"]} \ No newline at end of file diff --git a/examples/test-data/genesis/benchmark/bank.json b/examples/test-data/genesis/benchmark/bank.json new file mode 100644 index 000000000..d03fcc10f --- /dev/null +++ b/examples/test-data/genesis/benchmark/bank.json @@ -0,0 +1,17 @@ +{ + "tokens": [ + { + "token_name": "sov-demo-token", + "address_and_balances": [ + [ + "sov1l6n2cku82yfqld30lanm2nfw43n2auc8clw7r5u5m6s7p8jrm4zqrr8r94", + 100000000 + ] + ], + "authorized_minters": [ + "sov1l6n2cku82yfqld30lanm2nfw43n2auc8clw7r5u5m6s7p8jrm4zqrr8r94" + ], + "salt": 0 + } + ] +} diff --git a/examples/test-data/genesis/benchmark/chain_state.json b/examples/test-data/genesis/benchmark/chain_state.json new file mode 100644 index 000000000..d439a6854 --- /dev/null +++ b/examples/test-data/genesis/benchmark/chain_state.json @@ -0,0 +1,7 @@ +{ + "initial_slot_height": 0, + "current_time": { + "secs": 0, + "nanos": 0 + } +} diff --git a/examples/test-data/genesis/benchmark/evm.json b/examples/test-data/genesis/benchmark/evm.json new file mode 100644 index 000000000..7f5b2b108 --- /dev/null +++ b/examples/test-data/genesis/benchmark/evm.json @@ -0,0 +1,25 @@ +{ + "data": [ + { + "address": "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", + "balance": "0xffffffffffffffff", + "code_hash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "code": "0x", + "nonce": 0 + } + ], + "chain_id": 1, + "limit_contract_code_size": null, + "spec": { + "0": "SHANGHAI" + }, + "coinbase": "0x0000000000000000000000000000000000000000", + "starting_base_fee": 7, + "block_gas_limit": 30000000, + "genesis_timestamp": 0, + "block_timestamp_delta": 1, + "base_fee_params": { + "max_change_denominator": 8, + "elasticity_multiplier": 2 + } +} diff --git a/examples/test-data/genesis/benchmark/nft.json b/examples/test-data/genesis/benchmark/nft.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/examples/test-data/genesis/benchmark/nft.json @@ -0,0 +1 @@ +{} diff --git a/examples/test-data/genesis/benchmark/sequencer_registry.json b/examples/test-data/genesis/benchmark/sequencer_registry.json new file mode 100644 index 000000000..fe2696ae5 --- /dev/null +++ b/examples/test-data/genesis/benchmark/sequencer_registry.json @@ -0,0 +1,9 @@ +{ + "seq_rollup_address": "sov1l6n2cku82yfqld30lanm2nfw43n2auc8clw7r5u5m6s7p8jrm4zqrr8r94", + "seq_da_address": "0000000000000000000000000000000000000000000000000000000000000000", + "coins_to_lock": { + "amount": 50, + "token_address": "sov1zsnx7n2wjvtkr0ttscfgt06pjca3v2e6stxeu49qwynavmk7a8xqlxkkjp" + }, + "is_preferred_sequencer": true +} diff --git a/examples/test-data/genesis/benchmark/value_setter.json b/examples/test-data/genesis/benchmark/value_setter.json new file mode 100644 index 000000000..4e209004a --- /dev/null +++ b/examples/test-data/genesis/benchmark/value_setter.json @@ -0,0 +1,3 @@ +{ + "admin": "sov1l6n2cku82yfqld30lanm2nfw43n2auc8clw7r5u5m6s7p8jrm4zqrr8r94" +} diff --git a/module-system/module-implementations/integration-tests/Cargo.toml b/module-system/module-implementations/integration-tests/Cargo.toml index ec91c155c..36491011b 100644 --- a/module-system/module-implementations/integration-tests/Cargo.toml +++ b/module-system/module-implementations/integration-tests/Cargo.toml @@ -23,7 +23,7 @@ sov-state = { path = "../../sov-state", features = ["native"] } sov-mock-zkvm = { path = "../../../adapters/mock-zkvm" } sov-schema-db = { path = "../../../full-node/db/sov-schema-db" } -sov-data-generators = { path = "../../utils/sov-data-generators" } +sov-data-generators = { path = "../../sov-data-generators" } sov-rollup-interface = { path = "../../../rollup-interface", features = ["native"] } sov-mock-da = { path = "../../../adapters/mock-da", features = ["native"] } sov-modules-stf-blueprint = { path = "../../sov-modules-stf-blueprint", features = ["native"] } diff --git a/module-system/module-implementations/sov-chain-state/Cargo.toml b/module-system/module-implementations/sov-chain-state/Cargo.toml index 0ed6809fe..93ae18be2 100644 --- a/module-system/module-implementations/sov-chain-state/Cargo.toml +++ b/module-system/module-implementations/sov-chain-state/Cargo.toml @@ -23,7 +23,7 @@ sov-state = { path = "../../sov-state", version = "0.3" } [dev-dependencies] tempfile = { workspace = true } -sov-data-generators = { path = "../../utils/sov-data-generators" } +sov-data-generators = { path = "../../sov-data-generators" } sov-chain-state = { path = ".", features = ["native"] } sov-mock-da = { path = "../../../adapters/mock-da" } sov-prover-storage-manager = { path = "../../../full-node/sov-prover-storage-manager", features = ["test-utils"] } diff --git a/module-system/module-implementations/sov-nft-module/Cargo.toml b/module-system/module-implementations/sov-nft-module/Cargo.toml index c96f8a550..a4bcceea8 100644 --- a/module-system/module-implementations/sov-nft-module/Cargo.toml +++ b/module-system/module-implementations/sov-nft-module/Cargo.toml @@ -29,7 +29,7 @@ tracing = { workspace = true, optional = true } [dev-dependencies] sov-nft-module = { version = "*", features = ["native"], path = "." } tempfile = { workspace = true } -sov-data-generators = { path = "../../utils/sov-data-generators" } +sov-data-generators = { path = "../../sov-data-generators" } sov-prover-storage-manager = { path = "../../../full-node/sov-prover-storage-manager", features = ["test-utils"] } diff --git a/module-system/sov-data-generators/Cargo.toml b/module-system/sov-data-generators/Cargo.toml new file mode 100644 index 000000000..b79c3176e --- /dev/null +++ b/module-system/sov-data-generators/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "sov-data-generators" +description = "A set of generator utils used to automatically produce and serialize transaction data" +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +version = { workspace = true } +resolver = "2" +publish = false + +[dependencies] +sov-modules-api = { path = "../sov-modules-api", features = ["native"] } +sov-modules-stf-blueprint = { path = "../sov-modules-stf-blueprint", features = ["native"] } +sov-value-setter = { path = "../module-implementations/examples/sov-value-setter", features = ["native"] } +sov-bank = { path = "../module-implementations/sov-bank", features = ["native"] } +sov-state = { path = "../sov-state" } +sov-mock-da = { path = "../../adapters/mock-da", features = ["native"] } + +borsh = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/module-system/utils/sov-data-generators/src/bank_data.rs b/module-system/sov-data-generators/src/bank_data.rs similarity index 82% rename from module-system/utils/sov-data-generators/src/bank_data.rs rename to module-system/sov-data-generators/src/bank_data.rs index 5089dee91..7a0e6b895 100644 --- a/module-system/utils/sov-data-generators/src/bank_data.rs +++ b/module-system/sov-data-generators/src/bank_data.rs @@ -5,7 +5,7 @@ use sov_modules_api::default_context::DefaultContext; use sov_modules_api::default_signature::private_key::DefaultPrivateKey; use sov_modules_api::transaction::Transaction; use sov_modules_api::utils::generate_address; -use sov_modules_api::{Context, EncodeCall, Module, PrivateKey, Spec}; +use sov_modules_api::{Context, EncodeCall, Module, PrivateKey, PublicKey, Spec}; use crate::{Message, MessageGenerator}; @@ -36,6 +36,7 @@ const DEFAULT_PVT_KEY: &str = "236e80cb222c4ed0431b093b3ac53e6aa7a2273fe1f4351cd const DEFAULT_CHAIN_ID: u64 = 0; const DEFAULT_GAS_TIP: u64 = 0; const DEFAULT_GAS_LIMIT: u64 = 0; +const DEFAULT_INIT_BALANCE: u64 = 1000000; pub fn get_default_token_address() -> ::Address { let minter_key = DefaultPrivateKey::from_hex(DEFAULT_PVT_KEY).unwrap(); @@ -60,13 +61,13 @@ impl Default for BankMessageGenerator { salt, initial_balance: 1000, minter_address, - minter_pkey: Rc::new(minter_key), + minter_pkey: Rc::new(minter_key.clone()), authorized_minters: Vec::from([minter_address]), }; Self { token_mint_txs: Vec::from([mint_data]), transfer_txs: Vec::from([TransferData { - sender_pkey: Rc::new(DefaultPrivateKey::from_hex(DEFAULT_PVT_KEY).unwrap()), + sender_pkey: Rc::new(minter_key), transfer_amount: 15, receiver_address: generate_address::("just_receiver"), token_address: get_token_address::( @@ -80,6 +81,86 @@ impl Default for BankMessageGenerator { } impl BankMessageGenerator { + /// Gets the default sender address and private key. + fn default_address_with_pkey() -> (::Address, DefaultPrivateKey) { + let pkey = DefaultPrivateKey::from_hex(DEFAULT_PVT_KEY).unwrap(); + let address = pkey.default_address(); + (address, pkey) + } + + /// Generates random transfers between the default sender and random receivers for default token parameters. + pub fn default_generate_random_transfers(n: u64) -> Self { + Self::generate_random_transfers( + n, + DEFAULT_TOKEN_NAME.to_owned(), + DEFAULT_SALT, + DefaultPrivateKey::from_hex(DEFAULT_PVT_KEY).unwrap(), + ) + } + + /// Generates random transfers between the default sender and random receivers. + pub fn generate_random_transfers( + n: u64, + token_name: String, + salt: u64, + sender_pk: DefaultPrivateKey, + ) -> Self { + let sa = sender_pk.default_address(); + let token_address = + sov_bank::get_token_address::(token_name.as_str(), sa.as_ref(), salt); + + let mut transfer_txs = vec![]; + for _ in 1..(n + 1) { + let priv_key = DefaultPrivateKey::generate(); + let address: ::Address = priv_key.pub_key().to_address(); + + transfer_txs.push(TransferData { + sender_pkey: Rc::new(sender_pk.clone()), + receiver_address: address, + token_address, + transfer_amount: 1, + }); + } + + BankMessageGenerator { + token_mint_txs: vec![], + transfer_txs, + } + } + + /// Generates a create token transaction for default token parameters. + pub fn default_generate_create_token() -> Self { + let (minter_address, pk) = Self::default_address_with_pkey(); + Self::generate_create_token( + DEFAULT_TOKEN_NAME.to_owned(), + DEFAULT_SALT, + pk.into(), + vec![minter_address], + DEFAULT_INIT_BALANCE, + ) + } + + /// Generates a create token transaction. + pub fn generate_create_token( + token_name: String, + salt: u64, + minter_pkey: std::rc::Rc, + authorized_minters: Vec<::Address>, + initial_balance: u64, + ) -> Self { + Self { + token_mint_txs: vec![MintData { + token_name, + salt, + initial_balance, + minter_address: minter_pkey.default_address(), + minter_pkey, + authorized_minters, + }], + transfer_txs: vec![], + } + } + pub fn create_invalid_transfer() -> Self { let minter_key = DefaultPrivateKey::from_hex(DEFAULT_PVT_KEY).unwrap(); let minter_address = minter_key.default_address(); diff --git a/module-system/utils/sov-data-generators/src/lib.rs b/module-system/sov-data-generators/src/lib.rs similarity index 93% rename from module-system/utils/sov-data-generators/src/lib.rs rename to module-system/sov-data-generators/src/lib.rs index 8cde62ae0..8465ff2f6 100644 --- a/module-system/utils/sov-data-generators/src/lib.rs +++ b/module-system/sov-data-generators/src/lib.rs @@ -123,4 +123,14 @@ pub trait MessageGenerator { } serialized_messages } + + fn create_blobs>(&self) -> Vec { + let txs: Vec> = self + .create_raw_txs::() + .into_iter() + .map(|tx| tx.data) + .collect(); + + txs.try_to_vec().unwrap() + } } diff --git a/module-system/utils/sov-data-generators/src/value_setter_data.rs b/module-system/sov-data-generators/src/value_setter_data.rs similarity index 100% rename from module-system/utils/sov-data-generators/src/value_setter_data.rs rename to module-system/sov-data-generators/src/value_setter_data.rs diff --git a/module-system/sov-modules-stf-blueprint/Cargo.toml b/module-system/sov-modules-stf-blueprint/Cargo.toml index 11f83e81c..0cacdf4e3 100644 --- a/module-system/sov-modules-stf-blueprint/Cargo.toml +++ b/module-system/sov-modules-stf-blueprint/Cargo.toml @@ -22,6 +22,7 @@ hex = { workspace = true } sov-rollup-interface = { path = "../../rollup-interface", version = "0.3" } sov-state = { path = "../sov-state", version = "0.3" } +sov-modules-core = { path = "../sov-modules-core", version = "0.3" } sov-modules-api = { path = "../sov-modules-api", version = "0.3" } sov-zk-cycle-macros = { path = "../../utils/zk-cycle-macros", version = "0.3", optional = true } sov-zk-cycle-utils = { path = "../../utils/zk-cycle-utils", version = "0.3", optional = true } diff --git a/module-system/sov-modules-stf-blueprint/src/stf_blueprint.rs b/module-system/sov-modules-stf-blueprint/src/stf_blueprint.rs index 5fbd78160..732d8fa48 100644 --- a/module-system/sov-modules-stf-blueprint/src/stf_blueprint.rs +++ b/module-system/sov-modules-stf-blueprint/src/stf_blueprint.rs @@ -5,6 +5,7 @@ use sov_modules_api::runtime::capabilities::KernelSlotHooks; use sov_modules_api::{ BasicAddress, BlobReaderTrait, Context, DaSpec, DispatchCall, GasUnit, StateCheckpoint, }; +use sov_modules_core::WorkingSet; use sov_rollup_interface::stf::{BatchReceipt, TransactionReceipt}; use tracing::{debug, error}; @@ -177,13 +178,77 @@ where let gas_elastic_price = [0, 0]; let mut sequencer_reward = 0u64; - // Dispatching transactions let mut tx_receipts = Vec::with_capacity(txs.len()); + + let mut batch_workspace = self.apply_txs( + txs, + messages, + &gas_elastic_price, + &mut tx_receipts, + batch_workspace, + &mut sequencer_reward, + ); + + // TODO: calculate the amount based of gas and fees + let sequencer_outcome = SequencerOutcome::Rewarded(sequencer_reward); + + if let Err(e) = self + .runtime + .end_blob_hook(sequencer_outcome.clone(), &mut batch_workspace) + { + // TODO: will be covered in https://github.com/Sovereign-Labs/sovereign-sdk/issues/421 + error!("Failed on `end_blob_hook`: {}", e); + }; + + ( + Ok(BatchReceipt { + batch_hash: blob.hash(), + tx_receipts, + inner: sequencer_outcome, + }), + batch_workspace.checkpoint(), + ) + } + + // Do all stateless checks and data formatting, that can be results in sequencer slashing + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] + fn pre_process_batch( + &self, + blob_data: &mut impl BlobReaderTrait, + ) -> Result< + ( + Vec>, + Vec<::Decodable>, + ), + SlashingReason, + > { + let batch = self.deserialize_batch(blob_data)?; + debug!("Deserialized batch with {} txs", batch.txs.len()); + + // Run the stateless verification, since it is stateless we don't commit. + let txs = self.verify_txs_stateless(batch)?; + + let messages = self.decode_txs(&txs)?; + + Ok((txs, messages)) + } + + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] + fn apply_txs( + &self, + txs: Vec>, + messages: Vec<::Decodable>, + gas_elastic_price: &[u64], + tx_receipts: &mut Vec>, + mut batch_workspace: WorkingSet, + sequencer_reward: &mut u64, + ) -> WorkingSet { + // Dispatching transactions for (TransactionAndRawHash { tx, raw_tx_hash }, msg) in txs.into_iter().zip(messages.into_iter()) { // Update the working set gas meter with the available funds - let gas_price = C::GasUnit::from_arbitrary_dimensions(&gas_elastic_price); + let gas_price = C::GasUnit::from_arbitrary_dimensions(gas_elastic_price); let gas_limit = tx.gas_limit(); let gas_tip = tx.gas_tip(); batch_workspace.set_gas(gas_limit, gas_price); @@ -227,7 +292,7 @@ where .saturating_add(gas_tip) .saturating_sub(remaining_gas); - sequencer_reward = sequencer_reward.saturating_add(gas_reward); + *sequencer_reward = sequencer_reward.saturating_add(gas_reward); debug!( "Tx {} sequencer reward: {}", hex::encode(raw_tx_hash), @@ -270,50 +335,11 @@ where .expect("inconsistent state: error in post_dispatch_tx_hook"); } - // TODO: calculate the amount based of gas and fees - let sequencer_outcome = SequencerOutcome::Rewarded(sequencer_reward); - - if let Err(e) = self - .runtime - .end_blob_hook(sequencer_outcome.clone(), &mut batch_workspace) - { - // TODO: will be covered in https://github.com/Sovereign-Labs/sovereign-sdk/issues/421 - error!("Failed on `end_blob_hook`: {}", e); - }; - - ( - Ok(BatchReceipt { - batch_hash: blob.hash(), - tx_receipts, - inner: sequencer_outcome, - }), - batch_workspace.checkpoint(), - ) - } - - // Do all stateless checks and data formatting, that can be results in sequencer slashing - fn pre_process_batch( - &self, - blob_data: &mut impl BlobReaderTrait, - ) -> Result< - ( - Vec>, - Vec<::Decodable>, - ), - SlashingReason, - > { - let batch = self.deserialize_batch(blob_data)?; - debug!("Deserialized batch with {} txs", batch.txs.len()); - - // Run the stateless verification, since it is stateless we don't commit. - let txs = self.verify_txs_stateless(batch)?; - - let messages = self.decode_txs(&txs)?; - - Ok((txs, messages)) + batch_workspace } // Attempt to deserialize batch, error results in sequencer slashing. + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] fn deserialize_batch( &self, blob_data: &mut impl BlobReaderTrait, @@ -335,6 +361,7 @@ where // Stateless verification of transaction, such as signature check // Single malformed transaction results in sequencer slashing. + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] fn verify_txs_stateless( &self, batch: Batch, @@ -350,6 +377,7 @@ where // Checks that runtime message can be decoded from transaction. // If a single message cannot be decoded, sequencer is slashed + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] fn decode_txs( &self, txs: &[TransactionAndRawHash], diff --git a/module-system/sov-modules-stf-blueprint/src/tx_verifier.rs b/module-system/sov-modules-stf-blueprint/src/tx_verifier.rs index cb0941a83..a44bb3d6c 100644 --- a/module-system/sov-modules-stf-blueprint/src/tx_verifier.rs +++ b/module-system/sov-modules-stf-blueprint/src/tx_verifier.rs @@ -5,6 +5,8 @@ use serde::{Deserialize, Serialize}; use sov_modules_api::transaction::Transaction; use sov_modules_api::{Context, Spec}; use sov_rollup_interface::digest::Digest; +#[cfg(all(target_os = "zkvm", feature = "bench"))] +use sov_zk_cycle_macros::cycle_tracker; use tracing::debug; type RawTxHash = [u8; 32]; @@ -22,9 +24,16 @@ pub struct RawTx { } impl RawTx { + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] fn hash(&self) -> [u8; 32] { ::Hasher::digest(&self.data).into() } + + #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] + fn deserialize(&self) -> Result, std::io::Error> { + let mut data = Cursor::new(&self.data); + Transaction::::deserialize_reader(&mut data) + } } pub(crate) fn verify_txs_stateless( @@ -34,8 +43,7 @@ pub(crate) fn verify_txs_stateless( debug!("Verifying {} transactions", raw_txs.len()); for raw_tx in raw_txs { let raw_tx_hash = raw_tx.hash::(); - let mut data = Cursor::new(&raw_tx.data); - let tx = Transaction::::deserialize_reader(&mut data)?; + let tx = raw_tx.deserialize()?; tx.verify()?; txs.push(TransactionAndRawHash { tx, raw_tx_hash }); } diff --git a/module-system/sov-state/src/zk_storage.rs b/module-system/sov-state/src/zk_storage.rs index ab4469192..b3becb155 100644 --- a/module-system/sov-state/src/zk_storage.rs +++ b/module-system/sov-state/src/zk_storage.rs @@ -36,6 +36,63 @@ impl ZkStorage { } } +#[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] +fn jmt_verify_existence( + prev_state_root: [u8; 32], + state_accesses: &OrderedReadsAndWrites, + witness: &S::Witness, +) -> Result<(), anyhow::Error> { + // For each value that's been read from the tree, verify the provided smt proof + for (key, read_value) in &state_accesses.ordered_reads { + let key_hash = KeyHash::with::(key.key.as_ref()); + // TODO: Switch to the batch read API once it becomes available + let proof: jmt::proof::SparseMerkleProof = witness.get_hint(); + + match read_value { + Some(val) => proof.verify_existence( + jmt::RootHash(prev_state_root), + key_hash, + val.value.as_ref(), + )?, + None => proof.verify_nonexistence(jmt::RootHash(prev_state_root), key_hash)?, + } + } + + Ok(()) +} + +#[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] +fn jmt_verify_update( + prev_state_root: [u8; 32], + state_accesses: OrderedReadsAndWrites, + witness: &S::Witness, +) -> [u8; 32] { + // Compute the jmt update from the write batch + let batch = state_accesses + .ordered_writes + .into_iter() + .map(|(key, value)| { + let key_hash = KeyHash::with::(key.key.as_ref()); + ( + key_hash, + value.map(|v| Arc::try_unwrap(v.value).unwrap_or_else(|arc| (*arc).clone())), + ) + }) + .collect::>(); + + let update_proof: jmt::proof::UpdateMerkleProof = witness.get_hint(); + let new_root: [u8; 32] = witness.get_hint(); + update_proof + .verify_update( + jmt::RootHash(prev_state_root), + jmt::RootHash(new_root), + batch, + ) + .expect("Updates must be valid"); + + new_root +} + impl Storage for ZkStorage { type Witness = S::Witness; type RuntimeConfig = (); @@ -52,7 +109,6 @@ impl Storage for ZkStorage { witness.get_hint() } - #[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)] fn compute_state_update( &self, state_accesses: OrderedReadsAndWrites, @@ -61,42 +117,9 @@ impl Storage for ZkStorage { let prev_state_root = witness.get_hint(); // For each value that's been read from the tree, verify the provided smt proof - for (key, read_value) in state_accesses.ordered_reads { - let key_hash = KeyHash::with::(key.key.as_ref()); - // TODO: Switch to the batch read API once it becomes available - let proof: jmt::proof::SparseMerkleProof = witness.get_hint(); - match read_value { - Some(val) => proof.verify_existence( - jmt::RootHash(prev_state_root), - key_hash, - val.value.as_ref(), - )?, - None => proof.verify_nonexistence(jmt::RootHash(prev_state_root), key_hash)?, - } - } + jmt_verify_existence::(prev_state_root, &state_accesses, witness)?; - // Compute the jmt update from the write batch - let batch = state_accesses - .ordered_writes - .into_iter() - .map(|(key, value)| { - let key_hash = KeyHash::with::(key.key.as_ref()); - ( - key_hash, - value.map(|v| Arc::try_unwrap(v.value).unwrap_or_else(|arc| (*arc).clone())), - ) - }) - .collect::>(); - - let update_proof: jmt::proof::UpdateMerkleProof = witness.get_hint(); - let new_root: [u8; 32] = witness.get_hint(); - update_proof - .verify_update( - jmt::RootHash(prev_state_root), - jmt::RootHash(new_root), - batch, - ) - .expect("Updates must be valid"); + let new_root = jmt_verify_update::(prev_state_root, state_accesses, witness); Ok((jmt::RootHash(new_root), ())) } diff --git a/module-system/utils/sov-data-generators/Cargo.toml b/module-system/utils/sov-data-generators/Cargo.toml deleted file mode 100644 index 8f30aba8a..000000000 --- a/module-system/utils/sov-data-generators/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "sov-data-generators" -description = "A set of generator utils used to automatically produce and serialize transaction data" -authors = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } - -version = { workspace = true } -resolver = "2" -publish = false - - -[dependencies] -sov-modules-api = { path = "../../sov-modules-api", features = ["native"] } -sov-modules-stf-blueprint = { path = "../../sov-modules-stf-blueprint", features = ["native"] } -sov-value-setter = { path = "../../module-implementations/examples/sov-value-setter", features = ["native"] } -sov-bank = { path = "../../module-implementations/sov-bank", features = ["native"] } -sov-state = { path = "../../sov-state" } -sov-mock-da = { path = "../../../adapters/mock-da", features = ["native"] } - -borsh = { workspace = true } - -[dev-dependencies] -proptest = { workspace = true }