From ac8fab3bee4fae3a6efecd52c73bb5f764c3d74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cem=20=C3=96zer?= Date: Tue, 14 Nov 2023 22:42:17 +0300 Subject: [PATCH 01/12] WIP --- .../sov-bank/tests/archival_query_test.rs | 119 ++++++++++++++++++ .../sov-modules-core/src/storage/mod.rs | 3 + .../sov-modules-macros/src/rpc/expose_rpc.rs | 2 +- module-system/sov-state/src/prover_storage.rs | 9 ++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 module-system/module-implementations/sov-bank/tests/archival_query_test.rs diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs new file mode 100644 index 000000000..33b163ecf --- /dev/null +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -0,0 +1,119 @@ +mod helpers; + +use helpers::*; + +use sov_bank::{Bank, BankConfig, CallMessage, Coins, get_genesis_token_address}; +use sov_modules_api::{Address, Context, Error, Module, Spec, WorkingSet}; +use sov_modules_api::utils::generate_address; +use sov_modules_api::default_context::DefaultContext; +use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; + + +#[test] +fn transfer_initial_token() { + let initial_balance = 100; + let bank_config = create_bank_config_with_token(3, initial_balance); + let tmpdir = tempfile::tempdir().unwrap(); + let prover_storage = ProverStorage::with_path(tmpdir.path()).unwrap(); + let mut working_set = WorkingSet::new(prover_storage.clone()); + let bank = Bank::default(); + bank.genesis(&bank_config, &mut working_set).unwrap(); + + let token_address = get_genesis_token_address::( + &bank_config.tokens[0].token_name, + bank_config.tokens[0].salt, + ); + let sender_address = bank_config.tokens[0].address_and_balances[0].0; + let receiver_address = bank_config.tokens[0].address_and_balances[1].0; + assert_ne!(sender_address, receiver_address); + + query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + + println!("Starting commit for genesis, i.e. slot 1"); + commit(working_set, prover_storage.clone()); + println!("Genesis commit complete"); + + let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); + + transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); + query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + + + println!("Starting commit for slot 2"); + commit(working_set, prover_storage.clone()); + println!("Commit complete for slot 2"); + + let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); + + transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); + query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + + println!("Starting commit for slot 3"); + commit(working_set, prover_storage.clone()); + println!("Commit complete for slot 3"); + + // let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); + // + // let archival_slot: u64 = 2; + // println!("Starting archival work"); + // working_set.work_on_slot(archival_slot); + // println!("work on slot {} set", archival_slot); + // + // query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); +} + +fn query_sender_receiver_balances(bank: &Bank, + token_address: Address, + sender_address: Address, + receiver_address: Address, + working_set: &mut WorkingSet) -> (u64, u64){ + let query_user_balance = + |user_address: Address, working_set: &mut WorkingSet| -> Option { + bank.get_balance_of(user_address, token_address, working_set) + }; + + let sender_balance = query_user_balance(sender_address, working_set).unwrap(); + let receiver_balance = query_user_balance(receiver_address, working_set).unwrap(); + println!("S: {}", &sender_balance); + println!("R: {}", &receiver_balance); + return (sender_balance, receiver_balance); +} + +fn transfer(bank: &Bank, + token_address: Address, + sender_address: Address, + receiver_address: Address, + working_set: &mut WorkingSet) { + let transfer_amount = 10; + let transfer_message = CallMessage::Transfer { + to: receiver_address, + coins: Coins { + amount: transfer_amount, + token_address, + }, + }; + + let sender_context = C::new(sender_address); + + bank.call(transfer_message, &sender_context, working_set) + .expect("Transfer call failed"); + println!("Transfer complete"); +} + +fn commit(working_set: WorkingSet, storage: ProverStorage) { + // Save checkpoint + let mut checkpoint = working_set.checkpoint(); + + let (cache_log, witness) = checkpoint.freeze(); + + let (_, authenticated_node_batch) = storage + .compute_state_update(cache_log, &witness) + .expect("jellyfish merkle tree update must succeed"); + + let mut working_set = checkpoint.to_revertable(); + + let accessory_log = working_set.checkpoint().freeze_non_provable(); + + storage + .commit(&authenticated_node_batch, &accessory_log); +} \ No newline at end of file diff --git a/module-system/sov-modules-core/src/storage/mod.rs b/module-system/sov-modules-core/src/storage/mod.rs index 049f4d5e6..51b36f7c8 100644 --- a/module-system/sov-modules-core/src/storage/mod.rs +++ b/module-system/sov-modules-core/src/storage/mod.rs @@ -289,4 +289,7 @@ pub trait NativeStorage: Storage { /// Get the root hash of the tree at the requested version fn get_root_hash(&self, version: Version) -> Result; + + /// From here-on, return values from the given version in `get()` and `get_accessory()`. Used for archival queries. + fn set_archival_version(&mut self, version: u64); } diff --git a/module-system/sov-modules-macros/src/rpc/expose_rpc.rs b/module-system/sov-modules-macros/src/rpc/expose_rpc.rs index 1f87c632b..118cde0bd 100644 --- a/module-system/sov-modules-macros/src/rpc/expose_rpc.rs +++ b/module-system/sov-modules-macros/src/rpc/expose_rpc.rs @@ -59,7 +59,7 @@ impl ExposeRpcMacro { impl #impl_generics ::std::clone::Clone for RpcStorage #ty_generics #where_clause { fn clone(&self) -> Self { Self { - storage: self.storage.clone(), + storage: ArchivalStorage::new(self.storage.clone()), _phantom: ::std::marker::PhantomData, } } diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index ca6614123..068d9e125 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -1,6 +1,7 @@ use std::marker::PhantomData; use std::path::Path; use std::sync::Arc; +use anyhow::ensure; use jmt::storage::{NodeBatch, TreeWriter}; use jmt::{JellyfishMerkleTree, KeyHash, Version}; @@ -19,6 +20,7 @@ use crate::MerkleProofSpec; pub struct ProverStorage { db: StateDB, native_db: NativeDB, + archival_version: Option, _phantom_hasher: PhantomData, } @@ -27,6 +29,7 @@ impl Clone for ProverStorage { Self { db: self.db.clone(), native_db: self.native_db.clone(), + archival_version: self.archival_version.clone(), _phantom_hasher: Default::default(), } } @@ -228,4 +231,10 @@ impl NativeStorage for ProverStorage { JellyfishMerkleTree::new(&self.db); temp_merkle.get_root_hash(version) } + + fn set_archival_version(&mut self, version: u64) { + ensure!(version < self.db.get_next_version(), + "The storage default read version can not be set to a version greater than the next version"); + self.archival_version = Some(u64) + } } From b62ab864364832c586bb5f81169360b7f9238b5f Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Fri, 17 Nov 2023 04:30:19 +0530 Subject: [PATCH 02/12] get rough test working, create branch from fork --- .../sov-bank/tests/archival_query_test.rs | 38 ++++++++++++------- .../sov-modules-core/src/storage/mod.rs | 2 +- .../sov-modules-macros/src/rpc/expose_rpc.rs | 2 +- module-system/sov-state/src/prover_storage.rs | 10 +++-- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index 33b163ecf..4a5850d03 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -7,6 +7,7 @@ use sov_modules_api::{Address, Context, Error, Module, Spec, WorkingSet}; use sov_modules_api::utils::generate_address; use sov_modules_api::default_context::DefaultContext; use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; +use sov_state::storage::NativeStorage; #[test] @@ -27,8 +28,8 @@ fn transfer_initial_token() { let receiver_address = bank_config.tokens[0].address_and_balances[1].0; assert_ne!(sender_address, receiver_address); - query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (100,100)); println!("Starting commit for genesis, i.e. slot 1"); commit(working_set, prover_storage.clone()); println!("Genesis commit complete"); @@ -36,8 +37,8 @@ fn transfer_initial_token() { let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); - query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (90,110)); println!("Starting commit for slot 2"); commit(working_set, prover_storage.clone()); @@ -46,20 +47,29 @@ fn transfer_initial_token() { let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); - query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (80,120)); println!("Starting commit for slot 3"); commit(working_set, prover_storage.clone()); println!("Commit complete for slot 3"); - // let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); - // - // let archival_slot: u64 = 2; - // println!("Starting archival work"); - // working_set.work_on_slot(archival_slot); - // println!("work on slot {} set", archival_slot); - // - // query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + + let archival_slot: u64 = 2; + println!("Archival reads at slot {}",archival_slot); + let mut versioned_prover_storage = prover_storage.clone(); + versioned_prover_storage.set_archival_version(archival_slot).expect("TODO: panic message"); + let mut working_set: WorkingSet = WorkingSet::new(versioned_prover_storage.clone()); + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (90,110)); + + let archival_slot: u64 = 1; + println!("Archival archival reads at slot {}",archival_slot); + let mut versioned_prover_storage = prover_storage.clone(); + versioned_prover_storage.set_archival_version(archival_slot).expect("TODO: panic message"); + let mut working_set: WorkingSet = WorkingSet::new(versioned_prover_storage.clone()); + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (100,100)); + } fn query_sender_receiver_balances(bank: &Bank, diff --git a/module-system/sov-modules-core/src/storage/mod.rs b/module-system/sov-modules-core/src/storage/mod.rs index 51b36f7c8..5b7e9e7d5 100644 --- a/module-system/sov-modules-core/src/storage/mod.rs +++ b/module-system/sov-modules-core/src/storage/mod.rs @@ -291,5 +291,5 @@ pub trait NativeStorage: Storage { fn get_root_hash(&self, version: Version) -> Result; /// From here-on, return values from the given version in `get()` and `get_accessory()`. Used for archival queries. - fn set_archival_version(&mut self, version: u64); + fn set_archival_version(&mut self, version: u64) -> anyhow::Result<()>; } diff --git a/module-system/sov-modules-macros/src/rpc/expose_rpc.rs b/module-system/sov-modules-macros/src/rpc/expose_rpc.rs index 118cde0bd..1f87c632b 100644 --- a/module-system/sov-modules-macros/src/rpc/expose_rpc.rs +++ b/module-system/sov-modules-macros/src/rpc/expose_rpc.rs @@ -59,7 +59,7 @@ impl ExposeRpcMacro { impl #impl_generics ::std::clone::Clone for RpcStorage #ty_generics #where_clause { fn clone(&self) -> Self { Self { - storage: ArchivalStorage::new(self.storage.clone()), + storage: self.storage.clone(), _phantom: ::std::marker::PhantomData, } } diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index 068d9e125..4848bd604 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -45,6 +45,7 @@ impl ProverStorage { Ok(Self { db: state_db, native_db, + archival_version: None, _phantom_hasher: Default::default(), }) } @@ -53,14 +54,16 @@ impl ProverStorage { Self { db, native_db, + archival_version: None, _phantom_hasher: Default::default(), } } fn read_value(&self, key: &StorageKey) -> Option { + let version= self.archival_version.unwrap_or(self.db.get_next_version()); match self .db - .get_value_option_by_key(self.db.get_next_version(), key.as_ref()) + .get_value_option_by_key(version, key.as_ref()) { Ok(value) => value.map(Into::into), // It is ok to panic here, we assume the db is available and consistent. @@ -232,9 +235,10 @@ impl NativeStorage for ProverStorage { temp_merkle.get_root_hash(version) } - fn set_archival_version(&mut self, version: u64) { + fn set_archival_version(&mut self, version: u64) -> anyhow::Result<()> { ensure!(version < self.db.get_next_version(), "The storage default read version can not be set to a version greater than the next version"); - self.archival_version = Some(u64) + self.archival_version = Some(version); + Ok(()) } } From a99481a03a7c3acb7e3a07bed1a6f0df80a50f99 Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Fri, 17 Nov 2023 19:55:27 +0530 Subject: [PATCH 03/12] test w/ working set and revert --- .../sov-bank/tests/archival_query_test.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index 4a5850d03..cda0ab342 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -2,9 +2,8 @@ mod helpers; use helpers::*; -use sov_bank::{Bank, BankConfig, CallMessage, Coins, get_genesis_token_address}; -use sov_modules_api::{Address, Context, Error, Module, Spec, WorkingSet}; -use sov_modules_api::utils::generate_address; +use sov_bank::{Bank, CallMessage, Coins, get_genesis_token_address}; +use sov_modules_api::{Address, Context, Module, WorkingSet}; use sov_modules_api::default_context::DefaultContext; use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; use sov_state::storage::NativeStorage; @@ -69,6 +68,16 @@ fn transfer_initial_token() { let mut working_set: WorkingSet = WorkingSet::new(versioned_prover_storage.clone()); let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); assert_eq!((sender_balance, receiver_balance), (100,100)); + println!("Transfer on archival"); + transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); + println!("Archival query for modified working set"); + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (90,110)); + + println!(" Move back from archival to current once again"); + let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); + let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); + assert_eq!((sender_balance, receiver_balance), (80,120)); } @@ -120,7 +129,7 @@ fn commit(working_set: WorkingSet, storage: ProverStorage Date: Tue, 21 Nov 2023 07:46:47 +0530 Subject: [PATCH 04/12] native archival storage --- full-node/db/sov-db/src/native_db.rs | 53 +++++- full-node/db/sov-db/src/schema/tables.rs | 49 +++++- .../sov-bank/tests/archival_query_test.rs | 153 +++++++++++++----- module-system/sov-state/src/prover_storage.rs | 15 +- 4 files changed, 205 insertions(+), 65 deletions(-) diff --git a/full-node/db/sov-db/src/native_db.rs b/full-node/db/sov-db/src/native_db.rs index 97e627b23..85ed3d34e 100644 --- a/full-node/db/sov-db/src/native_db.rs +++ b/full-node/db/sov-db/src/native_db.rs @@ -1,11 +1,14 @@ use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use sov_schema_db::{SchemaBatch, DB}; use crate::rocks_db_config::gen_rocksdb_options; use crate::schema::tables::{ModuleAccessoryState, NATIVE_TABLES}; -use crate::schema::types::StateKey; +use crate::schema::types::AccessoryKey; + +/// Specifies a particular version of the Accessory state. +pub type Version = u64; /// A typed wrapper around RocksDB for storing native-only accessory state. /// Internally, this is roughly just an [`Arc`]. @@ -14,6 +17,8 @@ pub struct NativeDB { /// The underlying RocksDB instance, wrapped in an [`Arc`] for convenience /// and [`DB`] for type safety. db: Arc, + /// The [`Version`] that will be used for the next batch of writes to the DB. + next_version: Arc>, } impl NativeDB { @@ -31,15 +36,23 @@ impl NativeDB { &gen_rocksdb_options(&Default::default(), false), )?; + let next_version = Self::last_version_written(&inner)?.unwrap_or_default() + 1; + Ok(Self { db: Arc::new(inner), + next_version: Arc::new(Mutex::new(next_version)), }) } /// Queries for a value in the [`NativeDB`], given a key. - pub fn get_value_option(&self, key: &StateKey) -> anyhow::Result>> { + pub fn get_value_option( + &self, + key: &AccessoryKey, + version: Option, + ) -> anyhow::Result>> { + let version_to_use = version.unwrap_or_else(|| self.get_next_version()); self.db - .get::(key) + .get::(&(key.clone(), version_to_use)) .map(Option::flatten) } @@ -48,12 +61,36 @@ impl NativeDB { &self, key_value_pairs: impl IntoIterator, Option>)>, ) -> anyhow::Result<()> { + let version = self.get_next_version(); let mut batch = SchemaBatch::default(); for (key, value) in key_value_pairs { - batch.put::(&key, &value)?; + batch.put::(&(key, version), &value)?; } self.db.write_schemas(batch) } + + /// Increment the `next_version` counter by 1. + pub fn inc_next_version(&self) { + let mut version = self.next_version.lock().unwrap(); + *version += 1; + } + + /// Get the current value of the `next_version` counter + pub fn get_next_version(&self) -> Version { + let version = self.next_version.lock().unwrap(); + *version + } + + fn last_version_written(db: &DB) -> anyhow::Result> { + let mut iter = db.iter::()?; + iter.seek_to_last(); + + let version = match iter.next() { + Some(Ok(((_, version), _))) => Some(version), + _ => None, + }; + Ok(version) + } } #[cfg(feature = "arbitrary")] @@ -146,7 +183,7 @@ mod tests { let value = b"bar".to_vec(); db.set_values(vec![(key.clone(), Some(value.clone()))]) .unwrap(); - assert_eq!(db.get_value_option(&key).unwrap(), Some(value)); + assert_eq!(db.get_value_option(&key, None).unwrap(), Some(value)); } #[test] @@ -156,7 +193,7 @@ mod tests { let key = b"deleted".to_vec(); db.set_values(vec![(key.clone(), None)]).unwrap(); - assert_eq!(db.get_value_option(&key).unwrap(), None); + assert_eq!(db.get_value_option(&key, None).unwrap(), None); } #[test] @@ -165,6 +202,6 @@ mod tests { let db = NativeDB::with_path(tmpdir.path()).unwrap(); let key = b"spam".to_vec(); - assert_eq!(db.get_value_option(&key).unwrap(), None); + assert_eq!(db.get_value_option(&key, None).unwrap(), None); } } diff --git a/full-node/db/sov-db/src/schema/tables.rs b/full-node/db/sov-db/src/schema/tables.rs index 6f267d971..d223f5f03 100644 --- a/full-node/db/sov-db/src/schema/tables.rs +++ b/full-node/db/sov-db/src/schema/tables.rs @@ -219,11 +219,6 @@ define_table_with_default_codec!( (SlotByHash) DbHash => SlotNumber ); -define_table_with_default_codec!( - /// Non-JMT state stored by a module for JSON-RPC use. - (ModuleAccessoryState) AccessoryKey => AccessoryStateValue -); - define_table_with_seek_key_codec!( /// The primary source for batch data (BatchByNumber) BatchNumber => StoredBatch @@ -332,3 +327,47 @@ define_table_with_default_codec!( /// which requires the ability to fetch values by hash. (KeyHashToKey) [u8;32] => StateKey ); + +define_table_without_codec!( + /// Non-JMT state stored by a module for JSON-RPC use. + (ModuleAccessoryState) (AccessoryKey, Version) => AccessoryStateValue +); + +impl KeyEncoder for (AccessoryKey, Version) { + fn encode_key(&self) -> sov_schema_db::schema::Result> { + let mut out = Vec::with_capacity(self.0.len() + std::mem::size_of::() + 8); + self.0 + .as_slice() + .serialize(&mut out) + .map_err(CodecError::from)?; + // Write the version in big-endian order so that sorting order is based on the most-significant bytes of the key + out.write_u64::(self.1) + .expect("serialization to vec is infallible"); + Ok(out) + } +} + +impl SeekKeyEncoder for (AccessoryKey, Version) { + fn encode_seek_key(&self) -> sov_schema_db::schema::Result> { + <(Vec, u64) as KeyEncoder>::encode_key(self) + } +} + +impl KeyDecoder for (AccessoryKey, Version) { + fn decode_key(data: &[u8]) -> sov_schema_db::schema::Result { + let mut cursor = maybestd::io::Cursor::new(data); + let key = Vec::::deserialize_reader(&mut cursor)?; + let version = cursor.read_u64::()?; + Ok((key, version)) + } +} + +impl ValueCodec for AccessoryStateValue { + fn encode_value(&self) -> sov_schema_db::schema::Result> { + self.try_to_vec().map_err(CodecError::from) + } + + fn decode_value(data: &[u8]) -> sov_schema_db::schema::Result { + Ok(Self::deserialize_reader(&mut &data[..])?) + } +} diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index cda0ab342..cbdb6baaa 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -1,13 +1,11 @@ mod helpers; use helpers::*; - -use sov_bank::{Bank, CallMessage, Coins, get_genesis_token_address}; -use sov_modules_api::{Address, Context, Module, WorkingSet}; +use sov_bank::{get_genesis_token_address, Bank, CallMessage, Coins}; use sov_modules_api::default_context::DefaultContext; -use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; +use sov_modules_api::{Address, Context, Module, WorkingSet}; use sov_state::storage::NativeStorage; - +use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; #[test] fn transfer_initial_token() { @@ -27,17 +25,35 @@ fn transfer_initial_token() { let receiver_address = bank_config.tokens[0].address_and_balances[1].0; assert_ne!(sender_address, receiver_address); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (100,100)); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (100, 100)); println!("Starting commit for genesis, i.e. slot 1"); commit(working_set, prover_storage.clone()); println!("Genesis commit complete"); let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); - transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (90,110)); + transfer( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (90, 110)); println!("Starting commit for slot 2"); commit(working_set, prover_storage.clone()); @@ -45,47 +61,95 @@ fn transfer_initial_token() { let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); - transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (80,120)); + transfer( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (80, 120)); println!("Starting commit for slot 3"); commit(working_set, prover_storage.clone()); println!("Commit complete for slot 3"); - let archival_slot: u64 = 2; - println!("Archival reads at slot {}",archival_slot); + println!("Archival reads at slot {}", archival_slot); let mut versioned_prover_storage = prover_storage.clone(); - versioned_prover_storage.set_archival_version(archival_slot).expect("TODO: panic message"); - let mut working_set: WorkingSet = WorkingSet::new(versioned_prover_storage.clone()); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (90,110)); + versioned_prover_storage + .set_archival_version(archival_slot) + .expect("TODO: panic message"); + let mut working_set: WorkingSet = + WorkingSet::new(versioned_prover_storage.clone()); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (90, 110)); let archival_slot: u64 = 1; - println!("Archival archival reads at slot {}",archival_slot); + println!("Archival archival reads at slot {}", archival_slot); let mut versioned_prover_storage = prover_storage.clone(); - versioned_prover_storage.set_archival_version(archival_slot).expect("TODO: panic message"); - let mut working_set: WorkingSet = WorkingSet::new(versioned_prover_storage.clone()); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (100,100)); + versioned_prover_storage + .set_archival_version(archival_slot) + .expect("TODO: panic message"); + let mut working_set: WorkingSet = + WorkingSet::new(versioned_prover_storage.clone()); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (100, 100)); println!("Transfer on archival"); - transfer(&bank, token_address, sender_address, receiver_address, &mut working_set); + transfer( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); println!("Archival query for modified working set"); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (90,110)); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (90, 110)); println!(" Move back from archival to current once again"); let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); - let (sender_balance, receiver_balance) = query_sender_receiver_balances(&bank, token_address, sender_address, receiver_address, &mut working_set); - assert_eq!((sender_balance, receiver_balance), (80,120)); - + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (80, 120)); } -fn query_sender_receiver_balances(bank: &Bank, - token_address: Address, - sender_address: Address, - receiver_address: Address, - working_set: &mut WorkingSet) -> (u64, u64){ +fn query_sender_receiver_balances( + bank: &Bank, + token_address: Address, + sender_address: Address, + receiver_address: Address, + working_set: &mut WorkingSet, +) -> (u64, u64) { let query_user_balance = |user_address: Address, working_set: &mut WorkingSet| -> Option { bank.get_balance_of(user_address, token_address, working_set) @@ -95,14 +159,16 @@ fn query_sender_receiver_balances(bank: &Bank, let receiver_balance = query_user_balance(receiver_address, working_set).unwrap(); println!("S: {}", &sender_balance); println!("R: {}", &receiver_balance); - return (sender_balance, receiver_balance); + (sender_balance, receiver_balance) } -fn transfer(bank: &Bank, - token_address: Address, - sender_address: Address, - receiver_address: Address, - working_set: &mut WorkingSet) { +fn transfer( + bank: &Bank, + token_address: Address, + sender_address: Address, + receiver_address: Address, + working_set: &mut WorkingSet, +) { let transfer_amount = 10; let transfer_message = CallMessage::Transfer { to: receiver_address, @@ -133,6 +199,5 @@ fn commit(working_set: WorkingSet, storage: ProverStorage Clone for ProverStorage { Self { db: self.db.clone(), native_db: self.native_db.clone(), - archival_version: self.archival_version.clone(), + archival_version: self.archival_version, _phantom_hasher: Default::default(), } } @@ -60,11 +60,8 @@ impl ProverStorage { } fn read_value(&self, key: &StorageKey) -> Option { - let version= self.archival_version.unwrap_or(self.db.get_next_version()); - match self - .db - .get_value_option_by_key(version, key.as_ref()) - { + let version = self.archival_version.unwrap_or(self.db.get_next_version()); + match self.db.get_value_option_by_key(version, key.as_ref()) { Ok(value) => value.map(Into::into), // It is ok to panic here, we assume the db is available and consistent. Err(e) => panic!("Unable to read value from db: {e}"), @@ -97,8 +94,9 @@ impl Storage for ProverStorage { #[cfg(feature = "native")] fn get_accessory(&self, key: &StorageKey) -> Option { + let version = self.archival_version.unwrap_or(self.db.get_next_version()); self.native_db - .get_value_option(key.as_ref()) + .get_value_option(key.as_ref(), Some(version)) .unwrap() .map(Into::into) } @@ -194,6 +192,7 @@ impl Storage for ProverStorage { .expect("native db write must succeed"); self.db.inc_next_version(); + self.native_db.inc_next_version(); } fn open_proof( From 6a3a4082cf640fc4bb896fe87c23be1180d8ef22 Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Tue, 21 Nov 2023 11:32:48 +0530 Subject: [PATCH 05/12] added test case for native storage query --- .../sov-bank/tests/archival_query_test.rs | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index cbdb6baaa..ee639ff2d 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -4,7 +4,7 @@ use helpers::*; use sov_bank::{get_genesis_token_address, Bank, CallMessage, Coins}; use sov_modules_api::default_context::DefaultContext; use sov_modules_api::{Address, Context, Module, WorkingSet}; -use sov_state::storage::NativeStorage; +use sov_state::storage::{NativeStorage, StateReaderAndWriter, StorageKey, StorageValue}; use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; #[test] @@ -141,6 +141,73 @@ fn transfer_initial_token() { &mut working_set, ); assert_eq!((sender_balance, receiver_balance), (80, 120)); + + // Accessory tests + + transfer( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (70, 130)); + let mut accessory_state = working_set.accessory_state(); + accessory_state.set(&StorageKey::from("k"), StorageValue::from(b"v1".to_vec())); + let val = accessory_state.get(&StorageKey::from("k")).unwrap(); + assert_eq!("v1", String::from_utf8(val.value().to_vec()).unwrap()); + + commit(working_set, prover_storage.clone()); + + // next block + + let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); + transfer( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + let (sender_balance, receiver_balance) = query_sender_receiver_balances( + &bank, + token_address, + sender_address, + receiver_address, + &mut working_set, + ); + assert_eq!((sender_balance, receiver_balance), (60, 140)); + let mut accessory_state = working_set.accessory_state(); + accessory_state.set(&StorageKey::from("k"), StorageValue::from(b"v2".to_vec())); + let val = accessory_state.get(&StorageKey::from("k")).unwrap(); + assert_eq!("v2", String::from_utf8(val.value().to_vec()).unwrap()); + + commit(working_set, prover_storage.clone()); + + // archival versioned state query + + let archival_slot= 4; + let mut versioned_prover_storage = prover_storage.clone(); + versioned_prover_storage + .set_archival_version(archival_slot) + .expect("TODO: panic message"); + let mut working_set: WorkingSet = + WorkingSet::new(versioned_prover_storage.clone()); + let mut accessory_state = working_set.accessory_state(); + let val = accessory_state.get(&StorageKey::from("k")).unwrap(); + + assert_eq!("v1", String::from_utf8(val.value().to_vec()).unwrap()); + + commit(working_set, prover_storage.clone()); + println!("{:?}",val); + } fn query_sender_receiver_balances( @@ -178,7 +245,7 @@ fn transfer( }, }; - let sender_context = C::new(sender_address); + let sender_context = C::new(sender_address,1); bank.call(transfer_message, &sender_context, working_set) .expect("Transfer call failed"); From 8f0e0bce32cef12de19ce875c249a3a04d1a33ac Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Tue, 21 Nov 2023 11:34:03 +0530 Subject: [PATCH 06/12] formatting --- .../sov-bank/tests/archival_query_test.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index ee639ff2d..a8fb5d623 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -167,7 +167,7 @@ fn transfer_initial_token() { commit(working_set, prover_storage.clone()); // next block - + let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); transfer( &bank, @@ -193,7 +193,7 @@ fn transfer_initial_token() { // archival versioned state query - let archival_slot= 4; + let archival_slot = 4; let mut versioned_prover_storage = prover_storage.clone(); versioned_prover_storage .set_archival_version(archival_slot) @@ -206,8 +206,7 @@ fn transfer_initial_token() { assert_eq!("v1", String::from_utf8(val.value().to_vec()).unwrap()); commit(working_set, prover_storage.clone()); - println!("{:?}",val); - + println!("{:?}", val); } fn query_sender_receiver_balances( @@ -245,7 +244,7 @@ fn transfer( }, }; - let sender_context = C::new(sender_address,1); + let sender_context = C::new(sender_address, 1); bank.call(transfer_message, &sender_context, working_set) .expect("Transfer call failed"); From f53d2cf33c11f825e362518876047a9adc9809fb Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Tue, 21 Nov 2023 17:51:50 +0530 Subject: [PATCH 07/12] fix typo --- full-node/db/sov-db/src/native_db.rs | 1 - .../module-implementations/sov-evm/src/hooks.rs | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/full-node/db/sov-db/src/native_db.rs b/full-node/db/sov-db/src/native_db.rs index 85ed3d34e..97deb0232 100644 --- a/full-node/db/sov-db/src/native_db.rs +++ b/full-node/db/sov-db/src/native_db.rs @@ -37,7 +37,6 @@ impl NativeDB { )?; let next_version = Self::last_version_written(&inner)?.unwrap_or_default() + 1; - Ok(Self { db: Arc::new(inner), next_version: Arc::new(Mutex::new(next_version)), diff --git a/module-system/module-implementations/sov-evm/src/hooks.rs b/module-system/module-implementations/sov-evm/src/hooks.rs index 52a6034a5..67d610189 100644 --- a/module-system/module-implementations/sov-evm/src/hooks.rs +++ b/module-system/module-implementations/sov-evm/src/hooks.rs @@ -156,13 +156,13 @@ where pub fn finalize_hook( &self, root_hash: &<::Storage as Storage>::Root, - accesorry_working_set: &mut AccessoryWorkingSet, + accessory_working_set: &mut AccessoryWorkingSet, ) { - let expected_block_number = self.blocks.len(accesorry_working_set) as u64; + let expected_block_number = self.blocks.len(accessory_working_set) as u64; let mut block = self .pending_head - .get(accesorry_working_set) + .get(accessory_working_set) .unwrap_or_else(|| { panic!( "Pending head must be set to block {}, but was empty", @@ -181,12 +181,12 @@ where let sealed_block = block.seal(); - self.blocks.push(&sealed_block, accesorry_working_set); + self.blocks.push(&sealed_block, accessory_working_set); self.block_hashes.set( &sealed_block.header.hash, &sealed_block.header.number, - accesorry_working_set, + accessory_working_set, ); - self.pending_head.delete(accesorry_working_set); + self.pending_head.delete(accessory_working_set); } } From 1bd0ade427f6665f5a2353f71184da25f307fc36 Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Tue, 21 Nov 2023 23:44:55 +0530 Subject: [PATCH 08/12] fix broken tests --- full-node/db/sov-db/src/native_db.rs | 18 +++++++++++++++--- module-system/sov-state/src/prover_storage.rs | 4 +++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/full-node/db/sov-db/src/native_db.rs b/full-node/db/sov-db/src/native_db.rs index 97deb0232..a1152dfe6 100644 --- a/full-node/db/sov-db/src/native_db.rs +++ b/full-node/db/sov-db/src/native_db.rs @@ -50,9 +50,21 @@ impl NativeDB { version: Option, ) -> anyhow::Result>> { let version_to_use = version.unwrap_or_else(|| self.get_next_version()); - self.db - .get::(&(key.clone(), version_to_use)) - .map(Option::flatten) + let mut iter = self.db.iter::()?; + iter.seek_for_prev(&(key.to_vec(), version_to_use))?; + let found = iter.next(); + match found { + Some(result) => { + let ((found_key, found_version), value) = result?; + if &found_key == key { + anyhow::ensure!(found_version <= version_to_use, "Bug! iterator isn't returning expected values. expected a version <= {version_to_use:} but found {found_version:}"); + Ok(value) + } else { + Ok(None) + } + } + None => Ok(None), + } } /// Sets a sequence of key-value pairs in the [`NativeDB`]. The write is atomic. diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index 51d5770ae..56ab8f484 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -94,7 +94,9 @@ impl Storage for ProverStorage { #[cfg(feature = "native")] fn get_accessory(&self, key: &StorageKey) -> Option { - let version = self.archival_version.unwrap_or(self.db.get_next_version()); + let version = self + .archival_version + .unwrap_or(self.native_db.get_next_version() - 1); self.native_db .get_value_option(key.as_ref(), Some(version)) .unwrap() From 457a300b2a8345b21f13374c531ea52e1f610b4b Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Thu, 23 Nov 2023 05:56:20 +0530 Subject: [PATCH 09/12] unify version --- full-node/db/sov-db/src/native_db.rs | 48 ++++--------------- .../sov-bank/tests/archival_query_test.rs | 2 +- module-system/sov-state/src/prover_storage.rs | 7 +-- 3 files changed, 15 insertions(+), 42 deletions(-) diff --git a/full-node/db/sov-db/src/native_db.rs b/full-node/db/sov-db/src/native_db.rs index a1152dfe6..4af782961 100644 --- a/full-node/db/sov-db/src/native_db.rs +++ b/full-node/db/sov-db/src/native_db.rs @@ -1,5 +1,5 @@ use std::path::Path; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use sov_schema_db::{SchemaBatch, DB}; @@ -17,8 +17,6 @@ pub struct NativeDB { /// The underlying RocksDB instance, wrapped in an [`Arc`] for convenience /// and [`DB`] for type safety. db: Arc, - /// The [`Version`] that will be used for the next batch of writes to the DB. - next_version: Arc>, } impl NativeDB { @@ -36,10 +34,8 @@ impl NativeDB { &gen_rocksdb_options(&Default::default(), false), )?; - let next_version = Self::last_version_written(&inner)?.unwrap_or_default() + 1; Ok(Self { db: Arc::new(inner), - next_version: Arc::new(Mutex::new(next_version)), }) } @@ -47,17 +43,16 @@ impl NativeDB { pub fn get_value_option( &self, key: &AccessoryKey, - version: Option, + version: Version, ) -> anyhow::Result>> { - let version_to_use = version.unwrap_or_else(|| self.get_next_version()); let mut iter = self.db.iter::()?; - iter.seek_for_prev(&(key.to_vec(), version_to_use))?; + iter.seek_for_prev(&(key.to_vec(), version))?; let found = iter.next(); match found { Some(result) => { let ((found_key, found_version), value) = result?; if &found_key == key { - anyhow::ensure!(found_version <= version_to_use, "Bug! iterator isn't returning expected values. expected a version <= {version_to_use:} but found {found_version:}"); + anyhow::ensure!(found_version <= version, "Bug! iterator isn't returning expected values. expected a version <= {version:} but found {found_version:}"); Ok(value) } else { Ok(None) @@ -71,37 +66,14 @@ impl NativeDB { pub fn set_values( &self, key_value_pairs: impl IntoIterator, Option>)>, + version: Version, ) -> anyhow::Result<()> { - let version = self.get_next_version(); let mut batch = SchemaBatch::default(); for (key, value) in key_value_pairs { batch.put::(&(key, version), &value)?; } self.db.write_schemas(batch) } - - /// Increment the `next_version` counter by 1. - pub fn inc_next_version(&self) { - let mut version = self.next_version.lock().unwrap(); - *version += 1; - } - - /// Get the current value of the `next_version` counter - pub fn get_next_version(&self) -> Version { - let version = self.next_version.lock().unwrap(); - *version - } - - fn last_version_written(db: &DB) -> anyhow::Result> { - let mut iter = db.iter::()?; - iter.seek_to_last(); - - let version = match iter.next() { - Some(Ok(((_, version), _))) => Some(version), - _ => None, - }; - Ok(version) - } } #[cfg(feature = "arbitrary")] @@ -192,9 +164,9 @@ mod tests { let key = b"foo".to_vec(); let value = b"bar".to_vec(); - db.set_values(vec![(key.clone(), Some(value.clone()))]) + db.set_values(vec![(key.clone(), Some(value.clone()))], 0) .unwrap(); - assert_eq!(db.get_value_option(&key, None).unwrap(), Some(value)); + assert_eq!(db.get_value_option(&key, 0).unwrap(), Some(value)); } #[test] @@ -203,8 +175,8 @@ mod tests { let db = NativeDB::with_path(tmpdir.path()).unwrap(); let key = b"deleted".to_vec(); - db.set_values(vec![(key.clone(), None)]).unwrap(); - assert_eq!(db.get_value_option(&key, None).unwrap(), None); + db.set_values(vec![(key.clone(), None)], 0).unwrap(); + assert_eq!(db.get_value_option(&key, 0).unwrap(), None); } #[test] @@ -213,6 +185,6 @@ mod tests { let db = NativeDB::with_path(tmpdir.path()).unwrap(); let key = b"spam".to_vec(); - assert_eq!(db.get_value_option(&key, None).unwrap(), None); + assert_eq!(db.get_value_option(&key, 0).unwrap(), None); } } diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index a8fb5d623..23098f8bf 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -193,7 +193,7 @@ fn transfer_initial_token() { // archival versioned state query - let archival_slot = 4; + let archival_slot = 3; let mut versioned_prover_storage = prover_storage.clone(); versioned_prover_storage .set_archival_version(archival_slot) diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index 56ab8f484..7c96fccbd 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -96,9 +96,9 @@ impl Storage for ProverStorage { fn get_accessory(&self, key: &StorageKey) -> Option { let version = self .archival_version - .unwrap_or(self.native_db.get_next_version() - 1); + .unwrap_or(self.db.get_next_version() - 1); self.native_db - .get_value_option(key.as_ref(), Some(version)) + .get_value_option(key.as_ref(), version) .unwrap() .map(Into::into) } @@ -173,6 +173,7 @@ impl Storage for ProverStorage { } fn commit(&self, state_update: &Self::StateUpdate, accessory_writes: &OrderedReadsAndWrites) { + let latest_version = self.db.get_next_version() - 1; for (key_hash, key) in state_update.key_preimages.iter() { // Clone should be cheap self.db @@ -190,11 +191,11 @@ impl Storage for ProverStorage { .ordered_writes .iter() .map(|(k, v_opt)| (k.key.to_vec(), v_opt.as_ref().map(|v| v.value.to_vec()))), + latest_version, ) .expect("native db write must succeed"); self.db.inc_next_version(); - self.native_db.inc_next_version(); } fn open_proof( From 13f2732a5236dc40c05f0859158206181480d00c Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Fri, 24 Nov 2023 11:37:09 +0530 Subject: [PATCH 10/12] big archival refactor --- examples/demo-rollup/src/eth.rs | 4 +- full-node/sov-sequencer/src/batch_builder.rs | 6 +- .../src/tests/helpers.rs | 6 +- .../sov-bank/tests/archival_query_test.rs | 28 +- .../tests/capability_tests.rs | 4 +- .../sov-modules-api/src/default_context.rs | 4 +- .../sov-modules-core/src/storage/mod.rs | 3 - .../tests/custom_codec_must_be_used.rs | 2 +- module-system/sov-state/src/lib.rs | 4 + module-system/sov-state/src/prover_storage.rs | 285 ++++++++++++++++-- .../sov-state/src/storage_manager.rs | 9 +- 11 files changed, 296 insertions(+), 59 deletions(-) diff --git a/examples/demo-rollup/src/eth.rs b/examples/demo-rollup/src/eth.rs index ccd2f1b2b..8da400ec9 100644 --- a/examples/demo-rollup/src/eth.rs +++ b/examples/demo-rollup/src/eth.rs @@ -7,7 +7,7 @@ use sov_ethereum::GasPriceOracleConfig; use sov_modules_api::default_context::DefaultContext; use sov_modules_api::default_signature::private_key::DefaultPrivateKey; use sov_rollup_interface::services::da::DaService; -use sov_state::ProverStorage; +use sov_state::Storages; const TX_SIGNER_PRIV_KEY_PATH: &str = "../test-data/keys/tx_signer_private_key.json"; @@ -25,7 +25,7 @@ fn read_sov_tx_signer_priv_key() -> Result { // register ethereum methods. pub(crate) fn register_ethereum( da_service: Da, - storage: ProverStorage, + storage: Storages, methods: &mut jsonrpsee::RpcModule<()>, ) -> Result<(), anyhow::Error> { let eth_rpc_config = { diff --git a/full-node/sov-sequencer/src/batch_builder.rs b/full-node/sov-sequencer/src/batch_builder.rs index 041bf5f37..4e99734ae 100644 --- a/full-node/sov-sequencer/src/batch_builder.rs +++ b/full-node/sov-sequencer/src/batch_builder.rs @@ -184,7 +184,7 @@ mod tests { use sov_modules_api::transaction::Transaction; use sov_modules_api::{Context, DispatchCall, EncodeCall, Genesis, MessageCodec, PrivateKey}; use sov_rollup_interface::services::batch_builder::BatchBuilder; - use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; + use sov_state::{DefaultStorageSpec, ProverStorage, Storage, Storages}; use sov_value_setter::{CallMessage, ValueSetter, ValueSetterConfig}; use tempfile::TempDir; @@ -235,7 +235,7 @@ mod tests { tmpdir: &TempDir, ) -> ( FiFoStrictBatchBuilder>, - ProverStorage, + Storages, ) { let storage = ProverStorage::::with_path(tmpdir.path()).unwrap(); @@ -248,7 +248,7 @@ mod tests { (batch_builder, storage) } - fn setup_runtime(storage: ProverStorage, admin: Option) { + fn setup_runtime(storage: Storages, admin: Option) { let runtime = TestRuntime::::default(); let mut working_set = WorkingSet::new(storage.clone()); diff --git a/module-system/module-implementations/sov-attester-incentives/src/tests/helpers.rs b/module-system/module-implementations/sov-attester-incentives/src/tests/helpers.rs index e6eed8be8..81bf43a11 100644 --- a/module-system/module-implementations/sov-attester-incentives/src/tests/helpers.rs +++ b/module-system/module-implementations/sov-attester-incentives/src/tests/helpers.rs @@ -9,7 +9,7 @@ use sov_modules_api::hooks::SlotHooks; use sov_modules_api::utils::generate_address; use sov_modules_api::{Address, Genesis, Spec, ValidityConditionChecker, WorkingSet}; use sov_state::storage::{NativeStorage, Storage, StorageProof}; -use sov_state::{DefaultStorageSpec, ProverStorage}; +use sov_state::{DefaultStorageSpec, Storages}; use crate::AttesterIncentives; @@ -25,7 +25,7 @@ pub const INIT_HEIGHT: u64 = 0; /// Consumes and commit the existing working set on the underlying storage /// `storage` must be the underlying storage defined on the working set for this method to work. pub(crate) fn commit_get_new_working_set( - storage: &ProverStorage, + storage: &Storages, working_set: WorkingSet, ) -> (jmt::RootHash, WorkingSet) { let (reads_writes, witness) = working_set.checkpoint().freeze(); @@ -141,7 +141,7 @@ pub(crate) struct ExecutionSimulationVars { pub(crate) fn execution_simulation>( rounds: u8, module: &AttesterIncentives, - storage: &ProverStorage, + storage: &Storages, attester_address: ::Address, mut working_set: WorkingSet, ) -> ( diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index 23098f8bf..b46ea0585 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -4,8 +4,8 @@ use helpers::*; use sov_bank::{get_genesis_token_address, Bank, CallMessage, Coins}; use sov_modules_api::default_context::DefaultContext; use sov_modules_api::{Address, Context, Module, WorkingSet}; -use sov_state::storage::{NativeStorage, StateReaderAndWriter, StorageKey, StorageValue}; -use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; +use sov_state::storage::{StateReaderAndWriter, StorageKey, StorageValue}; +use sov_state::{DefaultStorageSpec, ProverStorage, Storage, Storages}; #[test] fn transfer_initial_token() { @@ -82,12 +82,9 @@ fn transfer_initial_token() { let archival_slot: u64 = 2; println!("Archival reads at slot {}", archival_slot); - let mut versioned_prover_storage = prover_storage.clone(); - versioned_prover_storage - .set_archival_version(archival_slot) - .expect("TODO: panic message"); + let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); let mut working_set: WorkingSet = - WorkingSet::new(versioned_prover_storage.clone()); + WorkingSet::new(archival_storage); let (sender_balance, receiver_balance) = query_sender_receiver_balances( &bank, token_address, @@ -99,12 +96,9 @@ fn transfer_initial_token() { let archival_slot: u64 = 1; println!("Archival archival reads at slot {}", archival_slot); - let mut versioned_prover_storage = prover_storage.clone(); - versioned_prover_storage - .set_archival_version(archival_slot) - .expect("TODO: panic message"); + let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); let mut working_set: WorkingSet = - WorkingSet::new(versioned_prover_storage.clone()); + WorkingSet::new(archival_storage); let (sender_balance, receiver_balance) = query_sender_receiver_balances( &bank, token_address, @@ -194,12 +188,8 @@ fn transfer_initial_token() { // archival versioned state query let archival_slot = 3; - let mut versioned_prover_storage = prover_storage.clone(); - versioned_prover_storage - .set_archival_version(archival_slot) - .expect("TODO: panic message"); - let mut working_set: WorkingSet = - WorkingSet::new(versioned_prover_storage.clone()); + let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); + let mut working_set: WorkingSet = WorkingSet::new(archival_storage); let mut accessory_state = working_set.accessory_state(); let val = accessory_state.get(&StorageKey::from("k")).unwrap(); @@ -251,7 +241,7 @@ fn transfer( println!("Transfer complete"); } -fn commit(working_set: WorkingSet, storage: ProverStorage) { +fn commit(working_set: WorkingSet, storage: Storages) { // Save checkpoint let mut checkpoint = working_set.checkpoint(); diff --git a/module-system/module-implementations/sov-blob-storage/tests/capability_tests.rs b/module-system/module-implementations/sov-blob-storage/tests/capability_tests.rs index 186d6bb17..e108e7f8b 100644 --- a/module-system/module-implementations/sov-blob-storage/tests/capability_tests.rs +++ b/module-system/module-implementations/sov-blob-storage/tests/capability_tests.rs @@ -10,7 +10,7 @@ use sov_modules_api::{ Address, BlobReaderTrait, Context, DaSpec, DispatchCall, MessageCodec, Module, Spec, WorkingSet, }; use sov_sequencer_registry::SequencerConfig; -use sov_state::{DefaultStorageSpec, ProverStorage, Storage}; +use sov_state::{DefaultStorageSpec, ProverStorage, Storage, Storages}; type C = DefaultContext; type B = MockBlob; @@ -663,7 +663,7 @@ struct TestRuntime { impl TestRuntime { pub fn pre_initialized( with_preferred_sequencer: bool, - ) -> (ProverStorage, Self, jmt::RootHash) { + ) -> (Storages, Self, jmt::RootHash) { use sov_modules_api::Genesis; let tmpdir = tempfile::tempdir().unwrap(); let storage = ProverStorage::with_path(tmpdir.path()).unwrap(); diff --git a/module-system/sov-modules-api/src/default_context.rs b/module-system/sov-modules-api/src/default_context.rs index 6c4c23a53..30b431596 100644 --- a/module-system/sov-modules-api/src/default_context.rs +++ b/module-system/sov-modules-api/src/default_context.rs @@ -4,7 +4,7 @@ use sha2::Digest; use sov_modules_core::{Address, Context, PublicKey, Spec, TupleGasUnit}; use sov_rollup_interface::RollupAddress; #[cfg(feature = "native")] -use sov_state::ProverStorage; +use sov_state::Storages; use sov_state::{ArrayWitness, DefaultStorageSpec, ZkStorage}; #[cfg(feature = "native")] @@ -23,7 +23,7 @@ pub struct DefaultContext { #[cfg(feature = "native")] impl Spec for DefaultContext { type Address = Address; - type Storage = ProverStorage; + type Storage = Storages; type PrivateKey = DefaultPrivateKey; type PublicKey = DefaultPublicKey; type Hasher = sha2::Sha256; diff --git a/module-system/sov-modules-core/src/storage/mod.rs b/module-system/sov-modules-core/src/storage/mod.rs index 5b7e9e7d5..049f4d5e6 100644 --- a/module-system/sov-modules-core/src/storage/mod.rs +++ b/module-system/sov-modules-core/src/storage/mod.rs @@ -289,7 +289,4 @@ pub trait NativeStorage: Storage { /// Get the root hash of the tree at the requested version fn get_root_hash(&self, version: Version) -> Result; - - /// From here-on, return values from the given version in `get()` and `get_accessory()`. Used for archival queries. - fn set_archival_version(&mut self, version: u64) -> anyhow::Result<()>; } diff --git a/module-system/sov-modules-macros/tests/custom_codec_must_be_used.rs b/module-system/sov-modules-macros/tests/custom_codec_must_be_used.rs index 27f84f2a9..6fc58afa6 100644 --- a/module-system/sov-modules-macros/tests/custom_codec_must_be_used.rs +++ b/module-system/sov-modules-macros/tests/custom_codec_must_be_used.rs @@ -57,7 +57,7 @@ impl StateValueCodec for CustomCodec { fn main() { let tempdir = tempfile::tempdir().unwrap(); - let storage: ProverStorage = ProverStorage::with_path(&tempdir).unwrap(); + let storage = ProverStorage::with_path(&tempdir).unwrap(); let module: TestModule = TestModule::default(); catch_unwind(|| { diff --git a/module-system/sov-state/src/lib.rs b/module-system/sov-state/src/lib.rs index 68ca97ad9..a19ed305e 100644 --- a/module-system/sov-state/src/lib.rs +++ b/module-system/sov-state/src/lib.rs @@ -10,8 +10,12 @@ mod prover_storage; mod witness; mod zk_storage; +#[cfg(feature = "native")] +pub use prover_storage::ArchivalStorage; #[cfg(feature = "native")] pub use prover_storage::ProverStorage; +#[cfg(feature = "native")] +pub use prover_storage::Storages; pub use zk_storage::ZkStorage; pub mod config; diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index 7c96fccbd..f622c6262 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -2,7 +2,7 @@ use std::marker::PhantomData; use std::path::Path; use std::sync::Arc; -use anyhow::ensure; +use anyhow::{bail, ensure, Error}; use jmt::storage::{NodeBatch, TreeWriter}; use jmt::{JellyfishMerkleTree, KeyHash, Version}; use sov_db::native_db::NativeDB; @@ -15,12 +15,119 @@ use sov_modules_core::{ use crate::config::Config; use crate::MerkleProofSpec; +/// placeholder +pub enum Storages { + /// placeholder + Prover(ProverStorage), + /// placeholder + Archival(ArchivalStorage), +} + +impl Clone for Storages { + fn clone(&self) -> Self { + match self { + Storages::Prover(prover_storage) => Storages::Prover(prover_storage.clone()), + Storages::Archival(archival_storage) => Storages::Archival(archival_storage.clone()), + } + } +} + +impl Storages { + /// placeholder + pub fn get_archival_storage(&self, version: u64) -> anyhow::Result> { + match self { + Storages::Prover(prover_storage) => Ok(Storages::Archival(prover_storage.get_archival_storage(version)?)), + Storages::Archival(archival_storage) => Ok(Storages::Archival(archival_storage.clone().set_archival_version(version)?)) + } + } +} + +impl Storage for Storages { + type Witness = S::Witness; + type RuntimeConfig = Config; + type Proof = jmt::proof::SparseMerkleProof; + type Root = jmt::RootHash; + type StateUpdate = ProverStateUpdate; + + fn with_config(config: Self::RuntimeConfig) -> anyhow::Result { + Ok(Storages::Prover(ProverStorage::with_config(config)?)) + } + + fn get(&self, key: &StorageKey, witness: &Self::Witness) -> Option { + match self { + Storages::Prover(prover_storage) => prover_storage.get(key, witness), + Storages::Archival(archival_storage) => archival_storage.get(key, witness), + } + } + + #[cfg(feature = "native")] + fn get_accessory(&self, key: &StorageKey) -> Option { + match self { + Storages::Prover(prover_storage) => prover_storage.get_accessory(key), + Storages::Archival(archival_storage) => archival_storage.get_accessory(key), + } + } + + fn compute_state_update( + &self, + state_accesses: OrderedReadsAndWrites, + witness: &Self::Witness, + ) -> Result<(Self::Root, Self::StateUpdate), Error> { + match self { + Storages::Prover(prover_storage) => { + prover_storage.compute_state_update(state_accesses, witness) + } + Storages::Archival(archival_storage) => { + archival_storage.compute_state_update(state_accesses, witness) + } + } + } + + fn commit(&self, node_batch: &Self::StateUpdate, accessory_update: &OrderedReadsAndWrites) { + match self { + Storages::Prover(prover_storage) => prover_storage.commit(node_batch, accessory_update), + Storages::Archival(archival_storage) => { + archival_storage.commit(node_batch, accessory_update) + } + } + } + + fn open_proof( + state_root: Self::Root, + proof: StorageProof, + ) -> Result<(StorageKey, Option), Error> { + ProverStorage::::open_proof(state_root, proof) + } + + fn is_empty(&self) -> bool { + match self { + Storages::Prover(prover_storage) => prover_storage.is_empty(), + Storages::Archival(archival_storage) => archival_storage.is_empty(), + } + } +} + +impl NativeStorage for Storages { + fn get_with_proof(&self, key: StorageKey) -> StorageProof { + match self { + Storages::Prover(prover_storage) => prover_storage.get_with_proof(key), + Storages::Archival(archival_storage) => archival_storage.get_with_proof(key), + } + } + + fn get_root_hash(&self, version: Version) -> Result { + match self { + Storages::Prover(prover_storage) => prover_storage.get_root_hash(version), + Storages::Archival(archival_storage) => archival_storage.get_root_hash(version), + } + } +} + /// A [`Storage`] implementation to be used by the prover in a native execution /// environment (outside of the zkVM). pub struct ProverStorage { db: StateDB, native_db: NativeDB, - archival_version: Option, _phantom_hasher: PhantomData, } @@ -29,7 +136,6 @@ impl Clone for ProverStorage { Self { db: self.db.clone(), native_db: self.native_db.clone(), - archival_version: self.archival_version, _phantom_hasher: Default::default(), } } @@ -38,41 +144,60 @@ impl Clone for ProverStorage { impl ProverStorage { /// Creates a new [`ProverStorage`] instance at the specified path, opening /// or creating the necessary RocksDB database(s) at the specified path. - pub fn with_path(path: impl AsRef) -> Result { + pub fn with_path(path: impl AsRef) -> Result, anyhow::Error> { let state_db = StateDB::with_path(&path)?; let native_db = NativeDB::with_path(&path)?; - Ok(Self { - db: state_db, - native_db, - archival_version: None, - _phantom_hasher: Default::default(), - }) + Ok( + Storages::Prover(Self { + db: state_db, + native_db, + _phantom_hasher: Default::default(), + }) + ) + } + + /// Creates a new [`ProverStorage`] instance at the specified path, opening + /// or creating the necessary RocksDB database(s) at the specified path. + pub fn with_path_prover(path: impl AsRef) -> Result { + let state_db = StateDB::with_path(&path)?; + let native_db = NativeDB::with_path(&path)?; + + Ok( + Self { + db: state_db, + native_db, + _phantom_hasher: Default::default(), + } + ) } pub(crate) fn with_db_handles(db: StateDB, native_db: NativeDB) -> Self { Self { db, native_db, - archival_version: None, _phantom_hasher: Default::default(), } } fn read_value(&self, key: &StorageKey) -> Option { - let version = self.archival_version.unwrap_or(self.db.get_next_version()); + let version = self.db.get_next_version(); match self.db.get_value_option_by_key(version, key.as_ref()) { Ok(value) => value.map(Into::into), // It is ok to panic here, we assume the db is available and consistent. Err(e) => panic!("Unable to read value from db: {e}"), } } + + /// placeholder + pub fn get_archival_storage(&self, version: Version) -> anyhow::Result> { + ArchivalStorage::new(self.clone(), version) + } } pub struct ProverStateUpdate { pub(crate) node_batch: NodeBatch, pub key_preimages: Vec<(KeyHash, CacheKey)>, - // pub accessory_update: OrderedReadsAndWrites, } impl Storage for ProverStorage { @@ -83,7 +208,15 @@ impl Storage for ProverStorage { type StateUpdate = ProverStateUpdate; fn with_config(config: Self::RuntimeConfig) -> Result { - Self::with_path(config.path.as_path()) + match Self::with_path(config.path.as_path()) { + Ok(storages) => { + match storages { + Storages::Prover(prover_storage) => Ok(prover_storage), + _ => bail!("Creating storage failed") + } + } + Err(_) => bail!("Creating storage failed") + } } fn get(&self, key: &StorageKey, witness: &Self::Witness) -> Option { @@ -94,9 +227,7 @@ impl Storage for ProverStorage { #[cfg(feature = "native")] fn get_accessory(&self, key: &StorageKey) -> Option { - let version = self - .archival_version - .unwrap_or(self.db.get_next_version() - 1); + let version = self.db.get_next_version() - 1; self.native_db .get_value_option(key.as_ref(), version) .unwrap() @@ -236,11 +367,123 @@ impl NativeStorage for ProverStorage { JellyfishMerkleTree::new(&self.db); temp_merkle.get_root_hash(version) } +} + +/// placeholder +pub struct ArchivalStorage { + prover_storage: ProverStorage, + archival_version: Version, +} + +impl ArchivalStorage { + /// placeholder + pub fn new(prover_storage: ProverStorage, version: Version) -> anyhow::Result { + ensure!(version < prover_storage.db.get_next_version(), + "The storage default read version can not be set to a version greater than the next version"); + Ok(Self { + prover_storage, + archival_version: version, + }) + } - fn set_archival_version(&mut self, version: u64) -> anyhow::Result<()> { - ensure!(version < self.db.get_next_version(), + /// placeholder + pub fn set_archival_version(mut self, version: Version) -> anyhow::Result { + ensure!(version < self.prover_storage.db.get_next_version(), "The storage default read version can not be set to a version greater than the next version"); - self.archival_version = Some(version); - Ok(()) + self.archival_version = version; + Ok(self) + } +} + +impl Clone for ArchivalStorage { + fn clone(&self) -> Self { + Self { + prover_storage: self.prover_storage.clone(), + archival_version: self.archival_version, + } + } +} + +impl Storage for ArchivalStorage { + type Witness = as Storage>::Witness; + type RuntimeConfig = as Storage>::RuntimeConfig; + type Proof = as Storage>::Proof; + type Root = as Storage>::Root; + type StateUpdate = as Storage>::StateUpdate; + + fn with_config(config: Self::RuntimeConfig) -> anyhow::Result { + ProverStorage::::with_config(config).map(|prover_storage| { + let archival_version = prover_storage.db.get_next_version(); + ArchivalStorage { + prover_storage, + archival_version, + } + }) + } + + fn get(&self, key: &StorageKey, witness: &Self::Witness) -> Option { + let val = match self + .prover_storage + .db + .get_value_option_by_key(self.archival_version, key.as_ref()) + { + Ok(value) => value.map(Into::into), + Err(e) => panic!("Unable to read value from db: {e}"), + }; + witness.add_hint(val.clone()); + val + } + + #[cfg(feature = "native")] + fn get_accessory(&self, key: &StorageKey) -> Option { + self.prover_storage + .native_db + .get_value_option(key.as_ref(), self.archival_version) + .unwrap() + .map(Into::into) + } + + fn compute_state_update( + &self, + _state_accesses: OrderedReadsAndWrites, + _witness: &Self::Witness, + ) -> Result<(Self::Root, Self::StateUpdate), Error> { + bail!("Archival storage cannot update"); + } + + fn commit(&self, _node_batch: &Self::StateUpdate, _accessory_update: &OrderedReadsAndWrites) {} + + fn open_proof( + state_root: Self::Root, + proof: StorageProof, + ) -> Result<(StorageKey, Option), Error> { + ProverStorage::::open_proof(state_root, proof) + } + + fn is_empty(&self) -> bool { + self.prover_storage.is_empty() + } +} + +impl NativeStorage for ArchivalStorage { + fn get_with_proof(&self, key: StorageKey) -> StorageProof { + let merkle = JellyfishMerkleTree::::new(&self.prover_storage.db); + let (val_opt, proof) = merkle + .get_with_proof( + KeyHash::with::(key.as_ref()), + self.archival_version, + ) + .unwrap(); + StorageProof { + key, + value: val_opt.map(StorageValue::from), + proof, + } + } + + fn get_root_hash(&self, version: Version) -> Result { + let temp_merkle: JellyfishMerkleTree<'_, StateDB, S::Hasher> = + JellyfishMerkleTree::new(&self.prover_storage.db); + temp_merkle.get_root_hash(version) } } diff --git a/module-system/sov-state/src/storage_manager.rs b/module-system/sov-state/src/storage_manager.rs index ac0e8072f..c7d9ecbfc 100644 --- a/module-system/sov-state/src/storage_manager.rs +++ b/module-system/sov-state/src/storage_manager.rs @@ -1,6 +1,6 @@ //! State manager for [`ProverStorage`] -use crate::{config, MerkleProofSpec, ProverStorage}; +use crate::{config, MerkleProofSpec, ProverStorage, Storages}; /// State manager for Prover and Zk Storage pub struct ProverStorageManager { @@ -24,9 +24,12 @@ impl ProverStorageManager { } impl sov_rollup_interface::storage::StorageManager for ProverStorageManager { - type NativeStorage = ProverStorage; + type NativeStorage = Storages; type NativeChangeSet = (); fn get_native_storage(&self) -> Self::NativeStorage { - ProverStorage::with_db_handles(self.state_db.clone(), self.native_db.clone()) + Storages::Prover(ProverStorage::with_db_handles( + self.state_db.clone(), + self.native_db.clone(), + )) } } From c1e4dc7fcbc3bf63ed31f4b4b57ae767d0d073cf Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Fri, 24 Nov 2023 11:55:50 +0530 Subject: [PATCH 11/12] formatting --- .../sov-bank/tests/archival_query_test.rs | 6 +-- module-system/sov-state/src/prover_storage.rs | 44 +++++++++---------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index b46ea0585..6cd7af79d 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -83,8 +83,7 @@ fn transfer_initial_token() { let archival_slot: u64 = 2; println!("Archival reads at slot {}", archival_slot); let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); - let mut working_set: WorkingSet = - WorkingSet::new(archival_storage); + let mut working_set: WorkingSet = WorkingSet::new(archival_storage); let (sender_balance, receiver_balance) = query_sender_receiver_balances( &bank, token_address, @@ -97,8 +96,7 @@ fn transfer_initial_token() { let archival_slot: u64 = 1; println!("Archival archival reads at slot {}", archival_slot); let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); - let mut working_set: WorkingSet = - WorkingSet::new(archival_storage); + let mut working_set: WorkingSet = WorkingSet::new(archival_storage); let (sender_balance, receiver_balance) = query_sender_receiver_balances( &bank, token_address, diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index f622c6262..eb061a1b9 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -36,8 +36,12 @@ impl Storages { /// placeholder pub fn get_archival_storage(&self, version: u64) -> anyhow::Result> { match self { - Storages::Prover(prover_storage) => Ok(Storages::Archival(prover_storage.get_archival_storage(version)?)), - Storages::Archival(archival_storage) => Ok(Storages::Archival(archival_storage.clone().set_archival_version(version)?)) + Storages::Prover(prover_storage) => Ok(Storages::Archival( + prover_storage.get_archival_storage(version)?, + )), + Storages::Archival(archival_storage) => Ok(Storages::Archival( + archival_storage.clone().set_archival_version(version)?, + )), } } } @@ -148,13 +152,11 @@ impl ProverStorage { let state_db = StateDB::with_path(&path)?; let native_db = NativeDB::with_path(&path)?; - Ok( - Storages::Prover(Self { - db: state_db, - native_db, - _phantom_hasher: Default::default(), - }) - ) + Ok(Storages::Prover(Self { + db: state_db, + native_db, + _phantom_hasher: Default::default(), + })) } /// Creates a new [`ProverStorage`] instance at the specified path, opening @@ -163,13 +165,11 @@ impl ProverStorage { let state_db = StateDB::with_path(&path)?; let native_db = NativeDB::with_path(&path)?; - Ok( - Self { - db: state_db, - native_db, - _phantom_hasher: Default::default(), - } - ) + Ok(Self { + db: state_db, + native_db, + _phantom_hasher: Default::default(), + }) } pub(crate) fn with_db_handles(db: StateDB, native_db: NativeDB) -> Self { @@ -209,13 +209,11 @@ impl Storage for ProverStorage { fn with_config(config: Self::RuntimeConfig) -> Result { match Self::with_path(config.path.as_path()) { - Ok(storages) => { - match storages { - Storages::Prover(prover_storage) => Ok(prover_storage), - _ => bail!("Creating storage failed") - } - } - Err(_) => bail!("Creating storage failed") + Ok(storages) => match storages { + Storages::Prover(prover_storage) => Ok(prover_storage), + _ => bail!("Creating storage failed"), + }, + Err(_) => bail!("Creating storage failed"), } } From 9df60cd3af9d3e2e15073e6449f741ca7dfce2f7 Mon Sep 17 00:00:00 2001 From: dubbelosix Date: Fri, 24 Nov 2023 14:08:39 +0530 Subject: [PATCH 12/12] some fixes --- .../sov-bank/tests/archival_query_test.rs | 14 -------------- module-system/sov-state/src/prover_storage.rs | 18 +++++------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs index 6cd7af79d..4a096806e 100644 --- a/module-system/module-implementations/sov-bank/tests/archival_query_test.rs +++ b/module-system/module-implementations/sov-bank/tests/archival_query_test.rs @@ -33,9 +33,7 @@ fn transfer_initial_token() { &mut working_set, ); assert_eq!((sender_balance, receiver_balance), (100, 100)); - println!("Starting commit for genesis, i.e. slot 1"); commit(working_set, prover_storage.clone()); - println!("Genesis commit complete"); let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); @@ -55,9 +53,7 @@ fn transfer_initial_token() { ); assert_eq!((sender_balance, receiver_balance), (90, 110)); - println!("Starting commit for slot 2"); commit(working_set, prover_storage.clone()); - println!("Commit complete for slot 2"); let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); @@ -76,12 +72,9 @@ fn transfer_initial_token() { &mut working_set, ); assert_eq!((sender_balance, receiver_balance), (80, 120)); - println!("Starting commit for slot 3"); commit(working_set, prover_storage.clone()); - println!("Commit complete for slot 3"); let archival_slot: u64 = 2; - println!("Archival reads at slot {}", archival_slot); let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); let mut working_set: WorkingSet = WorkingSet::new(archival_storage); let (sender_balance, receiver_balance) = query_sender_receiver_balances( @@ -94,7 +87,6 @@ fn transfer_initial_token() { assert_eq!((sender_balance, receiver_balance), (90, 110)); let archival_slot: u64 = 1; - println!("Archival archival reads at slot {}", archival_slot); let archival_storage = prover_storage.get_archival_storage(archival_slot).unwrap(); let mut working_set: WorkingSet = WorkingSet::new(archival_storage); let (sender_balance, receiver_balance) = query_sender_receiver_balances( @@ -105,7 +97,6 @@ fn transfer_initial_token() { &mut working_set, ); assert_eq!((sender_balance, receiver_balance), (100, 100)); - println!("Transfer on archival"); transfer( &bank, token_address, @@ -113,7 +104,6 @@ fn transfer_initial_token() { receiver_address, &mut working_set, ); - println!("Archival query for modified working set"); let (sender_balance, receiver_balance) = query_sender_receiver_balances( &bank, token_address, @@ -123,7 +113,6 @@ fn transfer_initial_token() { ); assert_eq!((sender_balance, receiver_balance), (90, 110)); - println!(" Move back from archival to current once again"); let mut working_set: WorkingSet = WorkingSet::new(prover_storage.clone()); let (sender_balance, receiver_balance) = query_sender_receiver_balances( &bank, @@ -190,11 +179,8 @@ fn transfer_initial_token() { let mut working_set: WorkingSet = WorkingSet::new(archival_storage); let mut accessory_state = working_set.accessory_state(); let val = accessory_state.get(&StorageKey::from("k")).unwrap(); - assert_eq!("v1", String::from_utf8(val.value().to_vec()).unwrap()); - commit(working_set, prover_storage.clone()); - println!("{:?}", val); } fn query_sender_receiver_balances( diff --git a/module-system/sov-state/src/prover_storage.rs b/module-system/sov-state/src/prover_storage.rs index eb061a1b9..a9ac298c4 100644 --- a/module-system/sov-state/src/prover_storage.rs +++ b/module-system/sov-state/src/prover_storage.rs @@ -44,6 +44,11 @@ impl Storages { )), } } + + /// placeholder + pub fn with_path(path: impl AsRef) -> Result, anyhow::Error> { + ProverStorage::with_path(path) + } } impl Storage for Storages { @@ -159,19 +164,6 @@ impl ProverStorage { })) } - /// Creates a new [`ProverStorage`] instance at the specified path, opening - /// or creating the necessary RocksDB database(s) at the specified path. - pub fn with_path_prover(path: impl AsRef) -> Result { - let state_db = StateDB::with_path(&path)?; - let native_db = NativeDB::with_path(&path)?; - - Ok(Self { - db: state_db, - native_db, - _phantom_hasher: Default::default(), - }) - } - pub(crate) fn with_db_handles(db: StateDB, native_db: NativeDB) -> Self { Self { db,