From e318636d202635866f47a1f1579d7be037dc9f67 Mon Sep 17 00:00:00 2001 From: Teng Zhang Date: Tue, 12 Mar 2024 17:17:44 -0700 Subject: [PATCH] [comparison-testing-tool] Add more features to the comparison testing tool (#11890) * add features * refactor * handling review comments * fix * always get aptos framework from git * simplify mismatch info * add failed cache * remove clean session * revert clean session * handle review comments * handle comments * comment followup * add gas meter to avoid hanging in loop * update package publish handling --- Cargo.lock | 1 + .../aptos-e2e-comparison-testing/Cargo.toml | 1 + .../src/data_collection.rs | 38 +- .../src/data_state_view.rs | 86 +++++ .../src/execution.rs | 350 +++++++++++++----- .../aptos-e2e-comparison-testing/src/lib.rs | 112 ++++-- .../aptos-e2e-comparison-testing/src/main.rs | 61 ++- .../src/online_execution.rs | 240 ++++++++++++ .../aptos-validator-interface/src/lib.rs | 49 +-- .../src/rest_interface.rs | 55 ++- .../src/storage_interface.rs | 9 + aptos-move/aptos-vm/src/gas.rs | 2 +- aptos-move/aptos-vm/src/lib.rs | 2 +- .../aptos-vm/src/move_vm_ext/warm_vm_cache.rs | 6 +- aptos-move/e2e-tests/src/executor.rs | 73 ++-- .../move-core/types/src/account_address.rs | 16 + types/src/state_store/state_key.rs | 13 + 17 files changed, 895 insertions(+), 219 deletions(-) create mode 100644 aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs create mode 100644 aptos-move/aptos-e2e-comparison-testing/src/online_execution.rs diff --git a/Cargo.lock b/Cargo.lock index 341fec332198e..9acb4cbef6941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -796,6 +796,7 @@ dependencies = [ "clap 4.4.14", "futures", "itertools 0.10.5", + "lru 0.7.8", "move-binary-format", "move-cli", "move-compiler", diff --git a/aptos-move/aptos-e2e-comparison-testing/Cargo.toml b/aptos-move/aptos-e2e-comparison-testing/Cargo.toml index cd142da6bb7f2..e57af6a499dd4 100644 --- a/aptos-move/aptos-e2e-comparison-testing/Cargo.toml +++ b/aptos-move/aptos-e2e-comparison-testing/Cargo.toml @@ -33,6 +33,7 @@ bcs = { workspace = true } clap = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +lru = { workspace = true } move-binary-format = { workspace = true } move-cli = { workspace = true } move-compiler = { workspace = true } diff --git a/aptos-move/aptos-e2e-comparison-testing/src/data_collection.rs b/aptos-move/aptos-e2e-comparison-testing/src/data_collection.rs index a07500f702421..837f495c22287 100644 --- a/aptos-move/aptos-e2e-comparison-testing/src/data_collection.rs +++ b/aptos-move/aptos-e2e-comparison-testing/src/data_collection.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - dump_and_compile_from_package_metadata, is_aptos_package, CompilationCache, DataManager, - IndexWriter, PackageInfo, TxnIndex, + data_state_view::DataStateView, dump_and_compile_from_package_metadata, is_aptos_package, + CompilationCache, DataManager, IndexWriter, PackageInfo, TxnIndex, }; use anyhow::{format_err, Result}; use aptos_framework::natives::code::PackageMetadata; @@ -16,9 +16,7 @@ use aptos_types::{ }, write_set::TOTAL_SUPPLY_STATE_KEY, }; -use aptos_validator_interface::{ - AptosValidatorInterface, DebuggerStateView, FilterCondition, RestDebuggerInterface, -}; +use aptos_validator_interface::{AptosValidatorInterface, FilterCondition, RestDebuggerInterface}; use aptos_vm::{AptosVM, VMExecutor}; use move_core_types::account_address::AccountAddress; use std::{ @@ -45,6 +43,7 @@ impl DataCollection { skip_publish_txns: bool, dump_write_set: bool, skip_source_code: bool, + target_account: Option, ) -> Self { Self { debugger, @@ -55,6 +54,7 @@ impl DataCollection { skip_failed_txns, skip_publish_txns, check_source_code: !skip_source_code, + target_account, }, } } @@ -67,6 +67,7 @@ impl DataCollection { skip_publish_txns: bool, dump_write_set: bool, skip_source_code: bool, + target_account: Option, ) -> Result { Ok(Self::new( Arc::new(RestDebuggerInterface::new(rest_client)), @@ -76,21 +77,22 @@ impl DataCollection { skip_publish_txns, dump_write_set, skip_source_code, + target_account, )) } fn execute_transactions_at_version_with_state_view( txns: Vec, - debugger_stateview: &DebuggerStateView, + debugger_state_view: &DataStateView, ) -> Result> { let sig_verified_txns: Vec = txns.into_iter().map(|x| x.into()).collect::>(); // check whether total supply can be retrieved // used for debugging the aggregator panic issue, will be removed later // FIXME(#10412): remove the assert - let val = debugger_stateview.get_state_value(TOTAL_SUPPLY_STATE_KEY.deref()); + let val = debugger_state_view.get_state_value(TOTAL_SUPPLY_STATE_KEY.deref()); assert!(val.is_ok() && val.unwrap().is_some()); - AptosVM::execute_block_no_limit(&sig_verified_txns, debugger_stateview) + AptosVM::execute_block_no_limit(&sig_verified_txns, debugger_state_view) .map_err(|err| format_err!("Unexpected VM Error: {:?}", err)) } @@ -114,15 +116,14 @@ impl DataCollection { package_name: package_name.clone(), upgrade_number, }; - + if compilation_cache.failed_packages_v1.contains(&package_info) { + return None; + } if !is_aptos_package(&package_name) && !compilation_cache .compiled_package_map .contains_key(&package_info) { - if compilation_cache.failed_packages.contains(&package_info) { - return None; - } let res = dump_and_compile_from_package_metadata( package_info.clone(), current_dir, @@ -131,7 +132,7 @@ impl DataCollection { None, ); if res.is_err() { - println!("compile package failed at:{}", version); + eprintln!("{} at: {}", res.unwrap_err(), version); return None; } } @@ -167,7 +168,7 @@ impl DataCollection { let index_writer = Arc::new(Mutex::new(IndexWriter::new(&self.current_dir))); let mut cur_version = begin; - + let mut module_registry_map = HashMap::new(); while cur_version < begin + limit { let batch = if cur_version + self.batch_size <= begin + limit { self.batch_size @@ -176,7 +177,12 @@ impl DataCollection { }; let res_txns = self .debugger - .get_and_filter_committed_transactions(cur_version, batch, self.filter_condition) + .get_and_filter_committed_transactions( + cur_version, + batch, + self.filter_condition, + &mut module_registry_map, + ) .await; // if error happens when collecting txns, log the version range if res_txns.is_err() { @@ -198,7 +204,7 @@ impl DataCollection { let index = index_writer.clone(); let state_view = - DebuggerStateView::new_with_data_reads(self.debugger.clone(), version); + DataStateView::new_with_data_reads(self.debugger.clone(), version); let txn_execution_thread = tokio::task::spawn_blocking(move || { let epoch_result_res = diff --git a/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs b/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs new file mode 100644 index 0000000000000..f5c81a5dc56e3 --- /dev/null +++ b/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs @@ -0,0 +1,86 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use aptos_language_e2e_tests::data_store::FakeDataStore; +use aptos_types::{ + state_store::{ + state_key::StateKey, state_storage_usage::StateStorageUsage, state_value::StateValue, + Result as StateViewResult, TStateView, + }, + transaction::Version, +}; +use aptos_validator_interface::{AptosValidatorInterface, DebuggerStateView}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +pub struct DataStateView { + debugger_view: DebuggerStateView, + code_data: Option, + data_read_state_keys: Option>>>, +} +use std::ops::DerefMut; + +impl DataStateView { + pub fn new( + db: Arc, + version: Version, + code_data: FakeDataStore, + ) -> Self { + Self { + debugger_view: DebuggerStateView::new(db, version), + code_data: Some(code_data), + data_read_state_keys: None, + } + } + + pub fn new_with_data_reads( + db: Arc, + version: Version, + ) -> Self { + Self { + debugger_view: DebuggerStateView::new(db, version), + code_data: None, + data_read_state_keys: Some(Arc::new(Mutex::new(HashMap::new()))), + } + } + + pub fn get_state_keys(self) -> Arc>> { + self.data_read_state_keys.unwrap() + } +} + +impl TStateView for DataStateView { + type Key = StateKey; + + fn get_state_value(&self, state_key: &StateKey) -> StateViewResult> { + if let Some(code) = &self.code_data { + if code.contains_key(state_key) { + return code.get_state_value(state_key).map_err(Into::into); + } + } + let ret = self.debugger_view.get_state_value(state_key); + if let Some(reads) = &self.data_read_state_keys { + if !state_key.is_aptos_code() + && !reads.lock().unwrap().contains_key(state_key) + && ret.is_ok() + { + let val = ret?; + if val.is_some() { + reads + .lock() + .unwrap() + .deref_mut() + .insert(state_key.clone(), val.clone().unwrap()); + }; + return Ok(val); + } + } + ret + } + + fn get_usage(&self) -> StateViewResult { + unreachable!() + } +} diff --git a/aptos-move/aptos-e2e-comparison-testing/src/execution.rs b/aptos-move/aptos-e2e-comparison-testing/src/execution.rs index 5f0e061e44bc5..9e88073be6929 100644 --- a/aptos-move/aptos-e2e-comparison-testing/src/execution.rs +++ b/aptos-move/aptos-e2e-comparison-testing/src/execution.rs @@ -3,8 +3,8 @@ use crate::{ check_aptos_packages_availability, compile_aptos_packages, compile_package, - generate_compiled_blob, is_aptos_package, DataManager, IndexReader, PackageInfo, TxnIndex, - APTOS_COMMONS, + data_state_view::DataStateView, generate_compiled_blob, is_aptos_package, CompilationCache, + DataManager, IndexReader, PackageInfo, TxnIndex, APTOS_COMMONS, }; use anyhow::Result; use aptos_framework::APTOS_PACKAGES; @@ -16,18 +16,22 @@ use aptos_types::{ vm_status::VMStatus, write_set::WriteSet, }; +use aptos_validator_interface::AptosValidatorInterface; use aptos_vm::{data_cache::AsMoveResolver, transaction_metadata::TransactionMetadata}; use clap::ValueEnum; use itertools::Itertools; use move_core_types::{account_address::AccountAddress, language_storage::ModuleId}; use move_package::CompilerVersion; -use std::{collections::HashMap, path::PathBuf}; +use std::{cmp, collections::HashMap, path::PathBuf, sync::Arc}; fn load_packages_to_executor( executor: &mut FakeExecutor, package_info: &PackageInfo, compiled_package_cache: &HashMap>>, ) { + if !compiled_package_cache.contains_key(package_info) { + return; + } let compiled_package = compiled_package_cache.get(package_info).unwrap(); for (module_id, module_blob) in compiled_package { executor.add_module(module_id, module_blob.clone()); @@ -85,11 +89,15 @@ impl Default for ExecutionMode { pub struct Execution { input_path: PathBuf, - execution_mode: ExecutionMode, - bytecode_version: u32, + pub execution_mode: ExecutionMode, + pub bytecode_version: u32, } impl Execution { + pub fn output_result_str(&self, msg: String) { + eprintln!("{}", msg); + } + pub fn new(input_path: PathBuf, execution_mode: ExecutionMode) -> Self { Self { input_path, @@ -104,15 +112,20 @@ impl Execution { return Err(anyhow::Error::msg("aptos packages are missing")); } - let mut compiled_package_cache: HashMap>> = - HashMap::new(); - let mut compiled_package_cache_v2: HashMap>> = - HashMap::new(); + let mut compiled_cache = CompilationCache::default(); if self.execution_mode.is_v1_or_compare() { - compile_aptos_packages(&aptos_commons_path, &mut compiled_package_cache, false)?; + compile_aptos_packages( + &aptos_commons_path, + &mut compiled_cache.compiled_package_cache_v1, + false, + )?; } if self.execution_mode.is_v2_or_compare() { - compile_aptos_packages(&aptos_commons_path, &mut compiled_package_cache_v2, true)?; + compile_aptos_packages( + &aptos_commons_path, + &mut compiled_cache.compiled_package_cache_v2, + true, + )?; } // prepare data @@ -135,22 +148,23 @@ impl Execution { let mut cur_version = ver.unwrap(); let mut i = 0; while i < num_txns_to_execute { - let res = self.execute_one_txn( - cur_version, - &data_manager, - &mut compiled_package_cache, - &mut compiled_package_cache_v2, - ); + let res = self.execute_one_txn(cur_version, &data_manager, &mut compiled_cache); if res.is_err() { - println!( + self.output_result_str(format!( "execution at version:{} failed, skip to the next txn", cur_version - ); + )); } - if let Some(ver) = index_reader.get_next_version() { - cur_version = ver; - } else { - break; + let mut ver_res = index_reader.get_next_version(); + while ver_res.is_err() { + ver_res = index_reader.get_next_version(); + } + if ver_res.is_ok() { + if let Some(ver) = ver_res.unwrap() { + cur_version = ver; + } else { + break; + } } i += 1; } @@ -160,8 +174,7 @@ impl Execution { fn compile_code( &self, txn_idx: &TxnIndex, - compiled_package_cache: &mut HashMap>>, - compiled_package_cache_v2: &mut HashMap>>, + compiled_cache: &mut CompilationCache, ) -> Result<()> { if !txn_idx.package_info.is_compilable() { return Err(anyhow::Error::msg("not compilable")); @@ -171,18 +184,68 @@ impl Execution { if !package_dir.exists() { return Err(anyhow::Error::msg("source code is not available")); } + let mut v1_failed = false; + let mut v2_failed = false; if self.execution_mode.is_v1_or_compare() - && !compiled_package_cache.contains_key(&package_info) + && !compiled_cache + .compiled_package_cache_v1 + .contains_key(&package_info) { - let compiled_res = compile_package(package_dir.clone(), &package_info, None)?; - generate_compiled_blob(&package_info, &compiled_res, compiled_package_cache); + if compiled_cache.failed_packages_v1.contains(&package_info) { + v1_failed = true; + } else { + let compiled_res_v1 = compile_package( + package_dir.clone(), + &package_info, + Some(CompilerVersion::V1), + ); + if let Ok(compiled_res) = compiled_res_v1 { + generate_compiled_blob( + &package_info, + &compiled_res, + &mut compiled_cache.compiled_package_cache_v1, + ); + } else { + v1_failed = true; + compiled_cache + .failed_packages_v1 + .insert(package_info.clone()); + } + } } if self.execution_mode.is_v2_or_compare() - && !compiled_package_cache_v2.contains_key(&package_info) + && !compiled_cache + .compiled_package_cache_v2 + .contains_key(&package_info) { - let compiled_res = - compile_package(package_dir, &package_info, Some(CompilerVersion::V2))?; - generate_compiled_blob(&package_info, &compiled_res, compiled_package_cache_v2); + if compiled_cache.failed_packages_v2.contains(&package_info) { + v2_failed = true; + } else { + let compiled_res_v2 = + compile_package(package_dir, &package_info, Some(CompilerVersion::V2)); + if let Ok(compiled_res) = compiled_res_v2 { + generate_compiled_blob( + &package_info, + &compiled_res, + &mut compiled_cache.compiled_package_cache_v2, + ); + } else { + v2_failed = true; + compiled_cache + .failed_packages_v2 + .insert(package_info.clone()); + } + } + } + if v1_failed || v2_failed { + let mut err_msg = "compilation failed at ".to_string(); + if v1_failed { + err_msg = format!("{} v1", err_msg); + } + if v2_failed { + err_msg = format!("{} v2", err_msg); + } + return Err(anyhow::Error::msg(err_msg)); } Ok(()) } @@ -191,182 +254,267 @@ impl Execution { &self, cur_version: Version, data_manager: &DataManager, - compiled_package_cache: &mut HashMap>>, - compiled_package_cache_v2: &mut HashMap>>, + compiled_cache: &mut CompilationCache, ) -> Result<()> { if let Some(txn_idx) = data_manager.get_txn_index(cur_version) { // compile the code if the source code is available if txn_idx.package_info.is_compilable() && !is_aptos_package(&txn_idx.package_info.package_name) { - let compiled_result = - self.compile_code(&txn_idx, compiled_package_cache, compiled_package_cache_v2); + let compiled_result = self.compile_code(&txn_idx, compiled_cache); if compiled_result.is_err() { - println!( + self.output_result_str(format!( "compilation failed for the package:{} at version:{}", txn_idx.package_info.package_name, cur_version - ); + )); return compiled_result; } } // read the state data; let state = data_manager.get_state(cur_version); - let state_view = state.as_move_resolver(); - let mut features = Features::fetch_config(&state_view).unwrap_or_default(); - if self.bytecode_version == 6 { - features.enable(FeatureFlag::VM_BINARY_FORMAT_V6); - } // execute and compare self.execute_and_compare( cur_version, - &state, - &features, + state, &txn_idx, - compiled_package_cache, - compiled_package_cache_v2, + &compiled_cache.compiled_package_cache_v1, + &compiled_cache.compiled_package_cache_v2, + None, ); } Ok(()) } - fn execute_and_compare( + pub(crate) fn execute_and_compare( &self, cur_version: Version, - state: &FakeDataStore, - features: &Features, + state: FakeDataStore, txn_idx: &TxnIndex, compiled_package_cache: &HashMap>>, compiled_package_cache_v2: &HashMap>>, + debugger: Option>, ) { let mut package_cache_main = compiled_package_cache; let package_cache_other = compiled_package_cache_v2; + let mut v2_flag = false; if self.execution_mode.is_v2() { package_cache_main = compiled_package_cache_v2; + v2_flag = true; } let res_main_opt = self.execute_code( - state, - features, + cur_version, + state.clone(), &txn_idx.package_info, &txn_idx.txn, package_cache_main, + debugger.clone(), + v2_flag, ); if self.execution_mode.is_compare() { let res_other_opt = self.execute_code( + cur_version, state, - features, &txn_idx.package_info, &txn_idx.txn, package_cache_other, + debugger.clone(), + true, ); - Self::print_mismatches(cur_version, &res_main_opt.unwrap(), &res_other_opt.unwrap()); + self.print_mismatches(cur_version, &res_main_opt.unwrap(), &res_other_opt.unwrap()); } else { let res = res_main_opt.unwrap(); if let Ok(res_ok) = res { - println!( + self.output_result_str(format!( "version:{}\nwrite set:{:?}\n events:{:?}\n", cur_version, res_ok.0, res_ok.1 - ); + )); } else { - println!( + self.output_result_str(format!( "execution error {} at version: {}, error", res.unwrap_err(), cur_version - ); + )); } } } fn execute_code( &self, - state: &FakeDataStore, - features: &Features, + version: Version, + state: FakeDataStore, package_info: &PackageInfo, txn: &Transaction, compiled_package_cache: &HashMap>>, + debugger_opt: Option>, + v2_flag: bool, ) -> Option), VMStatus>> { let executor = FakeExecutor::no_genesis(); let mut executor = executor.set_not_parallel(); - *executor.data_store_mut() = state.clone(); + *executor.data_store_mut() = state; if let Transaction::UserTransaction(signed_trans) = txn { let sender = signed_trans.sender(); let payload = signed_trans.payload(); if let TransactionPayload::EntryFunction(entry_function) = payload { - // always load 0x1 modules + // Always load 0x1 modules load_aptos_packages_to_executor(&mut executor, compiled_package_cache); - // Load other modules + // Load modules if package_info.is_compilable() { load_packages_to_executor(&mut executor, package_info, compiled_package_cache); } let mut senders = vec![sender]; senders.extend(TransactionMetadata::new(signed_trans).secondary_signers); - return Some(executor.try_exec_entry_with_features( - senders, - entry_function, - features, - )); - } else if let TransactionPayload::Multisig(multi_sig) = payload { - assert!(multi_sig.transaction_payload.is_some()); - println!("Multisig transaction is not supported yet"); + let enable_v7 = |features: &mut Features| { + if v2_flag { + features.enable(FeatureFlag::VM_BINARY_FORMAT_V7); + } else { + features.enable(FeatureFlag::VM_BINARY_FORMAT_V6); + } + }; + if let Some(debugger) = debugger_opt { + let data_view = + DataStateView::new(debugger, version, executor.data_store().clone()); + let mut features = + Features::fetch_config(&data_view.as_move_resolver()).unwrap_or_default(); + enable_v7(&mut features); + return Some(executor.try_exec_entry_with_state_view( + senders, + entry_function, + &data_view.as_move_resolver(), + features, + )); + } else { + let mut features = + Features::fetch_config(&executor.data_store().clone().as_move_resolver()) + .unwrap_or_default(); + enable_v7(&mut features); + return Some(executor.try_exec_entry_with_state_view( + senders, + entry_function, + &executor.data_store().clone().as_move_resolver(), + features, + )); + } } } - None + if let Some(debugger) = debugger_opt { + let data_view = DataStateView::new(debugger, version, executor.data_store().clone()); + Some( + executor + .execute_transaction_block_with_state_view([txn.clone()].to_vec(), &data_view) + .map(|res| res[0].clone().into()), + ) + } else { + Some( + executor + .execute_transaction_block(vec![txn.clone()]) + .map(|res| res[0].clone().into()), + ) + } } fn print_mismatches( + &self, cur_version: u64, res_1: &Result<(WriteSet, Vec), VMStatus>, res_2: &Result<(WriteSet, Vec), VMStatus>, ) { match (res_1, res_2) { (Err(e1), Err(e2)) => { - if e1 != e2 { - println!("error is different at {}", cur_version); - println!("error {} is raised from V1", e1); - println!("error {} is raised from V2", e2); + if e1.message() != e2.message() || e1.status_code() != e2.status_code() { + self.output_result_str(format!( + "error is different at version: {}", + cur_version + )); + self.output_result_str(format!("error {} is raised from V1", e1)); + self.output_result_str(format!("error {} is raised from V2", e2)); } }, - (Err(e), Ok(res)) => { - println!("error {} is raised from V1 at {}", e, cur_version); - println!( - "output from V2 at version:{}\nwrite set:{:?}\n events:{:?}\n", - cur_version, res.0, res.1 - ); + (Err(_), Ok(_)) => { + self.output_result_str(format!( + "V1 returns error while V2 does not at version: {}", + cur_version + )); }, - (Ok(res), Err(e)) => { - println!("error {} is raised from V2 at {}", e, cur_version); - println!( - "output from V1 at version:{}\nwrite set:{:?}\n events:{:?}\n", - cur_version, res.0, res.1 - ); + (Ok(_), Err(_)) => { + self.output_result_str(format!( + "V2 returns error while V1 does not at version: {}", + cur_version + )); }, (Ok(res_1), Ok(res_2)) => { // compare events - for idx in 0..res_1.1.len() { + let mut event_error = false; + if res_1.1.len() != res_2.1.len() { + event_error = true; + } + for idx in 0..cmp::min(res_1.1.len(), res_2.1.len()) { let event_1 = &res_1.1[idx]; let event_2 = &res_2.1[idx]; if event_1 != event_2 { - println!("event is different at version {}", cur_version); - println!("event raised from V1: {} at index:{}", event_1, idx); - println!("event raised from V2: {} at index:{}", event_2, idx); + event_error = true; + self.output_result_str(format!( + "event raised from V1: {} at index: {}", + event_1, idx + )); + self.output_result_str(format!( + "event raised from V2: {} at index: {}", + event_2, idx + )); } } + if event_error { + self.output_result_str(format!( + "event is different at version: {}", + cur_version + )); + } // compare write set + let mut write_set_error = false; let res_1_write_set_vec = res_1.0.iter().collect_vec(); let res_2_write_set_vec = res_2.0.iter().collect_vec(); - for idx in 0..res_1_write_set_vec.len() { - let write_set_1 = res_1_write_set_vec[0]; - let write_set_2 = res_2_write_set_vec[0]; + if res_1_write_set_vec.len() != res_2_write_set_vec.len() { + write_set_error = true; + } + for idx in 0..cmp::min(res_1_write_set_vec.len(), res_2_write_set_vec.len()) { + let write_set_1 = res_1_write_set_vec[idx]; + let write_set_2 = res_2_write_set_vec[idx]; if write_set_1.0 != write_set_2.0 { - println!("write set key is different at version {}", cur_version); - println!("state key at V1: {:?} at index:{}", write_set_1.0, idx); - println!("state key at V2: {:?} at index:{}", write_set_2.0, idx); + write_set_error = true; + self.output_result_str(format!( + "write set key is different at version: {}, index: {}", + cur_version, idx + )); + self.output_result_str(format!( + "state key at V1: {:?} at index: {}", + write_set_1.0, idx + )); + self.output_result_str(format!( + "state key at V2: {:?} at index: {}", + write_set_2.0, idx + )); } if write_set_1.1 != write_set_2.1 { - println!("write set value is different at version {}", cur_version); - println!("state value at V1: {:?} at index {}", write_set_1.1, idx); - println!("state value at V2: {:?} at index {}", write_set_2.1, idx); + write_set_error = true; + self.output_result_str(format!( + "write set value is different at version: {}, index: {}", + cur_version, idx + )); + self.output_result_str(format!( + "state value at V1: {:?} at index: {}", + write_set_1.1, idx + )); + self.output_result_str(format!( + "state value at V2: {:?} at index: {}", + write_set_2.1, idx + )); } } + if write_set_error { + self.output_result_str(format!( + "write set is different at version: {}", + cur_version + )); + } }, } } diff --git a/aptos-move/aptos-e2e-comparison-testing/src/lib.rs b/aptos-move/aptos-e2e-comparison-testing/src/lib.rs index 114d680bd89ae..18d295bbd2af3 100644 --- a/aptos-move/aptos-e2e-comparison-testing/src/lib.rs +++ b/aptos-move/aptos-e2e-comparison-testing/src/lib.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use aptos_framework::{ - natives::code::PackageMetadata, unzip_metadata_str, BuildOptions, BuiltPackage, APTOS_PACKAGES, + natives::code::PackageMetadata, unzip_metadata_str, BuiltPackage, APTOS_PACKAGES, }; use aptos_language_e2e_tests::data_store::FakeDataStore; use aptos_types::{ @@ -24,7 +24,9 @@ use std::{ use tempfile::TempDir; mod data_collection; +mod data_state_view; mod execution; +mod online_execution; pub use data_collection::*; pub use execution::*; @@ -38,6 +40,7 @@ use move_package::{ }, CompilerVersion, }; +pub use online_execution::*; const APTOS_PACKAGES_DIR_NAMES: [&str; 5] = [ "aptos-framework", @@ -110,7 +113,7 @@ impl IndexWriter { } pub fn write_err(&mut self, err_msg: &str) { - self.index_writer + self.err_logger .write_fmt(format_args!("{}\n", err_msg)) .unwrap(); self.err_logger.flush().unwrap(); @@ -145,7 +148,10 @@ impl IndexReader { pub fn _load_all_versions(&mut self) { loop { let next_val = self.get_next_version(); - if let Some(val) = next_val { + if next_val.is_err() { + continue; + } + if let Some(val) = next_val.unwrap() { self._version_cache.push(val); } else { break; @@ -153,19 +159,27 @@ impl IndexReader { } } - pub fn get_next_version(&mut self) -> Option { + pub fn get_next_version(&mut self) -> Result, ()> { let mut cur_idx = String::new(); let num_bytes = self.index_reader.read_line(&mut cur_idx).unwrap(); if num_bytes == 0 { - return None; + return Ok(None); + } + let indx = cur_idx.trim().parse(); + if indx.is_ok() { + Ok(indx.ok()) + } else { + Err(()) } - Some(cur_idx.trim().parse().unwrap()) } pub fn get_next_version_ge(&mut self, version: u64) -> Option { loop { let next_val = self.get_next_version(); - if let Some(val) = next_val { + if next_val.is_err() { + continue; + } + if let Some(val) = next_val.unwrap() { if val >= version { return Some(val); } @@ -315,7 +329,11 @@ fn check_aptos_packages_availability(path: PathBuf) -> bool { } pub async fn prepare_aptos_packages(path: PathBuf) { - if !path.exists() { + let mut success = true; + if path.exists() { + success = std::fs::remove_dir_all(path.clone()).is_ok(); + } + if success { std::fs::create_dir_all(path.clone()).unwrap(); download_aptos_packages(&path).await.unwrap(); } @@ -324,11 +342,14 @@ pub async fn prepare_aptos_packages(path: PathBuf) { #[derive(Default)] struct CompilationCache { compiled_package_map: HashMap, - failed_packages: HashSet, + failed_packages_v1: HashSet, + failed_packages_v2: HashSet, + compiled_package_cache_v1: HashMap>>, + compiled_package_cache_v2: HashMap>>, } #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Hash)] -struct PackageInfo { +pub(crate) struct PackageInfo { address: AccountAddress, package_name: String, upgrade_number: Option, @@ -360,7 +381,7 @@ impl PackageInfo { } #[derive(Debug, Clone, Serialize, Deserialize)] -struct TxnIndex { +pub(crate) struct TxnIndex { version: u64, package_info: PackageInfo, txn: Transaction, @@ -374,7 +395,7 @@ fn generate_compiled_blob( if compiled_blobs.contains_key(package_info) { return; } - let root_modules = compiled_package.all_modules(); + let root_modules = &compiled_package.root_compiled_units; let mut blob_map = HashMap::new(); for compiled_module in root_modules { if let CompiledUnitEnum::Module(module) = &compiled_module.unit { @@ -392,10 +413,10 @@ fn compile_aptos_packages( ) -> anyhow::Result<()> { for package in APTOS_PACKAGES { let root_package_dir = aptos_commons_path.join(get_aptos_dir(package).unwrap()); - let compiler_verion = if v2_flag { + let compiler_version = if v2_flag { Some(CompilerVersion::V2) } else { - None + Some(CompilerVersion::V1) }; // For simplicity, all packages including aptos token are stored under 0x1 in the map let package_info = PackageInfo { @@ -403,7 +424,7 @@ fn compile_aptos_packages( package_name: package.to_string(), upgrade_number: None, }; - let compiled_package = compile_package(root_package_dir, &package_info, compiler_verion); + let compiled_package = compile_package(root_package_dir, &package_info, compiler_version); if let Ok(built_package) = compiled_package { generate_compiled_blob(&package_info, &built_package, compiled_package_map); } else { @@ -432,7 +453,10 @@ fn compile_package( if let Ok(built_package) = compiled_package { Ok(built_package.package) } else { - Err(anyhow::Error::msg("compilation failed")) + Err(anyhow::Error::msg(format!( + "compilation failed for compiler: {:?}", + compiler_verion + ))) } } @@ -441,10 +465,10 @@ fn dump_and_compile_from_package_metadata( root_dir: PathBuf, dep_map: &HashMap<(AccountAddress, String), PackageMetadata>, compilation_cache: &mut CompilationCache, - compiler_verion: Option, + execution_mode: Option, ) -> anyhow::Result<()> { let root_package_dir = root_dir.join(format!("{}", package_info,)); - if compilation_cache.failed_packages.contains(&package_info) { + if compilation_cache.failed_packages_v1.contains(&package_info) { return Err(anyhow::Error::msg("compilation failed")); } if !root_package_dir.exists() { @@ -515,7 +539,7 @@ fn dump_and_compile_from_package_metadata( root_dir.clone(), dep_map, compilation_cache, - compiler_verion, + execution_mode, )?; } break; @@ -528,23 +552,47 @@ fn dump_and_compile_from_package_metadata( std::fs::write(toml_path, manifest.to_string()).unwrap(); // step 5: test whether the code can be compiled - if let std::collections::hash_map::Entry::Vacant(e) = compilation_cache + if !compilation_cache .compiled_package_map - .entry(package_info.clone()) + .contains_key(&package_info) { - let mut build_options = BuildOptions::default(); - build_options - .named_addresses - .insert(package_info.package_name.clone(), package_info.address); - build_options.compiler_version = compiler_verion; - let compiled_package = BuiltPackage::build(root_package_dir, build_options); - if let Ok(built_package) = compiled_package { - e.insert(built_package.package); + let package_v1 = compile_package( + root_package_dir.clone(), + &package_info, + Some(CompilerVersion::V1), + ); + if let Ok(built_package) = package_v1 { + if execution_mode.is_some_and(|mode| mode.is_v1_or_compare()) { + generate_compiled_blob( + &package_info, + &built_package, + &mut compilation_cache.compiled_package_cache_v1, + ); + } + compilation_cache + .compiled_package_map + .insert(package_info.clone(), built_package); } else { - if !compilation_cache.failed_packages.contains(&package_info) { - compilation_cache.failed_packages.insert(package_info); + if !compilation_cache.failed_packages_v1.contains(&package_info) { + compilation_cache.failed_packages_v1.insert(package_info); + } + return Err(anyhow::Error::msg("compilation failed at v1")); + } + if execution_mode.is_some_and(|mode| mode.is_v2_or_compare()) { + let package_v2 = + compile_package(root_package_dir, &package_info, Some(CompilerVersion::V2)); + if let Ok(built_package) = package_v2 { + generate_compiled_blob( + &package_info, + &built_package, + &mut compilation_cache.compiled_package_cache_v2, + ); + } else { + if !compilation_cache.failed_packages_v1.contains(&package_info) { + compilation_cache.failed_packages_v1.insert(package_info); + } + return Err(anyhow::Error::msg("compilation failed at v2")); } - return Err(anyhow::Error::msg("compilation failed")); } } Ok(()) diff --git a/aptos-move/aptos-e2e-comparison-testing/src/main.rs b/aptos-move/aptos-e2e-comparison-testing/src/main.rs index c1d7429822a8e..0d50666fe345f 100644 --- a/aptos-move/aptos-e2e-comparison-testing/src/main.rs +++ b/aptos-move/aptos-e2e-comparison-testing/src/main.rs @@ -3,21 +3,21 @@ use anyhow::Result; use aptos_comparison_testing::{ - prepare_aptos_packages, DataCollection, Execution, ExecutionMode, APTOS_COMMONS, + prepare_aptos_packages, DataCollection, Execution, ExecutionMode, OnlineExecutor, APTOS_COMMONS, }; use aptos_rest_client::Client; use clap::{Parser, Subcommand}; +use move_core_types::account_address::AccountAddress; use std::path::PathBuf; use url::Url; -const BATCH_SIZE: u64 = 100; +const BATCH_SIZE: u64 = 500; #[derive(Subcommand)] pub enum Cmd { /// Collect and dump the data Dump { - /// Endpoint url to obtain the txn data, - /// e.g. `https://api.mainnet.aptoslabs.com/v1` for mainnet. + /// Endpoint url to obtain the txn data, e.g. `https://api.mainnet.aptoslabs.com/v1` for mainnet. /// To avoid rate limiting, users need to apply for API key from `https://developers.aptoslabs.com/` /// and set the env variable X_API_KEY using the obtained key endpoint: String, @@ -35,6 +35,29 @@ pub enum Cmd { /// Dump the write set of txns #[clap(long, default_value_t = false)] dump_write_set: bool, + /// With this set, only dump transactions that are sent to this account + #[clap(long)] + target_account: Option, + }, + /// Collect and execute txns without dumping the state data + Online { + /// Endpoint url to obtain the txn data, + /// e.g. `https://api.mainnet.aptoslabs.com/v1` for mainnet. + /// To avoid rate limiting, users need to apply for API key from `https://developers.aptoslabs.com/` + /// and set the env variable X_API_KEY using the obtained key + endpoint: String, + /// Path to the dumped data + output_path: Option, + /// Do not dump failed txns + #[clap(long, default_value_t = false)] + skip_failed_txns: bool, + /// Do not dump publish txns + #[clap(long, default_value_t = false)] + skip_publish_txns: bool, + /// Whether to execute against V1, V2 alone or both compilers for comparison + /// Used when execution_only is true + #[clap(long)] + execution_mode: Option, }, /// Execution of txns Execute { @@ -72,6 +95,7 @@ async fn main() -> Result<()> { skip_publish_txns, skip_source_code_check: skip_source_code, dump_write_set, + target_account, } => { let batch_size = BATCH_SIZE; let output = if let Some(path) = output_path { @@ -93,11 +117,40 @@ async fn main() -> Result<()> { skip_publish_txns, dump_write_set, skip_source_code, + target_account, )?; data_collector .dump_data(args.begin_version, args.limit) .await?; }, + Cmd::Online { + endpoint, + output_path, + skip_failed_txns, + skip_publish_txns, + execution_mode, + } => { + let batch_size = BATCH_SIZE; + let output = if let Some(path) = output_path { + path + } else { + PathBuf::from(".") + }; + if !output.exists() { + std::fs::create_dir_all(output.as_path()).unwrap(); + } + prepare_aptos_packages(output.join(APTOS_COMMONS)).await; + let online = OnlineExecutor::new_with_rest_client( + Client::new(Url::parse(&endpoint)?), + output.clone(), + batch_size, + skip_failed_txns, + skip_publish_txns, + execution_mode.unwrap_or_default(), + endpoint, + )?; + online.execute(args.begin_version, args.limit).await?; + }, Cmd::Execute { input_path, execution_mode, diff --git a/aptos-move/aptos-e2e-comparison-testing/src/online_execution.rs b/aptos-move/aptos-e2e-comparison-testing/src/online_execution.rs new file mode 100644 index 0000000000000..166f2ff1e808f --- /dev/null +++ b/aptos-move/aptos-e2e-comparison-testing/src/online_execution.rs @@ -0,0 +1,240 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + compile_aptos_packages, dump_and_compile_from_package_metadata, is_aptos_package, + CompilationCache, ExecutionMode, IndexWriter, PackageInfo, TxnIndex, APTOS_COMMONS, +}; +use anyhow::Result; +use aptos_framework::natives::code::PackageMetadata; +use aptos_language_e2e_tests::data_store::FakeDataStore; +use aptos_rest_client::Client; +use aptos_types::transaction::Version; +use aptos_validator_interface::{AptosValidatorInterface, FilterCondition, RestDebuggerInterface}; +use move_core_types::account_address::AccountAddress; +use std::{ + collections::HashMap, + path::PathBuf, + sync::{Arc, Mutex}, +}; +use url::Url; + +pub struct OnlineExecutor { + debugger: Arc, + current_dir: PathBuf, + batch_size: u64, + filter_condition: FilterCondition, + execution_mode: ExecutionMode, + endpoint: String, +} + +impl OnlineExecutor { + pub fn new( + debugger: Arc, + current_dir: PathBuf, + batch_size: u64, + skip_failed_txns: bool, + skip_publish_txns: bool, + execution_mode: ExecutionMode, + endpoint: String, + ) -> Self { + Self { + debugger, + current_dir, + batch_size, + filter_condition: FilterCondition { + skip_failed_txns, + skip_publish_txns, + check_source_code: true, + target_account: None, + }, + execution_mode, + endpoint, + } + } + + pub fn new_with_rest_client( + rest_client: Client, + current_dir: PathBuf, + batch_size: u64, + skip_failed_txns: bool, + skip_publish_txns: bool, + execution_mode: ExecutionMode, + endpoint: String, + ) -> Result { + Ok(Self::new( + Arc::new(RestDebuggerInterface::new(rest_client)), + current_dir, + batch_size, + skip_failed_txns, + skip_publish_txns, + execution_mode, + endpoint, + )) + } + + fn dump_and_check_src( + version: Version, + address: AccountAddress, + package_name: String, + map: HashMap<(AccountAddress, String), PackageMetadata>, + compilation_cache: &mut CompilationCache, + execution_mode: Option, + current_dir: PathBuf, + ) -> Option { + let upgrade_number = if is_aptos_package(&package_name) { + None + } else { + let package = map.get(&(address, package_name.clone())).unwrap(); + Some(package.upgrade_number) + }; + + let package_info = PackageInfo { + address, + package_name: package_name.clone(), + upgrade_number, + }; + if compilation_cache.failed_packages_v1.contains(&package_info) { + return None; + } + if !is_aptos_package(&package_name) + && !compilation_cache + .compiled_package_map + .contains_key(&package_info) + { + let res = dump_and_compile_from_package_metadata( + package_info.clone(), + current_dir, + &map, + compilation_cache, + execution_mode, + ); + if res.is_err() { + eprintln!("{} at:{}", res.unwrap_err(), version); + return None; + } + } + Some(package_info) + } + + pub async fn execute(&self, begin: Version, limit: u64) -> Result<()> { + println!("begin executing events"); + let compilation_cache = Arc::new(Mutex::new(CompilationCache::default())); + let index_writer = Arc::new(Mutex::new(IndexWriter::new(&self.current_dir))); + + let aptos_commons_path = self.current_dir.join(APTOS_COMMONS); + if self.execution_mode.is_v1_or_compare() { + compile_aptos_packages( + &aptos_commons_path, + &mut compilation_cache.lock().unwrap().compiled_package_cache_v1, + false, + )?; + } + if self.execution_mode.is_v2_or_compare() { + compile_aptos_packages( + &aptos_commons_path, + &mut compilation_cache.lock().unwrap().compiled_package_cache_v2, + true, + )?; + } + + let mut cur_version = begin; + let mut module_registry_map = HashMap::new(); + while cur_version < begin + limit { + let batch = if cur_version + self.batch_size <= begin + limit { + self.batch_size + } else { + begin + limit - cur_version + }; + let res_txns = self + .debugger + .get_and_filter_committed_transactions( + cur_version, + batch, + self.filter_condition, + &mut module_registry_map, + ) + .await; + // if error happens when collecting txns, log the version range + if res_txns.is_err() { + index_writer.lock().unwrap().write_err(&format!( + "{}:{}:{:?}", + cur_version, + batch, + res_txns.unwrap_err() + )); + cur_version += batch; + continue; + } + let txns = res_txns.unwrap_or_default(); + if !txns.is_empty() { + let mut txn_execution_ths = vec![]; + for (version, txn, source_code_data) in txns { + println!("get txn at version:{}", version); + + let compilation_cache = compilation_cache.clone(); + let current_dir = self.current_dir.clone(); + let execution_mode = self.execution_mode; + let endpoint = self.endpoint.clone(); + + let txn_execution_thread = tokio::task::spawn_blocking(move || { + let executor = crate::Execution::new(current_dir.clone(), execution_mode); + + let mut version_idx = TxnIndex { + version, + txn: txn.clone(), + package_info: PackageInfo::non_compilable_info(), + }; + + // handle source code + if let Some((address, package_name, map)) = source_code_data { + let execution_mode_opt = Some(execution_mode); + let package_info_opt = Self::dump_and_check_src( + version, + address, + package_name, + map, + &mut compilation_cache.lock().unwrap(), + execution_mode_opt, + current_dir.clone(), + ); + if package_info_opt.is_none() { + return; + } + + version_idx.package_info = package_info_opt.unwrap(); + + let state_store = FakeDataStore::default(); + + let cache_v1 = compilation_cache + .lock() + .unwrap() + .compiled_package_cache_v1 + .clone(); + let cache_v2 = compilation_cache + .lock() + .unwrap() + .compiled_package_cache_v2 + .clone(); + + let client = Client::new(Url::parse(&endpoint).unwrap()); + let debugger = Arc::new(RestDebuggerInterface::new(client)); + executor.execute_and_compare( + version, + state_store, + &version_idx, + &cache_v1, + &cache_v2, + Some(debugger), + ); + } + }); + txn_execution_ths.push(txn_execution_thread); + } + futures::future::join_all(txn_execution_ths).await; + } + cur_version += batch; + } + Ok(()) + } +} diff --git a/aptos-move/aptos-validator-interface/src/lib.rs b/aptos-move/aptos-validator-interface/src/lib.rs index e0ca74b0295d8..c3f5611ef9caf 100644 --- a/aptos-move/aptos-validator-interface/src/lib.rs +++ b/aptos-move/aptos-validator-interface/src/lib.rs @@ -22,9 +22,9 @@ use aptos_types::{ }; use lru::LruCache; use move_binary_format::file_format::CompiledModule; +use move_core_types::language_storage::ModuleId; use std::{ collections::HashMap, - ops::DerefMut, sync::{Arc, Mutex}, }; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; @@ -34,6 +34,7 @@ pub struct FilterCondition { pub skip_failed_txns: bool, pub skip_publish_txns: bool, pub check_source_code: bool, + pub target_account: Option, } // TODO(skedia) Clean up this interfact to remove account specific logic and move to state store @@ -63,6 +64,14 @@ pub trait AptosValidatorInterface: Sync { start: Version, limit: u64, filter_condition: FilterCondition, + package_cache: &mut HashMap< + ModuleId, + ( + AccountAddress, + String, + HashMap<(AccountAddress, String), PackageMetadata>, + ), + >, ) -> Result< Vec<( u64, @@ -151,7 +160,6 @@ pub struct DebuggerStateView { )>, >, version: Version, - data_read_state_keys: Option>>>, } async fn handler_thread<'a>( @@ -201,20 +209,6 @@ impl DebuggerStateView { Self { query_sender: Mutex::new(query_sender), version, - data_read_state_keys: None, - } - } - - pub fn new_with_data_reads( - db: Arc, - version: Version, - ) -> Self { - let (fake_query_sender, thread_receiver) = unbounded_channel(); - tokio::spawn(async move { handler_thread(db, thread_receiver).await }); - Self { - query_sender: Mutex::new(fake_query_sender), - version, - data_read_state_keys: Some(Arc::new(Mutex::new(HashMap::new()))), } } @@ -228,28 +222,7 @@ impl DebuggerStateView { query_handler_locked .send((state_key.clone(), version, tx)) .unwrap(); - let ret = rx.recv()?; - if let Some(reads) = &self.data_read_state_keys { - if !reads.lock().unwrap().contains_key(state_key) && ret.is_ok() { - let val = ret?.clone(); - if val.is_some() { - reads - .lock() - .unwrap() - .deref_mut() - .insert(state_key.clone(), val.clone().unwrap()); - } - Ok(val) - } else { - ret - } - } else { - ret - } - } - - pub fn get_state_keys(self) -> Arc>> { - self.data_read_state_keys.unwrap() + rx.recv()? } } diff --git a/aptos-move/aptos-validator-interface/src/rest_interface.rs b/aptos-move/aptos-validator-interface/src/rest_interface.rs index b909fb1157ca7..7d2aa66440802 100644 --- a/aptos-move/aptos-validator-interface/src/rest_interface.rs +++ b/aptos-move/aptos-validator-interface/src/rest_interface.rs @@ -117,6 +117,14 @@ async fn check_and_obtain_source_code( addr: &AccountAddress, version: Version, transaction: &Transaction, + package_cache: &mut HashMap< + ModuleId, + ( + AccountAddress, + String, + HashMap<(AccountAddress, String), PackageMetadata>, + ), + >, txns: &mut Vec<( u64, Transaction, @@ -149,6 +157,14 @@ async fn check_and_obtain_source_code( if let Some(target_package) = target_package_opt { let mut map = HashMap::new(); if APTOS_PACKAGES.contains(&target_package.name.as_str()) { + package_cache.insert( + m.clone(), + ( + AccountAddress::ONE, + target_package.name.clone(), // all aptos packages are stored under 0x1 + HashMap::new(), + ), + ); txns.push(( version, transaction.clone(), @@ -168,6 +184,7 @@ async fn check_and_obtain_source_code( .await { map.insert((*addr, target_package.clone().name), target_package.clone()); + package_cache.insert(m.clone(), (*addr, target_package.name.clone(), map.clone())); txns.push(( version, transaction.clone(), @@ -252,6 +269,14 @@ impl AptosValidatorInterface for RestDebuggerInterface { start: Version, limit: u64, filter_condition: FilterCondition, + package_cache: &mut HashMap< + ModuleId, + ( + AccountAddress, + String, + HashMap<(AccountAddress, String), PackageMetadata>, + ), + >, ) -> Result< Vec<( u64, @@ -298,16 +323,38 @@ impl AptosValidatorInterface for RestDebuggerInterface { if let Some(entry_function) = extract_entry_fun(payload) { let m = entry_function.module(); let addr = m.address(); - if filter_condition.skip_publish_txns - && entry_function.function().as_str() == "publish_package_txn" + if filter_condition.target_account.is_some() + && filter_condition.target_account.unwrap() != *addr { continue; } + if entry_function.function().as_str() == "publish_package_txn" { + if filter_condition.skip_publish_txns { + continue; + } + // For publish txn, we remove all items in the package_cache where module_id.address is the sender of this txn + // to update the new package in the cache. + package_cache.retain(|k, _| k.address != signed_trans.sender()); + } if !filter_condition.check_source_code { txns.push((version, txn.clone(), None)); + } else if package_cache.contains_key(m) { + txns.push(( + version, + txn.clone(), + Some(package_cache.get(m).unwrap().clone()), + )); } else { - check_and_obtain_source_code(&self.0, m, addr, version, txn, &mut txns) - .await?; + check_and_obtain_source_code( + &self.0, + m, + addr, + version, + txn, + package_cache, + &mut txns, + ) + .await?; } } } diff --git a/aptos-move/aptos-validator-interface/src/storage_interface.rs b/aptos-move/aptos-validator-interface/src/storage_interface.rs index f9ed2da2a90a4..e6b4b2cd4d9dc 100644 --- a/aptos-move/aptos-validator-interface/src/storage_interface.rs +++ b/aptos-move/aptos-validator-interface/src/storage_interface.rs @@ -17,6 +17,7 @@ use aptos_types::{ state_store::{state_key::StateKey, state_key_prefix::StateKeyPrefix, state_value::StateValue}, transaction::{Transaction, TransactionInfo, Version}, }; +use move_core_types::language_storage::ModuleId; use std::{collections::HashMap, path::Path, sync::Arc}; pub struct DBDebuggerInterface(Arc); @@ -95,6 +96,14 @@ impl AptosValidatorInterface for DBDebuggerInterface { _start: Version, _limit: u64, _filter_condition: FilterCondition, + _package_cache: &mut HashMap< + ModuleId, + ( + AccountAddress, + String, + HashMap<(AccountAddress, String), PackageMetadata>, + ), + >, ) -> Result< Vec<( u64, diff --git a/aptos-move/aptos-vm/src/gas.rs b/aptos-move/aptos-vm/src/gas.rs index ee9ebd087b23c..0c0ef735816b9 100644 --- a/aptos-move/aptos-vm/src/gas.rs +++ b/aptos-move/aptos-vm/src/gas.rs @@ -44,7 +44,7 @@ pub(crate) fn get_gas_config_from_storage( } } -pub(crate) fn get_gas_parameters( +pub fn get_gas_parameters( features: &Features, config_storage: &impl ConfigStorage, ) -> ( diff --git a/aptos-move/aptos-vm/src/lib.rs b/aptos-move/aptos-vm/src/lib.rs index d9ebfafcafaeb..7709bf9e21e6f 100644 --- a/aptos-move/aptos-vm/src/lib.rs +++ b/aptos-move/aptos-vm/src/lib.rs @@ -108,7 +108,7 @@ pub mod data_cache; pub mod aptos_vm; pub mod block_executor; mod errors; -mod gas; +pub mod gas; mod keyless_validation; pub mod move_vm_ext; pub mod natives; diff --git a/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs b/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs index 52134d2f9e5a7..5e8fb0c22d483 100644 --- a/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs +++ b/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs @@ -8,7 +8,7 @@ use aptos_framework::natives::code::PackageRegistry; use aptos_infallible::RwLock; use aptos_metrics_core::TimerHelper; use aptos_native_interface::SafeNativeBuilder; -use aptos_types::on_chain_config::OnChainConfig; +use aptos_types::on_chain_config::{FeatureFlag, Features, OnChainConfig}; use bytes::Bytes; use move_binary_format::errors::{Location, PartialVMError, VMResult}; use move_core_types::{ @@ -101,6 +101,7 @@ struct WarmVmId { natives: Bytes, vm_config: Bytes, core_packages_registry: Option, + bin_v7_enabled: bool, } impl WarmVmId { @@ -117,6 +118,9 @@ impl WarmVmId { natives, vm_config: Self::vm_config_bytes(vm_config), core_packages_registry: Self::core_packages_id_bytes(resolver)?, + bin_v7_enabled: Features::fetch_config(resolver) + .unwrap_or_default() + .is_enabled(FeatureFlag::VM_BINARY_FORMAT_V7), }) } diff --git a/aptos-move/e2e-tests/src/executor.rs b/aptos-move/e2e-tests/src/executor.rs index 20a874405c28f..5519df46ed6dd 100644 --- a/aptos-move/e2e-tests/src/executor.rs +++ b/aptos-move/e2e-tests/src/executor.rs @@ -42,7 +42,7 @@ use aptos_types::{ FeatureFlag, Features, OnChainConfig, TimedFeatureOverride, TimedFeaturesBuilder, ValidatorSet, Version, }, - state_store::{state_key::StateKey, state_value::StateValue, TStateView}, + state_store::{state_key::StateKey, state_value::StateValue, StateView, TStateView}, transaction::{ signature_verified_transaction::{ into_signature_verified_block, SignatureVerifiedTransaction, @@ -57,7 +57,8 @@ use aptos_types::{ use aptos_vm::{ block_executor::{AptosTransactionOutput, BlockAptosVM}, data_cache::AsMoveResolver, - move_vm_ext::{MoveVmExt, SessionId}, + gas::get_gas_parameters, + move_vm_ext::{AptosMoveResolver, MoveVmExt, SessionId}, verifier, AptosVM, VMValidator, }; use aptos_vm_genesis::{generate_genesis_change_set_for_testing_with_count, GenesisOptions}; @@ -482,11 +483,12 @@ impl FakeExecutor { } } - fn execute_transaction_block_impl( + fn execute_transaction_block_impl_with_state_view( &self, txn_block: &[SignatureVerifiedTransaction], onchain_config: BlockExecutorConfigFromOnchain, sequential: bool, + state_view: &(impl StateView + Sync), ) -> Result, VMStatus> { let config = BlockExecutorConfig { local: BlockExecutorLocalConfig { @@ -503,15 +505,16 @@ impl FakeExecutor { BlockAptosVM::execute_block::<_, NoOpTransactionCommitHook>( self.executor_thread_pool.clone(), txn_block, - &self.data_store, + &state_view, config, None, ).map(BlockOutput::into_transaction_outputs_forced) } - pub fn execute_transaction_block( + pub fn execute_transaction_block_with_state_view( &self, txn_block: Vec, + state_view: &(impl StateView + Sync), ) -> Result, VMStatus> { let mut trace_map: (usize, Vec, Vec) = TraceSeqMapping::default(); @@ -540,17 +543,23 @@ impl FakeExecutor { let onchain_config = BlockExecutorConfigFromOnchain::on_but_large_for_test(); let sequential_output = if mode != ExecutorMode::ParallelOnly { - Some(self.execute_transaction_block_impl( + Some(self.execute_transaction_block_impl_with_state_view( &sig_verified_block, onchain_config.clone(), true, + state_view, )) } else { None }; let parallel_output = if mode != ExecutorMode::SequentialOnly { - Some(self.execute_transaction_block_impl(&sig_verified_block, onchain_config, false)) + Some(self.execute_transaction_block_impl_with_state_view( + &sig_verified_block, + onchain_config, + false, + state_view, + )) } else { None }; @@ -600,6 +609,13 @@ impl FakeExecutor { output } + pub fn execute_transaction_block( + &self, + txn_block: Vec, + ) -> Result, VMStatus> { + self.execute_transaction_block_with_state_view(txn_block, &self.data_store) + } + pub fn execute_transaction(&self, txn: SignedTransaction) -> TransactionOutput { let txn_block = vec![txn]; let mut outputs = self @@ -1078,34 +1094,48 @@ impl FakeExecutor { self.exec_module(&Self::module(module_name), function_name, type_params, args) } - pub fn try_exec_entry_with_features( + pub fn try_exec_entry_with_state_view( &mut self, senders: Vec, entry_fn: &EntryFunction, - features: &Features, + state_view: &impl AptosMoveResolver, + features: Features, ) -> Result<(WriteSet, Vec), VMStatus> { - let resolver = self.data_store.as_move_resolver(); + let ( + gas_params_res, + storage_gas_params, + native_gas_params, + misc_gas_params, + gas_feature_version, + ) = get_gas_parameters(&features, state_view); + + let gas_params = gas_params_res.unwrap(); + let mut gas_meter = + MemoryTrackedGasMeter::new(StandardGasMeter::new(StandardGasAlgebra::new( + gas_feature_version, + gas_params.clone().vm, + storage_gas_params.unwrap(), + 10000000000000, + ))); let timed_features = TimedFeaturesBuilder::enable_all() .with_override_profile(TimedFeatureOverride::Testing) .build(); - + let struct_constructors = features.is_enabled(FeatureFlag::STRUCT_CONSTRUCTORS); let vm = MoveVmExt::new( - NativeGasParameters::zeros(), - MiscGasParameters::zeros(), + native_gas_params, + misc_gas_params, LATEST_GAS_FEATURE_VERSION, self.chain_id, - features.clone(), + features, timed_features, - &resolver, - features.is_aggregator_v2_delayed_fields_enabled(), + state_view, + false, ) .unwrap(); - let mut session = vm.new_session(&resolver, SessionId::void()); - + let mut session = vm.new_session(state_view, SessionId::void()); let function = session.load_function(entry_fn.module(), entry_fn.function(), entry_fn.ty_args())?; - let struct_constructors = self.features.is_enabled(FeatureFlag::STRUCT_CONSTRUCTORS); let args = verifier::transaction_arg_validation::validate_combine_signer_and_txn_args( &mut session, senders, @@ -1119,15 +1149,16 @@ impl FakeExecutor { entry_fn.function(), entry_fn.ty_args().to_vec(), args, - &mut UnmeteredGasMeter, + &mut gas_meter, ) .map_err(|e| e.into_vm_status())?; - let change_set = session + let mut change_set = session .finish(&ChangeSetConfigs::unlimited_at_gas_feature_version( LATEST_GAS_FEATURE_VERSION, )) .expect("Failed to generate txn effects"); + change_set.try_materialize_aggregator_v1_delta_set(state_view)?; let (write_set, events) = change_set .try_into_storage_change_set() .expect("Failed to convert to ChangeSet") diff --git a/third_party/move/move-core/types/src/account_address.rs b/third_party/move/move-core/types/src/account_address.rs index f1f341833b55e..dc24ddb2a4676 100644 --- a/third_party/move/move-core/types/src/account_address.rs +++ b/third_party/move/move-core/types/src/account_address.rs @@ -15,6 +15,8 @@ use std::{convert::TryFrom, fmt, str::FromStr}; pub struct AccountAddress([u8; AccountAddress::LENGTH]); impl AccountAddress { + /// Hex address: 0x4 + pub const FOUR: Self = Self::get_hex_address_four(); /// The number of bytes in an address. /// Default to 16 bytes, can be set to 20 bytes with address20 feature. pub const LENGTH: usize = 32; @@ -22,6 +24,8 @@ impl AccountAddress { pub const MAX_ADDRESS: Self = Self([0xFF; Self::LENGTH]); /// Hex address: 0x1 pub const ONE: Self = Self::get_hex_address_one(); + /// Hex address: 0x3 + pub const THREE: Self = Self::get_hex_address_three(); /// Hex address: 0x2 pub const TWO: Self = Self::get_hex_address_two(); /// Hex address: 0x0 @@ -43,6 +47,18 @@ impl AccountAddress { Self(addr) } + const fn get_hex_address_three() -> Self { + let mut addr = [0u8; AccountAddress::LENGTH]; + addr[AccountAddress::LENGTH - 1] = 3u8; + Self(addr) + } + + const fn get_hex_address_four() -> Self { + let mut addr = [0u8; AccountAddress::LENGTH]; + addr[AccountAddress::LENGTH - 1] = 4u8; + Self(addr) + } + pub fn random() -> Self { let mut rng = OsRng; let buf: [u8; Self::LENGTH] = rng.gen(); diff --git a/types/src/state_store/state_key.rs b/types/src/state_store/state_key.rs index cb20282dfe37c..63461092a1039 100644 --- a/types/src/state_store/state_key.rs +++ b/types/src/state_store/state_key.rs @@ -154,6 +154,19 @@ impl StateKey { pub fn get_shard_id(&self) -> u8 { CryptoHash::hash(self).nibble(0) } + + pub fn is_aptos_code(&self) -> bool { + use move_core_types::account_address::AccountAddress; + match self.inner() { + StateKeyInner::AccessPath(access_path) => { + access_path.is_code() + && (access_path.address == AccountAddress::ONE + || access_path.address == AccountAddress::THREE + || access_path.address == AccountAddress::FOUR) + }, + _ => false, + } + } } impl StateKeyInner {