From f5d98d758d362d97b6a4abedce5731e4e684b950 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 4 Dec 2024 10:27:43 +0800 Subject: [PATCH 01/16] test(rpc): rpc unit tests with db snapshot --- src/chain/store/chain_store.rs | 2 +- src/chain/store/index.rs | 13 +- src/cli_shared/snapshot.rs | 3 + src/db/memory.rs | 39 ++ src/rpc/mod.rs | 402 +++++++++--------- src/rpc/reflect/mod.rs | 26 +- src/shim/address.rs | 2 +- src/tool/subcommands/api_cmd.rs | 13 + src/tool/subcommands/api_cmd/test_snapshot.rs | 161 +++++++ .../subcommands/api_cmd/test_snapshots.txt | 2 + 10 files changed, 450 insertions(+), 213 deletions(-) create mode 100644 src/tool/subcommands/api_cmd/test_snapshot.rs create mode 100644 src/tool/subcommands/api_cmd/test_snapshots.txt diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index 61b2ae5b374b..a3b6592dbddc 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -234,7 +234,7 @@ where &self .settings .require_obj::(HEAD_KEY) - .expect("failed to load heaviest tipset"), + .expect("failed to load heaviest tipset key"), ) .expect("failed to load heaviest tipset") } diff --git a/src/chain/store/index.rs b/src/chain/store/index.rs index a9383e396e32..17d59dceceb5 100644 --- a/src/chain/store/index.rs +++ b/src/chain/store/index.rs @@ -7,6 +7,7 @@ use crate::beacon::{BeaconEntry, IGNORE_DRAND_VAR}; use crate::blocks::{Tipset, TipsetKey}; use crate::metrics; use crate::shim::clock::ChainEpoch; +use crate::utils::misc::env::is_env_truthy; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use lru::LruCache; @@ -47,11 +48,13 @@ impl ChainIndex { /// Loads a tipset from memory given the tipset keys and cache. Semantically /// identical to [`Tipset::load`] but the result is cached. pub fn load_tipset(&self, tsk: &TipsetKey) -> Result>, Error> { - if let Some(ts) = self.ts_cache.lock().get(tsk) { - metrics::LRU_CACHE_HIT - .get_or_create(&metrics::values::TIPSET) - .inc(); - return Ok(Some(ts.clone())); + if !is_env_truthy("FOREST_TIPSET_CACHE_DISABLED") { + if let Some(ts) = self.ts_cache.lock().get(tsk) { + metrics::LRU_CACHE_HIT + .get_or_create(&metrics::values::TIPSET) + .inc(); + return Ok(Some(ts.clone())); + } } let ts_opt = Tipset::load(&self.db, tsk)?.map(Arc::new); diff --git a/src/cli_shared/snapshot.rs b/src/cli_shared/snapshot.rs index 87de6098d47d..2c9e3e929009 100644 --- a/src/cli_shared/snapshot.rs +++ b/src/cli_shared/snapshot.rs @@ -134,6 +134,9 @@ fn parse_content_disposition(value: &reqwest::header::HeaderValue) -> Option anyhow::Result { + if !directory.is_dir() { + std::fs::create_dir_all(directory)?; + } let dst_path = directory.join(filename); let destination = dst_path.display(); event!(target: "forest::snapshot", tracing::Level::INFO, %url, %destination, "downloading snapshot"); diff --git a/src/db/memory.rs b/src/db/memory.rs index 6d0abe23b00c..d813b2eb9b4a 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -11,6 +11,7 @@ use cid::Cid; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use parking_lot::RwLock; +use std::ops::Deref; use super::{EthMappingsStore, SettingsStore}; @@ -22,6 +23,44 @@ pub struct MemoryDB { eth_mappings_db: RwLock>>, } +impl MemoryDB { + #[allow(dead_code)] + pub fn serialize(&self) -> anyhow::Result> { + let blockchain_db = self.blockchain_db.read(); + let blockchain_persistent_db = self.blockchain_persistent_db.read(); + let settings_db = self.settings_db.read(); + let eth_mappings_db = self.eth_mappings_db.read(); + let tuple = ( + blockchain_db.deref(), + blockchain_persistent_db.deref(), + settings_db.deref(), + eth_mappings_db.deref(), + ); + Ok(fvm_ipld_encoding::to_vec(&tuple)?) + } + + pub fn deserialize_from(bytes: &[u8]) -> anyhow::Result { + let (blockchain_db, blockchain_persistent_db, settings_db, eth_mappings_db) = + fvm_ipld_encoding::from_slice(bytes)?; + Ok(Self { + blockchain_db: RwLock::new(blockchain_db), + blockchain_persistent_db: RwLock::new(blockchain_persistent_db), + settings_db: RwLock::new(settings_db), + eth_mappings_db: RwLock::new(eth_mappings_db), + }) + } + + pub fn deserialize_from_legacy(bytes: &[u8]) -> anyhow::Result { + let (blockchain_db, settings_db, eth_mappings_db) = fvm_ipld_encoding::from_slice(bytes)?; + Ok(Self { + blockchain_db: RwLock::new(blockchain_db), + blockchain_persistent_db: Default::default(), + settings_db: RwLock::new(settings_db), + eth_mappings_db: RwLock::new(eth_mappings_db), + }) + } +} + impl GarbageCollectable for MemoryDB { fn get_keys(&self) -> anyhow::Result { let mut set = CidHashSet::new(); diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index cb1c9856f0e9..4eb3ddf90645 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -20,6 +20,7 @@ mod error; mod reflect; pub mod types; pub use methods::*; +use serde::{Deserialize, Serialize}; /// Protocol or transport-specific error pub use jsonrpsee::core::ClientError; @@ -31,237 +32,238 @@ pub use jsonrpsee::core::ClientError; /// trait. /// /// All methods should be entered here. +#[macro_export] macro_rules! for_each_method { ($callback:path) => { // auth vertical - $callback!(crate::rpc::auth::AuthNew); - $callback!(crate::rpc::auth::AuthVerify); + $callback!($crate::rpc::auth::AuthNew); + $callback!($crate::rpc::auth::AuthVerify); // beacon vertical - $callback!(crate::rpc::beacon::BeaconGetEntry); + $callback!($crate::rpc::beacon::BeaconGetEntry); // chain vertical - $callback!(crate::rpc::chain::ChainExport); - $callback!(crate::rpc::chain::ChainGetBlock); - $callback!(crate::rpc::chain::ChainGetBlockMessages); - $callback!(crate::rpc::chain::ChainGetEvents); - $callback!(crate::rpc::chain::ChainGetGenesis); - $callback!(crate::rpc::chain::ChainGetMessage); - $callback!(crate::rpc::chain::ChainGetMessagesInTipset); - $callback!(crate::rpc::chain::ChainGetMinBaseFee); - $callback!(crate::rpc::chain::ChainGetParentMessages); - $callback!(crate::rpc::chain::ChainGetParentReceipts); - $callback!(crate::rpc::chain::ChainGetPath); - $callback!(crate::rpc::chain::ChainGetTipSet); - $callback!(crate::rpc::chain::ChainGetTipSetAfterHeight); - $callback!(crate::rpc::chain::ChainGetTipSetByHeight); - $callback!(crate::rpc::chain::ChainHasObj); - $callback!(crate::rpc::chain::ChainHead); - $callback!(crate::rpc::chain::ChainReadObj); - $callback!(crate::rpc::chain::ChainSetHead); - $callback!(crate::rpc::chain::ChainStatObj); - $callback!(crate::rpc::chain::ChainTipSetWeight); + $callback!($crate::rpc::chain::ChainExport); + $callback!($crate::rpc::chain::ChainGetBlock); + $callback!($crate::rpc::chain::ChainGetBlockMessages); + $callback!($crate::rpc::chain::ChainGetEvents); + $callback!($crate::rpc::chain::ChainGetGenesis); + $callback!($crate::rpc::chain::ChainGetMessage); + $callback!($crate::rpc::chain::ChainGetMessagesInTipset); + $callback!($crate::rpc::chain::ChainGetMinBaseFee); + $callback!($crate::rpc::chain::ChainGetParentMessages); + $callback!($crate::rpc::chain::ChainGetParentReceipts); + $callback!($crate::rpc::chain::ChainGetPath); + $callback!($crate::rpc::chain::ChainGetTipSet); + $callback!($crate::rpc::chain::ChainGetTipSetAfterHeight); + $callback!($crate::rpc::chain::ChainGetTipSetByHeight); + $callback!($crate::rpc::chain::ChainHasObj); + $callback!($crate::rpc::chain::ChainHead); + $callback!($crate::rpc::chain::ChainReadObj); + $callback!($crate::rpc::chain::ChainSetHead); + $callback!($crate::rpc::chain::ChainStatObj); + $callback!($crate::rpc::chain::ChainTipSetWeight); // common vertical - $callback!(crate::rpc::common::Session); - $callback!(crate::rpc::common::Shutdown); - $callback!(crate::rpc::common::StartTime); - $callback!(crate::rpc::common::Version); + $callback!($crate::rpc::common::Session); + $callback!($crate::rpc::common::Shutdown); + $callback!($crate::rpc::common::StartTime); + $callback!($crate::rpc::common::Version); // eth vertical - $callback!(crate::rpc::eth::EthAccounts); - $callback!(crate::rpc::eth::EthAddressToFilecoinAddress); - $callback!(crate::rpc::eth::EthBlockNumber); - $callback!(crate::rpc::eth::EthCall); - $callback!(crate::rpc::eth::EthChainId); - $callback!(crate::rpc::eth::EthEstimateGas); - $callback!(crate::rpc::eth::EthFeeHistory); - $callback!(crate::rpc::eth::EthGasPrice); - $callback!(crate::rpc::eth::EthGetBalance); - $callback!(crate::rpc::eth::EthGetBlockByHash); - $callback!(crate::rpc::eth::EthGetBlockByNumber); - $callback!(crate::rpc::eth::EthGetBlockReceipts); - $callback!(crate::rpc::eth::EthGetBlockReceiptsLimited); - $callback!(crate::rpc::eth::EthGetBlockTransactionCountByHash); - $callback!(crate::rpc::eth::EthGetBlockTransactionCountByNumber); - $callback!(crate::rpc::eth::EthGetCode); - $callback!(crate::rpc::eth::EthGetLogs); - $callback!(crate::rpc::eth::EthGetMessageCidByTransactionHash); - $callback!(crate::rpc::eth::EthGetStorageAt); - $callback!(crate::rpc::eth::EthGetTransactionByHash); - $callback!(crate::rpc::eth::EthGetTransactionByHashLimited); - $callback!(crate::rpc::eth::EthGetTransactionCount); - $callback!(crate::rpc::eth::EthGetTransactionHashByCid); - $callback!(crate::rpc::eth::EthGetTransactionByBlockNumberAndIndex); - $callback!(crate::rpc::eth::EthGetTransactionByBlockHashAndIndex); - $callback!(crate::rpc::eth::EthMaxPriorityFeePerGas); - $callback!(crate::rpc::eth::EthProtocolVersion); - $callback!(crate::rpc::eth::EthGetTransactionReceipt); - $callback!(crate::rpc::eth::EthGetTransactionReceiptLimited); - $callback!(crate::rpc::eth::EthNewFilter); - $callback!(crate::rpc::eth::EthNewPendingTransactionFilter); - $callback!(crate::rpc::eth::EthNewBlockFilter); - $callback!(crate::rpc::eth::EthUninstallFilter); - $callback!(crate::rpc::eth::EthSyncing); - $callback!(crate::rpc::eth::Web3ClientVersion); - $callback!(crate::rpc::eth::EthSendRawTransaction); + $callback!($crate::rpc::eth::EthAccounts); + $callback!($crate::rpc::eth::EthAddressToFilecoinAddress); + $callback!($crate::rpc::eth::EthBlockNumber); + $callback!($crate::rpc::eth::EthCall); + $callback!($crate::rpc::eth::EthChainId); + $callback!($crate::rpc::eth::EthEstimateGas); + $callback!($crate::rpc::eth::EthFeeHistory); + $callback!($crate::rpc::eth::EthGasPrice); + $callback!($crate::rpc::eth::EthGetBalance); + $callback!($crate::rpc::eth::EthGetBlockByHash); + $callback!($crate::rpc::eth::EthGetBlockByNumber); + $callback!($crate::rpc::eth::EthGetBlockReceipts); + $callback!($crate::rpc::eth::EthGetBlockReceiptsLimited); + $callback!($crate::rpc::eth::EthGetBlockTransactionCountByHash); + $callback!($crate::rpc::eth::EthGetBlockTransactionCountByNumber); + $callback!($crate::rpc::eth::EthGetCode); + $callback!($crate::rpc::eth::EthGetLogs); + $callback!($crate::rpc::eth::EthGetMessageCidByTransactionHash); + $callback!($crate::rpc::eth::EthGetStorageAt); + $callback!($crate::rpc::eth::EthGetTransactionByHash); + $callback!($crate::rpc::eth::EthGetTransactionByHashLimited); + $callback!($crate::rpc::eth::EthGetTransactionCount); + $callback!($crate::rpc::eth::EthGetTransactionHashByCid); + $callback!($crate::rpc::eth::EthGetTransactionByBlockNumberAndIndex); + $callback!($crate::rpc::eth::EthGetTransactionByBlockHashAndIndex); + $callback!($crate::rpc::eth::EthMaxPriorityFeePerGas); + $callback!($crate::rpc::eth::EthProtocolVersion); + $callback!($crate::rpc::eth::EthGetTransactionReceipt); + $callback!($crate::rpc::eth::EthGetTransactionReceiptLimited); + $callback!($crate::rpc::eth::EthNewFilter); + $callback!($crate::rpc::eth::EthNewPendingTransactionFilter); + $callback!($crate::rpc::eth::EthNewBlockFilter); + $callback!($crate::rpc::eth::EthUninstallFilter); + $callback!($crate::rpc::eth::EthSyncing); + $callback!($crate::rpc::eth::Web3ClientVersion); + $callback!($crate::rpc::eth::EthSendRawTransaction); // gas vertical - $callback!(crate::rpc::gas::GasEstimateFeeCap); - $callback!(crate::rpc::gas::GasEstimateGasLimit); - $callback!(crate::rpc::gas::GasEstimateGasPremium); - $callback!(crate::rpc::gas::GasEstimateMessageGas); + $callback!($crate::rpc::gas::GasEstimateFeeCap); + $callback!($crate::rpc::gas::GasEstimateGasLimit); + $callback!($crate::rpc::gas::GasEstimateGasPremium); + $callback!($crate::rpc::gas::GasEstimateMessageGas); // market vertical - $callback!(crate::rpc::market::MarketAddBalance); + $callback!($crate::rpc::market::MarketAddBalance); // miner vertical - $callback!(crate::rpc::miner::MinerCreateBlock); - $callback!(crate::rpc::miner::MinerGetBaseInfo); + $callback!($crate::rpc::miner::MinerCreateBlock); + $callback!($crate::rpc::miner::MinerGetBaseInfo); // mpool vertical - $callback!(crate::rpc::mpool::MpoolBatchPush); - $callback!(crate::rpc::mpool::MpoolBatchPushUntrusted); - $callback!(crate::rpc::mpool::MpoolGetNonce); - $callback!(crate::rpc::mpool::MpoolPending); - $callback!(crate::rpc::mpool::MpoolPush); - $callback!(crate::rpc::mpool::MpoolPushMessage); - $callback!(crate::rpc::mpool::MpoolPushUntrusted); - $callback!(crate::rpc::mpool::MpoolSelect); + $callback!($crate::rpc::mpool::MpoolBatchPush); + $callback!($crate::rpc::mpool::MpoolBatchPushUntrusted); + $callback!($crate::rpc::mpool::MpoolGetNonce); + $callback!($crate::rpc::mpool::MpoolPending); + $callback!($crate::rpc::mpool::MpoolPush); + $callback!($crate::rpc::mpool::MpoolPushMessage); + $callback!($crate::rpc::mpool::MpoolPushUntrusted); + $callback!($crate::rpc::mpool::MpoolSelect); // msig vertical - $callback!(crate::rpc::msig::MsigGetAvailableBalance); - $callback!(crate::rpc::msig::MsigGetPending); - $callback!(crate::rpc::msig::MsigGetVested); - $callback!(crate::rpc::msig::MsigGetVestingSchedule); + $callback!($crate::rpc::msig::MsigGetAvailableBalance); + $callback!($crate::rpc::msig::MsigGetPending); + $callback!($crate::rpc::msig::MsigGetVested); + $callback!($crate::rpc::msig::MsigGetVestingSchedule); // net vertical - $callback!(crate::rpc::net::NetAddrsListen); - $callback!(crate::rpc::net::NetAgentVersion); - $callback!(crate::rpc::net::NetAutoNatStatus); - $callback!(crate::rpc::net::NetConnect); - $callback!(crate::rpc::net::NetDisconnect); - $callback!(crate::rpc::net::NetFindPeer); - $callback!(crate::rpc::net::NetInfo); - $callback!(crate::rpc::net::NetListening); - $callback!(crate::rpc::net::NetPeers); - $callback!(crate::rpc::net::NetProtectAdd); - $callback!(crate::rpc::net::NetProtectList); - $callback!(crate::rpc::net::NetProtectRemove); - $callback!(crate::rpc::net::NetVersion); + $callback!($crate::rpc::net::NetAddrsListen); + $callback!($crate::rpc::net::NetAgentVersion); + $callback!($crate::rpc::net::NetAutoNatStatus); + $callback!($crate::rpc::net::NetConnect); + $callback!($crate::rpc::net::NetDisconnect); + $callback!($crate::rpc::net::NetFindPeer); + $callback!($crate::rpc::net::NetInfo); + $callback!($crate::rpc::net::NetListening); + $callback!($crate::rpc::net::NetPeers); + $callback!($crate::rpc::net::NetProtectAdd); + $callback!($crate::rpc::net::NetProtectList); + $callback!($crate::rpc::net::NetProtectRemove); + $callback!($crate::rpc::net::NetVersion); // node vertical - $callback!(crate::rpc::node::NodeStatus); + $callback!($crate::rpc::node::NodeStatus); // state vertical - $callback!(crate::rpc::state::StateAccountKey); - $callback!(crate::rpc::state::StateCall); - $callback!(crate::rpc::state::StateCirculatingSupply); - $callback!(crate::rpc::state::StateCompute); - $callback!(crate::rpc::state::StateDealProviderCollateralBounds); - $callback!(crate::rpc::state::StateFetchRoot); - $callback!(crate::rpc::state::StateGetActor); - $callback!(crate::rpc::state::StateGetAllAllocations); - $callback!(crate::rpc::state::StateGetAllClaims); - $callback!(crate::rpc::state::StateGetAllocation); - $callback!(crate::rpc::state::StateGetAllocationForPendingDeal); - $callback!(crate::rpc::state::StateGetAllocationIdForPendingDeal); - $callback!(crate::rpc::state::StateGetAllocations); - $callback!(crate::rpc::state::StateGetBeaconEntry); - $callback!(crate::rpc::state::StateGetClaim); - $callback!(crate::rpc::state::StateGetClaims); - $callback!(crate::rpc::state::StateGetNetworkParams); - $callback!(crate::rpc::state::StateGetRandomnessDigestFromBeacon); - $callback!(crate::rpc::state::StateGetRandomnessDigestFromTickets); - $callback!(crate::rpc::state::StateGetRandomnessFromBeacon); - $callback!(crate::rpc::state::StateGetRandomnessFromTickets); - $callback!(crate::rpc::state::StateGetReceipt); - $callback!(crate::rpc::state::StateListActors); - $callback!(crate::rpc::state::StateListMessages); - $callback!(crate::rpc::state::StateListMiners); - $callback!(crate::rpc::state::StateLookupID); - $callback!(crate::rpc::state::StateLookupRobustAddress); - $callback!(crate::rpc::state::StateMarketBalance); - $callback!(crate::rpc::state::StateMarketDeals); - $callback!(crate::rpc::state::StateMarketParticipants); - $callback!(crate::rpc::state::StateMarketStorageDeal); - $callback!(crate::rpc::state::StateMinerActiveSectors); - $callback!(crate::rpc::state::StateMinerAllocated); - $callback!(crate::rpc::state::StateMinerAvailableBalance); - $callback!(crate::rpc::state::StateMinerDeadlines); - $callback!(crate::rpc::state::StateMinerFaults); - $callback!(crate::rpc::state::StateMinerInfo); - $callback!(crate::rpc::state::StateMinerInitialPledgeCollateral); - $callback!(crate::rpc::state::StateMinerPartitions); - $callback!(crate::rpc::state::StateMinerPower); - $callback!(crate::rpc::state::StateMinerPreCommitDepositForPower); - $callback!(crate::rpc::state::StateMinerProvingDeadline); - $callback!(crate::rpc::state::StateMinerRecoveries); - $callback!(crate::rpc::state::StateMinerSectorAllocated); - $callback!(crate::rpc::state::StateMinerSectorCount); - $callback!(crate::rpc::state::StateMinerSectors); - $callback!(crate::rpc::state::StateNetworkName); - $callback!(crate::rpc::state::StateNetworkVersion); - $callback!(crate::rpc::state::StateReadState); - $callback!(crate::rpc::state::StateReplay); - $callback!(crate::rpc::state::StateSearchMsg); - $callback!(crate::rpc::state::StateSearchMsgLimited); - $callback!(crate::rpc::state::StateSectorExpiration); - $callback!(crate::rpc::state::StateSectorGetInfo); - $callback!(crate::rpc::state::StateSectorPartition); - $callback!(crate::rpc::state::StateSectorPreCommitInfo); - $callback!(crate::rpc::state::StateSectorPreCommitInfoV0); - $callback!(crate::rpc::state::StateVerifiedClientStatus); - $callback!(crate::rpc::state::StateVerifiedRegistryRootKey); - $callback!(crate::rpc::state::StateVerifierStatus); - $callback!(crate::rpc::state::StateVMCirculatingSupplyInternal); - $callback!(crate::rpc::state::StateWaitMsg); - $callback!(crate::rpc::state::StateWaitMsgV0); + $callback!($crate::rpc::state::StateAccountKey); + $callback!($crate::rpc::state::StateCall); + $callback!($crate::rpc::state::StateCirculatingSupply); + $callback!($crate::rpc::state::StateCompute); + $callback!($crate::rpc::state::StateDealProviderCollateralBounds); + $callback!($crate::rpc::state::StateFetchRoot); + $callback!($crate::rpc::state::StateGetActor); + $callback!($crate::rpc::state::StateGetAllAllocations); + $callback!($crate::rpc::state::StateGetAllClaims); + $callback!($crate::rpc::state::StateGetAllocation); + $callback!($crate::rpc::state::StateGetAllocationForPendingDeal); + $callback!($crate::rpc::state::StateGetAllocationIdForPendingDeal); + $callback!($crate::rpc::state::StateGetAllocations); + $callback!($crate::rpc::state::StateGetBeaconEntry); + $callback!($crate::rpc::state::StateGetClaim); + $callback!($crate::rpc::state::StateGetClaims); + $callback!($crate::rpc::state::StateGetNetworkParams); + $callback!($crate::rpc::state::StateGetRandomnessDigestFromBeacon); + $callback!($crate::rpc::state::StateGetRandomnessDigestFromTickets); + $callback!($crate::rpc::state::StateGetRandomnessFromBeacon); + $callback!($crate::rpc::state::StateGetRandomnessFromTickets); + $callback!($crate::rpc::state::StateGetReceipt); + $callback!($crate::rpc::state::StateListActors); + $callback!($crate::rpc::state::StateListMessages); + $callback!($crate::rpc::state::StateListMiners); + $callback!($crate::rpc::state::StateLookupID); + $callback!($crate::rpc::state::StateLookupRobustAddress); + $callback!($crate::rpc::state::StateMarketBalance); + $callback!($crate::rpc::state::StateMarketDeals); + $callback!($crate::rpc::state::StateMarketParticipants); + $callback!($crate::rpc::state::StateMarketStorageDeal); + $callback!($crate::rpc::state::StateMinerActiveSectors); + $callback!($crate::rpc::state::StateMinerAllocated); + $callback!($crate::rpc::state::StateMinerAvailableBalance); + $callback!($crate::rpc::state::StateMinerDeadlines); + $callback!($crate::rpc::state::StateMinerFaults); + $callback!($crate::rpc::state::StateMinerInfo); + $callback!($crate::rpc::state::StateMinerInitialPledgeCollateral); + $callback!($crate::rpc::state::StateMinerPartitions); + $callback!($crate::rpc::state::StateMinerPower); + $callback!($crate::rpc::state::StateMinerPreCommitDepositForPower); + $callback!($crate::rpc::state::StateMinerProvingDeadline); + $callback!($crate::rpc::state::StateMinerRecoveries); + $callback!($crate::rpc::state::StateMinerSectorAllocated); + $callback!($crate::rpc::state::StateMinerSectorCount); + $callback!($crate::rpc::state::StateMinerSectors); + $callback!($crate::rpc::state::StateNetworkName); + $callback!($crate::rpc::state::StateNetworkVersion); + $callback!($crate::rpc::state::StateReadState); + $callback!($crate::rpc::state::StateReplay); + $callback!($crate::rpc::state::StateSearchMsg); + $callback!($crate::rpc::state::StateSearchMsgLimited); + $callback!($crate::rpc::state::StateSectorExpiration); + $callback!($crate::rpc::state::StateSectorGetInfo); + $callback!($crate::rpc::state::StateSectorPartition); + $callback!($crate::rpc::state::StateSectorPreCommitInfo); + $callback!($crate::rpc::state::StateSectorPreCommitInfoV0); + $callback!($crate::rpc::state::StateVerifiedClientStatus); + $callback!($crate::rpc::state::StateVerifiedRegistryRootKey); + $callback!($crate::rpc::state::StateVerifierStatus); + $callback!($crate::rpc::state::StateVMCirculatingSupplyInternal); + $callback!($crate::rpc::state::StateWaitMsg); + $callback!($crate::rpc::state::StateWaitMsgV0); // sync vertical - $callback!(crate::rpc::sync::SyncCheckBad); - $callback!(crate::rpc::sync::SyncMarkBad); - $callback!(crate::rpc::sync::SyncState); - $callback!(crate::rpc::sync::SyncSubmitBlock); + $callback!($crate::rpc::sync::SyncCheckBad); + $callback!($crate::rpc::sync::SyncMarkBad); + $callback!($crate::rpc::sync::SyncState); + $callback!($crate::rpc::sync::SyncSubmitBlock); // wallet vertical - $callback!(crate::rpc::wallet::WalletBalance); - $callback!(crate::rpc::wallet::WalletDefaultAddress); - $callback!(crate::rpc::wallet::WalletDelete); - $callback!(crate::rpc::wallet::WalletExport); - $callback!(crate::rpc::wallet::WalletHas); - $callback!(crate::rpc::wallet::WalletImport); - $callback!(crate::rpc::wallet::WalletList); - $callback!(crate::rpc::wallet::WalletNew); - $callback!(crate::rpc::wallet::WalletSetDefault); - $callback!(crate::rpc::wallet::WalletSign); - $callback!(crate::rpc::wallet::WalletSignMessage); - $callback!(crate::rpc::wallet::WalletValidateAddress); - $callback!(crate::rpc::wallet::WalletVerify); + $callback!($crate::rpc::wallet::WalletBalance); + $callback!($crate::rpc::wallet::WalletDefaultAddress); + $callback!($crate::rpc::wallet::WalletDelete); + $callback!($crate::rpc::wallet::WalletExport); + $callback!($crate::rpc::wallet::WalletHas); + $callback!($crate::rpc::wallet::WalletImport); + $callback!($crate::rpc::wallet::WalletList); + $callback!($crate::rpc::wallet::WalletNew); + $callback!($crate::rpc::wallet::WalletSetDefault); + $callback!($crate::rpc::wallet::WalletSign); + $callback!($crate::rpc::wallet::WalletSignMessage); + $callback!($crate::rpc::wallet::WalletValidateAddress); + $callback!($crate::rpc::wallet::WalletVerify); // f3 - $callback!(crate::rpc::f3::F3GetCertificate); - $callback!(crate::rpc::f3::F3GetECPowerTable); - $callback!(crate::rpc::f3::F3GetF3PowerTable); - $callback!(crate::rpc::f3::F3IsRunning); - $callback!(crate::rpc::f3::F3GetProgress); - $callback!(crate::rpc::f3::F3GetManifest); - $callback!(crate::rpc::f3::F3ListParticipants); - $callback!(crate::rpc::f3::F3GetLatestCertificate); - $callback!(crate::rpc::f3::F3GetOrRenewParticipationTicket); - $callback!(crate::rpc::f3::F3Participate); - $callback!(crate::rpc::f3::GetHead); - $callback!(crate::rpc::f3::GetParent); - $callback!(crate::rpc::f3::GetParticipatingMinerIDs); - $callback!(crate::rpc::f3::GetPowerTable); - $callback!(crate::rpc::f3::GetTipset); - $callback!(crate::rpc::f3::GetTipsetByEpoch); - $callback!(crate::rpc::f3::Finalize); - $callback!(crate::rpc::f3::ProtectPeer); - $callback!(crate::rpc::f3::SignMessage); + $callback!($crate::rpc::f3::F3GetCertificate); + $callback!($crate::rpc::f3::F3GetECPowerTable); + $callback!($crate::rpc::f3::F3GetF3PowerTable); + $callback!($crate::rpc::f3::F3IsRunning); + $callback!($crate::rpc::f3::F3GetProgress); + $callback!($crate::rpc::f3::F3GetManifest); + $callback!($crate::rpc::f3::F3ListParticipants); + $callback!($crate::rpc::f3::F3GetLatestCertificate); + $callback!($crate::rpc::f3::F3GetOrRenewParticipationTicket); + $callback!($crate::rpc::f3::F3Participate); + $callback!($crate::rpc::f3::GetHead); + $callback!($crate::rpc::f3::GetParent); + $callback!($crate::rpc::f3::GetParticipatingMinerIDs); + $callback!($crate::rpc::f3::GetPowerTable); + $callback!($crate::rpc::f3::GetTipset); + $callback!($crate::rpc::f3::GetTipsetByEpoch); + $callback!($crate::rpc::f3::Finalize); + $callback!($crate::rpc::f3::ProtectPeer); + $callback!($crate::rpc::f3::SignMessage); // misc - $callback!(crate::rpc::misc::GetActorEventsRaw); + $callback!($crate::rpc::misc::GetActorEventsRaw); }; } pub(crate) use for_each_method; @@ -431,6 +433,14 @@ impl RPCState { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RpcCallSnapshot { + pub name: String, + pub params: Option, + pub response: serde_json::Value, + pub db: String, +} + #[derive(Clone)] struct PerConnection { methods: Methods, diff --git a/src/rpc/reflect/mod.rs b/src/rpc/reflect/mod.rs index 1185a135428a..ead3e5e23abc 100644 --- a/src/rpc/reflect/mod.rs +++ b/src/rpc/reflect/mod.rs @@ -149,6 +149,21 @@ pub trait RpcMethodExt: RpcMethod { )), } } + + fn parse_params( + params_raw: Option>, + calling_convention: ParamStructure, + ) -> anyhow::Result { + Ok(Self::Params::parse( + params_raw + .map(|s| serde_json::from_str(s.as_ref())) + .transpose()?, + Self::PARAM_NAMES, + calling_convention, + Self::N_REQUIRED_PARAMS, + )?) + } + /// Generate a full `OpenRPC` method definition for this endpoint. fn openrpc<'de>( gen: &mut SchemaGenerator, @@ -213,17 +228,8 @@ pub trait RpcMethodExt: RpcMethod { ); module.register_async_method(Self::NAME, move |params, ctx, _extensions| async move { - let raw = params - .as_str() - .map(serde_json::from_str) - .transpose() + let params = Self::parse_params(params.as_str(), calling_convention) .map_err(|e| Error::invalid_params(e, None))?; - let params = Self::Params::parse( - raw, - Self::PARAM_NAMES, - calling_convention, - Self::N_REQUIRED_PARAMS, - )?; let ok = Self::handle(ctx, params).await?; Result::<_, jsonrpsee::types::ErrorObjectOwned>::Ok(ok.into_lotus_json()) }) diff --git a/src/shim/address.rs b/src/shim/address.rs index 3e5493f60f4d..778fd1d8a461 100644 --- a/src/shim/address.rs +++ b/src/shim/address.rs @@ -49,7 +49,7 @@ thread_local! { /// /// The thread-local network variable is initialized to the value of the global network. This global /// network variable is set once when Forest has figured out which network it is using. -pub struct CurrentNetwork(); +pub struct CurrentNetwork; impl CurrentNetwork { pub fn get() -> Network { FromPrimitive::from_u8(LOCAL_NETWORK.with(|ident| ident.load(Ordering::Acquire))) diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 745f2699f0b9..e753952b10d3 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -1,6 +1,8 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +mod test_snapshot; + use crate::blocks::{ElectionProof, Ticket, Tipset}; use crate::db::car::ManyCar; use crate::eth::{EthChainId as EthChainIdType, SAFE_EPOCH_DELAY}; @@ -149,6 +151,10 @@ pub enum ApiCommands { #[arg(long)] include_ignored: bool, }, + Test { + #[arg(num_args = 1.., required = true)] + files: Vec, + }, } impl ApiCommands { @@ -255,6 +261,13 @@ impl ApiCommands { println!(); } } + Self::Test { files } => { + for path in files { + print!("Running RPC test with snapshot {} ...", path.display()); + test_snapshot::run_test_from_snapshot(&path).await?; + println!(" Success"); + } + } } Ok(()) } diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs new file mode 100644 index 000000000000..851b9f30b308 --- /dev/null +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -0,0 +1,161 @@ +// Copyright 2019-2024 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::filter::EthEventHandler; +use crate::{ + chain::ChainStore, + chain_sync::{network_context::SyncNetworkContext, SyncConfig, SyncStage}, + db::MemoryDB, + genesis::{get_network_name_from_genesis, read_genesis_header}, + libp2p::{NetworkMessage, PeerManager}, + lotus_json::HasLotusJson, + message_pool::{MessagePool, MpoolRpcProvider}, + networks::ChainConfig, + rpc::{RPCState, RpcCallSnapshot, RpcMethod as _, RpcMethodExt as _}, + shim::address::{CurrentNetwork, Network}, + state_manager::StateManager, + KeyStore, KeyStoreConfig, +}; +use base64::prelude::*; +use openrpc_types::ParamStructure; +use parking_lot::RwLock; +use std::{path::Path, sync::Arc}; +use tokio::{sync::mpsc, task::JoinSet}; + +pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { + CurrentNetwork::set_global(Network::Testnet); + let mut run = false; + let snapshot_bytes = std::fs::read(path)?; + let snapshot_bytes = if let Ok(bytes) = zstd::decode_all(snapshot_bytes.as_slice()) { + bytes + } else { + snapshot_bytes + }; + let snapshot: RpcCallSnapshot = serde_json::from_slice(snapshot_bytes.as_slice())?; + let db_bytes = BASE64_STANDARD.decode(&snapshot.db)?; + let db = Arc::new(match MemoryDB::deserialize_from(db_bytes.as_slice()) { + Ok(db) => db, + Err(_) => MemoryDB::deserialize_from_legacy(db_bytes.as_slice())?, + }); + let chain_config = Arc::new(ChainConfig::calibnet()); + let (ctx, _, _) = ctx(db, chain_config).await?; + let params_raw = if let Some(params) = &snapshot.params { + Some(serde_json::to_string(params)?) + } else { + None + }; + + macro_rules! run_test { + ($ty:ty) => { + if snapshot.name.as_str() == <$ty>::NAME { + let params = <$ty>::parse_params(params_raw.clone(), ParamStructure::Either)?; + let result = <$ty>::handle(ctx.clone(), params).await?; + assert_eq!(snapshot.response, result.into_lotus_json_value()?); + run = true; + } + }; + } + + crate::for_each_method!(run_test); + + assert!(run, "RPC method not found"); + + Ok(()) +} + +async fn ctx( + db: Arc, + chain_config: Arc, +) -> anyhow::Result<( + Arc>, + flume::Receiver, + tokio::sync::mpsc::Receiver<()>, +)> { + let (network_send, network_rx) = flume::bounded(5); + let (tipset_send, _) = flume::bounded(5); + let sync_config = Arc::new(SyncConfig::default()); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db).await?; + + let chain_store = Arc::new( + ChainStore::new( + db.clone(), + db.clone(), + db, + chain_config.clone(), + genesis_header.clone(), + ) + .unwrap(), + ); + + let state_manager = + Arc::new(StateManager::new(chain_store.clone(), chain_config, sync_config).unwrap()); + let network_name = get_network_name_from_genesis(&genesis_header, &state_manager)?; + let message_pool = MessagePool::new( + MpoolRpcProvider::new(chain_store.publisher().clone(), state_manager.clone()), + network_name.clone(), + network_send.clone(), + Default::default(), + state_manager.chain_config().clone(), + &mut JoinSet::new(), + )?; + + let peer_manager = Arc::new(PeerManager::default()); + let sync_network_context = + SyncNetworkContext::new(network_send, peer_manager, state_manager.blockstore_owned()); + let (shutdown, shutdown_recv) = mpsc::channel(1); + let rpc_state = Arc::new(RPCState { + state_manager, + keystore: Arc::new(tokio::sync::RwLock::new(KeyStore::new( + KeyStoreConfig::Memory, + )?)), + mpool: Arc::new(message_pool), + bad_blocks: Default::default(), + sync_state: Arc::new(RwLock::new(Default::default())), + eth_event_handler: Arc::new(EthEventHandler::new()), + sync_network_context, + network_name, + start_time: chrono::Utc::now(), + shutdown, + tipset_send, + }); + rpc_state.sync_state.write().set_stage(SyncStage::Idle); + Ok((rpc_state, network_rx, shutdown_recv)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::daemon::db_util::download_to; + use itertools::Itertools as _; + use url::Url; + + #[tokio::test] + async fn rpc_regression_tests() { + let urls = include_str!("test_snapshots.txt") + .trim() + .split("\n") + .filter_map(|n| { + Url::parse( + format!( + "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/rpc_test/{n}" + ) + .as_str(), + ) + .ok() + }) + .collect_vec(); + for url in urls { + print!("Testing {url} ..."); + let tmp_dir = tempfile::tempdir().unwrap(); + let tmp = tempfile::NamedTempFile::new_in(&tmp_dir) + .unwrap() + .into_temp_path(); + println!("start downloading at {}", tmp.display()); + download_to(&url, &tmp).await.unwrap(); + println!("done downloading {}", tmp.display()); + run_test_from_snapshot(&tmp).await.unwrap(); + println!(" succeeded."); + } + } +} diff --git a/src/tool/subcommands/api_cmd/test_snapshots.txt b/src/tool/subcommands/api_cmd/test_snapshots.txt new file mode 100644 index 000000000000..e2da66fd254b --- /dev/null +++ b/src/tool/subcommands/api_cmd/test_snapshots.txt @@ -0,0 +1,2 @@ +f3_gettipsetbyepoch_1730952732441851.json.zst +filecoin_statelistactors_1730953255032189.json.zst From 91e4716f58016f6e6644e66a33e6da48479b23fc Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 11 Dec 2024 20:45:14 +0800 Subject: [PATCH 02/16] feat(tool): forest-tool api generate-test-snapshot --- src/chain/store/index.rs | 13 +- src/db/memory.rs | 31 +- src/rpc/mod.rs | 393 +++++++++--------- src/rpc/reflect/mod.rs | 26 +- src/tool/subcommands/api_cmd.rs | 29 ++ .../api_cmd/generate_test_snapshot.rs | 234 +++++++++++ 6 files changed, 513 insertions(+), 213 deletions(-) create mode 100644 src/tool/subcommands/api_cmd/generate_test_snapshot.rs diff --git a/src/chain/store/index.rs b/src/chain/store/index.rs index a9383e396e32..17d59dceceb5 100644 --- a/src/chain/store/index.rs +++ b/src/chain/store/index.rs @@ -7,6 +7,7 @@ use crate::beacon::{BeaconEntry, IGNORE_DRAND_VAR}; use crate::blocks::{Tipset, TipsetKey}; use crate::metrics; use crate::shim::clock::ChainEpoch; +use crate::utils::misc::env::is_env_truthy; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use lru::LruCache; @@ -47,11 +48,13 @@ impl ChainIndex { /// Loads a tipset from memory given the tipset keys and cache. Semantically /// identical to [`Tipset::load`] but the result is cached. pub fn load_tipset(&self, tsk: &TipsetKey) -> Result>, Error> { - if let Some(ts) = self.ts_cache.lock().get(tsk) { - metrics::LRU_CACHE_HIT - .get_or_create(&metrics::values::TIPSET) - .inc(); - return Ok(Some(ts.clone())); + if !is_env_truthy("FOREST_TIPSET_CACHE_DISABLED") { + if let Some(ts) = self.ts_cache.lock().get(tsk) { + metrics::LRU_CACHE_HIT + .get_or_create(&metrics::values::TIPSET) + .inc(); + return Ok(Some(ts.clone())); + } } let ts_opt = Tipset::load(&self.db, tsk)?.map(Arc::new); diff --git a/src/db/memory.rs b/src/db/memory.rs index 6d0abe23b00c..ae32d2a14163 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -1,6 +1,7 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use super::{EthMappingsStore, SettingsStore}; use crate::cid_collections::CidHashSet; use crate::db::{GarbageCollectable, PersistentStore}; use crate::libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; @@ -11,8 +12,7 @@ use cid::Cid; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use parking_lot::RwLock; - -use super::{EthMappingsStore, SettingsStore}; +use std::ops::Deref; #[derive(Debug, Default)] pub struct MemoryDB { @@ -22,6 +22,33 @@ pub struct MemoryDB { eth_mappings_db: RwLock>>, } +impl MemoryDB { + pub fn serialize(&self) -> anyhow::Result> { + let blockchain_db = self.blockchain_db.read(); + let blockchain_persistent_db = self.blockchain_persistent_db.read(); + let settings_db = self.settings_db.read(); + let eth_mappings_db = self.eth_mappings_db.read(); + let tuple = ( + blockchain_db.deref(), + blockchain_persistent_db.deref(), + settings_db.deref(), + eth_mappings_db.deref(), + ); + Ok(fvm_ipld_encoding::to_vec(&tuple)?) + } + + pub fn deserialize_from(bytes: &[u8]) -> anyhow::Result { + let (blockchain_db, blockchain_persistent_db, settings_db, eth_mappings_db) = + fvm_ipld_encoding::from_slice(bytes)?; + Ok(Self { + blockchain_db: RwLock::new(blockchain_db), + blockchain_persistent_db: RwLock::new(blockchain_persistent_db), + settings_db: RwLock::new(settings_db), + eth_mappings_db: RwLock::new(eth_mappings_db), + }) + } +} + impl GarbageCollectable for MemoryDB { fn get_keys(&self) -> anyhow::Result { let mut set = CidHashSet::new(); diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index cb1c9856f0e9..f5244f081bb3 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -31,237 +31,238 @@ pub use jsonrpsee::core::ClientError; /// trait. /// /// All methods should be entered here. +#[macro_export] macro_rules! for_each_method { ($callback:path) => { // auth vertical - $callback!(crate::rpc::auth::AuthNew); - $callback!(crate::rpc::auth::AuthVerify); + $callback!($crate::rpc::auth::AuthNew); + $callback!($crate::rpc::auth::AuthVerify); // beacon vertical - $callback!(crate::rpc::beacon::BeaconGetEntry); + $callback!($crate::rpc::beacon::BeaconGetEntry); // chain vertical - $callback!(crate::rpc::chain::ChainExport); - $callback!(crate::rpc::chain::ChainGetBlock); - $callback!(crate::rpc::chain::ChainGetBlockMessages); - $callback!(crate::rpc::chain::ChainGetEvents); - $callback!(crate::rpc::chain::ChainGetGenesis); - $callback!(crate::rpc::chain::ChainGetMessage); - $callback!(crate::rpc::chain::ChainGetMessagesInTipset); - $callback!(crate::rpc::chain::ChainGetMinBaseFee); - $callback!(crate::rpc::chain::ChainGetParentMessages); - $callback!(crate::rpc::chain::ChainGetParentReceipts); - $callback!(crate::rpc::chain::ChainGetPath); - $callback!(crate::rpc::chain::ChainGetTipSet); - $callback!(crate::rpc::chain::ChainGetTipSetAfterHeight); - $callback!(crate::rpc::chain::ChainGetTipSetByHeight); - $callback!(crate::rpc::chain::ChainHasObj); - $callback!(crate::rpc::chain::ChainHead); - $callback!(crate::rpc::chain::ChainReadObj); - $callback!(crate::rpc::chain::ChainSetHead); - $callback!(crate::rpc::chain::ChainStatObj); - $callback!(crate::rpc::chain::ChainTipSetWeight); + $callback!($crate::rpc::chain::ChainExport); + $callback!($crate::rpc::chain::ChainGetBlock); + $callback!($crate::rpc::chain::ChainGetBlockMessages); + $callback!($crate::rpc::chain::ChainGetEvents); + $callback!($crate::rpc::chain::ChainGetGenesis); + $callback!($crate::rpc::chain::ChainGetMessage); + $callback!($crate::rpc::chain::ChainGetMessagesInTipset); + $callback!($crate::rpc::chain::ChainGetMinBaseFee); + $callback!($crate::rpc::chain::ChainGetParentMessages); + $callback!($crate::rpc::chain::ChainGetParentReceipts); + $callback!($crate::rpc::chain::ChainGetPath); + $callback!($crate::rpc::chain::ChainGetTipSet); + $callback!($crate::rpc::chain::ChainGetTipSetAfterHeight); + $callback!($crate::rpc::chain::ChainGetTipSetByHeight); + $callback!($crate::rpc::chain::ChainHasObj); + $callback!($crate::rpc::chain::ChainHead); + $callback!($crate::rpc::chain::ChainReadObj); + $callback!($crate::rpc::chain::ChainSetHead); + $callback!($crate::rpc::chain::ChainStatObj); + $callback!($crate::rpc::chain::ChainTipSetWeight); // common vertical - $callback!(crate::rpc::common::Session); - $callback!(crate::rpc::common::Shutdown); - $callback!(crate::rpc::common::StartTime); - $callback!(crate::rpc::common::Version); + $callback!($crate::rpc::common::Session); + $callback!($crate::rpc::common::Shutdown); + $callback!($crate::rpc::common::StartTime); + $callback!($crate::rpc::common::Version); // eth vertical - $callback!(crate::rpc::eth::EthAccounts); - $callback!(crate::rpc::eth::EthAddressToFilecoinAddress); - $callback!(crate::rpc::eth::EthBlockNumber); - $callback!(crate::rpc::eth::EthCall); - $callback!(crate::rpc::eth::EthChainId); - $callback!(crate::rpc::eth::EthEstimateGas); - $callback!(crate::rpc::eth::EthFeeHistory); - $callback!(crate::rpc::eth::EthGasPrice); - $callback!(crate::rpc::eth::EthGetBalance); - $callback!(crate::rpc::eth::EthGetBlockByHash); - $callback!(crate::rpc::eth::EthGetBlockByNumber); - $callback!(crate::rpc::eth::EthGetBlockReceipts); - $callback!(crate::rpc::eth::EthGetBlockReceiptsLimited); - $callback!(crate::rpc::eth::EthGetBlockTransactionCountByHash); - $callback!(crate::rpc::eth::EthGetBlockTransactionCountByNumber); - $callback!(crate::rpc::eth::EthGetCode); - $callback!(crate::rpc::eth::EthGetLogs); - $callback!(crate::rpc::eth::EthGetMessageCidByTransactionHash); - $callback!(crate::rpc::eth::EthGetStorageAt); - $callback!(crate::rpc::eth::EthGetTransactionByHash); - $callback!(crate::rpc::eth::EthGetTransactionByHashLimited); - $callback!(crate::rpc::eth::EthGetTransactionCount); - $callback!(crate::rpc::eth::EthGetTransactionHashByCid); - $callback!(crate::rpc::eth::EthGetTransactionByBlockNumberAndIndex); - $callback!(crate::rpc::eth::EthGetTransactionByBlockHashAndIndex); - $callback!(crate::rpc::eth::EthMaxPriorityFeePerGas); - $callback!(crate::rpc::eth::EthProtocolVersion); - $callback!(crate::rpc::eth::EthGetTransactionReceipt); - $callback!(crate::rpc::eth::EthGetTransactionReceiptLimited); - $callback!(crate::rpc::eth::EthNewFilter); - $callback!(crate::rpc::eth::EthNewPendingTransactionFilter); - $callback!(crate::rpc::eth::EthNewBlockFilter); - $callback!(crate::rpc::eth::EthUninstallFilter); - $callback!(crate::rpc::eth::EthSyncing); - $callback!(crate::rpc::eth::Web3ClientVersion); - $callback!(crate::rpc::eth::EthSendRawTransaction); + $callback!($crate::rpc::eth::EthAccounts); + $callback!($crate::rpc::eth::EthAddressToFilecoinAddress); + $callback!($crate::rpc::eth::EthBlockNumber); + $callback!($crate::rpc::eth::EthCall); + $callback!($crate::rpc::eth::EthChainId); + $callback!($crate::rpc::eth::EthEstimateGas); + $callback!($crate::rpc::eth::EthFeeHistory); + $callback!($crate::rpc::eth::EthGasPrice); + $callback!($crate::rpc::eth::EthGetBalance); + $callback!($crate::rpc::eth::EthGetBlockByHash); + $callback!($crate::rpc::eth::EthGetBlockByNumber); + $callback!($crate::rpc::eth::EthGetBlockReceipts); + $callback!($crate::rpc::eth::EthGetBlockReceiptsLimited); + $callback!($crate::rpc::eth::EthGetBlockTransactionCountByHash); + $callback!($crate::rpc::eth::EthGetBlockTransactionCountByNumber); + $callback!($crate::rpc::eth::EthGetCode); + $callback!($crate::rpc::eth::EthGetLogs); + $callback!($crate::rpc::eth::EthGetMessageCidByTransactionHash); + $callback!($crate::rpc::eth::EthGetStorageAt); + $callback!($crate::rpc::eth::EthGetTransactionByHash); + $callback!($crate::rpc::eth::EthGetTransactionByHashLimited); + $callback!($crate::rpc::eth::EthGetTransactionCount); + $callback!($crate::rpc::eth::EthGetTransactionHashByCid); + $callback!($crate::rpc::eth::EthGetTransactionByBlockNumberAndIndex); + $callback!($crate::rpc::eth::EthGetTransactionByBlockHashAndIndex); + $callback!($crate::rpc::eth::EthMaxPriorityFeePerGas); + $callback!($crate::rpc::eth::EthProtocolVersion); + $callback!($crate::rpc::eth::EthGetTransactionReceipt); + $callback!($crate::rpc::eth::EthGetTransactionReceiptLimited); + $callback!($crate::rpc::eth::EthNewFilter); + $callback!($crate::rpc::eth::EthNewPendingTransactionFilter); + $callback!($crate::rpc::eth::EthNewBlockFilter); + $callback!($crate::rpc::eth::EthUninstallFilter); + $callback!($crate::rpc::eth::EthSyncing); + $callback!($crate::rpc::eth::Web3ClientVersion); + $callback!($crate::rpc::eth::EthSendRawTransaction); // gas vertical - $callback!(crate::rpc::gas::GasEstimateFeeCap); - $callback!(crate::rpc::gas::GasEstimateGasLimit); - $callback!(crate::rpc::gas::GasEstimateGasPremium); - $callback!(crate::rpc::gas::GasEstimateMessageGas); + $callback!($crate::rpc::gas::GasEstimateFeeCap); + $callback!($crate::rpc::gas::GasEstimateGasLimit); + $callback!($crate::rpc::gas::GasEstimateGasPremium); + $callback!($crate::rpc::gas::GasEstimateMessageGas); // market vertical - $callback!(crate::rpc::market::MarketAddBalance); + $callback!($crate::rpc::market::MarketAddBalance); // miner vertical - $callback!(crate::rpc::miner::MinerCreateBlock); - $callback!(crate::rpc::miner::MinerGetBaseInfo); + $callback!($crate::rpc::miner::MinerCreateBlock); + $callback!($crate::rpc::miner::MinerGetBaseInfo); // mpool vertical - $callback!(crate::rpc::mpool::MpoolBatchPush); - $callback!(crate::rpc::mpool::MpoolBatchPushUntrusted); - $callback!(crate::rpc::mpool::MpoolGetNonce); - $callback!(crate::rpc::mpool::MpoolPending); - $callback!(crate::rpc::mpool::MpoolPush); - $callback!(crate::rpc::mpool::MpoolPushMessage); - $callback!(crate::rpc::mpool::MpoolPushUntrusted); - $callback!(crate::rpc::mpool::MpoolSelect); + $callback!($crate::rpc::mpool::MpoolBatchPush); + $callback!($crate::rpc::mpool::MpoolBatchPushUntrusted); + $callback!($crate::rpc::mpool::MpoolGetNonce); + $callback!($crate::rpc::mpool::MpoolPending); + $callback!($crate::rpc::mpool::MpoolPush); + $callback!($crate::rpc::mpool::MpoolPushMessage); + $callback!($crate::rpc::mpool::MpoolPushUntrusted); + $callback!($crate::rpc::mpool::MpoolSelect); // msig vertical - $callback!(crate::rpc::msig::MsigGetAvailableBalance); - $callback!(crate::rpc::msig::MsigGetPending); - $callback!(crate::rpc::msig::MsigGetVested); - $callback!(crate::rpc::msig::MsigGetVestingSchedule); + $callback!($crate::rpc::msig::MsigGetAvailableBalance); + $callback!($crate::rpc::msig::MsigGetPending); + $callback!($crate::rpc::msig::MsigGetVested); + $callback!($crate::rpc::msig::MsigGetVestingSchedule); // net vertical - $callback!(crate::rpc::net::NetAddrsListen); - $callback!(crate::rpc::net::NetAgentVersion); - $callback!(crate::rpc::net::NetAutoNatStatus); - $callback!(crate::rpc::net::NetConnect); - $callback!(crate::rpc::net::NetDisconnect); - $callback!(crate::rpc::net::NetFindPeer); - $callback!(crate::rpc::net::NetInfo); - $callback!(crate::rpc::net::NetListening); - $callback!(crate::rpc::net::NetPeers); - $callback!(crate::rpc::net::NetProtectAdd); - $callback!(crate::rpc::net::NetProtectList); - $callback!(crate::rpc::net::NetProtectRemove); - $callback!(crate::rpc::net::NetVersion); + $callback!($crate::rpc::net::NetAddrsListen); + $callback!($crate::rpc::net::NetAgentVersion); + $callback!($crate::rpc::net::NetAutoNatStatus); + $callback!($crate::rpc::net::NetConnect); + $callback!($crate::rpc::net::NetDisconnect); + $callback!($crate::rpc::net::NetFindPeer); + $callback!($crate::rpc::net::NetInfo); + $callback!($crate::rpc::net::NetListening); + $callback!($crate::rpc::net::NetPeers); + $callback!($crate::rpc::net::NetProtectAdd); + $callback!($crate::rpc::net::NetProtectList); + $callback!($crate::rpc::net::NetProtectRemove); + $callback!($crate::rpc::net::NetVersion); // node vertical - $callback!(crate::rpc::node::NodeStatus); + $callback!($crate::rpc::node::NodeStatus); // state vertical - $callback!(crate::rpc::state::StateAccountKey); - $callback!(crate::rpc::state::StateCall); - $callback!(crate::rpc::state::StateCirculatingSupply); - $callback!(crate::rpc::state::StateCompute); - $callback!(crate::rpc::state::StateDealProviderCollateralBounds); - $callback!(crate::rpc::state::StateFetchRoot); - $callback!(crate::rpc::state::StateGetActor); - $callback!(crate::rpc::state::StateGetAllAllocations); - $callback!(crate::rpc::state::StateGetAllClaims); - $callback!(crate::rpc::state::StateGetAllocation); - $callback!(crate::rpc::state::StateGetAllocationForPendingDeal); - $callback!(crate::rpc::state::StateGetAllocationIdForPendingDeal); - $callback!(crate::rpc::state::StateGetAllocations); - $callback!(crate::rpc::state::StateGetBeaconEntry); - $callback!(crate::rpc::state::StateGetClaim); - $callback!(crate::rpc::state::StateGetClaims); - $callback!(crate::rpc::state::StateGetNetworkParams); - $callback!(crate::rpc::state::StateGetRandomnessDigestFromBeacon); - $callback!(crate::rpc::state::StateGetRandomnessDigestFromTickets); - $callback!(crate::rpc::state::StateGetRandomnessFromBeacon); - $callback!(crate::rpc::state::StateGetRandomnessFromTickets); - $callback!(crate::rpc::state::StateGetReceipt); - $callback!(crate::rpc::state::StateListActors); - $callback!(crate::rpc::state::StateListMessages); - $callback!(crate::rpc::state::StateListMiners); - $callback!(crate::rpc::state::StateLookupID); - $callback!(crate::rpc::state::StateLookupRobustAddress); - $callback!(crate::rpc::state::StateMarketBalance); - $callback!(crate::rpc::state::StateMarketDeals); - $callback!(crate::rpc::state::StateMarketParticipants); - $callback!(crate::rpc::state::StateMarketStorageDeal); - $callback!(crate::rpc::state::StateMinerActiveSectors); - $callback!(crate::rpc::state::StateMinerAllocated); - $callback!(crate::rpc::state::StateMinerAvailableBalance); - $callback!(crate::rpc::state::StateMinerDeadlines); - $callback!(crate::rpc::state::StateMinerFaults); - $callback!(crate::rpc::state::StateMinerInfo); - $callback!(crate::rpc::state::StateMinerInitialPledgeCollateral); - $callback!(crate::rpc::state::StateMinerPartitions); - $callback!(crate::rpc::state::StateMinerPower); - $callback!(crate::rpc::state::StateMinerPreCommitDepositForPower); - $callback!(crate::rpc::state::StateMinerProvingDeadline); - $callback!(crate::rpc::state::StateMinerRecoveries); - $callback!(crate::rpc::state::StateMinerSectorAllocated); - $callback!(crate::rpc::state::StateMinerSectorCount); - $callback!(crate::rpc::state::StateMinerSectors); - $callback!(crate::rpc::state::StateNetworkName); - $callback!(crate::rpc::state::StateNetworkVersion); - $callback!(crate::rpc::state::StateReadState); - $callback!(crate::rpc::state::StateReplay); - $callback!(crate::rpc::state::StateSearchMsg); - $callback!(crate::rpc::state::StateSearchMsgLimited); - $callback!(crate::rpc::state::StateSectorExpiration); - $callback!(crate::rpc::state::StateSectorGetInfo); - $callback!(crate::rpc::state::StateSectorPartition); - $callback!(crate::rpc::state::StateSectorPreCommitInfo); - $callback!(crate::rpc::state::StateSectorPreCommitInfoV0); - $callback!(crate::rpc::state::StateVerifiedClientStatus); - $callback!(crate::rpc::state::StateVerifiedRegistryRootKey); - $callback!(crate::rpc::state::StateVerifierStatus); - $callback!(crate::rpc::state::StateVMCirculatingSupplyInternal); - $callback!(crate::rpc::state::StateWaitMsg); - $callback!(crate::rpc::state::StateWaitMsgV0); + $callback!($crate::rpc::state::StateAccountKey); + $callback!($crate::rpc::state::StateCall); + $callback!($crate::rpc::state::StateCirculatingSupply); + $callback!($crate::rpc::state::StateCompute); + $callback!($crate::rpc::state::StateDealProviderCollateralBounds); + $callback!($crate::rpc::state::StateFetchRoot); + $callback!($crate::rpc::state::StateGetActor); + $callback!($crate::rpc::state::StateGetAllAllocations); + $callback!($crate::rpc::state::StateGetAllClaims); + $callback!($crate::rpc::state::StateGetAllocation); + $callback!($crate::rpc::state::StateGetAllocationForPendingDeal); + $callback!($crate::rpc::state::StateGetAllocationIdForPendingDeal); + $callback!($crate::rpc::state::StateGetAllocations); + $callback!($crate::rpc::state::StateGetBeaconEntry); + $callback!($crate::rpc::state::StateGetClaim); + $callback!($crate::rpc::state::StateGetClaims); + $callback!($crate::rpc::state::StateGetNetworkParams); + $callback!($crate::rpc::state::StateGetRandomnessDigestFromBeacon); + $callback!($crate::rpc::state::StateGetRandomnessDigestFromTickets); + $callback!($crate::rpc::state::StateGetRandomnessFromBeacon); + $callback!($crate::rpc::state::StateGetRandomnessFromTickets); + $callback!($crate::rpc::state::StateGetReceipt); + $callback!($crate::rpc::state::StateListActors); + $callback!($crate::rpc::state::StateListMessages); + $callback!($crate::rpc::state::StateListMiners); + $callback!($crate::rpc::state::StateLookupID); + $callback!($crate::rpc::state::StateLookupRobustAddress); + $callback!($crate::rpc::state::StateMarketBalance); + $callback!($crate::rpc::state::StateMarketDeals); + $callback!($crate::rpc::state::StateMarketParticipants); + $callback!($crate::rpc::state::StateMarketStorageDeal); + $callback!($crate::rpc::state::StateMinerActiveSectors); + $callback!($crate::rpc::state::StateMinerAllocated); + $callback!($crate::rpc::state::StateMinerAvailableBalance); + $callback!($crate::rpc::state::StateMinerDeadlines); + $callback!($crate::rpc::state::StateMinerFaults); + $callback!($crate::rpc::state::StateMinerInfo); + $callback!($crate::rpc::state::StateMinerInitialPledgeCollateral); + $callback!($crate::rpc::state::StateMinerPartitions); + $callback!($crate::rpc::state::StateMinerPower); + $callback!($crate::rpc::state::StateMinerPreCommitDepositForPower); + $callback!($crate::rpc::state::StateMinerProvingDeadline); + $callback!($crate::rpc::state::StateMinerRecoveries); + $callback!($crate::rpc::state::StateMinerSectorAllocated); + $callback!($crate::rpc::state::StateMinerSectorCount); + $callback!($crate::rpc::state::StateMinerSectors); + $callback!($crate::rpc::state::StateNetworkName); + $callback!($crate::rpc::state::StateNetworkVersion); + $callback!($crate::rpc::state::StateReadState); + $callback!($crate::rpc::state::StateReplay); + $callback!($crate::rpc::state::StateSearchMsg); + $callback!($crate::rpc::state::StateSearchMsgLimited); + $callback!($crate::rpc::state::StateSectorExpiration); + $callback!($crate::rpc::state::StateSectorGetInfo); + $callback!($crate::rpc::state::StateSectorPartition); + $callback!($crate::rpc::state::StateSectorPreCommitInfo); + $callback!($crate::rpc::state::StateSectorPreCommitInfoV0); + $callback!($crate::rpc::state::StateVerifiedClientStatus); + $callback!($crate::rpc::state::StateVerifiedRegistryRootKey); + $callback!($crate::rpc::state::StateVerifierStatus); + $callback!($crate::rpc::state::StateVMCirculatingSupplyInternal); + $callback!($crate::rpc::state::StateWaitMsg); + $callback!($crate::rpc::state::StateWaitMsgV0); // sync vertical - $callback!(crate::rpc::sync::SyncCheckBad); - $callback!(crate::rpc::sync::SyncMarkBad); - $callback!(crate::rpc::sync::SyncState); - $callback!(crate::rpc::sync::SyncSubmitBlock); + $callback!($crate::rpc::sync::SyncCheckBad); + $callback!($crate::rpc::sync::SyncMarkBad); + $callback!($crate::rpc::sync::SyncState); + $callback!($crate::rpc::sync::SyncSubmitBlock); // wallet vertical - $callback!(crate::rpc::wallet::WalletBalance); - $callback!(crate::rpc::wallet::WalletDefaultAddress); - $callback!(crate::rpc::wallet::WalletDelete); - $callback!(crate::rpc::wallet::WalletExport); - $callback!(crate::rpc::wallet::WalletHas); - $callback!(crate::rpc::wallet::WalletImport); - $callback!(crate::rpc::wallet::WalletList); - $callback!(crate::rpc::wallet::WalletNew); - $callback!(crate::rpc::wallet::WalletSetDefault); - $callback!(crate::rpc::wallet::WalletSign); - $callback!(crate::rpc::wallet::WalletSignMessage); - $callback!(crate::rpc::wallet::WalletValidateAddress); - $callback!(crate::rpc::wallet::WalletVerify); + $callback!($crate::rpc::wallet::WalletBalance); + $callback!($crate::rpc::wallet::WalletDefaultAddress); + $callback!($crate::rpc::wallet::WalletDelete); + $callback!($crate::rpc::wallet::WalletExport); + $callback!($crate::rpc::wallet::WalletHas); + $callback!($crate::rpc::wallet::WalletImport); + $callback!($crate::rpc::wallet::WalletList); + $callback!($crate::rpc::wallet::WalletNew); + $callback!($crate::rpc::wallet::WalletSetDefault); + $callback!($crate::rpc::wallet::WalletSign); + $callback!($crate::rpc::wallet::WalletSignMessage); + $callback!($crate::rpc::wallet::WalletValidateAddress); + $callback!($crate::rpc::wallet::WalletVerify); // f3 - $callback!(crate::rpc::f3::F3GetCertificate); - $callback!(crate::rpc::f3::F3GetECPowerTable); - $callback!(crate::rpc::f3::F3GetF3PowerTable); - $callback!(crate::rpc::f3::F3IsRunning); - $callback!(crate::rpc::f3::F3GetProgress); - $callback!(crate::rpc::f3::F3GetManifest); - $callback!(crate::rpc::f3::F3ListParticipants); - $callback!(crate::rpc::f3::F3GetLatestCertificate); - $callback!(crate::rpc::f3::F3GetOrRenewParticipationTicket); - $callback!(crate::rpc::f3::F3Participate); - $callback!(crate::rpc::f3::GetHead); - $callback!(crate::rpc::f3::GetParent); - $callback!(crate::rpc::f3::GetParticipatingMinerIDs); - $callback!(crate::rpc::f3::GetPowerTable); - $callback!(crate::rpc::f3::GetTipset); - $callback!(crate::rpc::f3::GetTipsetByEpoch); - $callback!(crate::rpc::f3::Finalize); - $callback!(crate::rpc::f3::ProtectPeer); - $callback!(crate::rpc::f3::SignMessage); + $callback!($crate::rpc::f3::F3GetCertificate); + $callback!($crate::rpc::f3::F3GetECPowerTable); + $callback!($crate::rpc::f3::F3GetF3PowerTable); + $callback!($crate::rpc::f3::F3IsRunning); + $callback!($crate::rpc::f3::F3GetProgress); + $callback!($crate::rpc::f3::F3GetManifest); + $callback!($crate::rpc::f3::F3ListParticipants); + $callback!($crate::rpc::f3::F3GetLatestCertificate); + $callback!($crate::rpc::f3::F3GetOrRenewParticipationTicket); + $callback!($crate::rpc::f3::F3Participate); + $callback!($crate::rpc::f3::GetHead); + $callback!($crate::rpc::f3::GetParent); + $callback!($crate::rpc::f3::GetParticipatingMinerIDs); + $callback!($crate::rpc::f3::GetPowerTable); + $callback!($crate::rpc::f3::GetTipset); + $callback!($crate::rpc::f3::GetTipsetByEpoch); + $callback!($crate::rpc::f3::Finalize); + $callback!($crate::rpc::f3::ProtectPeer); + $callback!($crate::rpc::f3::SignMessage); // misc - $callback!(crate::rpc::misc::GetActorEventsRaw); + $callback!($crate::rpc::misc::GetActorEventsRaw); }; } pub(crate) use for_each_method; diff --git a/src/rpc/reflect/mod.rs b/src/rpc/reflect/mod.rs index 5d4f789a2220..026f710bf95c 100644 --- a/src/rpc/reflect/mod.rs +++ b/src/rpc/reflect/mod.rs @@ -150,6 +150,21 @@ pub trait RpcMethodExt: RpcMethod { )), } } + + fn parse_params( + params_raw: Option>, + calling_convention: ParamStructure, + ) -> anyhow::Result { + Ok(Self::Params::parse( + params_raw + .map(|s| serde_json::from_str(s.as_ref())) + .transpose()?, + Self::PARAM_NAMES, + calling_convention, + Self::N_REQUIRED_PARAMS, + )?) + } + /// Generate a full `OpenRPC` method definition for this endpoint. fn openrpc<'de>( gen: &mut SchemaGenerator, @@ -214,17 +229,8 @@ pub trait RpcMethodExt: RpcMethod { ); module.register_async_method(Self::NAME, move |params, ctx, _extensions| async move { - let raw = params - .as_str() - .map(serde_json::from_str) - .transpose() + let params = Self::parse_params(params.as_str(), calling_convention) .map_err(|e| Error::invalid_params(e, None))?; - let params = Self::Params::parse( - raw, - Self::PARAM_NAMES, - calling_convention, - Self::N_REQUIRED_PARAMS, - )?; let ok = Self::handle(ctx, params).await?; Result::<_, jsonrpsee::types::ErrorObjectOwned>::Ok(ok.into_lotus_json()) }) diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 5bcc3cae821f..9495f48f5b3c 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -1,6 +1,8 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +mod generate_test_snapshot; + use crate::blocks::{ElectionProof, Ticket, Tipset}; use crate::db::car::ManyCar; use crate::eth::{EthChainId as EthChainIdType, SAFE_EPOCH_DELAY}; @@ -143,6 +145,16 @@ pub enum ApiCommands { #[arg(long)] dump_dir: Option, }, + GenerateTestSnapshot { + #[arg(num_args = 1.., required = true)] + test_dump_files: Vec, + /// Path to the database folder that powers a Forest node + #[arg(long, required = true)] + db: PathBuf, + /// Filecoin network chain + #[arg(long, required = true)] + chain: NetworkChain, + }, DumpTests { #[command(flatten)] create_tests_args: CreateTestsArgs, @@ -217,6 +229,23 @@ impl ApiCommands { .await?; } } + Self::GenerateTestSnapshot { + test_dump_files, + db, + chain, + } => { + std::env::set_var("FOREST_TIPSET_CACHE_DISABLED", "1"); + for test_dump_file in test_dump_files { + let test_dump = serde_json::from_reader(std::fs::File::open(&test_dump_file)?)?; + print!( + "Running RPC test with snapshot {} ...", + test_dump_file.display() + ); + generate_test_snapshot::run_test_with_dump(&test_dump, &db, &chain).await?; + println!(" Success"); + } + } + Self::DumpTests { create_tests_args, path, diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs new file mode 100644 index 000000000000..650f39411dbb --- /dev/null +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -0,0 +1,234 @@ +// Copyright 2019-2024 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::*; +use crate::{ + chain::ChainStore, + chain_sync::{network_context::SyncNetworkContext, SyncConfig, SyncStage}, + daemon::db_util::load_all_forest_cars, + db::{ + db_engine::open_db, parity_db::ParityDb, EthMappingsStore, MemoryDB, SettingsStore, + CAR_DB_DIR_NAME, + }, + genesis::{get_network_name_from_genesis, read_genesis_header}, + libp2p::{NetworkMessage, PeerManager}, + libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite, Block64}, + message_pool::{MessagePool, MpoolRpcProvider}, + networks::ChainConfig, + shim::address::CurrentNetwork, + state_manager::StateManager, + KeyStore, KeyStoreConfig, +}; +use fvm_shared4::address::Network; +use openrpc_types::ParamStructure; +use parking_lot::RwLock; +use rpc::{eth::filter::EthEventHandler, RPCState, RpcMethod as _}; +use tokio::{sync::mpsc, task::JoinSet}; + +pub async fn run_test_with_dump( + test_dump: &TestDump, + db_root: &Path, + chain: &NetworkChain, +) -> anyhow::Result>>> { + if chain.is_testnet() { + CurrentNetwork::set_global(Network::Testnet); + } + let mut run = false; + let chain_config = Arc::new(ChainConfig::calibnet()); + let db = load_db(db_root)?; + let (ctx, _, _) = ctx(db.clone(), chain_config).await?; + let params_raw = Some(serde_json::to_string(&test_dump.request.params)?); + + macro_rules! run_test { + ($ty:ty) => { + if test_dump.request.method_name.as_ref() == <$ty>::NAME { + let params = <$ty>::parse_params(params_raw.clone(), ParamStructure::Either)?; + let result = <$ty>::handle(ctx.clone(), params).await?; + assert_eq!( + test_dump.forest_response, + Ok(result.into_lotus_json_value()?) + ); + run = true; + } + }; + } + + crate::for_each_method!(run_test); + + anyhow::ensure!(run, "RPC method not found"); + + Ok(db) +} + +fn load_db(db_root: &Path) -> anyhow::Result>>> { + let db_writer = open_db(db_root.into(), Default::default())?; + let db = ManyCar::new(db_writer); + let forest_car_db_dir = db_root.join(CAR_DB_DIR_NAME); + load_all_forest_cars(&db, &forest_car_db_dir)?; + Ok(Arc::new(ReadOpsTrackingStore::new(db))) +} + +async fn ctx( + db: Arc>>, + chain_config: Arc, +) -> anyhow::Result<( + Arc>>>, + flume::Receiver, + tokio::sync::mpsc::Receiver<()>, +)> { + let (network_send, network_rx) = flume::bounded(5); + let (tipset_send, _) = flume::bounded(5); + let sync_config = Arc::new(SyncConfig::default()); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db).await?; + + let chain_store = Arc::new( + ChainStore::new( + db.clone(), + db.clone(), + db, + chain_config.clone(), + genesis_header.clone(), + ) + .unwrap(), + ); + + let state_manager = + Arc::new(StateManager::new(chain_store.clone(), chain_config, sync_config).unwrap()); + let network_name = get_network_name_from_genesis(&genesis_header, &state_manager)?; + let message_pool = MessagePool::new( + MpoolRpcProvider::new(chain_store.publisher().clone(), state_manager.clone()), + network_name.clone(), + network_send.clone(), + Default::default(), + state_manager.chain_config().clone(), + &mut JoinSet::new(), + )?; + + let peer_manager = Arc::new(PeerManager::default()); + let sync_network_context = + SyncNetworkContext::new(network_send, peer_manager, state_manager.blockstore_owned()); + let (shutdown, shutdown_recv) = mpsc::channel(1); + let rpc_state = Arc::new(RPCState { + state_manager, + keystore: Arc::new(tokio::sync::RwLock::new(KeyStore::new( + KeyStoreConfig::Memory, + )?)), + mpool: Arc::new(message_pool), + bad_blocks: Default::default(), + sync_state: Arc::new(RwLock::new(Default::default())), + eth_event_handler: Arc::new(EthEventHandler::new()), + sync_network_context, + network_name, + start_time: chrono::Utc::now(), + shutdown, + tipset_send, + }); + rpc_state.sync_state.write().set_stage(SyncStage::Idle); + Ok((rpc_state, network_rx, shutdown_recv)) +} + +/// A [`Blockstore`] wrapper that tracks read operations to the inner [`Blockstore`] with an [`MemoryDB`] +pub struct ReadOpsTrackingStore { + inner: T, + pub tracked: Arc, +} + +impl ReadOpsTrackingStore { + pub fn new(inner: T) -> Self { + Self { + inner, + tracked: Arc::new(Default::default()), + } + } + + pub fn inner(&self) -> &T { + &self.inner + } +} + +impl Blockstore for ReadOpsTrackingStore { + fn get(&self, k: &Cid) -> anyhow::Result>> { + let result = self.inner.get(k)?; + if let Some(v) = &result { + self.tracked.put_keyed(k, v.as_slice())?; + } + Ok(result) + } + + fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> { + self.inner.put_keyed(k, block) + } +} + +impl SettingsStore for ReadOpsTrackingStore { + fn read_bin(&self, key: &str) -> anyhow::Result>> { + let result = self.inner.read_bin(key)?; + if let Some(v) = &result { + SettingsStore::write_bin(&self.tracked, key, v.as_slice())?; + } + Ok(result) + } + + fn write_bin(&self, key: &str, value: &[u8]) -> anyhow::Result<()> { + self.inner.write_bin(key, value) + } + + fn exists(&self, key: &str) -> anyhow::Result { + let result = self.inner.read_bin(key)?; + if let Some(v) = &result { + SettingsStore::write_bin(&self.tracked, key, v.as_slice())?; + } + Ok(result.is_some()) + } + + fn setting_keys(&self) -> anyhow::Result> { + // HACKHACK: may need some care + self.inner.setting_keys() + } +} + +impl BitswapStoreRead for ReadOpsTrackingStore { + fn contains(&self, cid: &Cid) -> anyhow::Result { + // HACKHACK: may need some care + self.inner.contains(cid) + } + + fn get(&self, cid: &Cid) -> anyhow::Result>> { + // HACKHACK: may need some care + self.inner.get(cid) + } +} + +impl BitswapStoreReadWrite for ReadOpsTrackingStore { + type Hashes = ::Hashes; + + fn insert(&self, block: &Block64) -> anyhow::Result<()> { + self.inner.insert(block) + } +} + +impl EthMappingsStore for ReadOpsTrackingStore { + fn read_bin(&self, key: &EthHash) -> anyhow::Result>> { + // HACKHACK: may need some care + self.inner.read_bin(key) + } + + fn write_bin(&self, key: &EthHash, value: &[u8]) -> anyhow::Result<()> { + self.inner.write_bin(key, value) + } + + fn exists(&self, key: &EthHash) -> anyhow::Result { + // HACKHACK: may need some care + self.inner.exists(key) + } + + fn get_message_cids(&self) -> anyhow::Result> { + // HACKHACK: may need some care + self.inner.get_message_cids() + } + + fn delete(&self, keys: Vec) -> anyhow::Result<()> { + self.inner.delete(keys) + } +} From 2f72b9bd6d2fe840b6278b0eddff055f0175186a Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Dec 2024 08:15:23 +0800 Subject: [PATCH 03/16] fix build errors --- src/tool/subcommands/api_cmd/test_snapshot.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index 851b9f30b308..2634ea64c44a 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -1,7 +1,6 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use super::filter::EthEventHandler; use crate::{ chain::ChainStore, chain_sync::{network_context::SyncNetworkContext, SyncConfig, SyncStage}, @@ -11,7 +10,9 @@ use crate::{ lotus_json::HasLotusJson, message_pool::{MessagePool, MpoolRpcProvider}, networks::ChainConfig, - rpc::{RPCState, RpcCallSnapshot, RpcMethod as _, RpcMethodExt as _}, + rpc::{ + eth::filter::EthEventHandler, RPCState, RpcCallSnapshot, RpcMethod as _, RpcMethodExt as _, + }, shim::address::{CurrentNetwork, Network}, state_manager::StateManager, KeyStore, KeyStoreConfig, From fd80f988d7bbba3d8831af7692e2f8e37f9be1c4 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Dec 2024 17:11:36 +0800 Subject: [PATCH 04/16] cc --- src/db/memory.rs | 44 +++++------- src/rpc/mod.rs | 9 --- src/tool/subcommands/api_cmd.rs | 67 ++++++++++++++----- .../api_cmd/generate_test_snapshot.rs | 27 ++++---- src/tool/subcommands/api_cmd/test_snapshot.rs | 34 ++++++---- 5 files changed, 100 insertions(+), 81 deletions(-) diff --git a/src/db/memory.rs b/src/db/memory.rs index ae32d2a14163..2f2888aa5e98 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -16,8 +16,8 @@ use std::ops::Deref; #[derive(Debug, Default)] pub struct MemoryDB { - blockchain_db: RwLock, Vec>>, - blockchain_persistent_db: RwLock, Vec>>, + blockchain_db: RwLock>>, + blockchain_persistent_db: RwLock>>, settings_db: RwLock>>, eth_mappings_db: RwLock>>, } @@ -52,9 +52,8 @@ impl MemoryDB { impl GarbageCollectable for MemoryDB { fn get_keys(&self) -> anyhow::Result { let mut set = CidHashSet::new(); - for key in self.blockchain_db.read().keys() { - let cid = Cid::try_from(key.as_slice())?; - set.insert(cid); + for &key in self.blockchain_db.read().keys() { + set.insert(key); } Ok(set) } @@ -63,17 +62,11 @@ impl GarbageCollectable for MemoryDB { let mut db = self.blockchain_db.write(); let mut deleted = 0; db.retain(|key, _| { - let cid = Cid::try_from(key.as_slice()); - match cid { - Ok(cid) => { - let retain = !keys.contains(&cid); - if !retain { - deleted += 1; - } - retain - } - _ => true, + let retain = !keys.contains(key); + if !retain { + deleted += 1; } + retain }); Ok(deleted) } @@ -138,22 +131,15 @@ impl EthMappingsStore for MemoryDB { impl Blockstore for MemoryDB { fn get(&self, k: &Cid) -> anyhow::Result>> { - Ok(self - .blockchain_db + Ok(self.blockchain_db.read().get(k).cloned().or(self + .blockchain_persistent_db .read() - .get(&k.to_bytes()) - .cloned() - .or(self - .blockchain_persistent_db - .read() - .get(&k.to_bytes()) - .cloned())) + .get(k) + .cloned())) } fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> { - self.blockchain_db - .write() - .insert(k.to_bytes(), block.to_vec()); + self.blockchain_db.write().insert(*k, block.to_vec()); Ok(()) } } @@ -162,14 +148,14 @@ impl PersistentStore for MemoryDB { fn put_keyed_persistent(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> { self.blockchain_persistent_db .write() - .insert(k.to_bytes(), block.to_vec()); + .insert(*k, block.to_vec()); Ok(()) } } impl BitswapStoreRead for MemoryDB { fn contains(&self, cid: &Cid) -> anyhow::Result { - Ok(self.blockchain_db.read().contains_key(&cid.to_bytes())) + Ok(self.blockchain_db.read().contains_key(cid)) } fn get(&self, cid: &Cid) -> anyhow::Result>> { diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 4eb3ddf90645..f5244f081bb3 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -20,7 +20,6 @@ mod error; mod reflect; pub mod types; pub use methods::*; -use serde::{Deserialize, Serialize}; /// Protocol or transport-specific error pub use jsonrpsee::core::ClientError; @@ -433,14 +432,6 @@ impl RPCState { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RpcCallSnapshot { - pub name: String, - pub params: Option, - pub response: serde_json::Value, - pub db: String, -} - #[derive(Clone)] struct PerConnection { methods: Methods, diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 57adb538770c..3f8f36b2ea58 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -33,7 +33,7 @@ use crate::shim::{ use crate::tool::offline_server::start_offline_server; use crate::utils::UrlFromMultiAddr; use ahash::HashMap; -use anyhow::{bail, ensure}; +use anyhow::{bail, ensure, Context as _}; use bls_signatures::Serialize as _; use cid::Cid; use clap::{Subcommand, ValueEnum}; @@ -58,6 +58,7 @@ use std::{ time::Duration, }; use tabled::{builder::Builder, settings::Style}; +use test_snapshot::RpcTestSnapshot; use tokio::sync::Semaphore; use tracing::debug; @@ -155,6 +156,9 @@ pub enum ApiCommands { /// Filecoin network chain #[arg(long, required = true)] chain: NetworkChain, + #[arg(long, required = true)] + /// Folder into which test snapshots are dumped + out_dir: PathBuf, }, DumpTests { #[command(flatten)] @@ -238,19 +242,59 @@ impl ApiCommands { test_dump_files, db, chain, + out_dir, } => { std::env::set_var("FOREST_TIPSET_CACHE_DISABLED", "1"); + if !out_dir.is_dir() { + std::fs::create_dir_all(&out_dir)?; + } + let tracking_db = generate_test_snapshot::load_db(&db)?; for test_dump_file in test_dump_files { + let out_path = out_dir + .join(test_dump_file.file_name().context("Infallible")?) + .with_extension("rpcsnap.json"); let test_dump = serde_json::from_reader(std::fs::File::open(&test_dump_file)?)?; - print!( - "Running RPC test with snapshot {} ...", - test_dump_file.display() - ); - generate_test_snapshot::run_test_with_dump(&test_dump, &db, &chain).await?; - println!(" Success"); + print!("Generating RPC snapshot at {} ...", out_path.display()); + match generate_test_snapshot::run_test_with_dump( + &test_dump, + tracking_db.clone(), + &chain, + ) + .await + { + Ok(_) => { + let snapshot = { + let db = tracking_db.tracker.serialize()?; + RpcTestSnapshot { + name: test_dump.request.method_name.to_string(), + params: test_dump.request.params, + response: test_dump.forest_response, + db, + } + }; + + std::fs::write(&out_path, serde_json::to_string_pretty(&snapshot)?)?; + println!(" Succeeded"); + } + Err(e) => { + println!(" Failed: {e}"); + } + }; + } + } + Self::Test { files } => { + for path in files { + print!("Running RPC test with snapshot {} ...", path.display()); + match test_snapshot::run_test_from_snapshot(&path).await { + Ok(_) => { + println!(" Succeeded"); + } + Err(e) => { + println!(" Failed: {e}"); + } + }; } } - Self::DumpTests { create_tests_args, path, @@ -294,13 +338,6 @@ impl ApiCommands { println!(); } } - Self::Test { files } => { - for path in files { - print!("Running RPC test with snapshot {} ...", path.display()); - test_snapshot::run_test_from_snapshot(&path).await?; - println!(" Success"); - } - } } Ok(()) } diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 7b5dba24ca45..90b83b43a0aa 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -27,18 +27,16 @@ use tokio::{sync::mpsc, task::JoinSet}; pub async fn run_test_with_dump( test_dump: &TestDump, - db_root: &Path, + db: Arc>>, chain: &NetworkChain, -) -> anyhow::Result>>> { +) -> anyhow::Result<()> { if chain.is_testnet() { CurrentNetwork::set_global(Network::Testnet); } let mut run = false; let chain_config = Arc::new(ChainConfig::calibnet()); - let db = load_db(db_root)?; - let (ctx, _, _) = ctx(db.clone(), chain_config).await?; + let (ctx, _, _) = ctx(db, chain_config).await?; let params_raw = Some(serde_json::to_string(&test_dump.request.params)?); - macro_rules! run_test { ($ty:ty) => { if test_dump.request.method_name.as_ref() == <$ty>::NAME { @@ -52,15 +50,14 @@ pub async fn run_test_with_dump( } }; } - crate::for_each_method!(run_test); - anyhow::ensure!(run, "RPC method not found"); - - Ok(db) + Ok(()) } -fn load_db(db_root: &Path) -> anyhow::Result>>> { +pub(super) fn load_db( + db_root: &Path, +) -> anyhow::Result>>> { let db_writer = open_db(db_root.into(), Default::default())?; let db = ManyCar::new(db_writer); let forest_car_db_dir = db_root.join(CAR_DB_DIR_NAME); @@ -131,14 +128,14 @@ async fn ctx( /// A [`Blockstore`] wrapper that tracks read operations to the inner [`Blockstore`] with an [`MemoryDB`] pub struct ReadOpsTrackingStore { inner: T, - pub tracked: Arc, + pub tracker: Arc, } impl ReadOpsTrackingStore { pub fn new(inner: T) -> Self { Self { inner, - tracked: Arc::new(Default::default()), + tracker: Arc::new(Default::default()), } } } @@ -147,7 +144,7 @@ impl Blockstore for ReadOpsTrackingStore { fn get(&self, k: &Cid) -> anyhow::Result>> { let result = self.inner.get(k)?; if let Some(v) = &result { - self.tracked.put_keyed(k, v.as_slice())?; + self.tracker.put_keyed(k, v.as_slice())?; } Ok(result) } @@ -161,7 +158,7 @@ impl SettingsStore for ReadOpsTrackingStore { fn read_bin(&self, key: &str) -> anyhow::Result>> { let result = self.inner.read_bin(key)?; if let Some(v) = &result { - SettingsStore::write_bin(&self.tracked, key, v.as_slice())?; + SettingsStore::write_bin(&self.tracker, key, v.as_slice())?; } Ok(result) } @@ -173,7 +170,7 @@ impl SettingsStore for ReadOpsTrackingStore { fn exists(&self, key: &str) -> anyhow::Result { let result = self.inner.read_bin(key)?; if let Some(v) = &result { - SettingsStore::write_bin(&self.tracked, key, v.as_slice())?; + SettingsStore::write_bin(&self.tracker, key, v.as_slice())?; } Ok(result.is_some()) } diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index a4032451318c..c8cd87b3b82a 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -10,19 +10,26 @@ use crate::{ lotus_json::HasLotusJson, message_pool::{MessagePool, MpoolRpcProvider}, networks::ChainConfig, - rpc::{ - eth::filter::EthEventHandler, RPCState, RpcCallSnapshot, RpcMethod as _, RpcMethodExt as _, - }, + rpc::{eth::filter::EthEventHandler, RPCState, RpcMethod as _, RpcMethodExt as _}, shim::address::{CurrentNetwork, Network}, state_manager::StateManager, KeyStore, KeyStoreConfig, }; -use base64::prelude::*; use openrpc_types::ParamStructure; use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; use std::{path::Path, sync::Arc}; use tokio::{sync::mpsc, task::JoinSet}; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RpcTestSnapshot { + pub name: String, + pub params: serde_json::Value, + pub response: Result, + #[serde(with = "crate::lotus_json::base64_standard")] + pub db: Vec, +} + pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { CurrentNetwork::set_global(Network::Testnet); let mut run = false; @@ -32,23 +39,24 @@ pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { } else { snapshot_bytes }; - let snapshot: RpcCallSnapshot = serde_json::from_slice(snapshot_bytes.as_slice())?; - let db_bytes = BASE64_STANDARD.decode(&snapshot.db)?; - let db = Arc::new(MemoryDB::deserialize_from(db_bytes.as_slice())?); + let snapshot: RpcTestSnapshot = serde_json::from_slice(snapshot_bytes.as_slice())?; + let db = Arc::new(MemoryDB::deserialize_from(snapshot.db.as_slice())?); let chain_config = Arc::new(ChainConfig::calibnet()); let (ctx, _, _) = ctx(db, chain_config).await?; - let params_raw = if let Some(params) = &snapshot.params { - Some(serde_json::to_string(params)?) - } else { - None + let params_raw = match serde_json::to_string(&snapshot.params)? { + s if s.is_empty() => None, + s => Some(s), }; macro_rules! run_test { ($ty:ty) => { if snapshot.name.as_str() == <$ty>::NAME { let params = <$ty>::parse_params(params_raw.clone(), ParamStructure::Either)?; - let result = <$ty>::handle(ctx.clone(), params).await?; - assert_eq!(snapshot.response, result.into_lotus_json_value()?); + let result = <$ty>::handle(ctx.clone(), params) + .await + .map_err(|e| e.to_string()) + .and_then(|r| r.into_lotus_json_value().map_err(|e| e.to_string())); + assert_eq!(snapshot.response, result); run = true; } }; From bd33685fc6cc090e69925262add0c3ce557aa703 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Dec 2024 17:19:49 +0800 Subject: [PATCH 05/16] update test cases --- src/tool/subcommands/api_cmd/test_snapshots.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tool/subcommands/api_cmd/test_snapshots.txt b/src/tool/subcommands/api_cmd/test_snapshots.txt index e2da66fd254b..38605336482f 100644 --- a/src/tool/subcommands/api_cmd/test_snapshots.txt +++ b/src/tool/subcommands/api_cmd/test_snapshots.txt @@ -1,2 +1,6 @@ -f3_gettipsetbyepoch_1730952732441851.json.zst -filecoin_statelistactors_1730953255032189.json.zst +filecoin_stategetallallocations_1733735079961566.rpcsnap.json.zst +filecoin_stategetallclaims_1733735080472880.rpcsnap.json.zst +filecoin_stategetallocation_1733735082114977.rpcsnap.json.zst +filecoin_stategetallocationforpendingdeal_1733735082149029.rpcsnap.json.zst +filecoin_stategetallocationidforpendingdeal_1733735082161045.rpcsnap.json.zst +filecoin_stategetallocations_1733735082163772.rpcsnap.json.zst From 0c7e42e85aca7d2f9223edcb530f12babfb46a0d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Dec 2024 17:35:12 +0800 Subject: [PATCH 06/16] track EthMappingsStore --- .../subcommands/api_cmd/generate_test_snapshot.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 90b83b43a0aa..7838923da6fe 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -203,8 +203,11 @@ impl BitswapStoreReadWrite for ReadOpsTrackingStore impl EthMappingsStore for ReadOpsTrackingStore { fn read_bin(&self, key: &EthHash) -> anyhow::Result>> { - // HACKHACK: may need some care - self.inner.read_bin(key) + let result = self.inner.read_bin(key)?; + if let Some(v) = &result { + EthMappingsStore::write_bin(&self.tracker, key, v.as_slice())?; + } + Ok(result) } fn write_bin(&self, key: &EthHash, value: &[u8]) -> anyhow::Result<()> { @@ -212,8 +215,11 @@ impl EthMappingsStore for ReadOpsTrackingStore { } fn exists(&self, key: &EthHash) -> anyhow::Result { - // HACKHACK: may need some care - self.inner.exists(key) + let result = self.inner.read_bin(key)?; + if let Some(v) = &result { + EthMappingsStore::write_bin(&self.tracker, key, v.as_slice())?; + } + Ok(result.is_some()) } fn get_message_cids(&self) -> anyhow::Result> { From 35c273a530d26457edeb32db5e8a4e567c57f7bc Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Dec 2024 17:45:13 +0800 Subject: [PATCH 07/16] for_each_rpc_method --- src/rpc/auth_layer.rs | 2 +- src/rpc/mod.rs | 10 +++++----- src/tool/subcommands/api_cmd/generate_test_snapshot.rs | 2 +- src/tool/subcommands/api_cmd/test_snapshot.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/rpc/auth_layer.rs b/src/rpc/auth_layer.rs index f98b63f0083d..867d3a342908 100644 --- a/src/rpc/auth_layer.rs +++ b/src/rpc/auth_layer.rs @@ -32,7 +32,7 @@ static METHOD_NAME2REQUIRED_PERMISSION: Lazy> = Lazy:: } }; } - super::for_each_method!(insert); + super::for_each_rpc_method!(insert); access.insert(chain::CHAIN_NOTIFY, Permission::Read); access.insert(CANCEL_METHOD_NAME, Permission::Read); diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index f5244f081bb3..574d2c65e9d1 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -32,7 +32,7 @@ pub use jsonrpsee::core::ClientError; /// /// All methods should be entered here. #[macro_export] -macro_rules! for_each_method { +macro_rules! for_each_rpc_method { ($callback:path) => { // auth vertical $callback!($crate::rpc::auth::AuthNew); @@ -265,7 +265,7 @@ macro_rules! for_each_method { $callback!($crate::rpc::misc::GetActorEventsRaw); }; } -pub(crate) use for_each_method; +pub(crate) use for_each_rpc_method; use tower_http::compression::CompressionLayer; use tower_http::sensitive_headers::SetSensitiveRequestHeadersLayer; @@ -292,7 +292,7 @@ pub mod prelude { }; } - for_each_method!(export); + for_each_rpc_method!(export); } /// All the methods live in their own folder @@ -578,7 +578,7 @@ where <$ty>::register_alias(&mut module).unwrap(); }; } - for_each_method!(register); + for_each_rpc_method!(register); module } @@ -630,7 +630,7 @@ pub fn openrpc(path: ApiPath, include: Option<&[&str]>) -> openrpc_types::OpenRP } }; } - for_each_method!(callback); + for_each_rpc_method!(callback); openrpc_types::OpenRPC { methods, components: Some(openrpc_types::Components { diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 7838923da6fe..1f9339fbef63 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -50,7 +50,7 @@ pub async fn run_test_with_dump( } }; } - crate::for_each_method!(run_test); + crate::for_each_rpc_method!(run_test); anyhow::ensure!(run, "RPC method not found"); Ok(()) } diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index c8cd87b3b82a..c4896a9f21e2 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -62,7 +62,7 @@ pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { }; } - crate::for_each_method!(run_test); + crate::for_each_rpc_method!(run_test); assert!(run, "RPC method not found"); From 18089ebc2684894464c5e68109368389d1970525 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 16 Dec 2024 16:12:08 +0800 Subject: [PATCH 08/16] switch to foreset CAR db --- src/db/memory.rs | 54 ++++++++++--------- src/tool/subcommands/api_cmd.rs | 4 +- .../api_cmd/generate_test_snapshot.rs | 38 +++++++++++-- src/tool/subcommands/api_cmd/test_snapshot.rs | 30 ++++++----- 4 files changed, 85 insertions(+), 41 deletions(-) diff --git a/src/db/memory.rs b/src/db/memory.rs index 2f2888aa5e98..b8d345123d99 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -1,18 +1,20 @@ // Copyright 2019-2024 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use super::{EthMappingsStore, SettingsStore}; +use super::{EthMappingsStore, SettingsStore, SettingsStoreExt}; +use crate::blocks::TipsetKey; use crate::cid_collections::CidHashSet; use crate::db::{GarbageCollectable, PersistentStore}; use crate::libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use crate::rpc::eth::types::EthHash; +use crate::utils::db::car_stream::CarBlock; use crate::utils::multihash::prelude::*; use ahash::HashMap; +use anyhow::Context as _; use cid::Cid; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; use parking_lot::RwLock; -use std::ops::Deref; #[derive(Debug, Default)] pub struct MemoryDB { @@ -23,29 +25,31 @@ pub struct MemoryDB { } impl MemoryDB { - pub fn serialize(&self) -> anyhow::Result> { - let blockchain_db = self.blockchain_db.read(); - let blockchain_persistent_db = self.blockchain_persistent_db.read(); - let settings_db = self.settings_db.read(); - let eth_mappings_db = self.eth_mappings_db.read(); - let tuple = ( - blockchain_db.deref(), - blockchain_persistent_db.deref(), - settings_db.deref(), - eth_mappings_db.deref(), - ); - Ok(fvm_ipld_encoding::to_vec(&tuple)?) - } - - pub fn deserialize_from(bytes: &[u8]) -> anyhow::Result { - let (blockchain_db, blockchain_persistent_db, settings_db, eth_mappings_db) = - fvm_ipld_encoding::from_slice(bytes)?; - Ok(Self { - blockchain_db: RwLock::new(blockchain_db), - blockchain_persistent_db: RwLock::new(blockchain_persistent_db), - settings_db: RwLock::new(settings_db), - eth_mappings_db: RwLock::new(eth_mappings_db), - }) + pub async fn export_forest_car( + &self, + writer: &mut W, + ) -> anyhow::Result<()> { + let roots = + SettingsStoreExt::read_obj::(self, crate::db::setting_keys::HEAD_KEY)? + .context("chain head is not tracked and cannot be exported")? + .into_cids(); + let blocks = { + let blockchain_db = self.blockchain_db.read(); + let blockchain_persistent_db = self.blockchain_persistent_db.read(); + blockchain_db + .iter() + .chain(blockchain_persistent_db.iter()) + .map(|(&cid, data)| { + anyhow::Ok(CarBlock { + cid, + data: data.clone(), + }) + }) + .collect_vec() + }; + let frames = + crate::db::car::forest::Encoder::compress_stream_default(futures::stream::iter(blocks)); + crate::db::car::forest::Encoder::write(writer, roots, frames).await } } diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 3f8f36b2ea58..b4f02a395834 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -264,7 +264,9 @@ impl ApiCommands { { Ok(_) => { let snapshot = { - let db = tracking_db.tracker.serialize()?; + tracking_db.ensure_chain_head_is_tracked()?; + let mut db = vec![]; + tracking_db.export_forest_car(&mut db).await?; RpcTestSnapshot { name: test_dump.request.method_name.to_string(), params: test_dump.request.params, diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 1f9339fbef63..ed8ac0223f32 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -3,12 +3,13 @@ use super::*; use crate::{ + blocks::{CachingBlockHeader, TipsetKey}, chain::ChainStore, chain_sync::{network_context::SyncNetworkContext, SyncConfig, SyncStage}, daemon::db_util::load_all_forest_cars, db::{ db_engine::open_db, parity_db::ParityDb, EthMappingsStore, MemoryDB, SettingsStore, - CAR_DB_DIR_NAME, + SettingsStoreExt, CAR_DB_DIR_NAME, }, genesis::{get_network_name_from_genesis, read_genesis_header}, libp2p::{NetworkMessage, PeerManager}, @@ -128,16 +129,47 @@ async fn ctx( /// A [`Blockstore`] wrapper that tracks read operations to the inner [`Blockstore`] with an [`MemoryDB`] pub struct ReadOpsTrackingStore { inner: T, - pub tracker: Arc, + tracker: Arc, } -impl ReadOpsTrackingStore { +impl ReadOpsTrackingStore +where + T: Blockstore + SettingsStore, +{ pub fn new(inner: T) -> Self { Self { inner, tracker: Arc::new(Default::default()), } } + + pub fn ensure_chain_head_is_tracked(&self) -> anyhow::Result<()> { + if !self.is_chain_head_tracked()? { + let _ = + SettingsStoreExt::read_obj::(self, crate::db::setting_keys::HEAD_KEY)? + .context("HEAD_KEY not found")? + .into_cids() + .into_iter() + .map(|key| CachingBlockHeader::load(self, key)) + .collect::>>>()? + .map(Tipset::new) + .transpose()? + .context("failed to load tipset")?; + } + + Ok(()) + } + + fn is_chain_head_tracked(&self) -> anyhow::Result { + SettingsStore::exists(&self.tracker, crate::db::setting_keys::HEAD_KEY) + } + + pub async fn export_forest_car( + &self, + writer: &mut W, + ) -> anyhow::Result<()> { + self.tracker.export_forest_car(writer).await + } } impl Blockstore for ReadOpsTrackingStore { diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index c4896a9f21e2..c0866e16028c 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -4,7 +4,10 @@ use crate::{ chain::ChainStore, chain_sync::{network_context::SyncNetworkContext, SyncConfig, SyncStage}, - db::MemoryDB, + db::{ + car::{AnyCar, ManyCar}, + MemoryDB, + }, genesis::{get_network_name_from_genesis, read_genesis_header}, libp2p::{NetworkMessage, PeerManager}, lotus_json::HasLotusJson, @@ -39,24 +42,29 @@ pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { } else { snapshot_bytes }; - let snapshot: RpcTestSnapshot = serde_json::from_slice(snapshot_bytes.as_slice())?; - let db = Arc::new(MemoryDB::deserialize_from(snapshot.db.as_slice())?); + let RpcTestSnapshot { + name: method_name, + params, + db: db_bytes, + response: expected_response, + } = serde_json::from_slice(snapshot_bytes.as_slice())?; + let db = Arc::new(ManyCar::new(MemoryDB::default()).with_read_only(AnyCar::new(db_bytes)?)?); let chain_config = Arc::new(ChainConfig::calibnet()); let (ctx, _, _) = ctx(db, chain_config).await?; - let params_raw = match serde_json::to_string(&snapshot.params)? { + let params_raw = match serde_json::to_string(¶ms)? { s if s.is_empty() => None, s => Some(s), }; macro_rules! run_test { ($ty:ty) => { - if snapshot.name.as_str() == <$ty>::NAME { + if method_name.as_str() == <$ty>::NAME { let params = <$ty>::parse_params(params_raw.clone(), ParamStructure::Either)?; let result = <$ty>::handle(ctx.clone(), params) .await .map_err(|e| e.to_string()) .and_then(|r| r.into_lotus_json_value().map_err(|e| e.to_string())); - assert_eq!(snapshot.response, result); + assert_eq!(expected_response, result); run = true; } }; @@ -70,10 +78,10 @@ pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { } async fn ctx( - db: Arc, + db: Arc>, chain_config: Arc, ) -> anyhow::Result<( - Arc>, + Arc>>, flume::Receiver, tokio::sync::mpsc::Receiver<()>, )> { @@ -143,10 +151,8 @@ mod tests { .split("\n") .filter_map(|n| { Url::parse( - format!( - "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/rpc_test/{n}" - ) - .as_str(), + format!("https://forest-snapshots.fra1.digitaloceanspaces.com/rpc_test/{n}") + .as_str(), ) .ok() }) From 1f2f7ff61c66d381cc9bdeba60fd3dc71c910ff3 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 17 Dec 2024 08:43:49 +0800 Subject: [PATCH 09/16] set_heaviest_tipset --- src/tool/subcommands/api_cmd/test_snapshot.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index c0866e16028c..6b6f19cbad36 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -90,18 +90,17 @@ async fn ctx( let sync_config = Arc::new(SyncConfig::default()); let genesis_header = read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db).await?; - let chain_store = Arc::new( ChainStore::new( db.clone(), db.clone(), - db, + db.clone(), chain_config.clone(), genesis_header.clone(), ) .unwrap(), ); - + chain_store.set_heaviest_tipset(db.heaviest_tipset()?.into())?; let state_manager = Arc::new(StateManager::new(chain_store.clone(), chain_config, sync_config).unwrap()); let network_name = get_network_name_from_genesis(&genesis_header, &state_manager)?; From 376ab9ec0fef68b8a95c53464e8ca5d21b10d4b7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 7 Jan 2025 23:03:12 +0800 Subject: [PATCH 10/16] fix copyright headers --- src/tool/subcommands/api_cmd/generate_test_snapshot.rs | 2 +- src/tool/subcommands/api_cmd/test_snapshot.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index ed8ac0223f32..ad7efc62f1f1 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2024 ChainSafe Systems +// Copyright 2019-2025 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT use super::*; diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index 6b6f19cbad36..d3816efbb5fd 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2024 ChainSafe Systems +// Copyright 2019-2025 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT use crate::{ From f20cb1da604c416b62a48048b26a3c5b94301e24 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 16:49:52 +0800 Subject: [PATCH 11/16] fix hackhack --- .../api_cmd/generate_test_snapshot.rs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index ad7efc62f1f1..06cece0a7f9a 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -208,20 +208,25 @@ impl SettingsStore for ReadOpsTrackingStore { } fn setting_keys(&self) -> anyhow::Result> { - // HACKHACK: may need some care self.inner.setting_keys() } } impl BitswapStoreRead for ReadOpsTrackingStore { fn contains(&self, cid: &Cid) -> anyhow::Result { - // HACKHACK: may need some care - self.inner.contains(cid) + let result = self.inner.get(cid)?; + if let Some(v) = &result { + Blockstore::put_keyed(&self.tracker, cid, v.as_slice())?; + } + Ok(result.is_some()) } fn get(&self, cid: &Cid) -> anyhow::Result>> { - // HACKHACK: may need some care - self.inner.get(cid) + let result = self.inner.get(cid)?; + if let Some(v) = &result { + Blockstore::put_keyed(&self.tracker, cid, v.as_slice())?; + } + Ok(result) } } @@ -235,11 +240,7 @@ impl BitswapStoreReadWrite for ReadOpsTrackingStore impl EthMappingsStore for ReadOpsTrackingStore { fn read_bin(&self, key: &EthHash) -> anyhow::Result>> { - let result = self.inner.read_bin(key)?; - if let Some(v) = &result { - EthMappingsStore::write_bin(&self.tracker, key, v.as_slice())?; - } - Ok(result) + self.inner.read_bin(key) } fn write_bin(&self, key: &EthHash, value: &[u8]) -> anyhow::Result<()> { @@ -247,15 +248,10 @@ impl EthMappingsStore for ReadOpsTrackingStore { } fn exists(&self, key: &EthHash) -> anyhow::Result { - let result = self.inner.read_bin(key)?; - if let Some(v) = &result { - EthMappingsStore::write_bin(&self.tracker, key, v.as_slice())?; - } - Ok(result.is_some()) + self.inner.exists(key) } fn get_message_cids(&self) -> anyhow::Result> { - // HACKHACK: may need some care self.inner.get_message_cids() } From 8ccbdd37864aae49f51c502025a6b069df3d1b40 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 17:12:53 +0800 Subject: [PATCH 12/16] construct ChainConfig from NetworkChain --- src/tool/subcommands/api_cmd.rs | 1 + src/tool/subcommands/api_cmd/generate_test_snapshot.rs | 2 +- src/tool/subcommands/api_cmd/test_snapshot.rs | 10 +++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 8127650b2f71..46e89d790f38 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -269,6 +269,7 @@ impl ApiCommands { let mut db = vec![]; tracking_db.export_forest_car(&mut db).await?; RpcTestSnapshot { + chain: chain.clone(), name: test_dump.request.method_name.to_string(), params: test_dump.request.params, response: test_dump.forest_response, diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 06cece0a7f9a..b74ed40d3f99 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -35,7 +35,7 @@ pub async fn run_test_with_dump( CurrentNetwork::set_global(Network::Testnet); } let mut run = false; - let chain_config = Arc::new(ChainConfig::calibnet()); + let chain_config = Arc::new(ChainConfig::from_chain(chain)); let (ctx, _, _) = ctx(db, chain_config).await?; let params_raw = Some(serde_json::to_string(&test_dump.request.params)?); macro_rules! run_test { diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index d3816efbb5fd..3f0175e506cd 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -12,7 +12,7 @@ use crate::{ libp2p::{NetworkMessage, PeerManager}, lotus_json::HasLotusJson, message_pool::{MessagePool, MpoolRpcProvider}, - networks::ChainConfig, + networks::{ChainConfig, NetworkChain}, rpc::{eth::filter::EthEventHandler, RPCState, RpcMethod as _, RpcMethodExt as _}, shim::address::{CurrentNetwork, Network}, state_manager::StateManager, @@ -26,6 +26,7 @@ use tokio::{sync::mpsc, task::JoinSet}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RpcTestSnapshot { + pub chain: NetworkChain, pub name: String, pub params: serde_json::Value, pub response: Result, @@ -34,7 +35,6 @@ pub struct RpcTestSnapshot { } pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { - CurrentNetwork::set_global(Network::Testnet); let mut run = false; let snapshot_bytes = std::fs::read(path)?; let snapshot_bytes = if let Ok(bytes) = zstd::decode_all(snapshot_bytes.as_slice()) { @@ -43,13 +43,17 @@ pub async fn run_test_from_snapshot(path: &Path) -> anyhow::Result<()> { snapshot_bytes }; let RpcTestSnapshot { + chain, name: method_name, params, db: db_bytes, response: expected_response, } = serde_json::from_slice(snapshot_bytes.as_slice())?; + if chain.is_testnet() { + CurrentNetwork::set_global(Network::Testnet); + } let db = Arc::new(ManyCar::new(MemoryDB::default()).with_read_only(AnyCar::new(db_bytes)?)?); - let chain_config = Arc::new(ChainConfig::calibnet()); + let chain_config = Arc::new(ChainConfig::from_chain(&chain)); let (ctx, _, _) = ctx(db, chain_config).await?; let params_raw = match serde_json::to_string(¶ms)? { s if s.is_empty() => None, From 0e21f0f1dcdce01cfab7df215c0eff280ac10d96 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 17:16:24 +0800 Subject: [PATCH 13/16] code docs --- src/tool/subcommands/api_cmd.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 46e89d790f38..f62c479dac0c 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -149,6 +149,7 @@ pub enum ApiCommands { dump_dir: Option, }, GenerateTestSnapshot { + /// Path to test dumps that are generated by `forest-tool api dump-tests` command #[arg(num_args = 1.., required = true)] test_dump_files: Vec, /// Path to the database folder that powers a Forest node @@ -171,6 +172,7 @@ pub enum ApiCommands { include_ignored: bool, }, Test { + /// Path to test snapshots that are generated by `forest-tool api generate-test-snapshot` command #[arg(num_args = 1.., required = true)] files: Vec, }, From b98c0ae07e03a7ae6ae1ac4d9454afbff63390ac Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 17:36:48 +0800 Subject: [PATCH 14/16] test_export_forest_car --- src/db/memory.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/db/memory.rs b/src/db/memory.rs index 632bb07b7f97..6bb4bdb3d358 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -174,3 +174,43 @@ impl BitswapStoreReadWrite for MemoryDB { self.put_keyed(block.cid(), block.data()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{car::ForestCar, setting_keys::HEAD_KEY}; + use fvm_ipld_encoding::DAG_CBOR; + use multihash_codetable::Code::Blake2b256; + use nunny::vec as nonempty; + + #[tokio::test] + async fn test_export_forest_car() { + let db = MemoryDB::default(); + let record1 = b"non-persistent"; + let key1 = Cid::new_v1(DAG_CBOR, Blake2b256.digest(record1.as_slice())); + db.put_keyed(&key1, record1.as_slice()).unwrap(); + + let record2 = b"persistent"; + let key2 = Cid::new_v1(DAG_CBOR, Blake2b256.digest(record2.as_slice())); + db.put_keyed_persistent(&key2, record2.as_slice()).unwrap(); + + let mut car_db_bytes = vec![]; + assert!(db + .export_forest_car(&mut car_db_bytes) + .await + .unwrap_err() + .to_string() + .contains("chain head is not tracked and cannot be exported")); + + db.write_obj(HEAD_KEY, &TipsetKey::from(nonempty![key1])) + .unwrap(); + + car_db_bytes.clear(); + db.export_forest_car(&mut car_db_bytes).await.unwrap(); + + let car = ForestCar::new(car_db_bytes).unwrap(); + assert_eq!(car.roots(), &nonempty![key1]); + assert!(car.has(&key1).unwrap()); + assert!(car.has(&key2).unwrap()); + } +} From e3a7ebf4e7bdcf9d84288592638146d5c389deb7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 18:42:16 +0800 Subject: [PATCH 15/16] local cache for rpc test snapshots --- Cargo.lock | 7 +++ Cargo.toml | 1 + src/tool/subcommands/api_cmd/test_snapshot.rs | 53 +++++++++++++++---- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c68eb0085c62..55313130a1ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,6 +3097,7 @@ dependencies = [ "libp2p-swarm-test", "libsecp256k1", "lru", + "md5", "memmap2 0.9.5", "memory-stats", "mimalloc", @@ -5939,6 +5940,12 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.4" diff --git a/Cargo.toml b/Cargo.toml index f428f4d9de83..764b53c1f91a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,6 +238,7 @@ glob = "0.3" http-range-header = "0.4" insta = { version = "1", features = ["yaml"] } libp2p-swarm-test = { workspace = true } +md5 = "0.7" num-bigint = { version = "0.4", features = ['quickcheck'] } petgraph = "0.7" predicates = "3" diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index 3f0175e506cd..aeed098b48cf 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -143,7 +143,8 @@ async fn ctx( #[cfg(test)] mod tests { use super::*; - use crate::daemon::db_util::download_to; + use crate::{daemon::db_util::download_to, utils::net::global_http_client}; + use directories::ProjectDirs; use itertools::Itertools as _; use url::Url; @@ -158,19 +159,49 @@ mod tests { .as_str(), ) .ok() + .map(|url| (n, url)) }) .collect_vec(); - for url in urls { - print!("Testing {url} ..."); - let tmp_dir = tempfile::tempdir().unwrap(); - let tmp = tempfile::NamedTempFile::new_in(&tmp_dir) - .unwrap() - .into_temp_path(); - println!("start downloading at {}", tmp.display()); - download_to(&url, &tmp).await.unwrap(); - println!("done downloading {}", tmp.display()); - run_test_from_snapshot(&tmp).await.unwrap(); + let project_dir = ProjectDirs::from("com", "ChainSafe", "Forest").unwrap(); + let cache_dir = project_dir.cache_dir().join("test").join("rpc-snapshots"); + for (filename, url) in urls { + let cache_file_path = cache_dir.join(filename); + let is_file_cached = match get_file_md5_etag(&cache_file_path) { + Some(file_etag) => { + let url_etag = get_digital_ocean_space_url_etag(url.clone()).await.unwrap(); + if Some(&file_etag) == url_etag.as_ref() { + true + } else { + println!( + "etag mismatch, file: {filename}, local: {file_etag}, remote: {}", + url_etag.unwrap_or_default() + ); + false + } + } + None => false, + }; + if !is_file_cached { + println!("Downloading from {url} to {}", cache_file_path.display()); + download_to(&url, &cache_file_path).await.unwrap(); + } + print!("Testing {filename} ..."); + run_test_from_snapshot(&cache_file_path).await.unwrap(); println!(" succeeded."); } } + + async fn get_digital_ocean_space_url_etag(url: Url) -> anyhow::Result> { + let response = global_http_client().head(url).send().await?; + Ok(response + .headers() + .get("etag") + .and_then(|v| v.to_str().ok().map(|v| v.replace('"', "").to_string()))) + } + + fn get_file_md5_etag(path: &Path) -> Option { + std::fs::read(path) + .ok() + .map(|bytes| format!("{:x}", md5::compute(bytes.as_slice()))) + } } From c469b880befa461cf285cfdd40d93de148a22c53 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 20:19:36 +0800 Subject: [PATCH 16/16] switch to a more popular md5 crate --- Cargo.lock | 12 ++++++++---- Cargo.toml | 2 +- src/tool/subcommands/api_cmd/test_snapshot.rs | 10 +++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55313130a1ed..6b177446b43f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,7 +3097,7 @@ dependencies = [ "libp2p-swarm-test", "libsecp256k1", "lru", - "md5", + "md-5", "memmap2 0.9.5", "memory-stats", "mimalloc", @@ -5941,10 +5941,14 @@ dependencies = [ ] [[package]] -name = "md5" -version = "0.7.0" +name = "md-5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] [[package]] name = "memchr" diff --git a/Cargo.toml b/Cargo.toml index 764b53c1f91a..0fed7d2bca8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,7 +238,7 @@ glob = "0.3" http-range-header = "0.4" insta = { version = "1", features = ["yaml"] } libp2p-swarm-test = { workspace = true } -md5 = "0.7" +md5 = { package = "md-5", version = "0.10" } num-bigint = { version = "0.4", features = ['quickcheck'] } petgraph = "0.7" predicates = "3" diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index aeed098b48cf..2a0f955063aa 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -146,6 +146,7 @@ mod tests { use crate::{daemon::db_util::download_to, utils::net::global_http_client}; use directories::ProjectDirs; use itertools::Itertools as _; + use md5::{Digest as _, Md5}; use url::Url; #[tokio::test] @@ -200,8 +201,11 @@ mod tests { } fn get_file_md5_etag(path: &Path) -> Option { - std::fs::read(path) - .ok() - .map(|bytes| format!("{:x}", md5::compute(bytes.as_slice()))) + std::fs::read(path).ok().map(|bytes| { + let mut hasher = Md5::new(); + hasher.update(bytes.as_slice()); + let hash = hasher.finalize(); + format!("{hash:x}") + }) } }