Merge pull request #45 from bitfinity-network/itsyaasir/feat/add-reth…

…-redundancy

feat: add reth redundancy

Cargo.toml

@@ -544,3 +544,4 @@ ic-certification = "2.3"
 hex = "0.4"
 lightspeed_scheduler = { version = "0.59.0", features = ["tracing"] }
 rlp = "0.5"
+

bin/reth/src/commands/bitfinity_import.rs

@@ -10,6 +10,7 @@ use reth_db::DatabaseEnv;
 
 use reth_consensus::Consensus;
 use reth_db::database::Database;
+use reth_downloaders::bitfinity_evm_client::RpcClientConfig;
 use reth_downloaders::{
     bitfinity_evm_client::{BitfinityEvmClient, CertificateCheckSettings},
     bodies::bodies::BodiesDownloaderBuilder,
@@ -133,9 +134,16 @@ impl BitfinityImportCommand {
 
         debug!(target: "reth::cli - BitfinityImportCommand", "Starting block: {}", start_block);
 
+        let rpc_config = RpcClientConfig {
+            primary_url: self.bitfinity.rpc_url.clone(),
+            backup_url: self.bitfinity.backup_rpc_url.clone(),
+            max_retries: self.bitfinity.max_retries,
+            retry_delay: Duration::from_secs(self.bitfinity.retry_delay_secs),
+        };
+
         let remote_client = Arc::new(
             BitfinityEvmClient::from_rpc_url(
-                &self.bitfinity.rpc_url,
+                rpc_config,
                 start_block,
                 self.bitfinity.end_block,
                 self.bitfinity.batch_size,

bin/reth/src/commands/node/mod.rs

@@ -65,7 +65,7 @@ pub struct NodeCommand<Ext: clap::Args + fmt::Debug = NoArgs> {
     /// Bitfinity Args
     #[command(flatten)]
     pub bitfinity: crate::args::BitfinityImportArgs,
-    
+
     /// All datadir related arguments
     #[command(flatten)]
     pub datadir: DatadirArgs,
@@ -155,7 +155,7 @@ impl<Ext: clap::Args + fmt::Debug> NodeCommand<Ext> {
         } = self;
 
         let chain = {
-            let mut chain = reth_downloaders::bitfinity_evm_client::BitfinityEvmClient::fetch_chain_spec(bitfinity.rpc_url.to_owned()).await?;
+            let mut chain = reth_downloaders::bitfinity_evm_client::BitfinityEvmClient::fetch_chain_spec_with_fallback(bitfinity.rpc_url.to_owned(), bitfinity.backup_rpc_url.clone()).await?;
             if let Some(send_raw_transaction_rpc_url) = &bitfinity.send_raw_transaction_rpc_url {
                 chain.bitfinity_evm_url = Some(send_raw_transaction_rpc_url.to_owned());
             }
@@ -197,7 +197,7 @@ impl<Ext: clap::Args + fmt::Debug> NodeCommand<Ext> {
         let builder = NodeBuilder::new(node_config)
             .with_database(database)
             .with_launch_context(ctx.task_executor);
-        
+
         launcher(builder, ext).await
     }
 }

bin/reth/tests/commands/bitfinity_import_it.rs

@@ -14,7 +14,7 @@ async fn bitfinity_test_should_import_data_from_evm() {
     let _log = init_logs();
     let evm_datasource_url = DEFAULT_EVM_DATASOURCE_URL;
     let (_temp_dir, mut import_data) =
-        bitfinity_import_config_data(evm_datasource_url, None).await.unwrap();
+        bitfinity_import_config_data(evm_datasource_url, None, None).await.unwrap();
 
     let end_block = 100;
     import_data.bitfinity_args.end_block = Some(end_block);
@@ -46,7 +46,7 @@ async fn bitfinity_test_should_import_with_small_batch_size() {
     let _log = init_logs();
     let evm_datasource_url = DEFAULT_EVM_DATASOURCE_URL;
     let (_temp_dir, mut import_data) =
-        bitfinity_import_config_data(evm_datasource_url, None).await.unwrap();
+        bitfinity_import_config_data(evm_datasource_url, None, None).await.unwrap();
 
     let end_block = 101;
     import_data.bitfinity_args.end_block = Some(end_block);
@@ -72,14 +72,13 @@ async fn bitfinity_test_should_import_with_small_batch_size() {
     }
 }
 
-
 #[tokio::test]
 async fn bitfinity_test_finalized_and_safe_query_params_works() {
     // Arrange
     let _log = init_logs();
     let evm_datasource_url = DEFAULT_EVM_DATASOURCE_URL;
     let (_temp_dir, mut import_data) =
-        bitfinity_import_config_data(evm_datasource_url, None).await.unwrap();
+        bitfinity_import_config_data(evm_datasource_url, None, None).await.unwrap();
 
     let end_block = 100;
     import_data.bitfinity_args.end_block = Some(end_block);
@@ -88,10 +87,52 @@ async fn bitfinity_test_finalized_and_safe_query_params_works() {
     // Act
     import_blocks(import_data.clone(), Duration::from_secs(20), true).await;
 
-    let latest_block = import_data.blockchain_db.block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Finalized).unwrap().unwrap();
+    let latest_block = import_data
+        .blockchain_db
+        .block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Finalized)
+        .unwrap()
+        .unwrap();
     assert_eq!(end_block, latest_block.number);
 
-    let safe_block = import_data.blockchain_db.block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Safe).unwrap().unwrap();
+    let safe_block = import_data
+        .blockchain_db
+        .block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Safe)
+        .unwrap()
+        .unwrap();
     assert_eq!(end_block, safe_block.number);
+}
+
+#[tokio::test]
+async fn bitfinity_test_should_import_data_from_evm_with_backup_rpc_url() {
+    // Arrange
+    let _log = init_logs();
+    let evm_datasource_url = "https://fake_rpc_url";
+    let backup_rpc_url = DEFAULT_EVM_DATASOURCE_URL;
+
+    let (_temp_dir, mut import_data) =
+        bitfinity_import_config_data(evm_datasource_url, Some(backup_rpc_url.to_owned()), None)
+            .await
+            .unwrap();
+
+    let end_block = 100;
+    import_data.bitfinity_args.end_block = Some(end_block);
+    import_data.bitfinity_args.batch_size = (end_block as usize) * 10;
+
+    // Act
+    import_blocks(import_data.clone(), Duration::from_secs(20), false).await;
 
-}
+    // Assert
+    {
+        let provider = import_data.provider_factory.provider().unwrap();
+        assert_eq!(end_block, provider.last_block_number().unwrap());
+
+        // create evm client
+        let evm_rpc_client = EthJsonRpcClient::new(ReqwestClient::new(backup_rpc_url.to_string()));
+
+        let remote_block = evm_rpc_client.get_block_by_number(end_block.into()).await.unwrap();
+        let local_block = provider.block_by_number(end_block).unwrap().unwrap();
+
+        assert_eq!(remote_block.hash.unwrap().0, local_block.header.hash_slow().0);
+        assert_eq!(remote_block.state_root.0, local_block.state_root.0);
+    }
+}

bin/reth/tests/commands/bitfinity_reset_evm_state_it.rs

@@ -39,7 +39,7 @@ async fn bitfinity_manual_test_should_reset_evm_state() {
     let end_block = 30_000;
     let data_dir = Some(format!("../../target/reth_{end_block}").into());
     let (_temp_dir, mut import_data) =
-        bitfinity_import_config_data(evm_datasource_url, data_dir).await.unwrap();
+        bitfinity_import_config_data(evm_datasource_url, None, data_dir).await.unwrap();
 
     let fetch_block_timeout_secs = std::cmp::max(20, end_block / 100);
 
@@ -99,7 +99,7 @@ async fn bitfinity_test_reset_should_extract_all_accounts_data() {
     let end_block = 30_000;
     let data_dir = Some(format!("../../target/reth_{end_block}").into());
     let (_temp_dir, mut import_data) =
-        bitfinity_import_config_data(evm_datasource_url, data_dir).await.unwrap();
+        bitfinity_import_config_data(evm_datasource_url, None, data_dir).await.unwrap();
 
     let fetch_block_timeout_secs = std::cmp::max(20, end_block / 100);
 

bin/reth/tests/commands/utils.rs

@@ -99,10 +99,16 @@ pub async fn import_blocks(
 /// If a `data_dir` is provided, it will be used, otherwise a temporary directory will be created.
 pub async fn bitfinity_import_config_data(
     evm_datasource_url: &str,
+    backup_evm_datasource_url: Option<String>,
     data_dir: Option<PathBuf>,
 ) -> eyre::Result<(TempDir, ImportData)> {
-    let chain =
-        Arc::new(BitfinityEvmClient::fetch_chain_spec(evm_datasource_url.to_owned()).await?);
+    let chain = Arc::new(
+        BitfinityEvmClient::fetch_chain_spec_with_fallback(
+            evm_datasource_url.to_owned(),
+            backup_evm_datasource_url.clone(),
+        )
+        .await?,
+    );
 
     let temp_dir = TempDir::new().unwrap();
 
@@ -144,9 +150,15 @@ pub async fn bitfinity_import_config_data(
         max_fetch_blocks: 10000,
         evmc_principal: LOCAL_EVM_CANISTER_ID.to_string(),
         ic_root_key: IC_MAINNET_KEY.to_string(),
+        backup_rpc_url: backup_evm_datasource_url,
+        max_retries: 3,
+        retry_delay_secs: 3,
     };
 
-    Ok((temp_dir, ImportData { data_dir, database, chain, provider_factory, blockchain_db, bitfinity_args }))
+    Ok((
+        temp_dir,
+        ImportData { data_dir, database, chain, provider_factory, blockchain_db, bitfinity_args },
+    ))
 }
 
 /// Waits until the block is imported.

crates/net/downloaders/Cargo.toml

@@ -61,6 +61,7 @@ ic-certification.workspace = true
 reth-chainspec.workspace = true
 rlp.workspace = true
 serde_json.workspace = true
+backon.workspace = true
 
 [dev-dependencies]
 reth-chainspec.workspace = true