Merge pull request #21 from ChainSafe/ec2/batchrunner

Ability to use the Sync algorithm from the zcash_client_backend crate

Cargo.toml

@@ -18,6 +18,7 @@ path = "examples/simple-sync.rs"
 name = "message-board-sync"
 path = "examples/message-board-sync.rs"
 
+
 [profile.release]
 # Tell `rustc` to optimize for small code size.
 opt-level = 3
@@ -35,6 +36,8 @@ native = ["dep:tokio", "tonic/channel", "tonic/gzip", "tonic/tls-webpki-roots"]
 sqlite-db = ["dep:zcash_client_sqlite"]
 console_error_panic_hook = ["dep:console_error_panic_hook"]
 
+sync2 = []
+
 [dependencies]
 ## Web dependencies
 wasm-bindgen = "0.2.84"
@@ -48,12 +51,12 @@ console_error_panic_hook = { version = "0.1.7", optional = true }
 tonic-web-wasm-client = "0.6.0"
 
 ## Zcash dependencies
-zcash_keys = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b", features = ["transparent-inputs", "orchard", "sapling", "unstable"] }
-zcash_client_backend = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b", default-features = false, features = ["lightwalletd-tonic", "wasm-bindgen"] }
-zcash_client_memory = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b", features = ["orchard"] }
-zcash_primitives = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b" }
-zcash_address = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b" }
-zcash_proofs = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b", default-features = false, features = ["bundled-prover"] }
+zcash_keys = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b", features = ["transparent-inputs", "orchard", "sapling", "unstable"] }
+zcash_client_backend = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b", default-features = false, features = ["sync", "lightwalletd-tonic", "wasm-bindgen"] }
+zcash_client_memory = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b", features = ["orchard"] }
+zcash_primitives = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b" }
+zcash_address = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b" }
+zcash_proofs = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b", default-features = false, features = ["bundled-prover"] }
 
 ## gRPC Web dependencies
 prost = { version = "0.12", default-features = false }
@@ -63,7 +66,7 @@ tonic = { version = "0.12", default-features = false, features = [
 
 # Used in Native tests
 tokio = { version = "1.0", features = ["rt", "macros", "rt-multi-thread"], optional = true }
-zcash_client_sqlite = { git = "https://github.com/ChainSafe/librustzcash", rev = "c97a3f6e60446523fafa63cdf77b6c2584ac9f3b", default-features = false, features = ["unstable", "orchard"], optional = true }
+zcash_client_sqlite = { git = "https://github.com/ChainSafe/librustzcash", rev = "0fdd2fbb992a6f84eba45f488ee74a75d08d449b", default-features = false, features = ["unstable", "orchard"], optional = true }
 
 getrandom = { version = "0.2", features = ["js"] }
 thiserror = "1.0.63"
@@ -85,3 +88,16 @@ tempfile = "3.12"
 
 [patch.crates-io]
 zip32 = { git = "https://github.com/zcash/zip32.git", branch = "diversifier_index_ord" }
+# TODO: Remove these once the PRs are merged
+shardtree = { git = "https://github.com/ec2/incrementalmerkletree.git", rev = "16eff253ad2575d48feec04f7387e6507a7dd698" }
+incrementalmerkletree = { git = "https://github.com/ec2/incrementalmerkletree.git", rev = "16eff253ad2575d48feec04f7387e6507a7dd698" }
+
+#[patch.'https://github.com/chainsafe/librustzcash']
+#zcash_address = { path = "../librustzcash/components/zcash_address" }
+#zcash_client_backend = { path = "../librustzcash/zcash_client_backend" }
+##zcash_client_sqlite = { path = "../librustzcash/zcash_client_sqlite" }
+#zcash_client_memory = { path = "../librustzcash/zcash_client_memory" }
+#zcash_keys = { path = "../librustzcash/zcash_keys" }
+#zcash_primitives = { path = "../librustzcash/zcash_primitives" }
+#zcash_proofs = { path = "../librustzcash/zcash_proofs" }
+#zcash_protocol = { path = "../librustzcash/components/zcash_protocol" }

examples/message-board-sync.rs

@@ -63,12 +63,20 @@ async fn main() {
     let id = w.import_ufvk(&ufvk, Some(2477329)).await.unwrap();
     tracing::info!("Created account with id: {}", id);
 
-    tracing::info!("Syncing wallet");
-    w.sync(|scanned_to, tip| {
-        println!("Scanned: {}/{}", scanned_to, tip);
-    })
-    .await
-    .unwrap();
+    #[cfg(not(feature = "sync2"))]
+    {
+        tracing::info!("Syncing wallet with our sync impl");
+        w.sync(|scanned_to, tip| {
+            println!("Scanned: {}/{}", scanned_to, tip);
+        })
+        .await
+        .unwrap();
+    }
+    #[cfg(feature = "sync2")]
+    {
+        tracing::info!("Syncing wallet with sync2");
+        w.sync2().await.unwrap();
+    }
 
     tracing::info!("Syncing complete :)");
 

examples/simple-sync.rs

@@ -59,12 +59,20 @@ async fn main() {
     let id = w.create_account(SEED, HD_INDEX, BIRTHDAY).await.unwrap();
     tracing::info!("Created account with id: {}", id);
 
-    tracing::info!("Syncing wallet");
-    w.sync(|scanned_to, tip| {
-        println!("Scanned: {}/{}", scanned_to, tip);
-    })
-    .await
-    .unwrap();
+    #[cfg(not(feature = "sync2"))]
+    {
+        tracing::info!("Syncing wallet with our sync impl");
+        w.sync(|scanned_to, tip| {
+            println!("Scanned: {}/{}", scanned_to, tip);
+        })
+        .await
+        .unwrap();
+    }
+    #[cfg(feature = "sync2")]
+    {
+        tracing::info!("Syncing wallet with sync2");
+        w.sync2().await.unwrap();
+    }
 
     tracing::info!("Syncing complete :)");
 

justfile

@@ -4,29 +4,26 @@ default:
 build:
     wasm-pack build -t web  --release --out-dir ./packages/webz-core -Z --no-default-features --features="wasm" build-std="panic_abort,std"
 
-## Wasm Tests
-test-web:
-    WASM_BINDGEN_TEST_TIMEOUT=99999 wasm-pack test --release --firefox --no-default-features --features="wasm" -Z build-std="panic_abort,std"
-
-test-message-board-web:
-    WASM_BINDGEN_TEST_TIMEOUT=99999 wasm-pack test --release --firefox --no-default-features --features="wasm" -Z build-std="panic_abort,std" --test message-board-sync
-
-test-simple-web:
-    WASM_BINDGEN_TEST_TIMEOUT=99999 wasm-pack test --release --firefox --no-default-features --features="wasm" -Z build-std="panic_abort,std" --test simple-sync-and-send
+# All Wasm Tests
+test-web *features:
+    WASM_BINDGEN_TEST_TIMEOUT=99999 wasm-pack test --release --firefox --no-default-features --features "wasm {{features}}" -Z build-std="panic_abort,std"
 
+# sync message board in the web: addigional args: sync2
+test-message-board-web *features:
+    WASM_BINDGEN_TEST_TIMEOUT=99999 wasm-pack test --release --firefox --no-default-features --features "wasm {{features}}" -Z build-std="panic_abort,std" --test message-board-sync
 
-## Native Examples
-example-simple:
-    cargo run -r --example simple-sync
+# simple example in the web: additional args: sync2
+test-simple-web *features:
+    WASM_BINDGEN_TEST_TIMEOUT=99999 wasm-pack test --release --firefox --no-default-features --features "wasm {{features}}" -Z build-std="panic_abort,std" --test simple-sync-and-send
 
-example-simple-sqlite:
-    cargo run -r --example simple-sync --features="sqlite-db"
 
-example-message-board:
-   cargo run -r --example message-board-sync
+# simple example: additional args: sync2, sqlite-db
+example-simple *features:
+   RUST_LOG="info,zcash_client_backend::sync=debug" cargo run -r --example simple-sync --features "native {{features}}"
 
-example-message-board-sqlite:
-  cargo run -r --example message-board-sync --features="sqlite-db"
+# sync the message board: additional args: sync2, sqlite-db
+example-message-board *features:
+  RUST_LOG=info,zcash_client_backend::sync=debug cargo run -r --example message-board-sync --features "native {{features}}"
 
 check:
     cargo check 

src/bindgen/wallet.rs

@@ -6,6 +6,7 @@ use wasm_bindgen::prelude::*;
 use tonic_web_wasm_client::Client;
 
 use zcash_address::ZcashAddress;
+use zcash_client_backend::proto::service::compact_tx_streamer_client::CompactTxStreamerClient;
 use zcash_client_memory::MemoryWalletDb;
 use zcash_keys::keys::UnifiedFullViewingKey;
 use zcash_primitives::consensus::{self, BlockHeight};
@@ -45,6 +46,14 @@ impl WebWallet {
             _ => Err(Error::InvalidNetwork(network.to_string())),
         }
     }
+
+    pub fn client(&mut self) -> &mut CompactTxStreamerClient<tonic_web_wasm_client::Client> {
+        self.inner.client()
+    }
+
+    pub fn inner_mut(&mut self) -> &mut MemoryWallet<tonic_web_wasm_client::Client> {
+        &mut self.inner
+    }
 }
 
 #[wasm_bindgen]
@@ -121,6 +130,11 @@ impl WebWallet {
         Ok(())
     }
 
+    /// Synchronize the wallet with the blockchain up to the tip using zcash_client_backend's algo
+    pub async fn sync2(&mut self) -> Result<(), Error> {
+        self.inner.sync2().await
+    }
+
     pub fn get_wallet_summary(&self) -> Result<Option<WalletSummary>, Error> {
         Ok(self.inner.get_wallet_summary()?.map(Into::into))
     }

src/error.rs

@@ -1,6 +1,7 @@
 // Copyright 2024 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use std::fmt::Display;
 use wasm_bindgen::JsValue;
 
 #[derive(thiserror::Error, Debug)]
@@ -47,6 +48,12 @@ pub enum Error {
     #[error("Failed to parse key: {0}")]
     KeyParseError(String),
 
+    // TODO: The error type from librustzcash backend is generic. Handle that later.
+    // Perhaps we make our error struct generic as well
+    // See: zcash_client_backend::sync::Error
+    #[error("Syncing Error: {0}")]
+    SyncError(String),
+
     #[cfg(feature = "sqlite-db")]
     #[error("Sqlite error: {0}")]
     SqliteError(#[from] zcash_client_sqlite::error::SqliteClientError),
@@ -79,3 +86,14 @@ impl From<zcash_client_backend::scanning::ScanError> for Error {
         Self::ScanError(e)
     }
 }
+
+impl<A, B, C> From<zcash_client_backend::sync::Error<A, B, C>> for Error
+where
+    A: Display,
+    B: Display,
+    C: Display,
+{
+    fn from(e: zcash_client_backend::sync::Error<A, B, C>) -> Self {
+        Self::SyncError(e.to_string())
+    }
+}

src/wallet.rs

@@ -31,7 +31,7 @@ use zcash_client_backend::scanning::{scan_block, Nullifiers, ScanningKeys};
 use zcash_client_backend::wallet::OvkPolicy;
 use zcash_client_backend::zip321::{Payment, TransactionRequest};
 use zcash_client_backend::ShieldedProtocol;
-use zcash_client_memory::MemoryWalletDb;
+use zcash_client_memory::{MemBlockCache, MemoryWalletDb};
 use zcash_keys::keys::{UnifiedFullViewingKey, UnifiedSpendingKey};
 use zcash_primitives::consensus::{self, BlockHeight, Network};
 use zcash_primitives::transaction::components::amount::NonNegativeAmount;
@@ -40,6 +40,7 @@ use zcash_primitives::transaction::TxId;
 use zcash_proofs::prover::LocalTxProver;
 
 use zcash_client_backend::proposal::Proposal;
+use zcash_client_backend::sync::run;
 
 const BATCH_SIZE: u32 = 10000;
 
@@ -85,8 +86,11 @@ where
     NoteRef: Copy + Eq + Ord + Debug,
     Error: From<<W as WalletRead>::Error>,
 
+    <W as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
+    <W as WalletCommitmentTrees>::Error: std::error::Error + Send + Sync + 'static,
+
     // GRPC connection Trait Bounds
-    T: GrpcService<tonic::body::BoxBody>,
+    T: GrpcService<tonic::body::BoxBody> + Clone,
     T::Error: Into<StdError>,
     T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
     <T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
@@ -106,6 +110,14 @@ where
         })
     }
 
+    pub fn db_mut(&mut self) -> &mut W {
+        &mut self.db
+    }
+
+    pub fn client(&mut self) -> &mut CompactTxStreamerClient<T> {
+        &mut self.client
+    }
+
     /// Add a new account to the wallet
     ///
     /// # Arguments
@@ -188,6 +200,22 @@ where
         })?)
     }
 
+    pub async fn sync2(&mut self) -> Result<(), Error> {
+        let mut client = self.client().clone();
+        // TODO: This should be held in the Wallet struct so we can download in parallel
+        let db_cache = MemBlockCache::new();
+
+        run(
+            &mut client,
+            &self.network.clone(),
+            &db_cache,
+            self.db_mut(),
+            BATCH_SIZE,
+        )
+        .await
+        .map_err(Into::into)
+    }
+
     /// Synchronize the wallet with the blockchain up to the tip
     /// The passed callback will be called for every batch of blocks processed with the current progress
     pub async fn sync(&mut self, callback: impl Fn(BlockHeight, BlockHeight)) -> Result<(), Error> {

tests/message-board-sync.rs

@@ -33,13 +33,21 @@ async fn test_message_board() {
     let id = w.import_ufvk(&ufvk_str, Some(2477329)).await.unwrap();
     tracing::info!("Created account with id: {}", id);
 
-    tracing::info!("Syncing wallet");
-    w.sync(&js_sys::Function::new_with_args(
-        "scanned_to, tip",
-        "console.log('Scanned: ', scanned_to, '/', tip)",
-    ))
-    .await
-    .unwrap();
+    #[cfg(not(feature = "sync2"))]
+    {
+        tracing::info!("Syncing wallet with our sync impl");
+        w.sync(&js_sys::Function::new_with_args(
+            "scanned_to, tip",
+            "console.log('Scanned: ', scanned_to, '/', tip)",
+        ))
+        .await
+        .unwrap();
+    }
+    #[cfg(feature = "sync2")]
+    {
+        tracing::info!("Syncing wallet with sync2");
+        w.sync2().await.unwrap();
+    }
     tracing::info!("Syncing complete :)");
 
     let summary = w.get_wallet_summary().unwrap();

tests/simple-sync-and-send.rs

@@ -25,13 +25,20 @@ async fn test_get_and_scan_range() {
     let id = w.create_account(SEED, HD_INDEX, BIRTHDAY).await.unwrap();
     tracing::info!("Created account with id: {}", id);
 
-    tracing::info!("Syncing wallet");
-    w.sync(&js_sys::Function::new_with_args(
-        "scanned_to, tip",
-        "console.log('Scanned: ', scanned_to, '/', tip)",
-    ))
-    .await
-    .unwrap();
+    #[cfg(not(feature = "sync2"))]
+    {
+        w.sync(&js_sys::Function::new_with_args(
+            "scanned_to, tip",
+            "console.log('Scanned: ', scanned_to, '/', tip)",
+        ))
+        .await
+        .unwrap();
+    }
+    #[cfg(feature = "sync2")]
+    {
+        tracing::info!("Syncing wallet with sync2");
+        w.sync2().await.unwrap();
+    }
     tracing::info!("Syncing complete :)");
 
     let summary = w.get_wallet_summary().unwrap();