diff --git a/crates/electrum/Cargo.toml b/crates/electrum/Cargo.toml index 4205f22946..7bdfeb0e02 100644 --- a/crates/electrum/Cargo.toml +++ b/crates/electrum/Cargo.toml @@ -12,7 +12,7 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bdk_chain = { path = "../chain", version = "0.13.0", default-features = false } +bdk_chain = { path = "../chain", version = "0.13.0" } electrum-client = { version = "0.19" } #rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] } diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 7fd81edcc7..ecd29d460f 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -2,6 +2,7 @@ use bdk_chain::{ bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, collections::{BTreeMap, HashMap, HashSet}, local_chain::CheckPoint, + spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult, TxCache}, tx_graph::TxGraph, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; @@ -12,9 +13,6 @@ use std::sync::Arc; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; -/// Type that maintains a cache of [`Arc`]-wrapped transactions. -pub type TxCache = HashMap>; - /// Combination of chain and transactions updates from electrum /// /// We have to update the chain and the txids at the same time since we anchor the txids to @@ -97,12 +95,10 @@ pub trait ElectrumExt { /// single batch request. fn full_scan( &self, - tx_cache: &mut TxCache, - prev_tip: CheckPoint, - keychain_spks: BTreeMap>, + request: FullScanRequest, stop_gap: usize, batch_size: usize, - ) -> Result<(ElectrumUpdate, BTreeMap), Error>; + ) -> Result, Error>; /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified /// and returns updates for [`bdk_chain`] data structures. @@ -123,28 +119,22 @@ pub trait ElectrumExt { /// [`full_scan`]: ElectrumExt::full_scan fn sync( &self, - tx_cache: &mut TxCache, - prev_tip: CheckPoint, - misc_spks: impl IntoIterator, - txids: impl IntoIterator, - outpoints: impl IntoIterator, + request: SyncRequest, batch_size: usize, - ) -> Result; + ) -> Result, Error>; } impl ElectrumExt for E { fn full_scan( &self, - tx_cache: &mut TxCache, - prev_tip: CheckPoint, - keychain_spks: BTreeMap>, + mut request: FullScanRequest, stop_gap: usize, batch_size: usize, - ) -> Result<(ElectrumUpdate, BTreeMap), Error> { - let mut request_spks = keychain_spks - .into_iter() - .map(|(k, s)| (k, s.into_iter())) - .collect::>(); + ) -> Result, Error> { + let mut request_spks = request.spks_by_keychain; + // .into_iter() + // .map(|(k, s)| (k, s.into_iter())) + // .collect::>(); // We keep track of already-scanned spks just in case a reorg happens and we need to do a // rescan. We need to keep track of this as iterators in `keychain_spks` are "unbounded" so @@ -154,8 +144,8 @@ impl ElectrumExt for E { // * val: (script_pubkey, has_tx_history). let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new(); - let (electrum_update, keychain_update) = loop { - let (tip, _) = construct_update_tip(self, prev_tip.clone())?; + let update = loop { + let (tip, _) = construct_update_tip(self, request.chain_tip.clone())?; let mut graph_update = TxGraph::::default(); let cps = tip .iter() @@ -168,7 +158,7 @@ impl ElectrumExt for E { scanned_spks.append(&mut populate_with_spks( self, &cps, - tx_cache, + &mut request.tx_cache, &mut graph_update, &mut scanned_spks .iter() @@ -182,7 +172,7 @@ impl ElectrumExt for E { populate_with_spks( self, &cps, - tx_cache, + &mut request.tx_cache, &mut graph_update, keychain_spks, stop_gap, @@ -213,41 +203,29 @@ impl ElectrumExt for E { }) .collect::>(); - break ( - ElectrumUpdate { - chain_update, - graph_update, - }, - keychain_update, - ); + break FullScanResult { + graph_update, + chain_update, + last_active_indices: keychain_update, + }; }; - Ok((electrum_update, keychain_update)) + Ok(update) } fn sync( &self, - tx_cache: &mut TxCache, - prev_tip: CheckPoint, - misc_spks: impl IntoIterator, - txids: impl IntoIterator, - outpoints: impl IntoIterator, + request: SyncRequest, batch_size: usize, - ) -> Result { - let spk_iter = misc_spks - .into_iter() - .enumerate() - .map(|(i, spk)| (i as u32, spk)); - - let (electrum_update, _) = self.full_scan( - tx_cache, - prev_tip.clone(), - [((), spk_iter)].into(), - usize::MAX, - batch_size, - )?; - - let (tip, _) = construct_update_tip(self, prev_tip)?; + ) -> Result, Error> { + let mut tx_cache = request.tx_cache.clone(); + + let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone()) + .cache_txs(request.tx_cache) + .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk))); + let full_scan_res = self.full_scan(full_scan_req, usize::MAX, batch_size)?; + + let (tip, _) = construct_update_tip(self, request.chain_tip)?; let cps = tip .iter() .take(10) @@ -255,13 +233,88 @@ impl ElectrumExt for E { .collect::>(); let mut tx_graph = TxGraph::::default(); - populate_with_txids(self, &cps, tx_cache, &mut tx_graph, txids)?; - populate_with_outpoints(self, &cps, &mut tx_graph, outpoints)?; + populate_with_txids(self, &cps, &mut tx_cache, &mut tx_graph, request.txids)?; + populate_with_outpoints(self, &cps, &mut tx_cache, &mut tx_graph, request.outpoints)?; - Ok(electrum_update) + Ok(SyncResult { + chain_update: full_scan_res.chain_update, + graph_update: full_scan_res.graph_update, + }) } } +/// Trait that extends [`SyncResult`] and [`FullScanResult`] functionality. +/// +/// Currently, only a single method exists that converts the update [`TxGraph`] to have an anchor +/// type of [`ConfirmationTimeHeightAnchor`]. +pub trait ElectrumResultExt { + /// New result type with a [`TxGraph`] that contains the [`ConfirmationTimeHeightAnchor`]. + type NewResult; + + /// Convert result type to have an update [`TxGraph`] that contains the [`ConfirmationTimeHeightAnchor`] . + fn try_into_confirmation_time_result( + self, + client: &impl ElectrumApi, + ) -> Result; +} + +impl ElectrumResultExt for FullScanResult { + type NewResult = FullScanResult; + + fn try_into_confirmation_time_result( + self, + client: &impl ElectrumApi, + ) -> Result { + Ok(FullScanResult:: { + graph_update: try_into_confirmation_time_result(self.graph_update, client)?, + chain_update: self.chain_update, + last_active_indices: self.last_active_indices, + }) + } +} + +impl ElectrumResultExt for SyncResult { + type NewResult = SyncResult; + + fn try_into_confirmation_time_result( + self, + client: &impl ElectrumApi, + ) -> Result { + Ok(SyncResult { + graph_update: try_into_confirmation_time_result(self.graph_update, client)?, + chain_update: self.chain_update, + }) + } +} + +fn try_into_confirmation_time_result( + graph_update: TxGraph, + client: &impl ElectrumApi, +) -> Result, Error> { + let relevant_heights = graph_update + .all_anchors() + .iter() + .map(|(a, _)| a.confirmation_height) + .collect::>(); + + let height_to_time = relevant_heights + .clone() + .into_iter() + .zip( + client + .batch_block_header(relevant_heights)? + .into_iter() + .map(|bh| bh.time as u64), + ) + .collect::>(); + + Ok(graph_update.map_anchors(|a| ConfirmationTimeHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + confirmation_time: height_to_time[&a.confirmation_height], + })) +} + /// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`. fn construct_update_tip( client: &impl ElectrumApi, @@ -380,6 +433,7 @@ fn determine_tx_anchor( fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, + tx_cache: &mut TxCache, tx_graph: &mut TxGraph, outpoints: impl IntoIterator, ) -> Result<(), Error> { @@ -415,9 +469,9 @@ fn populate_with_outpoints( let res_tx = match tx_graph.get_tx(res.tx_hash) { Some(tx) => tx, None => { - let res_tx = client.transaction_get(&res.tx_hash)?; - let _ = tx_graph.insert_tx(res_tx); - tx_graph.get_tx(res.tx_hash).expect("just inserted") + let res_tx = fetch_tx(client, tx_cache, res.tx_hash)?; + let _ = tx_graph.insert_tx(Arc::clone(&res_tx)); + res_tx } }; has_spending = res_tx diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index 4a47daea2e..40d639c5da 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -1,19 +1,20 @@ use std::{ - collections::BTreeMap, io::{self, Write}, sync::Mutex, }; use bdk_chain::{ - bitcoin::{constants::genesis_block, Address, Network, OutPoint, Txid}, + bitcoin::{constants::genesis_block, Address, Network, Txid}, + collections::BTreeSet, indexed_tx_graph::{self, IndexedTxGraph}, keychain, local_chain::{self, LocalChain}, + spk_client::{FullScanRequest, SyncRequest}, Append, ConfirmationHeightAnchor, }; use bdk_electrum::{ electrum_client::{self, Client, ElectrumApi}, - ElectrumExt, ElectrumUpdate, TxCache, + ElectrumExt, }; use example_cli::{ anyhow::{self, Context}, @@ -146,50 +147,48 @@ fn main() -> anyhow::Result<()> { }; let client = electrum_cmd.electrum_args().client(args.network)?; - let mut tx_cache = TxCache::new(); - let response = match electrum_cmd.clone() { + let (chain_update, mut graph_update, keychain_update) = match electrum_cmd.clone() { ElectrumCommands::Scan { stop_gap, scan_options, .. } => { - let (keychain_spks, tip) = { + let request = { let graph = &*graph.lock().unwrap(); let chain = &*chain.lock().unwrap(); - let keychain_spks = graph - .index - .all_unbounded_spk_iters() - .into_iter() - .map(|(keychain, iter)| { - let mut first = true; - let spk_iter = iter.inspect(move |(i, _)| { - if first { - eprint!("\nscanning {}: ", keychain); - first = false; + FullScanRequest::from_chain_tip(chain.tip()) + .cache_graph_txs(graph.graph()) + .set_spks_for_keychain( + Keychain::External, + graph.index.unbounded_spk_iter(&Keychain::External), + ) + .set_spks_for_keychain( + Keychain::Internal, + graph.index.unbounded_spk_iter(&Keychain::Internal), + ) + .inspect_spks_for_all_keychains({ + let mut once = BTreeSet::new(); + move |k, spk_i, _| { + if once.insert(k) { + eprint!("\nScanning {}: ", k); + } else { + eprint!("{} ", spk_i); } - - eprint!("{} ", i); let _ = io::stdout().flush(); - }); - (keychain, spk_iter) + } }) - .collect::>(); - - let tip = chain.tip(); - (keychain_spks, tip) }; - client - .full_scan::<_>( - &mut tx_cache, - tip, - keychain_spks, - stop_gap, - scan_options.batch_size, - ) - .context("scanning the blockchain")? + let res = client + .full_scan::<_>(request, stop_gap, scan_options.batch_size) + .context("scanning the blockchain")?; + ( + res.chain_update, + res.graph_update, + Some(res.last_active_indices), + ) } ElectrumCommands::Sync { mut unused_spks, @@ -202,7 +201,6 @@ fn main() -> anyhow::Result<()> { // Get a short lock on the tracker to get the spks we're interested in let graph = graph.lock().unwrap(); let chain = chain.lock().unwrap(); - let chain_tip = chain.tip().block_id(); if !(all_spks || unused_spks || utxos || unconfirmed) { unused_spks = true; @@ -212,18 +210,20 @@ fn main() -> anyhow::Result<()> { unused_spks = false; } - let mut spks: Box> = - Box::new(core::iter::empty()); + let chain_tip = chain.tip(); + let mut request = + SyncRequest::from_chain_tip(chain_tip.clone()).cache_graph_txs(graph.graph()); + if all_spks { let all_spks = graph .index .revealed_spks(..) .map(|(k, i, spk)| (k.to_owned(), i, spk.to_owned())) .collect::>(); - spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| { - eprintln!("scanning {}:{}", k, i); + request = request.chain_spks(all_spks.into_iter().map(|(k, spk_i, spk)| { + eprintln!("scanning {}: {}", k, spk_i); spk - }))); + })); } if unused_spks { let unused_spks = graph @@ -231,82 +231,61 @@ fn main() -> anyhow::Result<()> { .unused_spks() .map(|(k, i, spk)| (k, i, spk.to_owned())) .collect::>(); - spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| { - eprintln!( - "Checking if address {} {}:{} has been used", - Address::from_script(&spk, args.network).unwrap(), - k, - i, - ); - spk - }))); + request = + request.chain_spks(unused_spks.into_iter().map(move |(k, spk_i, spk)| { + eprintln!( + "Checking if address {} {}:{} has been used", + Address::from_script(&spk, args.network).unwrap(), + k, + spk_i, + ); + spk + })); } - let mut outpoints: Box> = Box::new(core::iter::empty()); - if utxos { let init_outpoints = graph.index.outpoints().iter().cloned(); let utxos = graph .graph() - .filter_chain_unspents(&*chain, chain_tip, init_outpoints) + .filter_chain_unspents(&*chain, chain_tip.block_id(), init_outpoints) .map(|(_, utxo)| utxo) .collect::>(); - - outpoints = Box::new( - utxos - .into_iter() - .inspect(|utxo| { - eprintln!( - "Checking if outpoint {} (value: {}) has been spent", - utxo.outpoint, utxo.txout.value - ); - }) - .map(|utxo| utxo.outpoint), - ); + request = request.chain_outpoints(utxos.into_iter().map(|utxo| { + eprintln!( + "Checking if outpoint {} (value: {}) has been spent", + utxo.outpoint, utxo.txout.value + ); + utxo.outpoint + })); }; - let mut txids: Box> = Box::new(core::iter::empty()); - if unconfirmed { let unconfirmed_txids = graph .graph() - .list_chain_txs(&*chain, chain_tip) + .list_chain_txs(&*chain, chain_tip.block_id()) .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed()) .map(|canonical_tx| canonical_tx.tx_node.txid) .collect::>(); - txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| { - eprintln!("Checking if {} is confirmed yet", txid); - })); + request = request.chain_txids( + unconfirmed_txids + .into_iter() + .inspect(|txid| eprintln!("Checking if {} is confirmed yet", txid)), + ); } - let electrum_update = client - .sync( - &mut tx_cache, - chain.tip(), - spks, - txids, - outpoints, - scan_options.batch_size, - ) + let res = client + .sync(request, scan_options.batch_size) .context("scanning the blockchain")?; // drop lock on graph and chain drop((graph, chain)); - (electrum_update, BTreeMap::new()) + (res.chain_update, res.graph_update, None) } }; - let ( - ElectrumUpdate { - chain_update, - mut graph_update, - }, - keychain_update, - ) = response; - let now = std::time::UNIX_EPOCH .elapsed() .expect("must get time") @@ -317,26 +296,17 @@ fn main() -> anyhow::Result<()> { let mut chain = chain.lock().unwrap(); let mut graph = graph.lock().unwrap(); - let chain = chain.apply_update(chain_update)?; - - let indexed_tx_graph = { - let mut changeset = - indexed_tx_graph::ChangeSet::::default(); - let (_, indexer) = graph.index.reveal_to_target_multi(&keychain_update); - changeset.append(indexed_tx_graph::ChangeSet { - indexer, - ..Default::default() - }); - changeset.append(graph.apply_update(graph_update.map_anchors(|a| { - ConfirmationHeightAnchor { - anchor_block: a.anchor_block, - confirmation_height: a.confirmation_height, - } - }))); - changeset - }; - - (chain, indexed_tx_graph) + let chain_changeset = chain.apply_update(chain_update)?; + + let mut indexed_tx_graph_changeset = + indexed_tx_graph::ChangeSet::::default(); + if let Some(keychain_update) = keychain_update { + let (_, keychain_changeset) = graph.index.reveal_to_target_multi(&keychain_update); + indexed_tx_graph_changeset.append(keychain_changeset.into()); + } + indexed_tx_graph_changeset.append(graph.apply_update(graph_update)); + + (chain_changeset, indexed_tx_graph_changeset) }; let mut db = db.lock().unwrap(); diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index 848dbe578e..b4330148f1 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -3,14 +3,13 @@ const SEND_AMOUNT: u64 = 5000; const STOP_GAP: usize = 50; const BATCH_SIZE: usize = 5; -use std::io::Write; use std::str::FromStr; use bdk::bitcoin::Address; -use bdk::wallet::Update; +use bdk::chain::collections::HashSet; use bdk::{bitcoin::Network, Wallet}; use bdk::{KeychainKind, SignOptions}; -use bdk_electrum::TxCache; +use bdk_electrum::ElectrumResultExt; use bdk_electrum::{ electrum_client::{self, ElectrumApi}, ElectrumExt, @@ -38,40 +37,25 @@ fn main() -> Result<(), anyhow::Error> { print!("Syncing..."); let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?; - let mut tx_cache = TxCache::new(); - - let prev_tip = wallet.latest_checkpoint(); - let keychain_spks = wallet - .all_unbounded_spk_iters() - .into_iter() - .map(|(k, k_spks)| { - let mut once = Some(()); - let mut stdout = std::io::stdout(); - let k_spks = k_spks - .inspect(move |(spk_i, _)| match once.take() { - Some(_) => print!("\nScanning keychain [{:?}]", k), - None => print!(" {:<3}", spk_i), - }) - .inspect(move |_| stdout.flush().expect("must flush")); - (k, k_spks) - }) - .collect(); - - let (update, keychain_update) = - client.full_scan(&mut tx_cache, prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?; - let mut update = update.into_confirmation_time_update(&client)?; - println!(); + let request = wallet.start_full_scan().inspect_spks_for_all_keychains({ + let mut once = HashSet::::new(); + move |k, spk_i, _| match once.insert(k) { + true => print!("\nScanning keychain [{:?}]", k), + false => print!(" {:<3}", spk_i), + } + }); + + let mut update = client + .full_scan(request, STOP_GAP, BATCH_SIZE)? + .try_into_confirmation_time_result(&client)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); let _ = update.graph_update.update_last_seen_unconfirmed(now); - let wallet_update = Update { - last_active_indices: keychain_update, - graph: update.graph_update, - chain: Some(update.chain_update), - }; - wallet.apply_update(wallet_update)?; + println!(); + + wallet.apply_update(update)?; wallet.commit()?; let balance = wallet.get_balance();