From 2fae4bb24433346b904d6077d2d0c3b230321e68 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 22 Jun 2024 13:51:02 +0100 Subject: [PATCH 01/56] ZIP 320 implementation. Co-authored-by: Kris Nuttycombe Co-authored-by: Jack Grigg Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 35 +- zcash_client_backend/proto/proposal.proto | 4 + zcash_client_backend/src/data_api.rs | 88 ++- zcash_client_backend/src/data_api/wallet.rs | 554 ++++++++++-------- .../src/data_api/wallet/input_selection.rs | 206 ++++++- zcash_client_backend/src/fees.rs | 2 + zcash_client_backend/src/fees/common.rs | 52 +- zcash_client_backend/src/fees/fixed.rs | 17 + zcash_client_backend/src/fees/standard.rs | 14 + zcash_client_backend/src/fees/zip317.rs | 48 +- zcash_client_backend/src/proposal.rs | 4 +- zcash_client_backend/src/proto/proposal.rs | 4 + zcash_client_backend/src/wallet.rs | 72 ++- zcash_client_sqlite/CHANGELOG.md | 21 + zcash_client_sqlite/src/error.rs | 19 +- zcash_client_sqlite/src/lib.rs | 86 ++- zcash_client_sqlite/src/testing/pool.rs | 286 ++++----- zcash_client_sqlite/src/wallet.rs | 29 +- zcash_client_sqlite/src/wallet/db.rs | 37 ++ zcash_client_sqlite/src/wallet/init.rs | 10 + .../src/wallet/init/migrations.rs | 4 + .../init/migrations/ephemeral_addresses.rs | 57 ++ .../init/migrations/receiving_key_scopes.rs | 2 +- zcash_client_sqlite/src/wallet/transparent.rs | 427 ++++++++++++-- zcash_primitives/CHANGELOG.md | 3 + zcash_primitives/src/legacy/keys.rs | 31 +- zcash_primitives/src/transaction/builder.rs | 4 +- 27 files changed, 1622 insertions(+), 494 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index b722f206f5..0e6fc3e339 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -6,6 +6,23 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Notable changes +`zcash_client_backend` now supports TEX (transparent-source-only) addresses as specified +in ZIP 320. Sending to one or more TEX addresses will automatically create a multi-step +proposal that uses two transactions. This is intended to be used in conjunction with +`zcash_client_sqlite` 0.11 or later. + +In order to take advantage of this support, client wallets will need to be able to send +multiple transactions created from `zcash_client_backend::data_api::wallet::create_proposed_transactions`. +This API was added in `zcash_client_backend` 0.11.0 but previously could only return a +single transaction. + +**Note:** This feature changes the use of transparent addresses in ways that are relevant +to security and access to funds, and that may interact with other wallet behaviour. In +particular it exposes new ephemeral transparent addresses belonging to the wallet, which +need to be scanned in order to recover funds if the first transaction of the proposal is +mined but the second is not, or if someone (e.g. the TEX-address recipient) sends back +funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for details. ### Added - `zcash_client_backend::data_api`: @@ -17,7 +34,8 @@ and this library adheres to Rust's notion of - `orchard::EmptyBundleView` - `zcash_client_backend::scanning`: - `testing` module -- `zcash_client_backend::sync` module, behind the `sync` feature flag +- `zcash_client_backend::sync` module, behind the `sync` feature flag. +- `zcash_client_backend::wallet::Recipient::map_ephemeral_transparent_outpoint` ### Changed - MSRV is now 1.70.0. @@ -33,6 +51,8 @@ and this library adheres to Rust's notion of have changed as a consequence of this extraction; please see the `zip321` CHANGELOG for details. - `zcash_client_backend::data_api`: + - `WalletRead` has a new `get_reserved_ephemeral_addresses` method. + - `WalletWrite` has a new `reserve_next_n_ephemeral_addresses` method. - `error::Error` has a new `Address` variant. - `wallet::input_selection::InputSelectorError` has a new `Address` variant. - `zcash_client_backend::proto::proposal::Proposal::{from_standard_proposal, @@ -46,12 +66,19 @@ and this library adheres to Rust's notion of if a memo is given for the transparent pool. Use `ChangeValue::shielded` to avoid this error case when creating a `ChangeValue` known to be for a shielded pool. + - `ChangeStrategy::compute_balance`: this trait method has two additional + parameters when the "transparent-inputs" feature is enabled. These are + used to specify amounts of additional transparent inputs and outputs, + for which `InputView` or `OutputView` instances may not be available. + Empty slices can be passed to obtain the previous behaviour. - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. - `zcash_client_backend::wallet::Recipient` variants have changed. Instead of - wrapping protocol-address types, the `Recipient` type now wraps a - `zcash_address::ZcashAddress`. This simplifies the process of tracking the - original address to which value was sent. + wrapping protocol-address types, the `External` and `InternalAccount` variants + now wrap a `zcash_address::ZcashAddress`. This simplifies the process of + tracking the original address to which value was sent. There is also a new + `EphemeralTransparent` variant, and an additional generic parameter for the + type of metadata associated with an ephemeral transparent outpoint. ### Removed - `zcash_client_backend::data_api`: diff --git a/zcash_client_backend/proto/proposal.proto b/zcash_client_backend/proto/proposal.proto index 950bb3406c..0935d66a1a 100644 --- a/zcash_client_backend/proto/proposal.proto +++ b/zcash_client_backend/proto/proposal.proto @@ -113,6 +113,10 @@ enum FeeRule { // The proposed change outputs and fee value. message TransactionBalance { // A list of change output values. + // + // Each `ChangeValue` for the transparent value pool must be consumed by + // a subsequent step. These represent ephemeral outputs that will each be + // given a unique t-address. repeated ChangeValue proposedChange = 1; // The fee to be paid by the proposed transaction, in zatoshis. uint64 feeRequired = 2; diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index fc89170786..dd1f6f6eb4 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -88,16 +88,16 @@ use zcash_primitives::{ consensus::BlockHeight, memo::{Memo, MemoBytes}, transaction::{ - components::amount::{BalanceError, NonNegativeAmount}, + components::{ + amount::{BalanceError, NonNegativeAmount}, + OutPoint, + }, Transaction, TxId, }, }; #[cfg(feature = "transparent-inputs")] -use { - crate::wallet::TransparentAddressMetadata, - zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint}, -}; +use {crate::wallet::TransparentAddressMetadata, zcash_primitives::legacy::TransparentAddress}; #[cfg(any(test, feature = "test-dependencies"))] use zcash_primitives::consensus::NetworkUpgrade; @@ -890,11 +890,12 @@ pub trait WalletRead { query: NullifierQuery, ) -> Result, Self::Error>; - /// Returns the set of all transparent receivers associated with the given account. + /// Returns the set of non-ephemeral transparent receivers associated with the given + /// account controlled by this wallet. /// - /// The set contains all transparent receivers that are known to have been derived - /// under this account. Wallets should scan the chain for UTXOs sent to these - /// receivers. + /// The set contains all non-ephemeral transparent receivers that are known to have + /// been derived under this account. Wallets should scan the chain for UTXOs sent to + /// these receivers. #[cfg(feature = "transparent-inputs")] fn get_transparent_receivers( &self, @@ -903,8 +904,11 @@ pub trait WalletRead { Ok(HashMap::new()) } - /// Returns a mapping from transparent receiver to not-yet-shielded UTXO balance, - /// for each address associated with a nonzero balance. + /// Returns a mapping from each transparent receiver associated with the specified account + /// to its not-yet-shielded UTXO balance as of the end of the block at the provided + /// `max_height`, when that balance is non-zero. + /// + /// Balances of ephemeral transparent addresses will not be included. #[cfg(feature = "transparent-inputs")] fn get_transparent_balances( &self, @@ -913,6 +917,21 @@ pub trait WalletRead { ) -> Result, Self::Error> { Ok(HashMap::new()) } + + /// Returns the set of reserved ephemeral transparent addresses associated with the + /// given account controlled by this wallet. + /// + /// The set contains all ephemeral transparent receivers that are known to have + /// been derived under this account. Wallets should scan the chain for UTXOs sent to + /// these receivers. + #[cfg(feature = "transparent-inputs")] + fn get_reserved_ephemeral_addresses( + &self, + _account: Self::AccountId, + _for_detection: bool, + ) -> Result>, Self::Error> { + Ok(HashMap::new()) + } } /// The relevance of a seed to a given wallet. @@ -1243,7 +1262,7 @@ impl<'a, AccountId> SentTransaction<'a, AccountId> { /// This type is capable of representing both shielded and transparent outputs. pub struct SentTransactionOutput { output_index: usize, - recipient: Recipient, + recipient: Recipient, value: NonNegativeAmount, memo: Option, } @@ -1260,7 +1279,7 @@ impl SentTransactionOutput { /// * `memo` - the memo that was sent with this output pub fn from_parts( output_index: usize, - recipient: Recipient, + recipient: Recipient, value: NonNegativeAmount, memo: Option, ) -> Self { @@ -1282,8 +1301,8 @@ impl SentTransactionOutput { self.output_index } /// Returns the recipient address of the transaction, or the account id and - /// resulting note for wallet-internal outputs. - pub fn recipient(&self) -> &Recipient { + /// resulting note/outpoint for wallet-internal outputs. + pub fn recipient(&self) -> &Recipient { &self.recipient } /// Returns the value of the newly created output. @@ -1542,6 +1561,26 @@ pub trait WalletWrite: WalletRead { /// /// There may be restrictions on heights to which it is possible to truncate. fn truncate_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; + + /// Reserves the next `n` available ephemeral addresses for the given account. + /// This cannot be undone, so as far as possible, errors associated with transaction + /// construction should have been reported before calling this method. + /// + /// To ensure that sufficient information is stored on-chain to allow recovering + /// funds sent back to any of the used addresses, a "gap limit" of 20 addresses + /// should be observed as described in [BIP 44]. + /// + /// Returns an error if there is insufficient space within the gap limit to allocate + /// the given number of addresses, or if the account identifier does not correspond + /// to a known account. + /// + /// [BIP 44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#user-content-Address_gap_limit + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + account_id: Self::AccountId, + n: u32, + ) -> Result, Self::Error>; } /// This trait describes a capability for manipulating wallet note commitment trees. @@ -1869,6 +1908,16 @@ pub mod testing { ) -> Result, Self::Error> { Ok(HashMap::new()) } + + #[cfg(feature = "transparent-inputs")] + fn get_reserved_ephemeral_addresses( + &self, + _account: Self::AccountId, + _for_detection: bool, + ) -> Result>, Self::Error> + { + Ok(HashMap::new()) + } } impl WalletWrite for MockWalletDb { @@ -1931,6 +1980,15 @@ pub mod testing { ) -> Result { Ok(0) } + + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + _account_id: Self::AccountId, + _n: u32, + ) -> Result, Self::Error> { + Err(()) + } } impl WalletCommitmentTrees for MockWalletDb { diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 2518998906..dc4d47f901 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -33,6 +33,7 @@ to a wallet-internal shielded address, as described in [ZIP 316](https://zips.z. //! [`TransactionRequest`]: crate::zip321::TransactionRequest //! [`propose_transfer`]: crate::data_api::wallet::propose_transfer +use core::convert::Infallible; use nonempty::NonEmpty; use rand_core::OsRng; use sapling::{ @@ -51,16 +52,19 @@ use crate::{ decrypt_transaction, fees::{self, DustOutputPolicy}, keys::UnifiedSpendingKey, - proposal::{self, Proposal, ProposalError}, + proposal::{Proposal, ProposalError, Step, StepOutputIndex}, wallet::{Note, OvkPolicy, Recipient}, zip321::{self, Payment}, PoolType, ShieldedProtocol, }; -use zcash_primitives::transaction::{ - builder::{BuildConfig, BuildResult, Builder}, - components::{amount::NonNegativeAmount, sapling::zip212_enforcement}, - fees::{zip317::FeeError as Zip317FeeError, FeeRule, StandardFeeRule}, - Transaction, TxId, +use zcash_primitives::{ + legacy::TransparentAddress, + transaction::{ + builder::{BuildConfig, BuildResult, Builder}, + components::{amount::NonNegativeAmount, sapling::zip212_enforcement, OutPoint}, + fees::{zip317::FeeError as Zip317FeeError, FeeRule, StandardFeeRule}, + Transaction, TxId, + }, }; use zcash_protocol::{ consensus::{self, BlockHeight, NetworkUpgrade}, @@ -70,11 +74,11 @@ use zip32::Scope; #[cfg(feature = "transparent-inputs")] use { + crate::{fees::ChangeValue, proposal::StepOutput, wallet::TransparentAddressMetadata}, input_selection::ShieldingSelector, - std::convert::Infallible, + std::collections::HashMap, zcash_keys::encoding::AddressCodec, - zcash_primitives::legacy::TransparentAddress, - zcash_primitives::transaction::components::{OutPoint, TxOut}, + zcash_primitives::transaction::components::TxOut, }; pub mod input_selection; @@ -597,6 +601,10 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { + // The set of transparent StepOutputs available and unused from prior steps. + #[cfg(feature = "transparent-inputs")] + let mut unused_transparent_outputs = HashMap::new(); + let mut step_results = Vec::with_capacity(proposal.steps().len()); for step in proposal.steps() { let step_result = create_proposed_transaction( @@ -610,10 +618,22 @@ where proposal.min_target_height(), &step_results, step, + #[cfg(feature = "transparent-inputs")] + &mut unused_transparent_outputs, )?; step_results.push((step, step_result)); } + // Change step outputs represent ephemeral outputs that must be referenced exactly once. + // (We don't support transparent change.) + #[cfg(feature = "transparent-inputs")] + if unused_transparent_outputs + .into_keys() + .any(|s: StepOutput| matches!(s.output_index(), StepOutputIndex::Change(_))) + { + return Err(Error::ProposalNotSupported); + } + Ok(NonEmpty::from_vec( step_results .iter() @@ -634,48 +654,51 @@ fn create_proposed_transaction( ovk_policy: OvkPolicy, fee_rule: &FeeRuleT, min_target_height: BlockHeight, - prior_step_results: &[(&proposal::Step, BuildResult)], - proposal_step: &proposal::Step, + prior_step_results: &[(&Step, BuildResult)], + proposal_step: &Step, + #[cfg(feature = "transparent-inputs")] unused_transparent_outputs: &mut HashMap< + StepOutput, + ( + TransparentAddress, + Option, + OutPoint, + ), + >, ) -> Result> where DbT: WalletWrite + WalletCommitmentTrees, ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - // TODO: Spending shielded outputs of prior multi-step transaction steps is not yet - // supported. Maybe support this at some point? Doing so would require a higher-level - // approach in the wallet that waits for transactions with shielded outputs to be - // mined and only then attempts to perform the next step. - for s_ref in proposal_step.prior_step_inputs() { - prior_step_results.get(s_ref.step_index()).map_or_else( - || { - // Return an error in case the step index doesn't match up with a step - Err(Error::Proposal(ProposalError::ReferenceError(*s_ref))) - }, - |step| match s_ref.output_index() { - proposal::StepOutputIndex::Payment(i) => { - let prior_pool = step - .0 - .payment_pools() - .get(&i) - .ok_or(Error::Proposal(ProposalError::ReferenceError(*s_ref)))?; - - if matches!(prior_pool, PoolType::Shielded(_)) { - Err(Error::ProposalNotSupported) - } else { - Ok(()) - } - } - proposal::StepOutputIndex::Change(_) => { - // Only shielded change is supported by zcash_client_backend, so multi-step - // transactions cannot yet spend prior transactions' change outputs. - Err(Error::ProposalNotSupported) - } - }, - )?; + #[cfg(feature = "transparent-inputs")] + let step_index = prior_step_results.len(); + + // Spending shielded outputs of prior multi-step transaction steps (either payments or change) + // is not supported. + // + // TODO: Maybe support this at some point? Doing so would require a higher-level approach in + // the wallet that waits for transactions with shielded outputs to be mined and only then + // attempts to perform the next step. + for input_ref in proposal_step.prior_step_inputs() { + let prior_pool = prior_step_results + .get(input_ref.step_index()) + .and_then(|(prior_step, _)| match input_ref.output_index() { + StepOutputIndex::Payment(i) => prior_step.payment_pools().get(&i).cloned(), + StepOutputIndex::Change(i) => prior_step + .balance() + .proposed_change() + .get(i) + .map(|change| change.output_pool()), + }) + .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))?; + + // Return an error on trying to spend a prior shielded output. + if matches!(prior_pool, PoolType::Shielded(_)) { + return Err(Error::ProposalNotSupported); + } } - let account = wallet_db + let account_id = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? .ok_or(Error::KeyNotRecognized)? @@ -762,7 +785,7 @@ where let orchard_anchor = None; // Create the transaction. The type of the proposal ensures that there - // are no possible transparent inputs, so we ignore those + // are no possible transparent inputs, so we ignore those here. let mut builder = Builder::new( params.clone(), min_target_height, @@ -772,104 +795,108 @@ where }, ); + #[cfg(feature = "transparent-inputs")] + let mut has_shielded_inputs = false; + for (sapling_key, sapling_note, merkle_path) in sapling_inputs.into_iter() { builder.add_sapling_spend(&sapling_key, sapling_note.clone(), merkle_path)?; + #[cfg(feature = "transparent-inputs")] + { + has_shielded_inputs = true; + } } #[cfg(feature = "orchard")] for (orchard_note, merkle_path) in orchard_inputs.into_iter() { builder.add_orchard_spend(usk.orchard(), *orchard_note, merkle_path.into())?; + #[cfg(feature = "transparent-inputs")] + { + has_shielded_inputs = true; + } } #[cfg(feature = "transparent-inputs")] - let utxos_spent = { - let known_addrs = wallet_db - .get_transparent_receivers(account) - .map_err(Error::DataSource)?; + let mut known_addrs = wallet_db + .get_transparent_receivers(account_id) + .map_err(Error::DataSource)?; + #[cfg(feature = "transparent-inputs")] + let mut ephemeral_added = false; + + #[cfg(feature = "transparent-inputs")] + let mut metadata_from_address = |addr: TransparentAddress| -> Result< + TransparentAddressMetadata, + ErrorT, + > { + match known_addrs.get(&addr) { + None if !ephemeral_added => { + // The ephemeral addresses are added lazily to avoid extra database operations + // in the common case. We don't need to include them in order to be able to + // construct ZIP 320 transactions, because in that case the ephemeral output + // is represented via a "change" reference to a previous step. However, we do + // need them in order to create a transaction from a proposal that explicitly + // spends an output from an ephemeral address. This need not set `for_detection` + // because we only need to be able to spend outputs already detected by this + // wallet instance. + ephemeral_added = true; + known_addrs.extend( + wallet_db + .get_reserved_ephemeral_addresses(account_id, false) + .map_err(Error::DataSource)? + .into_iter(), + ); + known_addrs.get(&addr) + } + result => result, + } + .ok_or(Error::AddressNotRecognized(addr))? + .clone() + .ok_or_else(|| Error::NoSpendingKey(addr.encode(params))) + }; + #[cfg(feature = "transparent-inputs")] + let utxos_spent = { let mut utxos_spent: Vec = vec![]; - let mut add_transparent_input = |addr: &TransparentAddress, + let mut add_transparent_input = |address_metadata: &TransparentAddressMetadata, outpoint: OutPoint, - utxo: TxOut| - -> Result< - (), - Error< - ::Error, - ::Error, - InputsErrT, - FeeRuleT::Error, - >, - > { - let address_metadata = known_addrs - .get(addr) - .ok_or(Error::AddressNotRecognized(*addr))? - .clone() - .ok_or_else(|| Error::NoSpendingKey(addr.encode(params)))?; - + txout: TxOut| + -> Result<(), ErrorT> { let secret_key = usk .transparent() .derive_secret_key(address_metadata.scope(), address_metadata.address_index()) - .unwrap(); + .expect("spending key derivation should not fail"); utxos_spent.push(outpoint.clone()); - builder.add_transparent_input(secret_key, outpoint, utxo)?; + builder.add_transparent_input(secret_key, outpoint, txout)?; Ok(()) }; for utxo in proposal_step.transparent_inputs() { add_transparent_input( - utxo.recipient_address(), + &metadata_from_address(*utxo.recipient_address())?, utxo.outpoint().clone(), utxo.txout().clone(), )?; } for input_ref in proposal_step.prior_step_inputs() { - match input_ref.output_index() { - proposal::StepOutputIndex::Payment(i) => { - // We know based upon the earlier check that this must be a transparent input, - // We also know that transparent outputs for that previous step were added to - // the transaction in payment index order, so we can use dead reckoning to - // figure out which output it ended up being. - let (prior_step, result) = &prior_step_results[input_ref.step_index()]; - let recipient_address = &prior_step - .transaction_request() - .payments() - .get(&i) - .expect("Payment step references are checked at construction") - .recipient_address() - .clone() - .convert_if_network(params.network_type())?; - - let recipient_taddr = match recipient_address { - Address::Transparent(t) => Some(t), - Address::Unified(uaddr) => uaddr.transparent(), - _ => None, - } - .ok_or(Error::ProposalNotSupported)?; - let outpoint = OutPoint::new( - result.transaction().txid().into(), - u32::try_from( - prior_step - .payment_pools() - .iter() - .filter(|(_, pool)| pool == &&PoolType::Transparent) - .take_while(|(j, _)| j <= &&i) - .count() - - 1, - ) - .expect("Transparent output index fits into a u32"), - ); - let utxo = &result - .transaction() - .transparent_bundle() - .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))? - .vout[outpoint.n() as usize]; - - add_transparent_input(recipient_taddr, outpoint, utxo.clone())?; - } - proposal::StepOutputIndex::Change(_) => unreachable!(), - } + // A referenced transparent step output must exist and be referenced *at most* once. + // (Exactly once in the case of ephemeral outputs.) + let (address, address_metadata_opt, outpoint) = unused_transparent_outputs + .remove(input_ref) + .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))?; + + let address_metadata = match address_metadata_opt { + Some(meta) => meta, + None => metadata_from_address(address)?, + }; + let txout = &prior_step_results[input_ref.step_index()] + .1 + .transaction() + .transparent_bundle() + .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))? + .vout[outpoint.n() as usize]; + + add_transparent_input(&address_metadata, outpoint, txout.clone())?; } utxos_spent }; @@ -922,108 +949,132 @@ where Some(sapling_dfvk.to_ovk(Scope::Internal)) }; + #[cfg(feature = "transparent-inputs")] + type TransparentMetadataT = TransparentAddressMetadata; + #[cfg(not(feature = "transparent-inputs"))] + type TransparentMetadataT = Infallible; + #[cfg(feature = "orchard")] - let mut orchard_output_meta = vec![]; - let mut sapling_output_meta = vec![]; - let mut transparent_output_meta = vec![]; - for (payment, output_pool) in proposal_step - .payment_pools() - .iter() - .map(|(idx, output_pool)| { - let payment = proposal_step - .transaction_request() - .payments() - .get(idx) - .expect( - "The mapping between payment index and payment is checked in step construction", - ); - (payment, output_pool) - }) - { - let recipient_address: Address = payment - .recipient_address() + let mut orchard_output_meta: Vec<( + Recipient<_, PoolType, _>, + NonNegativeAmount, + Option, + )> = vec![]; + let mut sapling_output_meta: Vec<( + Recipient<_, PoolType, _>, + NonNegativeAmount, + Option, + )> = vec![]; + let mut transparent_output_meta: Vec<( + Recipient<_, _, ()>, + TransparentAddress, + Option, + NonNegativeAmount, + StepOutputIndex, + )> = vec![]; + + for (&payment_index, output_pool) in proposal_step.payment_pools() { + let payment = proposal_step + .transaction_request() + .payments() + .get(&payment_index) + .expect( + "The mapping between payment index and payment is checked in step construction", + ); + let recipient_address = payment.recipient_address(); + + let mut add_sapling_output = |builder: &mut Builder<_, _>, + to: sapling::PaymentAddress| + -> Result<(), ErrorT> { + let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); + builder.add_sapling_output(sapling_external_ovk, to, payment.amount(), memo.clone())?; + sapling_output_meta.push(( + Recipient::External(recipient_address.clone(), PoolType::SAPLING), + payment.amount(), + Some(memo), + )); + Ok(()) + }; + + #[cfg(feature = "orchard")] + let mut add_orchard_output = |builder: &mut Builder<_, _>, + to: orchard::Address| + -> Result<(), ErrorT> { + let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); + builder.add_orchard_output( + orchard_external_ovk.clone(), + to, + payment.amount().into(), + memo.clone(), + )?; + orchard_output_meta.push(( + Recipient::External(recipient_address.clone(), PoolType::ORCHARD), + payment.amount(), + Some(memo), + )); + Ok(()) + }; + + #[allow(unused_mut)] + let mut add_transparent_output = |builder: &mut Builder<_, _>, + to: TransparentAddress| + -> Result<(), ErrorT> { + if payment.memo().is_some() { + return Err(Error::MemoForbidden); + } else { + builder.add_transparent_output(&to, payment.amount())?; + } + transparent_output_meta.push(( + Recipient::External(recipient_address.clone(), PoolType::TRANSPARENT), + to, + None, + payment.amount(), + StepOutputIndex::Payment(payment_index), + )); + Ok(()) + }; + + match recipient_address .clone() - .convert_if_network(params.network_type())?; - - match recipient_address { - Address::Unified(ua) => { - let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); - - match output_pool { - #[cfg(not(feature = "orchard"))] - PoolType::Shielded(ShieldedProtocol::Orchard) => { - return Err(Error::ProposalNotSupported); - } - #[cfg(feature = "orchard")] - PoolType::Shielded(ShieldedProtocol::Orchard) => { - builder.add_orchard_output( - orchard_external_ovk.clone(), - *ua.orchard().expect("The mapping between payment pool and receiver is checked in step construction"), - payment.amount().into(), - memo.clone(), - )?; - orchard_output_meta.push(( - Recipient::External(payment.recipient_address().clone(), *output_pool), - payment.amount(), - Some(memo), - )); - } - - PoolType::Shielded(ShieldedProtocol::Sapling) => { - builder.add_sapling_output( - sapling_external_ovk, - *ua.sapling().expect("The mapping between payment pool and receiver is checked in step construction"), - payment.amount(), - memo.clone(), - )?; - sapling_output_meta.push(( - Recipient::External(payment.recipient_address().clone(), *output_pool), - payment.amount(), - Some(memo), - )); - } - - PoolType::Transparent => { - if payment.memo().is_some() { - return Err(Error::MemoForbidden); - } else { - builder.add_transparent_output( - ua.transparent().expect("The mapping between payment pool and receiver is checked in step construction."), - payment.amount() - )?; - } - } + .convert_if_network(params.network_type())? + { + Address::Unified(ua) => match output_pool { + #[cfg(not(feature = "orchard"))] + PoolType::Shielded(ShieldedProtocol::Orchard) => { + return Err(Error::ProposalNotSupported); } - } - Address::Sapling(addr) => { - let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); - builder.add_sapling_output( - sapling_external_ovk, - addr, - payment.amount(), - memo.clone(), - )?; - sapling_output_meta.push(( - Recipient::External(payment.recipient_address().clone(), PoolType::SAPLING), - payment.amount(), - Some(memo), - )); + #[cfg(feature = "orchard")] + PoolType::Shielded(ShieldedProtocol::Orchard) => { + let to = *ua.orchard().expect("The mapping between payment pool and receiver is checked in step construction"); + add_orchard_output(&mut builder, to)?; + } + PoolType::Shielded(ShieldedProtocol::Sapling) => { + let to = *ua.sapling().expect("The mapping between payment pool and receiver is checked in step construction"); + add_sapling_output(&mut builder, to)?; + } + PoolType::Transparent => { + let to = *ua.transparent().expect("The mapping between payment pool and receiver is checked in step construction"); + add_transparent_output(&mut builder, to)?; + } + }, + Address::Sapling(to) => { + add_sapling_output(&mut builder, to)?; } Address::Transparent(to) => { - if payment.memo().is_some() { - return Err(Error::MemoForbidden); - } else { - builder.add_transparent_output(&to, payment.amount())?; - } - transparent_output_meta.push(( - Recipient::External(payment.recipient_address().clone(), PoolType::TRANSPARENT), - to, - payment.amount(), - )); + add_transparent_output(&mut builder, to)?; } + #[cfg(not(feature = "transparent-inputs"))] Address::Tex(_) => { return Err(Error::ProposalNotSupported); } + #[cfg(feature = "transparent-inputs")] + Address::Tex(data) => { + if has_shielded_inputs { + return Err(Error::ProposalNotSupported); + } + let to = TransparentAddress::PublicKeyHash(data); + add_transparent_output(&mut builder, to)?; + } } } @@ -1042,7 +1093,7 @@ where )?; sapling_output_meta.push(( Recipient::InternalAccount { - receiving_account: account, + receiving_account: account_id, external_address: None, note: output_pool, }, @@ -1064,7 +1115,7 @@ where )?; orchard_output_meta.push(( Recipient::InternalAccount { - receiving_account: account, + receiving_account: account_id, external_address: None, note: output_pool, }, @@ -1074,11 +1125,53 @@ where } } PoolType::Transparent => { + #[cfg(not(feature = "transparent-inputs"))] return Err(Error::UnsupportedChangeType(output_pool)); } } } + // This reserves the ephemeral addresses even if transaction construction fails. + // It is not worth the complexity of being able to unreserve them, because there + // are few failure modes after this point that would allow us to do so. + #[cfg(feature = "transparent-inputs")] + { + let ephemeral_outputs: Vec<(usize, &ChangeValue)> = proposal_step + .balance() + .proposed_change() + .iter() + .enumerate() + .filter(|(_, change_value)| matches!(change_value.output_pool(), PoolType::Transparent)) + .collect(); + let num_ephemeral_outputs = + u32::try_from(ephemeral_outputs.len()).map_err(|_| Error::ProposalNotSupported)?; + + let addresses_and_metadata = wallet_db + .reserve_next_n_ephemeral_addresses(account_id, num_ephemeral_outputs) + .map_err(Error::DataSource)?; + assert_eq!(addresses_and_metadata.len(), ephemeral_outputs.len()); + + for ((change_index, change_value), (ephemeral_address, address_metadata)) in + ephemeral_outputs.iter().zip(addresses_and_metadata) + { + // This is intended for an ephemeral transparent output, rather than a + // non-ephemeral transparent change output. We will report an error in + // `create_proposed_transactions` if a later step does not consume this output. + builder.add_transparent_output(&ephemeral_address, change_value.value())?; + transparent_output_meta.push(( + Recipient::EphemeralTransparent { + receiving_account: account_id, + ephemeral_address, + outpoint_metadata: (), + }, + ephemeral_address, + Some(address_metadata), + change_value.value(), + StepOutputIndex::Change(*change_index), + )) + } + } + // Build the transaction with the specified fee rule let build_result = builder.build(OsRng, spend_prover, output_prover, fee_rule)?; @@ -1146,29 +1239,30 @@ where SentTransactionOutput::from_parts(output_index, recipient, value, memo) }); - let transparent_outputs = - transparent_output_meta - .into_iter() - .map(|(recipient, addr, value)| { - let script = addr.script(); - let output_index = build_result - .transaction() - .transparent_bundle() - .and_then(|b| { - b.vout - .iter() - .enumerate() - .find(|(_, tx_out)| tx_out.script_pubkey == script) - }) - .map(|(index, _)| index) - .expect( - "An output should exist in the transaction for each transparent payment.", - ); + let txid: [u8; 32] = build_result.transaction().txid().into(); + assert_eq!( + transparent_output_meta.len(), + build_result + .transaction() + .transparent_bundle() + .map_or(0, |b| b.vout.len()), + ); - SentTransactionOutput::from_parts(output_index, recipient, value, None) - }); + #[allow(unused_variables)] + let transparent_outputs = transparent_output_meta.into_iter().enumerate().map( + |(n, (recipient, ephemeral_address, address_metadata_opt, value, step_output_index))| { + let outpoint = OutPoint::new(txid, n as u32); + let recipient = recipient.map_ephemeral_transparent_outpoint(|()| outpoint.clone()); + #[cfg(feature = "transparent-inputs")] + unused_transparent_outputs.insert( + StepOutput::new(step_index, step_output_index), + (ephemeral_address, address_metadata_opt, outpoint), + ); + SentTransactionOutput::from_parts(n, recipient, value, None) + }, + ); - let mut outputs = vec![]; + let mut outputs: Vec> = vec![]; #[cfg(feature = "orchard")] outputs.extend(orchard_outputs); outputs.extend(sapling_outputs); @@ -1178,7 +1272,7 @@ where .store_sent_tx(&SentTransaction { tx: build_result.transaction(), created: time::OffsetDateTime::now_utc(), - account, + account: account_id, outputs, fee_amount: proposal_step.balance().fee_required(), #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index e6579aa031..776dc98691 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -32,9 +32,14 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { - std::collections::BTreeSet, std::convert::Infallible, - zcash_primitives::legacy::TransparentAddress, - zcash_primitives::transaction::components::OutPoint, + crate::{ + fees::ChangeValue, + proposal::{Step, StepOutput, StepOutputIndex}, + zip321::Payment, + }, + std::collections::BTreeSet, + std::convert::Infallible, + zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint}, }; #[cfg(feature = "orchard")] @@ -364,6 +369,16 @@ where #[cfg(feature = "orchard")] let mut orchard_outputs = vec![]; let mut payment_pools = BTreeMap::new(); + + #[cfg(feature = "transparent-inputs")] + let mut tr1_transparent_outputs = vec![]; + #[cfg(feature = "transparent-inputs")] + let mut tr1_payments = vec![]; + #[cfg(feature = "transparent-inputs")] + let mut tr1_payment_pools = BTreeMap::new(); + #[cfg(feature = "transparent-inputs")] + let mut total_ephemeral = NonNegativeAmount::ZERO; + for (idx, payment) in transaction_request.payments() { let recipient_address: Address = payment .recipient_address() @@ -378,6 +393,30 @@ where script_pubkey: addr.script(), }); } + #[cfg(feature = "transparent-inputs")] + Address::Tex(data) => { + let p2pkh_addr = TransparentAddress::PublicKeyHash(data); + + tr1_payment_pools.insert(*idx, PoolType::TRANSPARENT); + tr1_transparent_outputs.push(TxOut { + value: payment.amount(), + script_pubkey: p2pkh_addr.script(), + }); + tr1_payments.push( + Payment::new( + payment.recipient_address().clone(), + payment.amount(), + None, + payment.label().cloned(), + payment.message().cloned(), + payment.other_params().to_vec(), + ) + .expect("cannot fail because memo is None"), + ); + total_ephemeral = (total_ephemeral + payment.amount()) + .ok_or_else(|| GreedyInputSelectorError::Balance(BalanceError::Overflow))?; + } + #[cfg(not(feature = "transparent-inputs"))] Address::Tex(_) => { return Err(InputSelectorError::Selection( GreedyInputSelectorError::UnsupportedTexAddress, @@ -417,10 +456,59 @@ where } } + #[cfg(feature = "transparent-inputs")] + let (ephemeral_output_amounts, tr1_balance_opt) = { + if tr1_transparent_outputs.is_empty() { + (vec![], None) + } else { + // The ephemeral input going into transaction 1 must be able to pay that + // transaction's fee, as well as the TEX address payments. + + let tr1_required_input_value = + match self.change_strategy.compute_balance::<_, DbT::NoteRef>( + params, + target_height, + &[] as &[WalletTransparentOutput], + &tr1_transparent_outputs, + &sapling::EmptyBundleView, + #[cfg(feature = "orchard")] + &orchard_fees::EmptyBundleView, + &self.dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[NonNegativeAmount::ZERO], + #[cfg(feature = "transparent-inputs")] + &[], + ) { + Err(ChangeError::InsufficientFunds { required, .. }) => required, + Ok(_) => NonNegativeAmount::ZERO, // shouldn't happen + Err(other) => return Err(other.into()), + }; + + let tr1_balance = self.change_strategy.compute_balance::<_, DbT::NoteRef>( + params, + target_height, + &[] as &[WalletTransparentOutput], + &tr1_transparent_outputs, + &sapling::EmptyBundleView, + #[cfg(feature = "orchard")] + &orchard_fees::EmptyBundleView, + &self.dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[tr1_required_input_value], + #[cfg(feature = "transparent-inputs")] + &[], + )?; + assert_eq!(tr1_balance.total(), tr1_balance.fee_required()); + + (vec![tr1_required_input_value], Some(tr1_balance)) + } + }; + let mut shielded_inputs = SpendableNotes::empty(); let mut prior_available = NonNegativeAmount::ZERO; let mut amount_required = NonNegativeAmount::ZERO; let mut exclude: Vec = vec![]; + // This loop is guaranteed to terminate because on each iteration we check that the amount // of funds selected is strictly increasing. The loop will either return a successful // result or the wallet will eventually run out of funds to select. @@ -434,12 +522,12 @@ where shielded_inputs.orchard_value()?, ); - // Use Sapling inputs if there are no Orchard outputs or there are not sufficient - // Orchard outputs to cover the amount required. + // Use Sapling inputs if there are no Orchard outputs or if there are insufficient + // funds from Orchard inputs to cover the amount required. let use_sapling = orchard_outputs.is_empty() || amount_required > orchard_input_total; - // Use Orchard inputs if there are insufficient Sapling funds to cover the amount - // reqiuired. + // Use Orchard inputs if there are insufficient funds from Sapling inputs to cover + // the amount required. let use_orchard = !use_sapling || amount_required > sapling_input_total; (use_sapling, use_orchard) @@ -466,10 +554,12 @@ where vec![] }; + // In the ZIP 320 case, this is the balance for transaction 0, taking into account + // the ephemeral output. let balance = self.change_strategy.compute_balance( params, target_height, - &Vec::::new(), + &[] as &[WalletTransparentOutput], &transparent_outputs, &( ::sapling::builder::BundleType::DEFAULT, @@ -483,22 +573,98 @@ where &orchard_outputs[..], ), &self.dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &ephemeral_output_amounts, ); match balance { Ok(balance) => { - return Proposal::single_step( - transaction_request, - payment_pools, - vec![], + // At this point, we have enough input value to pay for everything, so we will + // return at the end of this block. + + let shielded_inputs = NonEmpty::from_vec(shielded_inputs.into_vec(&SimpleNoteRetention { sapling: use_sapling, #[cfg(feature = "orchard")] orchard: use_orchard, })) - .map(|notes| ShieldedInputs::from_parts(anchor_height, notes)), + .map(|notes| ShieldedInputs::from_parts(anchor_height, notes)); + + #[cfg(feature = "transparent-inputs")] + if let Some(tr1_balance) = tr1_balance_opt { + // Construct two new `TransactionRequest`s: + // * `tr0` excludes the TEX outputs, and in their place includes + // a single additional "change" output to the transparent pool. + // * `tr1` spends from that change output to each TEX output. + + // The ephemeral output should always be at the last change index. + assert_eq!( + *balance.proposed_change().last().expect("nonempty"), + ChangeValue::transparent(ephemeral_output_amounts[0]) + ); + let ephemeral_stepoutput = StepOutput::new( + 0, + StepOutputIndex::Change(balance.proposed_change().len() - 1), + ); + + let tr0 = TransactionRequest::from_indexed( + transaction_request + .payments() + .iter() + .filter(|(idx, _payment)| !tr1_payment_pools.contains_key(idx)) + .map(|(k, v)| (*k, v.clone())) + .collect(), + ) + .expect("removing payments from a TransactionRequest preserves validity"); + + let mut steps = vec![]; + steps.push( + Step::from_parts( + &[], + tr0, + payment_pools, + vec![], + shielded_inputs, + vec![], + balance, + false, + ) + .map_err(InputSelectorError::Proposal)?, + ); + + let tr1 = + TransactionRequest::new(tr1_payments).expect("valid by construction"); + steps.push( + Step::from_parts( + &steps, + tr1, + tr1_payment_pools, + vec![], + None, + vec![ephemeral_stepoutput], + tr1_balance, + false, + ) + .map_err(InputSelectorError::Proposal)?, + ); + + return Proposal::multi_step( + self.change_strategy.fee_rule().clone(), + target_height, + NonEmpty::from_vec(steps).expect("steps is known to be nonempty"), + ) + .map_err(InputSelectorError::Proposal); + } + + return Proposal::single_step( + transaction_request, + payment_pools, + vec![], + shielded_inputs, balance, - (*self.change_strategy.fee_rule()).clone(), + self.change_strategy.fee_rule().clone(), target_height, false, ) @@ -592,11 +758,15 @@ where params, target_height, &transparent_inputs, - &Vec::::new(), + &[] as &[TxOut], &sapling::EmptyBundleView, #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); let balance = match trial_balance { @@ -609,11 +779,15 @@ where params, target_height, &transparent_inputs, - &Vec::::new(), + &[] as &[TxOut], &sapling::EmptyBundleView, #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], )? } Err(other) => { diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index f4fcfc2312..fcdbc9c524 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -328,6 +328,8 @@ pub trait ChangeStrategy { sapling: &impl sapling::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result>; } diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 4079046b6e..b77811abf4 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -5,7 +5,11 @@ use zcash_primitives::{ memo::MemoBytes, transaction::{ components::amount::{BalanceError, NonNegativeAmount}, - fees::{transparent, zip317::MINIMUM_FEE, FeeRule}, + fees::{ + transparent::{self, InputSize}, + zip317::{MINIMUM_FEE, P2PKH_STANDARD_OUTPUT_SIZE}, + FeeRule, + }, }, }; use zcash_protocol::ShieldedProtocol; @@ -49,6 +53,8 @@ pub(crate) fn calculate_net_flows( transparent_outputs: &[impl transparent::OutputView], sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, + ephemeral_input_amounts: &[NonNegativeAmount], + ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> where E: From + From, @@ -58,11 +64,13 @@ where let t_in = transparent_inputs .iter() .map(|t_in| t_in.coin().value) + .chain(ephemeral_input_amounts.iter().cloned()) .sum::>() .ok_or_else(overflow)?; let t_out = transparent_outputs .iter() .map(|t_out| t_out.value()) + .chain(ephemeral_output_amounts.iter().cloned()) .sum::>() .ok_or_else(overflow)?; let sapling_in = sapling @@ -156,6 +164,8 @@ pub(crate) fn single_change_output_balance< default_dust_threshold: NonNegativeAmount, change_memo: Option, fallback_change_pool: ShieldedProtocol, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> where E: From + From, @@ -163,12 +173,18 @@ where let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); let underflow = || ChangeError::StrategyError(E::from(BalanceError::Underflow)); + #[cfg(not(feature = "transparent-inputs"))] + let (ephemeral_input_amounts, ephemeral_output_amounts) = + (&[] as &[NonNegativeAmount], &[] as &[NonNegativeAmount]); + let net_flows = calculate_net_flows::( transparent_inputs, transparent_outputs, sapling, #[cfg(feature = "orchard")] orchard, + ephemeral_input_amounts, + ephemeral_output_amounts, )?; let total_in = net_flows .total_in() @@ -240,12 +256,29 @@ where // Note that using the `DustAction::AddDustToFee` policy inherently leaks // more information. + let transparent_input_sizes = transparent_inputs + .iter() + .map(|i| i.serialized_size()) + .chain( + ephemeral_input_amounts + .iter() + .map(|_| InputSize::STANDARD_P2PKH), + ); + let transparent_output_sizes = transparent_outputs + .iter() + .map(|i| i.serialized_size()) + .chain( + ephemeral_output_amounts + .iter() + .map(|_| P2PKH_STANDARD_OUTPUT_SIZE), + ); + let fee_without_change = fee_rule .fee_required( params, target_height, - transparent_inputs.iter().map(|i| i.serialized_size()), - transparent_outputs.iter().map(|i| i.serialized_size()), + transparent_input_sizes.clone(), + transparent_output_sizes.clone(), sapling_input_count, sapling_output_count, orchard_action_count, @@ -258,8 +291,8 @@ where .fee_required( params, target_height, - transparent_inputs.iter().map(|i| i.serialized_size()), - transparent_outputs.iter().map(|i| i.serialized_size()), + transparent_input_sizes, + transparent_output_sizes, sapling_input_count, sapling_output_count_with_change, orchard_action_count_with_change, @@ -274,7 +307,8 @@ where (total_out + fee_without_change).ok_or_else(overflow)?; let total_out_plus_fee_with_change = (total_out + fee_with_change).ok_or_else(overflow)?; - let (change, fee) = { + #[allow(unused_mut)] + let (mut change, fee) = { if transparent && total_in < total_out_plus_fee_without_change { // Case 1 for a tx with all transparent flows. return Err(ChangeError::InsufficientFunds { @@ -362,6 +396,12 @@ where } } }; + #[cfg(feature = "transparent-inputs")] + change.extend( + ephemeral_output_amounts + .iter() + .map(|&amount| ChangeValue::transparent(amount)), + ); TransactionBalance::new(change, fee).map_err(|_| overflow()) } diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index ee032ab013..1a340c7a21 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -19,6 +19,9 @@ use super::{ #[cfg(feature = "orchard")] use super::orchard as orchard_fees; +#[cfg(feature = "transparent-inputs")] +use super::NonNegativeAmount; + /// A change strategy that proposes change as a single output to the most current supported /// shielded pool and delegates fee calculation to the provided fee rule. pub struct SingleOutputChangeStrategy { @@ -63,6 +66,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> { single_change_output_balance( params, @@ -77,6 +82,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fee_rule().fixed_fee(), self.change_memo.clone(), self.fallback_change_pool, + #[cfg(feature = "transparent-inputs")] + ephemeral_input_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_output_amounts, ) } } @@ -132,6 +141,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -177,6 +190,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( diff --git a/zcash_client_backend/src/fees/standard.rs b/zcash_client_backend/src/fees/standard.rs index fd146b5105..c5fc6cb9f5 100644 --- a/zcash_client_backend/src/fees/standard.rs +++ b/zcash_client_backend/src/fees/standard.rs @@ -68,6 +68,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> { #[allow(deprecated)] match self.fee_rule() { @@ -85,6 +87,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, + #[cfg(feature = "transparent-inputs")] + ephemeral_input_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_output_amounts, ) .map_err(|e| e.map(Zip317FeeError::Balance)), StandardFeeRule::Zip313 => fixed::SingleOutputChangeStrategy::new( @@ -101,6 +107,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, + #[cfg(feature = "transparent-inputs")] + ephemeral_input_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_output_amounts, ) .map_err(|e| e.map(Zip317FeeError::Balance)), StandardFeeRule::Zip317 => zip317::SingleOutputChangeStrategy::new( @@ -117,6 +127,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, + #[cfg(feature = "transparent-inputs")] + ephemeral_input_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_output_amounts, ), } } diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 9ef62fbca6..63ce70c4af 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -17,7 +17,8 @@ use crate::ShieldedProtocol; use super::{ common::{calculate_net_flows, single_change_output_balance, single_change_output_policy}, - sapling as sapling_fees, ChangeError, ChangeStrategy, DustOutputPolicy, TransactionBalance, + sapling as sapling_fees, ChangeError, ChangeStrategy, DustOutputPolicy, NonNegativeAmount, + TransactionBalance, }; #[cfg(feature = "orchard")] @@ -67,7 +68,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> { + // We intentionally never count ephemeral inputs as dust. let mut transparent_dust: Vec<_> = transparent_inputs .iter() .filter_map(|i| { @@ -115,6 +119,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let t_non_dust = transparent_inputs.len() - transparent_dust.len(); let t_allowed_dust = transparent_outputs.len().saturating_sub(t_non_dust); + #[cfg(not(feature = "transparent-inputs"))] + let (ephemeral_input_amounts, ephemeral_output_amounts) = + (&[] as &[NonNegativeAmount], &[] as &[NonNegativeAmount]); + // We add one to either the Sapling or Orchard outputs for the (single) // change output. Note that this means that wallet-internal shielding // transactions are an opportunity to spend a dust note. @@ -124,6 +132,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling, #[cfg(feature = "orchard")] orchard, + ephemeral_input_amounts, + ephemeral_output_amounts, )?; let (_, sapling_change, orchard_change) = single_change_output_policy(&net_flows, self.fallback_change_pool); @@ -206,6 +216,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fee_rule.marginal_fee(), self.change_memo.clone(), self.fallback_change_pool, + #[cfg(feature = "transparent-inputs")] + ephemeral_input_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_output_amounts, ) } } @@ -268,6 +282,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -311,6 +329,10 @@ mod tests { ))][..], ), &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -363,6 +385,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -406,6 +432,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -449,6 +479,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -498,6 +532,10 @@ mod tests { DustAction::AllowDustChange, Some(NonNegativeAmount::const_from_u64(1000)), ), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -558,6 +596,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, dust_output_policy, + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); assert_matches!( @@ -610,6 +652,10 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), + #[cfg(feature = "transparent-inputs")] + &[], + #[cfg(feature = "transparent-inputs")] + &[], ); // We will get an error here, because the dust input now isn't free to add diff --git a/zcash_client_backend/src/proposal.rs b/zcash_client_backend/src/proposal.rs index 3c0c01215a..212b8ba721 100644 --- a/zcash_client_backend/src/proposal.rs +++ b/zcash_client_backend/src/proposal.rs @@ -292,14 +292,14 @@ impl Debug for Proposal { } /// A reference to either a payment or change output within a step. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum StepOutputIndex { Payment(usize), Change(usize), } /// A reference to the output of a step in a proposal. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct StepOutput { step_index: usize, output_index: StepOutputIndex, diff --git a/zcash_client_backend/src/proto/proposal.rs b/zcash_client_backend/src/proto/proposal.rs index a17b83bf8b..be8da848aa 100644 --- a/zcash_client_backend/src/proto/proposal.rs +++ b/zcash_client_backend/src/proto/proposal.rs @@ -116,6 +116,10 @@ pub mod proposed_input { #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionBalance { /// A list of change output values. + /// + /// Each `ChangeValue` for the transparent value pool must be consumed by + /// a subsequent step. These represent ephemeral outputs that will each be + /// given a unique t-address. #[prost(message, repeated, tag = "1")] pub proposed_change: ::prost::alloc::vec::Vec, /// The fee to be paid by the proposed transaction, in zatoshis. diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index 7d555b07f6..18540c8241 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -62,13 +62,19 @@ impl NoteId { } } -/// A type that represents the recipient of a transaction output: a recipient address (and, for -/// unified addresses, the pool to which the payment is sent) in the case of an outgoing output, or an -/// internal account ID and the pool to which funds were sent in the case of a wallet-internal -/// output. +/// A type that represents the recipient of a transaction output: +/// * a recipient address; +/// * for external unified addresses, the pool to which the payment is sent; +/// * for ephemeral transparent addresses, the internal account ID and metadata about the outpoint; +/// * for wallet-internal outputs, the internal account ID and metadata about the note. #[derive(Debug, Clone)] -pub enum Recipient { +pub enum Recipient { External(ZcashAddress, PoolType), + EphemeralTransparent { + receiving_account: AccountId, + ephemeral_address: TransparentAddress, + outpoint_metadata: O, + }, InternalAccount { receiving_account: AccountId, external_address: Option, @@ -76,10 +82,22 @@ pub enum Recipient { }, } -impl Recipient { - pub fn map_internal_account_note B>(self, f: F) -> Recipient { +impl Recipient { + pub fn map_internal_account_note B>( + self, + f: F, + ) -> Recipient { match self { Recipient::External(addr, pool) => Recipient::External(addr, pool), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + } => Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + }, Recipient::InternalAccount { receiving_account, external_address, @@ -91,12 +109,48 @@ impl Recipient { }, } } + + pub fn map_ephemeral_transparent_outpoint B>( + self, + f: F, + ) -> Recipient { + match self { + Recipient::External(addr, pool) => Recipient::External(addr, pool), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + } => Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata: f(outpoint_metadata), + }, + Recipient::InternalAccount { + receiving_account, + external_address, + note, + } => Recipient::InternalAccount { + receiving_account, + external_address, + note, + }, + } + } } -impl Recipient> { - pub fn internal_account_note_transpose_option(self) -> Option> { +impl Recipient, O> { + pub fn internal_account_note_transpose_option(self) -> Option> { match self { Recipient::External(addr, pool) => Some(Recipient::External(addr, pool)), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + } => Some(Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + }), Recipient::InternalAccount { receiving_account, external_address, diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 6c3e8df808..615e22aefc 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -6,7 +6,28 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Notable changes +`zcash_client_sqlite` now supports TEX (transparent-source-only) addresses as specified +in ZIP 320. Sending to one or more TEX addresses will automatically create a multi-step +proposal that uses two transactions. + +In order to take advantage of this support, client wallets will need to be able to send +multiple transactions created from `zcash_client_backend::data_api::wallet::create_proposed_transactions`. +This API was added in `zcash_client_backend` 0.11.0 but previously could only return a +single transaction. + +**Note:** This feature changes the use of transparent addresses in ways that are relevant +to security and access to funds, and that may interact with other wallet behaviour. In +particular it exposes new ephemeral transparent addresses belonging to the wallet, which +need to be scanned in order to recover funds if the first transaction of the proposal is +mined but the second is not, or if someone (e.g. the TEX-address recipient) sends back +funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for details. + ### Changed +- `zcash_client_sqlite::error::SqliteClientError` has a new `ReachedGapLimit` and + `EphemeralAddressReuse` variants when the "transparent-inputs" feature is enabled. +- The result of the `v_tx_outputs` SQL query could now include transparent outputs + with unknown height. - MSRV is now 1.70.0. - `zcash_client_sqlite::error::SqliteClientError` has changed variants: - Removed `HdwalletError`. diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index c0a10cac4b..e9c8d687c7 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -16,7 +16,7 @@ use crate::PRUNING_DEPTH; #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::encoding::TransparentCodecError, - zcash_primitives::legacy::TransparentAddress, + zcash_primitives::{legacy::TransparentAddress, transaction::TxId}, }; /// The primary error type for the SQLite wallet backend. @@ -112,6 +112,17 @@ pub enum SqliteClientError { /// An error occurred in computing wallet balance BalanceError(BalanceError), + + /// The proposal cannot be constructed until transactions with previously reserved + /// ephemeral address outputs have been mined. + #[cfg(feature = "transparent-inputs")] + ReachedGapLimit, + + /// An ephemeral address (given in string form) would be reused, or incorrectly used as an + /// external address. If it is known to have been used in a previous transaction, the txid + /// is given. + #[cfg(feature = "transparent-inputs")] + EphemeralAddressReuse(String, Option), } impl error::Error for SqliteClientError { @@ -162,6 +173,12 @@ impl fmt::Display for SqliteClientError { SqliteClientError::ChainHeightUnknown => write!(f, "Chain height unknown; please call `update_chain_tip`"), SqliteClientError::UnsupportedPoolType(t) => write!(f, "Pool type is not currently supported: {}", t), SqliteClientError::BalanceError(e) => write!(f, "Balance error: {}", e), + #[cfg(feature = "transparent-inputs")] + SqliteClientError::ReachedGapLimit => write!(f, "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined."), + #[cfg(feature = "transparent-inputs")] + SqliteClientError::EphemeralAddressReuse(address_str, Some(txid)) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), + #[cfg(feature = "transparent-inputs")] + SqliteClientError::EphemeralAddressReuse(address_str, None) => write!(f, "The ephemeral address {address_str} would be reused, or incorrectly used as an external address."), } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index a429eee12b..8f976ed642 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -88,7 +88,10 @@ use { #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::wallet::TransparentAddressMetadata, - zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint}, + zcash_primitives::{ + legacy::TransparentAddress, + transaction::components::{OutPoint, TxOut}, + }, }; #[cfg(feature = "unstable")] @@ -523,13 +526,27 @@ impl, P: consensus::Parameters> WalletRead for W account: AccountId, max_height: BlockHeight, ) -> Result, Self::Error> { - wallet::transparent::get_transparent_address_balances( + wallet::transparent::get_transparent_balances( self.conn.borrow(), &self.params, account, max_height, ) } + + #[cfg(feature = "transparent-inputs")] + fn get_reserved_ephemeral_addresses( + &self, + account: Self::AccountId, + for_detection: bool, + ) -> Result>, Self::Error> { + wallet::transparent::get_reserved_ephemeral_addresses( + self.conn.borrow(), + &self.params, + account, + for_detection, + ) + } } impl WalletWrite for WalletDb { @@ -1084,6 +1101,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1103,6 +1121,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1133,6 +1152,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, account_id, tx_ref, output.index(), @@ -1165,6 +1185,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1184,6 +1205,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1215,6 +1237,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, account_id, tx_ref, output.index(), @@ -1267,6 +1290,11 @@ impl WalletWrite for WalletDb .enumerate() { if let Some(address) = txout.recipient_address() { + // TODO: we really want to only mark outputs when a transaction has been + // *reliably* mined. + #[cfg(feature = "transparent-inputs")] + wallet::transparent::mark_ephemeral_address_as_mined(wdb.conn.0, &wdb.params, &address, tx_ref).map_err(SqliteClientError::from)?; + let receiver = Receiver::Transparent(address); #[cfg(feature = "transparent-inputs")] @@ -1286,6 +1314,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, account_id, tx_ref, output_index, @@ -1352,7 +1381,13 @@ impl WalletWrite for WalletDb } for output in sent_tx.outputs() { - wallet::insert_sent_output(wdb.conn.0, tx_ref, *sent_tx.account_id(), output)?; + wallet::insert_sent_output( + wdb.conn.0, + &wdb.params, + tx_ref, + *sent_tx.account_id(), + output, + )?; match output.recipient() { Recipient::InternalAccount { @@ -1396,7 +1431,39 @@ impl WalletWrite for WalletDb None, )?; } - _ => (), + #[cfg(feature = "transparent-inputs")] + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + } => { + wallet::transparent::put_transparent_output( + wdb.conn.0, + &wdb.params, + outpoint_metadata, + &TxOut { + value: output.value(), + script_pubkey: ephemeral_address.script(), + }, + None, + ephemeral_address, + *receiving_account, + )?; + wallet::transparent::mark_ephemeral_address_as_used( + wdb, + ephemeral_address, + tx_ref, + )?; + } + #[cfg(feature = "transparent-inputs")] + Recipient::External(zcash_address, PoolType::Transparent) => { + // Always reject sending to one of our ephemeral addresses. + wallet::transparent::check_address_is_not_ephemeral( + wdb, + &zcash_address.encode(), + )?; + } + _ => {} } } @@ -1409,6 +1476,17 @@ impl WalletWrite for WalletDb wallet::truncate_to_height(wdb.conn.0, &wdb.params, block_height) }) } + + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + account_id: Self::AccountId, + n: u32, + ) -> Result, Self::Error> { + self.transactionally(|wdb| { + wallet::transparent::reserve_next_n_ephemeral_addresses(wdb, account_id, n) + }) + } } impl WalletCommitmentTrees for WalletDb { diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 833eb52b00..17dd6dfeb6 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -62,13 +62,14 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { - zcash_client_backend::{ - fees::TransactionBalance, proposal::Step, wallet::WalletTransparentOutput, + zcash_client_backend::wallet::WalletTransparentOutput, + zcash_primitives::transaction::{ + components::{OutPoint, TxOut}, + fees::zip317, }, - zcash_primitives::transaction::components::{OutPoint, TxOut}, }; -#[cfg(any(feature = "transparent-inputs", feature = "orchard"))] +#[cfg(feature = "orchard")] use zcash_client_backend::PoolType; pub(crate) type OutputRecoveryError = Error< @@ -301,14 +302,8 @@ pub(crate) fn send_single_step_proposed_transfer() { #[cfg(feature = "transparent-inputs")] pub(crate) fn send_multi_step_proposed_transfer() { - use std::collections::BTreeSet; - - use nonempty::NonEmpty; - use zcash_client_backend::{ - fees::ChangeValue, - proposal::{Proposal, StepOutput, StepOutputIndex}, - }; - use zcash_primitives::{legacy::keys::IncomingViewingKey, transaction::TxId}; + use zcash_address::ZcashAddress; + use zcash_client_backend::fees::ChangeValue; let mut st = TestBuilder::new() .with_block_cache() @@ -318,151 +313,168 @@ pub(crate) fn send_multi_step_proposed_transfer() { let account = st.test_account().cloned().unwrap(); let dfvk = T::test_account_fvk(&st); - // Add funds to the wallet in a single note + let fee_rule = StandardFeeRule::Zip317; + let input_selector = GreedyInputSelector::new( + standard::SingleOutputChangeStrategy::new(fee_rule, None, T::SHIELDED_PROTOCOL), + DustOutputPolicy::default(), + ); + + let add_funds = |st: &mut TestState<_>, value| { + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); + + assert_eq!( + block_max_scanned(&st.wallet().conn, &st.wallet().params) + .unwrap() + .unwrap() + .block_height(), + h + ); + assert_eq!(st.get_spendable_balance(account.account_id(), 1), value); + }; + let value = NonNegativeAmount::const_from_u64(100000); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h, 1); + let amount = NonNegativeAmount::const_from_u64(50000); + + let run_test = |st: &mut TestState<_>, expected_index| { + // Add funds to the wallet. + add_funds(st, value); + + let expected_step0_fee = (zip317::MARGINAL_FEE * 3).unwrap(); + let expected_step1_fee = zip317::MINIMUM_FEE; + let expected_ephemeral = (amount + expected_step1_fee).unwrap(); + let expected_step0_change = + (value - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); + assert!(expected_step0_change.is_positive()); + + // Generate a ZIP 320 proposal, sending to the wallet's default transparent address + // expressed as a TEX address. + let tex_addr = match account.usk().default_transparent_address().0 { + TransparentAddress::PublicKeyHash(data) => Address::Tex(data), + _ => unreachable!(), + }; + let request = zip321::TransactionRequest::new(vec![Payment::without_memo( + tex_addr.to_zcash_address(&st.network()), + amount, + )]) + .unwrap(); - // Spendable balance matches total balance - assert_eq!(st.get_total_balance(account.account_id()), value); - assert_eq!(st.get_spendable_balance(account.account_id(), 1), value); + let proposal = st + .propose_transfer( + account.account_id(), + &input_selector, + request, + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); - assert_eq!( - block_max_scanned(&st.wallet().conn, &st.wallet().params) - .unwrap() - .unwrap() - .block_height(), - h - ); + let steps: Vec<_> = proposal.steps().iter().cloned().collect(); + assert_eq!(steps.len(), 2); - // Generate a single-step proposal. Then, instead of executing that proposal, - // we will use its only step as the first step in a multi-step proposal that - // spends the first step's output. + assert_eq!( + steps[0].balance().proposed_change(), + [ + ChangeValue::shielded(T::SHIELDED_PROTOCOL, expected_step0_change, None), + ChangeValue::transparent((amount + expected_step1_fee).unwrap()), + ] + ); + assert_eq!(steps[0].balance().fee_required(), expected_step0_fee); + assert_eq!(steps[1].balance().proposed_change(), []); + assert_eq!(steps[1].balance().fee_required(), expected_step1_fee); - // The first step will deshield to the wallet's default transparent address - let to0 = Address::Transparent(account.usk().default_transparent_address().0); - let request0 = zip321::TransactionRequest::new(vec![Payment::without_memo( - to0.to_zcash_address(&st.network()), - NonNegativeAmount::const_from_u64(50000), + let create_proposed_result = st.create_proposed_transactions::( + account.usk(), + OvkPolicy::Sender, + &proposal, + ); + assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); + let txids = create_proposed_result.unwrap(); + + // Verify that the stored sent outputs match what we're expecting. + let mut stmt_sent = st + .wallet() + .conn + .prepare( + "SELECT value, to_address, ephemeral_addresses.address, ephemeral_addresses.address_index + FROM sent_notes + JOIN transactions ON transactions.id_tx = sent_notes.tx + LEFT JOIN ephemeral_addresses ON ephemeral_addresses.used_in_tx = sent_notes.tx + WHERE transactions.txid = ? + ORDER BY value", + ) + .unwrap(); + + // Check that there are sent outputs with the correct values. + let confirmed_sent: Vec> = txids + .iter() + .map(|sent_txid| { + stmt_sent + .query(rusqlite::params![sent_txid.as_ref()]) + .unwrap() + .mapped(|row| { + let v: u32 = row.get(0)?; + let to_address: Option = row.get(1)?; + let ephemeral_address: Option = row.get(2)?; + let address_index: Option = row.get(3)?; + Ok((u64::from(v), to_address, ephemeral_address, address_index)) + }) + .collect::, _>>() + .unwrap() + }) + .collect(); + + assert!(expected_step0_change < expected_ephemeral); + assert_eq!(confirmed_sent.len(), 2); + assert_eq!(confirmed_sent[0].len(), 2); + assert_eq!( + confirmed_sent[0][0].0, + u64::try_from(expected_step0_change).unwrap() + ); + let (ephemeral_v, to_addr, ephemeral_addr, index) = confirmed_sent[0][1].clone(); + assert_eq!(ephemeral_v, u64::try_from(expected_ephemeral).unwrap()); + assert!(to_addr.is_some()); + assert_eq!(ephemeral_addr, to_addr); + assert_eq!(index, Some(expected_index)); + + assert_eq!(confirmed_sent[1].len(), 1); + assert_matches!( + confirmed_sent[1][0].clone(), + (sent_v, sent_to_addr, None, None) + if sent_v == u64::try_from(amount).unwrap() && sent_to_addr == Some(tex_addr.encode(&st.wallet().params))); + + (ephemeral_addr.unwrap(), txids.head) + }; + + // Each transfer should use a different ephemeral address. + let (ephemeral0, txid0) = run_test(&mut st, 0); + let (ephemeral1, _) = run_test(&mut st, 1); + assert!(ephemeral0 != ephemeral1); + + add_funds(&mut st, value); + let request = zip321::TransactionRequest::new(vec![Payment::without_memo( + ZcashAddress::try_from_encoded(&ephemeral0).expect("valid address"), + amount, )]) .unwrap(); - let fee_rule = StandardFeeRule::Zip317; - let input_selector = GreedyInputSelector::new( - standard::SingleOutputChangeStrategy::new(fee_rule, None, T::SHIELDED_PROTOCOL), - DustOutputPolicy::default(), - ); - let proposal0 = st + // Attempting to use the same address again should cause an error. + let proposal = st .propose_transfer( account.account_id(), &input_selector, - request0, + request, NonZeroU32::new(1).unwrap(), ) .unwrap(); - let min_target_height = proposal0.min_target_height(); - let step0 = &proposal0.steps().head; - - assert_eq!( - step0.balance().proposed_change(), - [ChangeValue::shielded( - T::SHIELDED_PROTOCOL, - NonNegativeAmount::const_from_u64(35000), - None - )] - ); - assert_eq!( - step0.balance().fee_required(), - NonNegativeAmount::const_from_u64(15000) - ); - - // We'll use an internal transparent address that hasn't been added to the wallet - // to simulate an external transparent recipient. - let to1 = Address::Transparent( - account - .usk() - .transparent() - .to_account_pubkey() - .derive_internal_ivk() - .unwrap() - .default_address() - .0, - ); - let request1 = zip321::TransactionRequest::new(vec![Payment::without_memo( - to1.to_zcash_address(&st.network()), - NonNegativeAmount::const_from_u64(40000), - )]) - .unwrap(); - - let step1 = Step::from_parts( - &[step0.clone()], - request1, - [(0, PoolType::TRANSPARENT)].into_iter().collect(), - vec![], - None, - vec![StepOutput::new(0, StepOutputIndex::Payment(0))], - TransactionBalance::new(vec![], NonNegativeAmount::const_from_u64(10000)).unwrap(), - false, - ) - .unwrap(); - - let proposal = Proposal::multi_step( - fee_rule, - min_target_height, - NonEmpty::from_vec(vec![step0.clone(), step1]).unwrap(), - ) - .unwrap(); - let create_proposed_result = st.create_proposed_transactions::( account.usk(), OvkPolicy::Sender, &proposal, ); - assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); - let txids = create_proposed_result.unwrap(); - - // Verify that the stored sent outputs match what we're expecting - let mut stmt_sent = st - .wallet() - .conn - .prepare( - "SELECT value - FROM sent_notes - JOIN transactions ON transactions.id_tx = sent_notes.tx - WHERE transactions.txid = ?", - ) - .unwrap(); - - // Check that there are sent outputs with the correct values for each transaction. - let confirmed_sent: Vec> = txids - .iter() - .map(|sent_txid| { - stmt_sent - .query(rusqlite::params![sent_txid.as_ref()]) - .unwrap() - .mapped(|row| { - let value: u32 = row.get(0)?; - Ok((sent_txid, value)) - }) - .collect::, _>>() - .unwrap() - }) - .collect(); - - assert_eq!( - confirmed_sent.get(0), - Some( - &[(&txids[0], 35000), (&txids[0], 50000)] - .iter() - .cloned() - .collect() - ), - ); - assert_eq!( - confirmed_sent.get(1), - Some(&[(&txids[1], 40000)].iter().cloned().collect()), - ); + assert_matches!( + &create_proposed_result, + Err(Error::DataSource(SqliteClientError::EphemeralAddressReuse(addr, Some(txid)))) if addr == &ephemeral0 && txid == &txid0); } #[allow(deprecated)] diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index d0cf7faf45..a1f9b15cfa 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -68,6 +68,7 @@ use incrementalmerkletree::{Marking, Retention}; use rusqlite::{self, named_params, OptionalExtension}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; +use zcash_keys::encoding::encode_transparent_address_p; use zip32::fingerprint::SeedFingerprint; use std::collections::{HashMap, HashSet}; @@ -102,7 +103,7 @@ use zcash_primitives::{ memo::{Memo, MemoBytes}, merkle_tree::read_commitment_tree, transaction::{ - components::{amount::NonNegativeAmount, Amount}, + components::{amount::NonNegativeAmount, Amount, OutPoint}, Transaction, TransactionData, TxId, }, }; @@ -2136,11 +2137,21 @@ pub(crate) fn put_tx_data( // A utility function for creation of parameters for use in `insert_sent_output` // and `put_sent_output` -fn recipient_params( - to: &Recipient, +fn recipient_params( + params: &P, + to: &Recipient, ) -> (Option, Option, PoolType) { match to { Recipient::External(addr, pool) => (Some(addr.encode()), None, *pool), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + .. + } => ( + Some(encode_transparent_address_p(params, ephemeral_address)), + Some(*receiving_account), + PoolType::TRANSPARENT, + ), Recipient::InternalAccount { receiving_account, external_address, @@ -2154,8 +2165,9 @@ fn recipient_params( } /// Records information about a transaction output that your wallet created. -pub(crate) fn insert_sent_output( +pub(crate) fn insert_sent_output( conn: &rusqlite::Connection, + params: &P, tx_ref: i64, from_account: AccountId, output: &SentTransactionOutput, @@ -2169,7 +2181,7 @@ pub(crate) fn insert_sent_output( :to_address, :to_account_id, :value, :memo)", )?; - let (to_address, to_account_id, pool_type) = recipient_params(output.recipient()); + let (to_address, to_account_id, pool_type) = recipient_params(params, output.recipient()); let sql_args = named_params![ ":tx": &tx_ref, ":output_pool": &pool_code(pool_type), @@ -2198,12 +2210,13 @@ pub(crate) fn insert_sent_output( /// - If `recipient` is an internal account, `output_index` is an index into the Sapling outputs of /// the transaction. #[allow(clippy::too_many_arguments)] -pub(crate) fn put_sent_output( +pub(crate) fn put_sent_output( conn: &rusqlite::Connection, + params: &P, from_account: AccountId, tx_ref: i64, output_index: usize, - recipient: &Recipient, + recipient: &Recipient, value: NonNegativeAmount, memo: Option<&MemoBytes>, ) -> Result<(), SqliteClientError> { @@ -2222,7 +2235,7 @@ pub(crate) fn put_sent_output( memo = IFNULL(:memo, memo)", )?; - let (to_address, to_account_id, pool_type) = recipient_params(recipient); + let (to_address, to_account_id, pool_type) = recipient_params(params, recipient); let sql_args = named_params![ ":tx": &tx_ref, ":output_pool": &pool_code(pool_type), diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 6e54eb3dcf..e6d840e1d0 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -76,6 +76,43 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( "account_id" ASC )"#; +/// Stores ephemeral transparent addresses used for ZIP 320. For each account, these addresses are +/// allocated sequentially by address index under custom scope 2 at the "change" level of the BIP 32 +/// address hierarchy. Only "reserved" ephemeral addresses, that is addresses that have been allocated +/// for use in a ZIP 320 transaction proposal, are stored in the table. Addresses are never removed. +/// New ones should only be reserved via the `WalletWrite::reserve_next_n_ephemeral_addresses` API. +/// All of the addresses in the table should be scanned for incoming funds. +/// +/// ### Columns +/// - `address` contains the string (Base58Check) encoding of a transparent P2PKH address. +/// - `used_in_tx` indicates that the address has been used by this wallet in a transaction (which +/// has not necessarily been mined yet). This should only be set once, when the txid is known. +/// - `mined_in_tx` is non-null iff the address has been observed in a mined transaction (which may +/// have been sent by this wallet or another one using the same seed, or by a TEX address recipient +/// sending back the funds). This is used to advance the "gap limit", as well as to heuristically +/// reduce the chance of address reuse collisions with another wallet using the same seed. +/// +/// Note that the fact that `used_in_tx` and `mined_in_tx` reference specific transactions is primarily +/// a debugging aid. We only really care which addresses have been used, and whether we can allocate +/// a new address within the gap limit. +pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" +CREATE TABLE ephemeral_addresses ( + account_id INTEGER NOT NULL, + address_index INTEGER NOT NULL, + address TEXT NOT NULL, + used_in_tx INTEGER, + mined_in_tx INTEGER, + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), + FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), + PRIMARY KEY (account_id, address_index) +) WITHOUT ROWID"#; +// "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. +pub(super) const INDEX_EPHEMERAL_ADDRESSES_ADDRESS: &str = r#" +CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( + address ASC +)"#; + /// Stores information about every block that the wallet has scanned. /// /// Note that this table does not contain any rows for blocks that the wallet might have diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 08838d0486..cfe6f73208 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -176,6 +176,14 @@ fn sqlite_client_error_to_wallet_migration_error(e: SqliteClientError) -> Wallet SqliteClientError::ChainHeightUnknown => { unreachable!("we don't call methods that require a known chain height") } + #[cfg(feature = "transparent-inputs")] + SqliteClientError::ReachedGapLimit => { + unreachable!("we don't do ephemeral address tracking") + } + #[cfg(feature = "transparent-inputs")] + SqliteClientError::EphemeralAddressReuse(_, _) => { + unreachable!("we don't do ephemeral address tracking") + } } } @@ -376,6 +384,7 @@ mod tests { db::TABLE_ACCOUNTS, db::TABLE_ADDRESSES, db::TABLE_BLOCKS, + db::TABLE_EPHEMERAL_ADDRESSES, db::TABLE_NULLIFIER_MAP, db::TABLE_ORCHARD_RECEIVED_NOTE_SPENDS, db::TABLE_ORCHARD_RECEIVED_NOTES, @@ -413,6 +422,7 @@ mod tests { db::INDEX_ACCOUNTS_UIVK, db::INDEX_HD_ACCOUNT, db::INDEX_ADDRESSES_ACCOUNTS, + db::INDEX_EPHEMERAL_ADDRESSES_ADDRESS, db::INDEX_NF_MAP_LOCATOR_IDX, db::INDEX_ORCHARD_RECEIVED_NOTES_ACCOUNT, db::INDEX_ORCHARD_RECEIVED_NOTES_TX, diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1f4befd996..f8c2cf7b7f 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -3,6 +3,7 @@ mod add_transaction_views; mod add_utxo_account; mod addresses_table; mod ensure_orchard_ua_receiver; +mod ephemeral_addresses; mod full_account_ids; mod initial_setup; mod nullifier_map; @@ -66,6 +67,8 @@ pub(super) fn all_migrations( // orchard_received_notes // / \ // ensure_orchard_ua_receiver utxos_to_txos + // | + // ephemeral_addresses vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -116,6 +119,7 @@ pub(super) fn all_migrations( params: params.clone(), }), Box::new(utxos_to_txos::Migration), + Box::new(ephemeral_addresses::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs new file mode 100644 index 0000000000..bab03be761 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -0,0 +1,57 @@ +//! The migration that records ephemeral addresses for each account. +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::utxos_to_txos; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0e1d4274_1f8e_44e2_909d_689a4bc2967b); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [utxos_to_txos::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Record ephemeral addresses for each account." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch( + "CREATE TABLE ephemeral_addresses ( + account_id INTEGER NOT NULL, + address_index INTEGER NOT NULL, + address TEXT NOT NULL, + used_in_tx INTEGER, + mined_in_tx INTEGER, + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), + FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), + PRIMARY KEY (account_id, address_index) + ) WITHOUT ROWID; + CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( + address ASC + );", + )?; + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index cd1f768455..722c609bb5 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -367,7 +367,7 @@ mod tests { usk0.transparent() .derive_external_secret_key(NonHardenedChildIndex::ZERO) .unwrap(), - transparent::OutPoint::new([1; 32], 0), + transparent::OutPoint::fake(), transparent::TxOut { value: NonNegativeAmount::const_from_u64(EXTERNAL_VALUE + INTERNAL_VALUE), script_pubkey: usk0 diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 181bc30c8b..70b21856f2 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -5,23 +5,28 @@ use std::collections::HashMap; use std::collections::HashSet; use zcash_client_backend::data_api::AccountBalance; use zcash_keys::address::Address; +use zcash_keys::keys::AddressGenerationError; use zip32::{DiversifierIndex, Scope}; use zcash_address::unified::{Encoding, Ivk, Uivk}; use zcash_client_backend::wallet::{TransparentAddressMetadata, WalletTransparentOutput}; -use zcash_keys::encoding::AddressCodec; +use zcash_keys::encoding::{encode_transparent_address_p, AddressCodec}; use zcash_primitives::{ legacy::{ - keys::{IncomingViewingKey, NonHardenedChildIndex}, + keys::{EphemeralIvk, IncomingViewingKey, NonHardenedChildIndex, TransparentKeyScope}, Script, TransparentAddress, }, - transaction::components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, + transaction::{ + components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, + TxId, + }, }; use zcash_protocol::consensus::{self, BlockHeight}; use crate::{error::SqliteClientError, AccountId, UtxoId}; +use crate::{SqlTransaction, WalletDb}; -use super::{chain_tip_height, get_account_ids}; +use super::{chain_tip_height, get_account, get_account_ids}; pub(crate) fn detect_spending_accounts<'a>( conn: &Connection, @@ -239,6 +244,9 @@ pub(crate) fn get_unspent_transparent_output( result } +/// Returns spendable transparent outputs that have been received by this wallet at the given +/// transparent address, as outputs of transactions in blocks mined at a height less than or +/// equal to the provided `max_height`. pub(crate) fn get_spendable_transparent_outputs( conn: &rusqlite::Connection, params: &P, @@ -301,10 +309,13 @@ pub(crate) fn get_spendable_transparent_outputs( Ok(utxos) } -/// Returns the unspent balance for each transparent address associated with the specified account, -/// such that the block that included the transaction was mined at a height less than or equal to -/// the provided `summary_height`. -pub(crate) fn get_transparent_address_balances( +/// Returns a mapping from each transparent receiver associated with the specified account +/// to its not-yet-shielded UTXO balance, including only the effects of transactions mined +/// at a block height less than or equal to `summary_height`. +/// +/// Only non-ephemeral transparent receivers with a non-zero balance at the summary height +/// will be included. +pub(crate) fn get_transparent_balances( conn: &rusqlite::Connection, params: &P, account: AccountId, @@ -438,63 +449,107 @@ pub(crate) fn put_received_transparent_utxo( params: &P, output: &WalletTransparentOutput, ) -> Result { + if let Some(receiving_account) = find_account_for_transparent_output(conn, params, output)? { + put_transparent_output( + conn, + params, + output.outpoint(), + output.txout(), + Some(output.height()), + output.recipient_address(), + receiving_account, + ) + } else { + // The UTXO was not for any of our transparent addresses. + Err(SqliteClientError::AddressNotRecognized( + *output.recipient_address(), + )) + } +} + +/// Attempts to determine the account that received the given transparent output. +/// +/// The following three locations in the wallet's key tree are searched: +/// - Transparent receivers that have been generated as part of a Unified Address. +/// - Transparent ephemeral addresses that have been reserved. +/// - "Legacy transparent addresses" (at BIP 44 address index 0 within an account). +/// +/// Returns `Ok(None)` if the transparent output's recipient address is not in any of the +/// above locations. This means the wallet considers the output "not interesting". +pub(crate) fn find_account_for_transparent_output( + conn: &rusqlite::Connection, + params: &P, + output: &WalletTransparentOutput, +) -> Result, SqliteClientError> { let address_str = output.recipient_address().encode(params); - let account_id = conn + + if let Some(account_id) = conn .query_row( "SELECT account_id FROM addresses WHERE cached_transparent_receiver_address = :address", named_params![":address": &address_str], |row| Ok(AccountId(row.get(0)?)), ) - .optional()?; + .optional()? + { + return Ok(Some(account_id)); + } - if let Some(account) = account_id { - Ok(put_transparent_output(conn, params, output, account)?) - } else { - // If the UTXO is received at the legacy transparent address (at BIP 44 address - // index 0 within its particular account, which we specifically ensure is returned - // from `get_transparent_receivers`), there may be no entry in the addresses table - // that can be used to tie the address to a particular account. In this case, we - // look up the legacy address for each account in the wallet, and check whether it - // matches the address for the received UTXO; if so, insert/update it directly. - get_account_ids(conn)? - .into_iter() - .find_map( - |account| match get_legacy_transparent_address(params, conn, account) { - Ok(Some((legacy_taddr, _))) if &legacy_taddr == output.recipient_address() => { - Some( - put_transparent_output(conn, params, output, account) - .map_err(SqliteClientError::from), - ) - } - Ok(_) => None, - Err(e) => Some(Err(e)), - }, - ) - // The UTXO was not for any of the legacy transparent addresses. - .unwrap_or_else(|| { - Err(SqliteClientError::AddressNotRecognized( - *output.recipient_address(), - )) - }) + // Note that this does not search ephemeral addresses that have not yet been reserved. + if let Some(account_id) = conn + .query_row( + "SELECT account_id FROM ephemeral_addresses WHERE address = :address", + named_params![":address": &address_str], + |row| Ok(AccountId(row.get(0)?)), + ) + .optional()? + { + return Ok(Some(account_id)); } + + // If the UTXO is received at the legacy transparent address (at BIP 44 address + // index 0 within its particular account, which we specifically ensure is returned + // from `get_transparent_receivers`), there may be no entry in the addresses table + // that can be used to tie the address to a particular account. In this case, we + // look up the legacy address for each account in the wallet, and check whether it + // matches the address for the received UTXO. + for account_id in get_account_ids(conn)? { + if let Some((legacy_taddr, _)) = get_legacy_transparent_address(params, conn, account_id)? { + if &legacy_taddr == output.recipient_address() { + return Ok(Some(account_id)); + } + } + } + Ok(None) } +/// Add a transparent output relevant to this wallet to the database. +/// +/// `output_height` may be None if this is an ephemeral output from a +/// transaction we created, that we do not yet know to have been mined. pub(crate) fn put_transparent_output( conn: &rusqlite::Connection, params: &P, - output: &WalletTransparentOutput, - received_by_account: AccountId, -) -> Result { + outpoint: &OutPoint, + txout: &TxOut, + output_height: Option, + address: &TransparentAddress, + receiving_account: AccountId, +) -> Result { + let output_height = output_height.map(u32::from); + // Check whether we have an entry in the blocks table for the output height; // if not, the transaction will be updated with its mined height when the // associated block is scanned. - let block = conn - .query_row( - "SELECT height FROM blocks WHERE height = :height", - named_params![":height": &u32::from(output.height())], - |row| row.get::<_, u32>(0), - ) - .optional()?; + let block = match output_height { + Some(height) => conn + .query_row( + "SELECT height FROM blocks WHERE height = :height", + named_params![":height": height], + |row| row.get::<_, u32>(0), + ) + .optional()?, + None => None, + }; let id_tx = conn.query_row( "INSERT INTO transactions (txid, block, mined_height) @@ -504,9 +559,9 @@ pub(crate) fn put_transparent_output( mined_height = :mined_height RETURNING id_tx", named_params![ - ":txid": &output.outpoint().hash().to_vec(), + ":txid": &outpoint.hash().to_vec(), ":block": block, - ":mined_height": u32::from(output.height()) + ":mined_height": output_height ], |row| row.get::<_, i64>(0), )?; @@ -533,15 +588,271 @@ pub(crate) fn put_transparent_output( let sql_args = named_params![ ":transaction_id": id_tx, - ":output_index": &output.outpoint().n(), - ":account_id": received_by_account.0, - ":address": &output.recipient_address().encode(params), - ":script": &output.txout().script_pubkey.0, - ":value_zat": &i64::from(Amount::from(output.txout().value)), - ":height": &u32::from(output.height()), + ":output_index": &outpoint.n(), + ":account_id": receiving_account.0, + ":address": &address.encode(params), + ":script": &txout.script_pubkey.0, + ":value_zat": &i64::from(Amount::from(txout.value)), + ":height": output_height, ]; - stmt_upsert_transparent_output.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) + let utxo_id = stmt_upsert_transparent_output + .query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId))?; + Ok(utxo_id) +} + +/// If `address` is one of our ephemeral addresses, mark it as having an output +/// in a transaction that we have just created. This has no effect if `address` is +/// not one of our ephemeral addresses. +/// +/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was +/// already used. +pub(crate) fn mark_ephemeral_address_as_used( + wdb: &mut WalletDb, P>, + ephemeral_address: &TransparentAddress, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let address_str = encode_transparent_address_p(&wdb.params, ephemeral_address); + ephemeral_address_check_internal(wdb, &address_str, false)?; + + wdb.conn.0.execute( + "UPDATE ephemeral_addresses SET used_in_tx = :used_in_tx WHERE address = :address", + named_params![":used_in_tx": &tx_ref, ":address": address_str], + )?; + Ok(()) +} + +/// Returns a `SqliteClientError::EphemeralAddressReuse` error if `address` is +/// an ephemeral transparent address. +pub(crate) fn check_address_is_not_ephemeral( + wdb: &mut WalletDb, P>, + address_str: &str, +) -> Result<(), SqliteClientError> { + ephemeral_address_check_internal(wdb, address_str, true) +} + +/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was +/// already used. If `reject_all_ephemeral` is set, return an error if the address +/// is ephemeral at all, regardless of reuse. +fn ephemeral_address_check_internal( + wdb: &mut WalletDb, P>, + address_str: &str, + reject_all_ephemeral: bool, +) -> Result<(), SqliteClientError> { + let res = wdb + .conn + .0 + .query_row( + "SELECT t.txid FROM ephemeral_addresses + LEFT OUTER JOIN transactions t + ON t.id_tx = COALESCE(used_in_tx, mined_in_tx) + WHERE address = :address", + named_params![":address": address_str], + |row| row.get::<_, Option>>(0), + ) + .optional()?; + + match res { + Some(Some(txid_bytes)) => { + let txid = TxId::from_bytes( + txid_bytes + .try_into() + .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, + ); + Err(SqliteClientError::EphemeralAddressReuse( + address_str.to_owned(), + Some(txid), + )) + } + Some(None) if reject_all_ephemeral => Err(SqliteClientError::EphemeralAddressReuse( + address_str.to_owned(), + None, + )), + _ => Ok(()), + } +} + +/// If `address` is one of our ephemeral addresses, mark it as having an output +/// in the given mined transaction (which may or may not be a transaction we sent). +/// This has no effect if `address` is not one of our ephemeral addresses. +pub(crate) fn mark_ephemeral_address_as_mined( + conn: &rusqlite::Connection, + params: &P, + address: &TransparentAddress, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let address_str = encode_transparent_address_p(params, address); + + conn.execute( + "UPDATE ephemeral_addresses SET mined_in_tx = :mined_in_tx WHERE address = :address", + named_params![":mined_in_tx": &tx_ref, ":address": address_str], + )?; + Ok(()) +} + +/// Returns the ephemeral transparent IVK for a given account ID. +pub(crate) fn get_ephemeral_ivk( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, +) -> Result { + use zcash_client_backend::data_api::Account; + + Ok(get_account(conn, params, account_id)? + .ok_or(SqliteClientError::AccountUnknown)? + .ufvk() + .and_then(|ufvk| ufvk.transparent()) + .ok_or(SqliteClientError::UnknownZip32Derivation)? + .derive_ephemeral_ivk()?) +} + +// Same as Bitcoin. +const GAP_LIMIT: i32 = 20; + +const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2) { + Some(s) => s, + None => unreachable!(), +}; + +/// Returns a vector with all ephemeral transparent addresses potentially belonging to this wallet. +/// If `for_detection` is true, this includes addresses for an additional GAP_LIMIT indices. +pub(crate) fn get_reserved_ephemeral_addresses( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, + for_detection: bool, +) -> Result>, SqliteClientError> { + let mut stmt = conn.prepare( + "SELECT address, address_index FROM ephemeral_addresses WHERE account_id = :account ORDER BY address_index", + )?; + let mut rows = stmt.query(named_params! { ":account": account_id.0 })?; + + let mut result = HashMap::new(); + let mut first_unused_index: Option = Some(0); + + while let Some(row) = rows.next()? { + let addr_str: String = row.get(0)?; + let raw_index: u32 = row.get(1)?; + first_unused_index = i32::try_from(raw_index) + .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? + .checked_add(1); + let address_index = NonHardenedChildIndex::from_index(raw_index).expect("just checked"); + result.insert( + TransparentAddress::decode(params, &addr_str)?, + Some(TransparentAddressMetadata::new( + EPHEMERAL_SCOPE, + address_index, + )), + ); + } + + if for_detection { + if let Some(first) = first_unused_index { + let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; + + for index in first..=first.saturating_add(GAP_LIMIT - 1) { + let address_index = + NonHardenedChildIndex::from_index(index as u32).expect("valid index"); + result.insert( + ephemeral_ivk.derive_address(address_index)?, + Some(TransparentAddressMetadata::new( + EPHEMERAL_SCOPE, + address_index, + )), + ); + } + } + } + Ok(result) +} + +/// Returns a vector with the next `n` previously unreserved ephemeral addresses for +/// the given account. +/// +/// # Errors +/// +/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. +/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and +/// it is not possible to derive new addresses for it. +/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses +/// within the gap limit after the last address in this account that is known to have an +/// output in a mined transaction. +/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, +/// if the limit on transparent address indices has been reached. +pub(crate) fn reserve_next_n_ephemeral_addresses( + wdb: &mut WalletDb, P>, + account_id: AccountId, + n: u32, +) -> Result, SqliteClientError> { + if n == 0 { + return Ok(vec![]); + } + assert!(n > 0); + + let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; + + let last_gap_index: i32 = wdb + .conn + .0 + .query_row( + "SELECT address_index FROM ephemeral_addresses + WHERE account_id = :account_id AND mined_in_tx IS NOT NULL + ORDER BY address_index DESC LIMIT 1", + named_params![":account_id": account_id.0], + |row| row.get::<_, u32>(0), + ) + .optional()? + .map_or(Ok(-1i32), |i| { + i32::try_from(i).map_err(|e| SqliteClientError::CorruptedData(e.to_string())) + })? + .saturating_add(GAP_LIMIT); + + let (first_index, last_index) = wdb + .conn + .0 + .query_row( + "SELECT address_index FROM ephemeral_addresses + WHERE account_id = :account_id + ORDER BY address_index DESC LIMIT 1", + named_params![":account_id": account_id.0], + |row| row.get::<_, u32>(0), + ) + .optional()? + .map_or(Ok(-1i32), |i| { + i32::try_from(i).map_err(|e| SqliteClientError::CorruptedData(e.to_string())) + }) + .map(|i: i32| i.checked_add(1).zip(i.checked_add(n.try_into().ok()?)))? + .ok_or(SqliteClientError::AddressGeneration( + AddressGenerationError::DiversifierSpaceExhausted, + ))?; + + assert!(last_index >= first_index); + if last_index > last_gap_index { + return Err(SqliteClientError::ReachedGapLimit); + } + + // used_in_tx and mined_in_tx are initially NULL + let mut stmt_insert_ephemeral_address = wdb.conn.0.prepare_cached( + "INSERT INTO ephemeral_addresses (account_id, address_index, address) + VALUES (:account_id, :address_index, :address)", + )?; + + (first_index..=last_index) + .map(|address_index| { + let child = NonHardenedChildIndex::from_index(address_index as u32) + .expect("valid by construction"); + let address = ephemeral_ivk.derive_address(child)?; + stmt_insert_ephemeral_address.execute(named_params![ + ":account_id": account_id.0, + ":address_index": address_index, + ":address": encode_transparent_address_p(&wdb.params, &address) + ])?; + Ok(( + address, + TransparentAddressMetadata::new(EPHEMERAL_SCOPE, child), + )) + }) + .collect() } #[cfg(test)] diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 9e2d78a6f8..259d505529 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -11,6 +11,9 @@ and this library adheres to Rust's notion of - `impl From for bip32::ChildNumber` - `impl From for bip32::ChildNumber` - `impl TryFrom for NonHardenedChildIndex` + - `EphemeralIvk` + - `AccountPubKey::derive_ephemeral_ivk` + - `TransparentKeyScope::custom` is now `const`. - `zcash_primitives::legacy::Script::serialized_size` - `zcash_primitives::transaction::fees::transparent`: - `InputSize` diff --git a/zcash_primitives/src/legacy/keys.rs b/zcash_primitives/src/legacy/keys.rs index 8357762cc1..346b87e07a 100644 --- a/zcash_primitives/src/legacy/keys.rs +++ b/zcash_primitives/src/legacy/keys.rs @@ -21,7 +21,7 @@ use super::TransparentAddress; pub struct TransparentKeyScope(u32); impl TransparentKeyScope { - pub fn custom(i: u32) -> Option { + pub const fn custom(i: u32) -> Option { if i < (1 << 31) { Some(TransparentKeyScope(i)) } else { @@ -223,6 +223,14 @@ impl AccountPubKey { .map(InternalIvk) } + /// Derives the public key at the "ephemeral" path + /// `m/44'/'/'/2`. + pub fn derive_ephemeral_ivk(&self) -> Result { + self.0 + .derive_child(ChildNumber::new(2, false)?) + .map(EphemeralIvk) + } + /// Derives the internal ovk and external ovk corresponding to this /// transparent fvk. As specified in [ZIP 316][transparent-ovk]. /// @@ -405,6 +413,27 @@ impl private::SealedChangeLevelKey for InternalIvk { impl IncomingViewingKey for InternalIvk {} +/// An incoming viewing key at the "ephemeral" path +/// `m/44'/'/'/2`. +/// +/// This allows derivation of ephemeral addresses for use within the wallet. +#[derive(Clone, Debug)] +pub struct EphemeralIvk(ExtendedPublicKey); + +impl private::SealedChangeLevelKey for EphemeralIvk { + const SCOPE: TransparentKeyScope = TransparentKeyScope(2); + + fn extended_pubkey(&self) -> &ExtendedPublicKey { + &self.0 + } + + fn from_extended_pubkey(key: ExtendedPublicKey) -> Self { + EphemeralIvk(key) + } +} + +impl IncomingViewingKey for EphemeralIvk {} + /// Internal outgoing viewing key used for autoshielding. pub struct InternalOvk([u8; 32]); diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 8304c19f80..b4f0c69507 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -538,6 +538,8 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< /// /// This fee is a function of the spends and outputs that have been added to the builder, /// pursuant to the specified [`FeeRule`]. + /// + /// Any ephemeral inputs or outputs are *not* taken into account. pub fn get_fee( &self, fee_rule: &FR, @@ -1003,7 +1005,7 @@ mod tests { .add_transparent_input( tsk.derive_external_secret_key(NonHardenedChildIndex::ZERO) .unwrap(), - OutPoint::new([0u8; 32], 1), + OutPoint::fake(), prev_coin, ) .unwrap(); From 0f3de63ae15a5dbfc9d449cfd2c98881cb45e4b1 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 22 Jun 2024 13:25:56 +0100 Subject: [PATCH 02/56] Apply documentation suggestions from code review. Co-authored-by: str4d Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 4 ++- zcash_client_backend/src/data_api.rs | 27 ++++++++++++++++++- zcash_client_backend/src/fees.rs | 6 +++++ zcash_client_sqlite/src/error.rs | 3 ++- zcash_client_sqlite/src/lib.rs | 3 ++- zcash_client_sqlite/src/wallet/transparent.rs | 13 +++++++++ 6 files changed, 52 insertions(+), 4 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 0e6fc3e339..7c7eaeb847 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -32,6 +32,8 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - `ChangeValue::{transparent, shielded}` - `sapling::EmptyBundleView` - `orchard::EmptyBundleView` +- `zcash_client_backend::proposal`: + - `impl Hash for {StepOutput, StepOutputIndex}` - `zcash_client_backend::scanning`: - `testing` module - `zcash_client_backend::sync` module, behind the `sync` feature flag. @@ -68,7 +70,7 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail shielded pool. - `ChangeStrategy::compute_balance`: this trait method has two additional parameters when the "transparent-inputs" feature is enabled. These are - used to specify amounts of additional transparent inputs and outputs, + used to specify amounts of additional transparent P2PKH inputs and outputs, for which `InputView` or `OutputView` instances may not be available. Empty slices can be passed to obtain the previous behaviour. - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index dd1f6f6eb4..234632ede9 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -896,6 +896,9 @@ pub trait WalletRead { /// The set contains all non-ephemeral transparent receivers that are known to have /// been derived under this account. Wallets should scan the chain for UTXOs sent to /// these receivers. + /// + /// Use [`Self::get_reserved_ephemeral_addresses`] to obtain the ephemeral transparent + /// receivers. #[cfg(feature = "transparent-inputs")] fn get_transparent_receivers( &self, @@ -923,7 +926,29 @@ pub trait WalletRead { /// /// The set contains all ephemeral transparent receivers that are known to have /// been derived under this account. Wallets should scan the chain for UTXOs sent to - /// these receivers. + /// these receivers, but do not need to do so regularly. Under expected usage, outputs + /// would only be detected with these receivers in the following situations: + /// + /// - This wallet created a payment to a ZIP 320 (TEX) address, but the second + /// transaction (that spent the output sent to the ephemeral address) did not get + /// mined before it expired. + /// - In this case the output will already be known to the wallet (because it + /// stores the transactions that it creates). + /// + /// - Another wallet app using the same seed phrase created a payment to a ZIP 320 + /// address, and this wallet queried for the ephemeral UTXOs after the first + /// transaction was mined but before the second transaction was mined. + /// - In this case, the output should not be considered unspent until the expiry + /// height of the transaction it was received in has passed. Wallets creating + /// payments to TEX addresses generally set the same expiry height for the first + /// and second transactions, meaning that this wallet does not need to observe + /// the second transaction to determine when it would have expired. + /// + /// - A TEX address recipient decided to return funds that the wallet had sent to + /// them. + /// + /// In all cases, the wallet should re-shield the unspent outputs, in a separate + /// transaction per ephemeral address, before re-spending the funds. #[cfg(feature = "transparent-inputs")] fn get_reserved_ephemeral_addresses( &self, diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index fcdbc9c524..3bf99f9fc4 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -318,6 +318,12 @@ pub trait ChangeStrategy { /// change outputs recommended by this operation. If insufficient funds are available to /// supply the requested outputs and required fees, implementations should return /// [`ChangeError::InsufficientFunds`]. + /// + #[cfg_attr( + feature = "transparent-inputs", + doc = "The `ephemeral_input_amounts` and `ephemeral_output_amounts` parameters + specify the amounts of additional transparent P2PKH inputs and outputs." + )] #[allow(clippy::too_many_arguments)] fn compute_balance( &self, diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index e9c8d687c7..e3f0915262 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -120,7 +120,8 @@ pub enum SqliteClientError { /// An ephemeral address (given in string form) would be reused, or incorrectly used as an /// external address. If it is known to have been used in a previous transaction, the txid - /// is given. + /// is given (if there is more than one such known transaction, the choice of txid is + /// unspecified). #[cfg(feature = "transparent-inputs")] EphemeralAddressReuse(String, Option), } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 8f976ed642..3f5f196b9b 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1291,7 +1291,8 @@ impl WalletWrite for WalletDb { if let Some(address) = txout.recipient_address() { // TODO: we really want to only mark outputs when a transaction has been - // *reliably* mined. + // *reliably* mined, because that is strictly more conservative in avoiding + // going over the gap limit. #[cfg(feature = "transparent-inputs")] wallet::transparent::mark_ephemeral_address_as_mined(wdb.conn.0, &wdb.params, &address, tx_ref).map_err(SqliteClientError::from)?; diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 70b21856f2..bad20c794e 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -639,6 +639,19 @@ fn ephemeral_address_check_internal( address_str: &str, reject_all_ephemeral: bool, ) -> Result<(), SqliteClientError> { + // It is intentional that we don't require `t.mined_height` to be non-null. + // That is, we conservatively treat an ephemeral address as potentially + // reused even if we think that the transaction where we had evidence of + // its use is at present unmined. This should never occur in supported + // situations where only a single correctly operating wallet instance is + // using a given seed, because such a wallet will not reuse an address that + // it ever reserved. + // + // `COALESCE(used_in_tx, mined_in_tx)` can only differ from `used_in_tx` + // if the address was reserved, an error occurred in transaction creation + // before calling `mark_ephemeral_address_as_used`, and then we observed + // the address to have been used in a mined transaction (presumably by + // another wallet instance, or due to a bug) anyway. let res = wdb .conn .0 From 549fe0b652e08d9d91960d329060a73dfb3f9562 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 22 Jun 2024 13:55:50 +0100 Subject: [PATCH 03/56] In `reserve_next_n_ephemeral_addresses`, exclude addresses observed in unmined transactions from consideration when calculating the end of the gap. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/db.rs | 5 +++-- zcash_client_sqlite/src/wallet/transparent.rs | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index e6d840e1d0..1a34a28f84 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -93,8 +93,9 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( /// reduce the chance of address reuse collisions with another wallet using the same seed. /// /// Note that the fact that `used_in_tx` and `mined_in_tx` reference specific transactions is primarily -/// a debugging aid. We only really care which addresses have been used, and whether we can allocate -/// a new address within the gap limit. +/// a debugging aid (although the latter allows us to account for whether the referenced transaction +/// is unmined). We only really care which addresses have been used, and whether we can allocate a +/// new address within the gap limit. pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" CREATE TABLE ephemeral_addresses ( account_id INTEGER NOT NULL, diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index bad20c794e..36339cc373 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -804,12 +804,19 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; + // The inner join with `transactions` excludes addresses for which + // `mined_in_tx` is NULL. The query also excludes addresses observed + // to have been mined in a transaction that we currently see as unmined. + // This is conservative in terms of avoiding violation of the gap + // invariant: it can only cause us to get to the end of the gap (and + // start reporting `ReachedGapLimit` errors) sooner. let last_gap_index: i32 = wdb .conn .0 .query_row( "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id AND mined_in_tx IS NOT NULL + JOIN transactions t ON t.id_tx = mined_in_tx + WHERE account_id = :account_id AND t.mined_height IS NOT NULL ORDER BY address_index DESC LIMIT 1", named_params![":account_id": account_id.0], |row| row.get::<_, u32>(0), From eb8846162c83d38632b21229cd1c346341f4b7d2 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 22 Jun 2024 14:53:56 +0100 Subject: [PATCH 04/56] Address review comment: `EphemeralIvk` should not implement `IncomingViewingKey`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/transparent.rs | 4 ++-- zcash_primitives/src/legacy/keys.rs | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 36339cc373..9e53a07e36 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -767,7 +767,7 @@ pub(crate) fn get_reserved_ephemeral_addresses( let address_index = NonHardenedChildIndex::from_index(index as u32).expect("valid index"); result.insert( - ephemeral_ivk.derive_address(address_index)?, + ephemeral_ivk.derive_ephemeral_address(address_index)?, Some(TransparentAddressMetadata::new( EPHEMERAL_SCOPE, address_index, @@ -861,7 +861,7 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( .map(|address_index| { let child = NonHardenedChildIndex::from_index(address_index as u32) .expect("valid by construction"); - let address = ephemeral_ivk.derive_address(child)?; + let address = ephemeral_ivk.derive_ephemeral_address(child)?; stmt_insert_ephemeral_address.execute(named_params![ ":account_id": account_id.0, ":address_index": address_index, diff --git a/zcash_primitives/src/legacy/keys.rs b/zcash_primitives/src/legacy/keys.rs index 346b87e07a..e969cd7330 100644 --- a/zcash_primitives/src/legacy/keys.rs +++ b/zcash_primitives/src/legacy/keys.rs @@ -432,7 +432,19 @@ impl private::SealedChangeLevelKey for EphemeralIvk { } } -impl IncomingViewingKey for EphemeralIvk {} +#[cfg(feature = "transparent-inputs")] +impl EphemeralIvk { + /// Derives a transparent address at the provided child index. + pub fn derive_ephemeral_address( + &self, + address_index: NonHardenedChildIndex, + ) -> Result { + use private::SealedChangeLevelKey; + let child_key = self.extended_pubkey().derive_child(address_index.into())?; + #[allow(deprecated)] + Ok(pubkey_to_address(child_key.public_key())) + } +} /// Internal outgoing viewing key used for autoshielding. pub struct InternalOvk([u8; 32]); From c6520cf6a680fa21fdcf57af5ff543f7006471d0 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 22 Jun 2024 19:20:36 +0100 Subject: [PATCH 05/56] Change the protobuf schema to explicitly specify whether a `ChangeValue` is ephemeral. This also fixes `try_into_standard_proposal` to allow decoding from the protobuf representation into a proposal that uses references to prior ephemeral transparent outputs, provided that the "transparent-inputs" feature is enabled. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 7 ++- zcash_client_backend/proto/proposal.proto | 29 +++++----- zcash_client_backend/src/data_api/wallet.rs | 51 ++++++++++------- .../src/data_api/wallet/input_selection.rs | 2 +- zcash_client_backend/src/fees.rs | 36 +++++++++--- zcash_client_backend/src/fees/common.rs | 2 +- zcash_client_backend/src/proto.rs | 31 ++++++++--- zcash_client_backend/src/proto/proposal.rs | 30 +++++----- zcash_client_sqlite/src/testing/pool.rs | 55 ++++++++++--------- 9 files changed, 154 insertions(+), 89 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 7c7eaeb847..7de96c9a9a 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -29,7 +29,7 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - `chain::BlockCache` trait, behind the `sync` feature flag. - `WalletRead::get_spendable_transparent_outputs`. - `zcash_client_backend::fees`: - - `ChangeValue::{transparent, shielded}` + - `ChangeValue::{ephemeral_transparent, shielded}` - `sapling::EmptyBundleView` - `orchard::EmptyBundleView` - `zcash_client_backend::proposal`: @@ -64,6 +64,9 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - The return type of `ChangeValue::output_pool`, and the type of the `output_pool` argument to `ChangeValue::new`, have changed from `ShieldedProtocol` to `zcash_protocol::PoolType`. + - `ChangeValue::new` takes an additional `is_ephemeral` parameter indicating + whether the value is ephemeral (i.e. for use in a subsequent proposal step) + or change. - The return type of `ChangeValue::new` is now optional; it returns `None` if a memo is given for the transparent pool. Use `ChangeValue::shielded` to avoid this error case when creating a `ChangeValue` known to be for a @@ -75,6 +78,8 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail Empty slices can be passed to obtain the previous behaviour. - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. +- `zcash_client_backend::proto::ProposalDecodingError` has a new variant + `InvalidEphemeralRecipient`. - `zcash_client_backend::wallet::Recipient` variants have changed. Instead of wrapping protocol-address types, the `External` and `InternalAccount` variants now wrap a `zcash_address::ZcashAddress`. This simplifies the process of diff --git a/zcash_client_backend/proto/proposal.proto b/zcash_client_backend/proto/proposal.proto index 0935d66a1a..db548c6e23 100644 --- a/zcash_client_backend/proto/proposal.proto +++ b/zcash_client_backend/proto/proposal.proto @@ -73,14 +73,14 @@ message ReceivedOutput { uint64 value = 4; } -// A reference a payment in a prior step of the proposal. This payment must +// A reference to a payment in a prior step of the proposal. This payment must // belong to the wallet. message PriorStepOutput { uint32 stepIndex = 1; uint32 paymentIndex = 2; } -// A reference a change output from a prior step of the proposal. +// A reference to a change or ephemeral output from a prior step of the proposal. message PriorStepChange { uint32 stepIndex = 1; uint32 changeIndex = 2; @@ -112,26 +112,29 @@ enum FeeRule { // The proposed change outputs and fee value. message TransactionBalance { - // A list of change output values. - // - // Each `ChangeValue` for the transparent value pool must be consumed by - // a subsequent step. These represent ephemeral outputs that will each be - // given a unique t-address. + // A list of change or ephemeral output values. repeated ChangeValue proposedChange = 1; // The fee to be paid by the proposed transaction, in zatoshis. uint64 feeRequired = 2; } -// A proposed change output. If the transparent value pool is selected, -// the `memo` field must be null. +// A proposed change or ephemeral output. If the transparent value pool is +// selected, the `memo` field must be null. +// +// When the `isEphemeral` field of a `ChangeValue` is set, it represents +// an ephemeral output, which must be spent by a subsequent step. This is +// only supported for transparent outputs. Each ephemeral output will be +// given a unique t-address. message ChangeValue { - // The value of a change output to be created, in zatoshis. + // The value of a change or ephemeral output to be created, in zatoshis. uint64 value = 1; - // The value pool in which the change output should be created. + // The value pool in which the change or ephemeral output should be created. ValuePool valuePool = 2; - // The optional memo that should be associated with the newly created change output. - // Memos must not be present for transparent change outputs. + // The optional memo that should be associated with the newly created output. + // Memos must not be present for transparent outputs. MemoBytes memo = 3; + // Whether this is to be an ephemeral output. + bool isEphemeral = 4; } // An object wrapper for memo bytes, to facilitate representing the diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index dc4d47f901..77e6387dcb 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -52,7 +52,7 @@ use crate::{ decrypt_transaction, fees::{self, DustOutputPolicy}, keys::UnifiedSpendingKey, - proposal::{Proposal, ProposalError, Step, StepOutputIndex}, + proposal::{Proposal, Step, StepOutputIndex}, wallet::{Note, OvkPolicy, Recipient}, zip321::{self, Payment}, PoolType, ShieldedProtocol, @@ -74,7 +74,11 @@ use zip32::Scope; #[cfg(feature = "transparent-inputs")] use { - crate::{fees::ChangeValue, proposal::StepOutput, wallet::TransparentAddressMetadata}, + crate::{ + fees::ChangeValue, + proposal::{ProposalError, StepOutput}, + wallet::TransparentAddressMetadata, + }, input_selection::ShieldingSelector, std::collections::HashMap, zcash_keys::encoding::AddressCodec, @@ -624,8 +628,10 @@ where step_results.push((step, step_result)); } - // Change step outputs represent ephemeral outputs that must be referenced exactly once. - // (We don't support transparent change.) + // Ephemeral outputs must be referenced exactly once. Currently this is all + // transparent outputs using `StepOutputIndex::Change`. + // TODO: if we support transparent change, this will need to be updated to + // not require it to be referenced by a later step. #[cfg(feature = "transparent-inputs")] if unused_transparent_outputs .into_keys() @@ -670,33 +676,40 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - #[cfg(feature = "transparent-inputs")] + #[allow(unused_variables)] let step_index = prior_step_results.len(); - // Spending shielded outputs of prior multi-step transaction steps (either payments or change) - // is not supported. + // We only support spending transparent payments or ephemeral outputs from a prior step. // - // TODO: Maybe support this at some point? Doing so would require a higher-level approach in - // the wallet that waits for transactions with shielded outputs to be mined and only then - // attempts to perform the next step. + // TODO: Maybe support spending prior shielded outputs at some point? Doing so would require + // a higher-level approach in the wallet that waits for transactions with shielded outputs to + // be mined and only then attempts to perform the next step. + #[cfg(feature = "transparent-inputs")] for input_ref in proposal_step.prior_step_inputs() { - let prior_pool = prior_step_results + let supported = prior_step_results .get(input_ref.step_index()) .and_then(|(prior_step, _)| match input_ref.output_index() { - StepOutputIndex::Payment(i) => prior_step.payment_pools().get(&i).cloned(), - StepOutputIndex::Change(i) => prior_step - .balance() - .proposed_change() - .get(i) - .map(|change| change.output_pool()), + StepOutputIndex::Payment(i) => prior_step + .payment_pools() + .get(&i) + .map(|&pool| pool == PoolType::TRANSPARENT), + StepOutputIndex::Change(i) => { + prior_step.balance().proposed_change().get(i).map(|change| { + change.is_ephemeral() && change.output_pool() == PoolType::TRANSPARENT + }) + } }) .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))?; - // Return an error on trying to spend a prior shielded output. - if matches!(prior_pool, PoolType::Shielded(_)) { + // Return an error on trying to spend a prior shielded output or non-ephemeral change output. + if !supported { return Err(Error::ProposalNotSupported); } } + #[cfg(not(feature = "transparent-inputs"))] + if !proposal_step.prior_step_inputs().is_empty() { + return Err(Error::ProposalNotSupported); + } let account_id = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 776dc98691..6a8ece442e 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -602,7 +602,7 @@ where // The ephemeral output should always be at the last change index. assert_eq!( *balance.proposed_change().last().expect("nonempty"), - ChangeValue::transparent(ephemeral_output_amounts[0]) + ChangeValue::ephemeral_transparent(ephemeral_output_amounts[0]) ); let ephemeral_stepoutput = StepOutput::new( 0, diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 3bf99f9fc4..7e5e966a5b 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -27,28 +27,44 @@ pub struct ChangeValue { output_pool: PoolType, value: NonNegativeAmount, memo: Option, + is_ephemeral: bool, } impl ChangeValue { - /// Constructs a new change value from its constituent parts. + /// Constructs a new change or ephemeral value from its constituent parts. + /// + /// Currently, the only supported combinations are change sent to a shielded + /// pool, or (if the "transparent-inputs" feature is enabled) an ephemeral + /// output to the transparent pool. pub fn new( output_pool: PoolType, value: NonNegativeAmount, memo: Option, + is_ephemeral: bool, ) -> Option { - (matches!(output_pool, PoolType::Shielded(_)) || memo.is_none()).then_some(Self { + match output_pool { + PoolType::Shielded(_) => !is_ephemeral, + #[cfg(feature = "transparent-inputs")] + PoolType::Transparent => is_ephemeral && memo.is_none(), + #[cfg(not(feature = "transparent-inputs"))] + PoolType::Transparent => false, + } + .then_some(Self { output_pool, value, memo, + is_ephemeral, }) } - /// Constructs a new change value that will be created as a transparent output. - pub fn transparent(value: NonNegativeAmount) -> Self { + /// Constructs a new ephemeral transparent output value. + #[cfg(feature = "transparent-inputs")] + pub fn ephemeral_transparent(value: NonNegativeAmount) -> Self { Self { output_pool: PoolType::TRANSPARENT, value, memo: None, + is_ephemeral: true, } } @@ -62,6 +78,7 @@ impl ChangeValue { output_pool: PoolType::Shielded(protocol), value, memo, + is_ephemeral: false, } } @@ -76,20 +93,25 @@ impl ChangeValue { Self::shielded(ShieldedProtocol::Orchard, value, memo) } - /// Returns the pool to which the change output should be sent. + /// Returns the pool to which the change or ephemeral output should be sent. pub fn output_pool(&self) -> PoolType { self.output_pool } - /// Returns the value of the change output to be created, in zatoshis. + /// Returns the value of the change or ephemeral output to be created, in zatoshis. pub fn value(&self) -> NonNegativeAmount { self.value } - /// Returns the memo to be associated with the change output. + /// Returns the memo to be associated with the output. pub fn memo(&self) -> Option<&MemoBytes> { self.memo.as_ref() } + + /// Whether this is to be an ephemeral output. + pub fn is_ephemeral(&self) -> bool { + self.is_ephemeral + } } /// The amount of change and fees required to make a transaction's inputs and diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index b77811abf4..1d40d4966b 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -400,7 +400,7 @@ where change.extend( ephemeral_output_amounts .iter() - .map(|&amount| ChangeValue::transparent(amount)), + .map(|&amount| ChangeValue::ephemeral_transparent(amount)), ); TransactionBalance::new(change, fee).map_err(|_| overflow()) diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index 915d1d39e3..f90967ae85 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -335,7 +335,7 @@ pub const PROPOSAL_SER_V1: u32 = 1; /// representation. #[derive(Debug, Clone)] pub enum ProposalDecodingError { - /// The encoded proposal contained no steps + /// The encoded proposal contained no steps. NoSteps, /// The ZIP 321 transaction request URI was invalid. Zip321(Zip321Error), @@ -366,6 +366,8 @@ pub enum ProposalDecodingError { TransparentMemo, /// Change outputs to the specified pool are not supported. InvalidChangeRecipient(PoolType), + /// Ephemeral outputs to the specified pool are not supported. + InvalidEphemeralRecipient(PoolType), } impl From for ProposalDecodingError { @@ -424,6 +426,11 @@ impl Display for ProposalDecodingError { "Change outputs to the {} pool are not supported.", pool_type ), + ProposalDecodingError::InvalidEphemeralRecipient(pool_type) => write!( + f, + "Ephemeral outputs to the {} pool are not supported.", + pool_type + ), } } } @@ -572,6 +579,7 @@ impl proposal::Proposal { memo: change.memo().map(|memo_bytes| proposal::MemoBytes { value: memo_bytes.as_slice().to_vec(), }), + is_ephemeral: change.is_ephemeral(), }) .collect(), fee_required: step.balance().fee_required().into(), @@ -662,7 +670,7 @@ impl proposal::Proposal { PoolType::Transparent => { #[cfg(not(feature = "transparent-inputs"))] return Err(ProposalDecodingError::ValuePoolNotSupported( - 1, + out.value_pool, )); #[cfg(feature = "transparent-inputs")] @@ -751,18 +759,27 @@ impl proposal::Proposal { .map_err(ProposalDecodingError::MemoInvalid) }) .transpose()?; - match cv.pool_type()? { - PoolType::Shielded(ShieldedProtocol::Sapling) => { + match (cv.pool_type()?, cv.is_ephemeral) { + (PoolType::Shielded(ShieldedProtocol::Sapling), false) => { Ok(ChangeValue::sapling(value, memo)) } #[cfg(feature = "orchard")] - PoolType::Shielded(ShieldedProtocol::Orchard) => { + (PoolType::Shielded(ShieldedProtocol::Orchard), false) => { Ok(ChangeValue::orchard(value, memo)) } - PoolType::Transparent if memo.is_some() => { + (PoolType::Transparent, _) if memo.is_some() => { Err(ProposalDecodingError::TransparentMemo) } - t => Err(ProposalDecodingError::InvalidChangeRecipient(t)), + #[cfg(feature = "transparent-inputs")] + (PoolType::Transparent, true) => { + Ok(ChangeValue::ephemeral_transparent(value)) + } + (pool, false) => { + Err(ProposalDecodingError::InvalidChangeRecipient(pool)) + } + (pool, true) => { + Err(ProposalDecodingError::InvalidEphemeralRecipient(pool)) + } } }) .collect::, _>>()?, diff --git a/zcash_client_backend/src/proto/proposal.rs b/zcash_client_backend/src/proto/proposal.rs index be8da848aa..1ea321afcd 100644 --- a/zcash_client_backend/src/proto/proposal.rs +++ b/zcash_client_backend/src/proto/proposal.rs @@ -72,7 +72,7 @@ pub struct ReceivedOutput { #[prost(uint64, tag = "4")] pub value: u64, } -/// A reference a payment in a prior step of the proposal. This payment must +/// A reference to a payment in a prior step of the proposal. This payment must /// belong to the wallet. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -82,7 +82,7 @@ pub struct PriorStepOutput { #[prost(uint32, tag = "2")] pub payment_index: u32, } -/// A reference a change output from a prior step of the proposal. +/// A reference to a change or ephemeral output from a prior step of the proposal. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PriorStepChange { @@ -115,32 +115,36 @@ pub mod proposed_input { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionBalance { - /// A list of change output values. - /// - /// Each `ChangeValue` for the transparent value pool must be consumed by - /// a subsequent step. These represent ephemeral outputs that will each be - /// given a unique t-address. + /// A list of change or ephemeral output values. #[prost(message, repeated, tag = "1")] pub proposed_change: ::prost::alloc::vec::Vec, /// The fee to be paid by the proposed transaction, in zatoshis. #[prost(uint64, tag = "2")] pub fee_required: u64, } -/// A proposed change output. If the transparent value pool is selected, -/// the `memo` field must be null. +/// A proposed change or ephemeral output. If the transparent value pool is +/// selected, the `memo` field must be null. +/// +/// When the `isEphemeral` field of a `ChangeValue` is set, it represents +/// an ephemeral output, which must be spent by a subsequent step. This is +/// only supported for transparent outputs. Each ephemeral output will be +/// given a unique t-address. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ChangeValue { - /// The value of a change output to be created, in zatoshis. + /// The value of a change or ephemeral output to be created, in zatoshis. #[prost(uint64, tag = "1")] pub value: u64, - /// The value pool in which the change output should be created. + /// The value pool in which the change or ephemeral output should be created. #[prost(enumeration = "ValuePool", tag = "2")] pub value_pool: i32, - /// The optional memo that should be associated with the newly created change output. - /// Memos must not be present for transparent change outputs. + /// The optional memo that should be associated with the newly created output. + /// Memos must not be present for transparent outputs. #[prost(message, optional, tag = "3")] pub memo: ::core::option::Option, + /// Whether this is to be an ephemeral output. + #[prost(bool, tag = "4")] + pub is_ephemeral: bool, } /// An object wrapper for memo bytes, to facilitate representing the /// `change_memo == None` case. diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 17dd6dfeb6..cb0292bd2a 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -302,7 +302,6 @@ pub(crate) fn send_single_step_proposed_transfer() { #[cfg(feature = "transparent-inputs")] pub(crate) fn send_multi_step_proposed_transfer() { - use zcash_address::ZcashAddress; use zcash_client_backend::fees::ChangeValue; let mut st = TestBuilder::new() @@ -313,12 +312,6 @@ pub(crate) fn send_multi_step_proposed_transfer() { let account = st.test_account().cloned().unwrap(); let dfvk = T::test_account_fvk(&st); - let fee_rule = StandardFeeRule::Zip317; - let input_selector = GreedyInputSelector::new( - standard::SingleOutputChangeStrategy::new(fee_rule, None, T::SHIELDED_PROTOCOL), - DustOutputPolicy::default(), - ); - let add_funds = |st: &mut TestState<_>, value| { let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); st.scan_cached_blocks(h, 1); @@ -353,34 +346,37 @@ pub(crate) fn send_multi_step_proposed_transfer() { TransparentAddress::PublicKeyHash(data) => Address::Tex(data), _ => unreachable!(), }; - let request = zip321::TransactionRequest::new(vec![Payment::without_memo( - tex_addr.to_zcash_address(&st.network()), - amount, - )]) - .unwrap(); + // As of this commit, change memos are not correctly handled in ZIP 320 proposals. + let change_memo = None; + // We use `st.propose_standard_transfer` here in order to also test round-trip + // serialization of the proposal. let proposal = st - .propose_transfer( + .propose_standard_transfer::( account.account_id(), - &input_selector, - request, + StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), + &tex_addr, + amount, + None, + change_memo.clone(), + T::SHIELDED_PROTOCOL, ) .unwrap(); let steps: Vec<_> = proposal.steps().iter().cloned().collect(); assert_eq!(steps.len(), 2); + assert_eq!(steps[0].balance().fee_required(), expected_step0_fee); + assert_eq!(steps[1].balance().fee_required(), expected_step1_fee); assert_eq!( steps[0].balance().proposed_change(), [ - ChangeValue::shielded(T::SHIELDED_PROTOCOL, expected_step0_change, None), - ChangeValue::transparent((amount + expected_step1_fee).unwrap()), + ChangeValue::shielded(T::SHIELDED_PROTOCOL, expected_step0_change, change_memo), + ChangeValue::ephemeral_transparent((amount + expected_step1_fee).unwrap()), ] ); - assert_eq!(steps[0].balance().fee_required(), expected_step0_fee); assert_eq!(steps[1].balance().proposed_change(), []); - assert_eq!(steps[1].balance().fee_required(), expected_step1_fee); let create_proposed_result = st.create_proposed_transactions::( account.usk(), @@ -451,19 +447,24 @@ pub(crate) fn send_multi_step_proposed_transfer() { assert!(ephemeral0 != ephemeral1); add_funds(&mut st, value); - let request = zip321::TransactionRequest::new(vec![Payment::without_memo( - ZcashAddress::try_from_encoded(&ephemeral0).expect("valid address"), - amount, - )]) - .unwrap(); + + let ephemeral_taddr = Address::decode(&st.wallet().params, &ephemeral0).expect("valid address"); + assert_matches!( + ephemeral_taddr, + Address::Transparent(TransparentAddress::PublicKeyHash(_)) + ); // Attempting to use the same address again should cause an error. let proposal = st - .propose_transfer( + .propose_standard_transfer::( account.account_id(), - &input_selector, - request, + StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), + &ephemeral_taddr, + amount, + None, + None, + T::SHIELDED_PROTOCOL, ) .unwrap(); From 2f521d78736be6f4b349fd48fbe590745d58f251 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sun, 23 Jun 2024 08:19:41 +0100 Subject: [PATCH 06/56] If a change memo is supplied, it should not be used in the second step of a ZIP 320 proposal. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 9 ++++--- .../src/data_api/wallet/input_selection.rs | 10 +++++++ zcash_client_backend/src/fees.rs | 1 + zcash_client_backend/src/fees/fixed.rs | 14 +++++++++- zcash_client_backend/src/fees/standard.rs | 7 +++++ zcash_client_backend/src/fees/zip317.rs | 26 ++++++++++++++++++- zcash_client_sqlite/src/testing/pool.rs | 5 ++-- 7 files changed, 64 insertions(+), 8 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 7de96c9a9a..49e2852859 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -71,11 +71,12 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail if a memo is given for the transparent pool. Use `ChangeValue::shielded` to avoid this error case when creating a `ChangeValue` known to be for a shielded pool. - - `ChangeStrategy::compute_balance`: this trait method has two additional + - `ChangeStrategy::compute_balance`: this trait method has three additional parameters when the "transparent-inputs" feature is enabled. These are - used to specify amounts of additional transparent P2PKH inputs and outputs, - for which `InputView` or `OutputView` instances may not be available. - Empty slices can be passed to obtain the previous behaviour. + used to specify whether the change memo should be ignored, and the amounts + of additional transparent P2PKH inputs and outputs. Passing `false` for + `ignore_change_memo` and empty slices for the other two arguments will + retain the previous behaviour. - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. - `zcash_client_backend::proto::ProposalDecodingError` has a new variant diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 6a8ece442e..bec5283ec1 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -475,6 +475,8 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] + true, + #[cfg(feature = "transparent-inputs")] &[NonNegativeAmount::ZERO], #[cfg(feature = "transparent-inputs")] &[], @@ -494,6 +496,8 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] + true, + #[cfg(feature = "transparent-inputs")] &[tr1_required_input_value], #[cfg(feature = "transparent-inputs")] &[], @@ -574,6 +578,8 @@ where ), &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &ephemeral_output_amounts, @@ -764,6 +770,8 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -785,6 +793,8 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 7e5e966a5b..a85acbdf28 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -356,6 +356,7 @@ pub trait ChangeStrategy { sapling: &impl sapling::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result>; diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index 1a340c7a21..f9ab9a9cae 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -66,9 +66,13 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> { + #[cfg(not(feature = "transparent-inputs"))] + let ignore_change_memo = false; + single_change_output_balance( params, &self.fee_rule, @@ -80,7 +84,11 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, self.fee_rule().fixed_fee(), - self.change_memo.clone(), + if ignore_change_memo { + None + } else { + self.change_memo.clone() + }, self.fallback_change_pool, #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts, @@ -142,6 +150,8 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -191,6 +201,8 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], diff --git a/zcash_client_backend/src/fees/standard.rs b/zcash_client_backend/src/fees/standard.rs index c5fc6cb9f5..8d53ecb66d 100644 --- a/zcash_client_backend/src/fees/standard.rs +++ b/zcash_client_backend/src/fees/standard.rs @@ -68,6 +68,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> { @@ -88,6 +89,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, #[cfg(feature = "transparent-inputs")] + ignore_change_memo, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts, #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts, @@ -108,6 +111,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, #[cfg(feature = "transparent-inputs")] + ignore_change_memo, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts, #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts, @@ -128,6 +133,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, #[cfg(feature = "transparent-inputs")] + ignore_change_memo, + #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts, #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts, diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 63ce70c4af..e6c8d7858e 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -68,6 +68,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, + #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], ) -> Result> { @@ -203,6 +204,9 @@ impl ChangeStrategy for SingleOutputChangeStrategy { } } + #[cfg(not(feature = "transparent-inputs"))] + let ignore_change_memo = false; + single_change_output_balance( params, &self.fee_rule, @@ -214,7 +218,11 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, self.fee_rule.marginal_fee(), - self.change_memo.clone(), + if ignore_change_memo { + None + } else { + self.change_memo.clone() + }, self.fallback_change_pool, #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts, @@ -283,6 +291,8 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -330,6 +340,8 @@ mod tests { ), &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -386,6 +398,8 @@ mod tests { &orchard_fees::EmptyBundleView, dust_output_policy, #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -433,6 +447,8 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -480,6 +496,8 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -533,6 +551,8 @@ mod tests { Some(NonNegativeAmount::const_from_u64(1000)), ), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -597,6 +617,8 @@ mod tests { &orchard_fees::EmptyBundleView, dust_output_policy, #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], @@ -653,6 +675,8 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] + false, + #[cfg(feature = "transparent-inputs")] &[], #[cfg(feature = "transparent-inputs")] &[], diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index cb0292bd2a..ed6cb158ed 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -302,6 +302,8 @@ pub(crate) fn send_single_step_proposed_transfer() { #[cfg(feature = "transparent-inputs")] pub(crate) fn send_multi_step_proposed_transfer() { + use std::str::FromStr; + use zcash_client_backend::fees::ChangeValue; let mut st = TestBuilder::new() @@ -346,8 +348,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { TransparentAddress::PublicKeyHash(data) => Address::Tex(data), _ => unreachable!(), }; - // As of this commit, change memos are not correctly handled in ZIP 320 proposals. - let change_memo = None; + let change_memo = Some(Memo::from_str("change").expect("valid memo").encode()); // We use `st.propose_standard_transfer` here in order to also test round-trip // serialization of the proposal. From 0f49daed5f2e070f1148045ed06299221df28084 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sun, 23 Jun 2024 09:45:48 +0100 Subject: [PATCH 07/56] `mark_ephemeral_address_as_mined` now prefers setting `mined_in_tx` to the transaction mined at an earlier height, out of the newly observed transaction and any already referenced one. This slightly reduces the chance of unnecessarily reaching the gap limit too early in some corner cases. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/error.rs | 8 ++--- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet/transparent.rs | 34 +++++++++++++++---- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index e3f0915262..aad3df1b7b 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -118,10 +118,10 @@ pub enum SqliteClientError { #[cfg(feature = "transparent-inputs")] ReachedGapLimit, - /// An ephemeral address (given in string form) would be reused, or incorrectly used as an - /// external address. If it is known to have been used in a previous transaction, the txid - /// is given (if there is more than one such known transaction, the choice of txid is - /// unspecified). + /// An ephemeral address would be reused, or incorrectly used as an external address. + /// The parameters are the address in string form, and if it is known to have been + /// used in one or more previous transactions, the txid of the earliest of those + /// transactions. #[cfg(feature = "transparent-inputs")] EphemeralAddressReuse(String, Option), } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 3f5f196b9b..dcbb145da1 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1294,7 +1294,7 @@ impl WalletWrite for WalletDb // *reliably* mined, because that is strictly more conservative in avoiding // going over the gap limit. #[cfg(feature = "transparent-inputs")] - wallet::transparent::mark_ephemeral_address_as_mined(wdb.conn.0, &wdb.params, &address, tx_ref).map_err(SqliteClientError::from)?; + wallet::transparent::mark_ephemeral_address_as_mined(wdb, &address, tx_ref)?; let receiver = Receiver::Transparent(address); diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 9e53a07e36..9b1a4cb4c1 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -687,18 +687,40 @@ fn ephemeral_address_check_internal( /// If `address` is one of our ephemeral addresses, mark it as having an output /// in the given mined transaction (which may or may not be a transaction we sent). -/// This has no effect if `address` is not one of our ephemeral addresses. +/// +/// `tx_ref` must be a valid transaction reference. This call has no effect if +/// `address` is not one of our ephemeral addresses. pub(crate) fn mark_ephemeral_address_as_mined( - conn: &rusqlite::Connection, - params: &P, + wdb: &mut WalletDb, P>, address: &TransparentAddress, tx_ref: i64, ) -> Result<(), SqliteClientError> { - let address_str = encode_transparent_address_p(params, address); + let address_str = encode_transparent_address_p(&wdb.params, address); + + // Figure out which transaction was mined earlier: `tx_ref`, or any existing + // tx referenced by `mined_in_tx` for the given address. Prefer the existing + // reference in case of a tie or if both transactions are unmined. + // This slightly reduces the chance of unnecessarily reaching the gap limit + // too early in some corner cases (because the earlier transaction is less + // likely to be unmined). + // + // The query should always return a value if `tx_ref` is valid. + let earlier_ref = wdb.conn.0.query_row( + "SELECT id_tx FROM transactions + LEFT OUTER JOIN ephemeral_addresses e + ON id_tx = e.mined_in_tx + WHERE id_tx = :tx_ref OR e.address = :address + ORDER BY mined_height ASC NULLS LAST, + tx_index ASC NULLS LAST, + e.mined_in_tx ASC NULLS LAST + LIMIT 1", + named_params![":tx_ref": &tx_ref, ":address": address_str], + |row| row.get::<_, i64>(0), + )?; - conn.execute( + wdb.conn.0.execute( "UPDATE ephemeral_addresses SET mined_in_tx = :mined_in_tx WHERE address = :address", - named_params![":mined_in_tx": &tx_ref, ":address": address_str], + named_params![":mined_in_tx": &earlier_ref, ":address": address_str], )?; Ok(()) } From 637ae925dae401cb56c23ade0f707ad5f5c1de15 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sun, 23 Jun 2024 10:02:19 +0100 Subject: [PATCH 08/56] Add a migration test for the `ephemeral_addresses` migration. Signed-off-by: Daira-Emma Hopwood --- .../src/wallet/init/migrations/ephemeral_addresses.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index bab03be761..346dd05fb5 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -55,3 +55,13 @@ impl RusqliteMigration for Migration { Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) } } + +#[cfg(test)] +mod tests { + use crate::wallet::init::migrations::tests::test_migrate; + + #[test] + fn migrate() { + test_migrate(&[super::MIGRATION_ID]); + } +} From 994f6ff3877527d458ab65e6fc33b1ee80f88304 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 24 Jun 2024 10:40:49 +0100 Subject: [PATCH 09/56] Change type of `n` in `reserve_next_n_ephemeral_addresses`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api.rs | 7 +++++-- zcash_client_backend/src/data_api/wallet.rs | 2 +- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet/transparent.rs | 4 +++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 234632ede9..4501ef87dc 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1599,12 +1599,14 @@ pub trait WalletWrite: WalletRead { /// the given number of addresses, or if the account identifier does not correspond /// to a known account. /// + /// Precondition: `n >= 0` + /// /// [BIP 44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#user-content-Address_gap_limit #[cfg(feature = "transparent-inputs")] fn reserve_next_n_ephemeral_addresses( &mut self, account_id: Self::AccountId, - n: u32, + n: i32, ) -> Result, Self::Error>; } @@ -2010,8 +2012,9 @@ pub mod testing { fn reserve_next_n_ephemeral_addresses( &mut self, _account_id: Self::AccountId, - _n: u32, + n: i32, ) -> Result, Self::Error> { + assert!(n >= 0); Err(()) } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 77e6387dcb..dd36b369bf 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1157,7 +1157,7 @@ where .filter(|(_, change_value)| matches!(change_value.output_pool(), PoolType::Transparent)) .collect(); let num_ephemeral_outputs = - u32::try_from(ephemeral_outputs.len()).map_err(|_| Error::ProposalNotSupported)?; + i32::try_from(ephemeral_outputs.len()).map_err(|_| Error::ProposalNotSupported)?; let addresses_and_metadata = wallet_db .reserve_next_n_ephemeral_addresses(account_id, num_ephemeral_outputs) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index dcbb145da1..8ffb097801 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1482,7 +1482,7 @@ impl WalletWrite for WalletDb fn reserve_next_n_ephemeral_addresses( &mut self, account_id: Self::AccountId, - n: u32, + n: i32, ) -> Result, Self::Error> { self.transactionally(|wdb| { wallet::transparent::reserve_next_n_ephemeral_addresses(wdb, account_id, n) diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 9b1a4cb4c1..cdd632d1fc 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -804,6 +804,8 @@ pub(crate) fn get_reserved_ephemeral_addresses( /// Returns a vector with the next `n` previously unreserved ephemeral addresses for /// the given account. /// +/// Precondition: `n >= 0` +/// /// # Errors /// /// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. @@ -817,7 +819,7 @@ pub(crate) fn get_reserved_ephemeral_addresses( pub(crate) fn reserve_next_n_ephemeral_addresses( wdb: &mut WalletDb, P>, account_id: AccountId, - n: u32, + n: i32, ) -> Result, SqliteClientError> { if n == 0 { return Ok(vec![]); From 25f07da47d191f8e8f7a469237f718854499c16f Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 24 Jun 2024 10:42:06 +0100 Subject: [PATCH 10/56] Add a constraint on the range of `ephemeral_addresses(address_index)`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/db.rs | 5 ++++- .../src/wallet/init/migrations/ephemeral_addresses.rs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 1a34a28f84..9dc3b1a0bc 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -106,8 +106,11 @@ CREATE TABLE ephemeral_addresses ( FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), - PRIMARY KEY (account_id, address_index) + PRIMARY KEY (account_id, address_index), + CONSTRAINT address_index_in_range CHECK (address_index >= 0 AND address_index <= 0x7FFFFFFF) ) WITHOUT ROWID"#; +// Hexadecimal integer literals were added in SQLite version 3.8.6 (2014-08-15). +// libsqlite3-sys requires at least version 3.14.0. // "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. pub(super) const INDEX_EPHEMERAL_ADDRESSES_ADDRESS: &str = r#" CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 346dd05fb5..886f9bd871 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -42,7 +42,8 @@ impl RusqliteMigration for Migration { FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), - PRIMARY KEY (account_id, address_index) + PRIMARY KEY (account_id, address_index), + CONSTRAINT address_index_in_range CHECK (address_index >= 0 AND address_index <= 0x7FFFFFFF) ) WITHOUT ROWID; CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( address ASC From e164b593297f4804dd9644ac0351bf404a77b8e6 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 24 Jun 2024 10:49:47 +0100 Subject: [PATCH 11/56] Move most ephemeral address index handling into helper functions in `zcash_client_sqlite::wallet::transparent::ephemeral`. Also report the account id and index for `SqliteClientError::ReachedGapLimit`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/wallet.rs | 1 + zcash_client_sqlite/src/error.rs | 11 +- zcash_client_sqlite/src/wallet/init.rs | 2 +- zcash_client_sqlite/src/wallet/transparent.rs | 129 ++++++------------ .../src/wallet/transparent/ephemeral.rs | 98 +++++++++++++ 5 files changed, 148 insertions(+), 93 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/transparent/ephemeral.rs diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index 18540c8241..f6897fac08 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -633,6 +633,7 @@ impl OvkPolicy { } /// Metadata related to the ZIP 32 derivation of a transparent address. +/// This is implicitly scoped to an account. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg(feature = "transparent-inputs")] pub struct TransparentAddressMetadata { diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index aad3df1b7b..5e4fc1c976 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -15,6 +15,7 @@ use crate::PRUNING_DEPTH; #[cfg(feature = "transparent-inputs")] use { + crate::AccountId, zcash_client_backend::encoding::TransparentCodecError, zcash_primitives::{legacy::TransparentAddress, transaction::TxId}, }; @@ -114,9 +115,10 @@ pub enum SqliteClientError { BalanceError(BalanceError), /// The proposal cannot be constructed until transactions with previously reserved - /// ephemeral address outputs have been mined. + /// ephemeral address outputs have been mined. The parameters are the account id and + /// the index that could not safely be reserved. #[cfg(feature = "transparent-inputs")] - ReachedGapLimit, + ReachedGapLimit(AccountId, u32), /// An ephemeral address would be reused, or incorrectly used as an external address. /// The parameters are the address in string form, and if it is known to have been @@ -175,7 +177,10 @@ impl fmt::Display for SqliteClientError { SqliteClientError::UnsupportedPoolType(t) => write!(f, "Pool type is not currently supported: {}", t), SqliteClientError::BalanceError(e) => write!(f, "Balance error: {}", e), #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit => write!(f, "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined."), + SqliteClientError::ReachedGapLimit(account_id, bad_index) => write!(f, + "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined. \ + The ephemeral address in account {account_id:?} at index {bad_index} could not be safely reserved.", + ), #[cfg(feature = "transparent-inputs")] SqliteClientError::EphemeralAddressReuse(address_str, Some(txid)) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index cfe6f73208..b14ecd9e5b 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -177,7 +177,7 @@ fn sqlite_client_error_to_wallet_migration_error(e: SqliteClientError) -> Wallet unreachable!("we don't call methods that require a known chain height") } #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit => { + SqliteClientError::ReachedGapLimit(_, _) => { unreachable!("we don't do ephemeral address tracking") } #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index cdd632d1fc..82e43cedeb 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -1,19 +1,24 @@ //! Functions for transparent input support in the wallet. +use std::cmp::max; +use std::collections::{HashMap, HashSet}; + use rusqlite::OptionalExtension; use rusqlite::{named_params, Connection, Row}; -use std::collections::HashMap; -use std::collections::HashSet; -use zcash_client_backend::data_api::AccountBalance; -use zcash_keys::address::Address; -use zcash_keys::keys::AddressGenerationError; use zip32::{DiversifierIndex, Scope}; use zcash_address::unified::{Encoding, Ivk, Uivk}; -use zcash_client_backend::wallet::{TransparentAddressMetadata, WalletTransparentOutput}; -use zcash_keys::encoding::{encode_transparent_address_p, AddressCodec}; +use zcash_client_backend::{ + data_api::AccountBalance, + keys::AddressGenerationError, + wallet::{TransparentAddressMetadata, WalletTransparentOutput}, +}; +use zcash_keys::{ + address::Address, + encoding::{encode_transparent_address_p, AddressCodec}, +}; use zcash_primitives::{ legacy::{ - keys::{EphemeralIvk, IncomingViewingKey, NonHardenedChildIndex, TransparentKeyScope}, + keys::{EphemeralIvk, IncomingViewingKey, NonHardenedChildIndex}, Script, TransparentAddress, }, transaction::{ @@ -28,6 +33,8 @@ use crate::{SqlTransaction, WalletDb}; use super::{chain_tip_height, get_account, get_account_ids}; +mod ephemeral; + pub(crate) fn detect_spending_accounts<'a>( conn: &Connection, spent: impl Iterator, @@ -494,7 +501,7 @@ pub(crate) fn find_account_for_transparent_output( return Ok(Some(account_id)); } - // Note that this does not search ephemeral addresses that have not yet been reserved. + // Search ephemeral addresses that have already been reserved. if let Some(account_id) = conn .query_row( "SELECT account_id FROM ephemeral_addresses WHERE address = :address", @@ -741,14 +748,6 @@ pub(crate) fn get_ephemeral_ivk( .derive_ephemeral_ivk()?) } -// Same as Bitcoin. -const GAP_LIMIT: i32 = 20; - -const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2) { - Some(s) => s, - None => unreachable!(), -}; - /// Returns a vector with all ephemeral transparent addresses potentially belonging to this wallet. /// If `for_detection` is true, this includes addresses for an additional GAP_LIMIT indices. pub(crate) fn get_reserved_ephemeral_addresses( @@ -771,30 +770,19 @@ pub(crate) fn get_reserved_ephemeral_addresses( first_unused_index = i32::try_from(raw_index) .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? .checked_add(1); - let address_index = NonHardenedChildIndex::from_index(raw_index).expect("just checked"); - result.insert( - TransparentAddress::decode(params, &addr_str)?, - Some(TransparentAddressMetadata::new( - EPHEMERAL_SCOPE, - address_index, - )), - ); + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address = TransparentAddress::decode(params, &addr_str)?; + result.insert(address, Some(ephemeral::metadata(address_index))); } if for_detection { if let Some(first) = first_unused_index { let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; - for index in first..=first.saturating_add(GAP_LIMIT - 1) { - let address_index = - NonHardenedChildIndex::from_index(index as u32).expect("valid index"); - result.insert( - ephemeral_ivk.derive_ephemeral_address(address_index)?, - Some(TransparentAddressMetadata::new( - EPHEMERAL_SCOPE, - address_index, - )), - ); + for raw_index in ephemeral::range_after(first, ephemeral::GAP_LIMIT) { + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; + result.insert(address, Some(ephemeral::metadata(address_index))); } } } @@ -827,52 +815,18 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( assert!(n > 0); let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; + let last_reserved_index = ephemeral::last_reserved_index(wdb.conn.0, account_id)?; + let last_safe_index = ephemeral::last_safe_index(wdb.conn.0, account_id)?; + let allocation = ephemeral::range_after(last_reserved_index, n); - // The inner join with `transactions` excludes addresses for which - // `mined_in_tx` is NULL. The query also excludes addresses observed - // to have been mined in a transaction that we currently see as unmined. - // This is conservative in terms of avoiding violation of the gap - // invariant: it can only cause us to get to the end of the gap (and - // start reporting `ReachedGapLimit` errors) sooner. - let last_gap_index: i32 = wdb - .conn - .0 - .query_row( - "SELECT address_index FROM ephemeral_addresses - JOIN transactions t ON t.id_tx = mined_in_tx - WHERE account_id = :account_id AND t.mined_height IS NOT NULL - ORDER BY address_index DESC LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - .map_or(Ok(-1i32), |i| { - i32::try_from(i).map_err(|e| SqliteClientError::CorruptedData(e.to_string())) - })? - .saturating_add(GAP_LIMIT); - - let (first_index, last_index) = wdb - .conn - .0 - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id - ORDER BY address_index DESC LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - .map_or(Ok(-1i32), |i| { - i32::try_from(i).map_err(|e| SqliteClientError::CorruptedData(e.to_string())) - }) - .map(|i: i32| i.checked_add(1).zip(i.checked_add(n.try_into().ok()?)))? - .ok_or(SqliteClientError::AddressGeneration( + if allocation.clone().count() < n.try_into().unwrap() { + return Err(SqliteClientError::AddressGeneration( AddressGenerationError::DiversifierSpaceExhausted, - ))?; - - assert!(last_index >= first_index); - if last_index > last_gap_index { - return Err(SqliteClientError::ReachedGapLimit); + )); + } + if *allocation.end() > last_safe_index { + let unsafe_index = max(*allocation.start(), last_safe_index.saturating_add(1)); + return Err(SqliteClientError::ReachedGapLimit(account_id, unsafe_index)); } // used_in_tx and mined_in_tx are initially NULL @@ -881,20 +835,17 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( VALUES (:account_id, :address_index, :address)", )?; - (first_index..=last_index) - .map(|address_index| { - let child = NonHardenedChildIndex::from_index(address_index as u32) - .expect("valid by construction"); - let address = ephemeral_ivk.derive_ephemeral_address(child)?; + allocation + .map(|raw_index| { + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; + stmt_insert_ephemeral_address.execute(named_params![ ":account_id": account_id.0, - ":address_index": address_index, + ":address_index": raw_index, ":address": encode_transparent_address_p(&wdb.params, &address) ])?; - Ok(( - address, - TransparentAddressMetadata::new(EPHEMERAL_SCOPE, child), - )) + Ok((address, ephemeral::metadata(address_index))) }) .collect() } diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs new file mode 100644 index 0000000000..7930d5bf3e --- /dev/null +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -0,0 +1,98 @@ +//! Functions for wallet support of ephemeral transparent addresses. +use std::ops::RangeInclusive; + +use rusqlite::{named_params, OptionalExtension}; + +use zcash_client_backend::wallet::TransparentAddressMetadata; +use zcash_primitives::legacy::keys::{NonHardenedChildIndex, TransparentKeyScope}; + +use crate::{error::SqliteClientError, AccountId}; + +/// The number of ephemeral addresses that can be safely reserved without observing any +/// of them to be mined. This is the same as the gap limit in Bitcoin. +pub(crate) const GAP_LIMIT: i32 = 20; + +// The custom scope used for derivation of ephemeral addresses. +// TODO: consider moving this to `zcash_primitives::legacy::keys`, or else +// provide a way to derive `ivk`s for custom scopes in general there, so that +// the constant isn't duplicated. +pub(crate) const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2) { + Some(s) => s, + None => unreachable!(), +}; + +// Returns `TransparentAddressMetadata` in the ephemeral scope for the +// given address index. +pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddressMetadata { + TransparentAddressMetadata::new(EPHEMERAL_SCOPE, address_index) +} + +/// Returns the last reserved ephemeral address index in the given account, +/// or -1 if the account has no reserved ephemeral addresses. +pub(crate) fn last_reserved_index( + conn: &rusqlite::Connection, + account_id: AccountId, +) -> Result { + match conn + .query_row( + "SELECT address_index FROM ephemeral_addresses + WHERE account_id = :account_id + ORDER BY address_index DESC + LIMIT 1", + named_params![":account_id": account_id.0], + |row| row.get::<_, i32>(0), + ) + .optional()? + { + Some(i) if i < 0 => Err(SqliteClientError::CorruptedData( + "negative index".to_owned(), + )), + Some(i) => Ok(i), + None => Ok(-1), + } +} + +/// Returns the last ephemeral address index in the given account that +/// would not violate the gap invariant if used. +pub(crate) fn last_safe_index( + conn: &rusqlite::Connection, + account_id: AccountId, +) -> Result { + // The inner join with `transactions` excludes addresses for which + // `mined_in_tx` is NULL. The query also excludes addresses observed + // to have been mined in a transaction that we currently see as unmined. + // This is conservative in terms of avoiding violation of the gap + // invariant: it can only cause us to get to the end of the gap sooner. + let last_mined_index: i32 = match conn + .query_row( + "SELECT address_index FROM ephemeral_addresses + JOIN transactions t ON t.id_tx = mined_in_tx + WHERE account_id = :account_id AND t.mined_height IS NOT NULL + ORDER BY address_index DESC + LIMIT 1", + named_params![":account_id": account_id.0], + |row| row.get::<_, i32>(0), + ) + .optional()? + { + Some(i) if i < 0 => Err(SqliteClientError::CorruptedData( + "negative index".to_owned(), + )), + Some(i) => Ok(i), + None => Ok(-1), + }?; + Ok(u32::try_from(last_mined_index.saturating_add(GAP_LIMIT)).unwrap()) +} + +/// Utility function to return an `InclusiveRange` that starts at `i + 1` +/// and is of length up to `n`. The range is truncated if necessary to end at +/// the maximum valid address index, `i32::MAX`. +/// +/// Precondition: `i >= -1 and n > 0` +pub(crate) fn range_after(i: i32, n: i32) -> RangeInclusive { + assert!(i >= -1); + assert!(n > 0); + let first = u32::try_from(i64::from(i) + 1).unwrap(); + let last = u32::try_from(i.saturating_add(n)).unwrap(); + first..=last +} From 914acb57ce418844e5501aca0c440f75c2fe1cf3 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 24 Jun 2024 13:04:45 +0100 Subject: [PATCH 12/56] Move most remaining code for wallet support of ephemeral addresses into `zcash_client_sqlite::wallet::transparent::ephemeral`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/lib.rs | 10 +- zcash_client_sqlite/src/wallet/transparent.rs | 261 +---------------- .../src/wallet/transparent/ephemeral.rs | 263 +++++++++++++++++- 3 files changed, 268 insertions(+), 266 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 8ffb097801..3e062b2bd3 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -540,7 +540,7 @@ impl, P: consensus::Parameters> WalletRead for W account: Self::AccountId, for_detection: bool, ) -> Result>, Self::Error> { - wallet::transparent::get_reserved_ephemeral_addresses( + wallet::transparent::ephemeral::get_reserved_ephemeral_addresses( self.conn.borrow(), &self.params, account, @@ -1294,7 +1294,7 @@ impl WalletWrite for WalletDb // *reliably* mined, because that is strictly more conservative in avoiding // going over the gap limit. #[cfg(feature = "transparent-inputs")] - wallet::transparent::mark_ephemeral_address_as_mined(wdb, &address, tx_ref)?; + wallet::transparent::ephemeral::mark_ephemeral_address_as_mined(wdb, &address, tx_ref)?; let receiver = Receiver::Transparent(address); @@ -1450,7 +1450,7 @@ impl WalletWrite for WalletDb ephemeral_address, *receiving_account, )?; - wallet::transparent::mark_ephemeral_address_as_used( + wallet::transparent::ephemeral::mark_ephemeral_address_as_used( wdb, ephemeral_address, tx_ref, @@ -1459,7 +1459,7 @@ impl WalletWrite for WalletDb #[cfg(feature = "transparent-inputs")] Recipient::External(zcash_address, PoolType::Transparent) => { // Always reject sending to one of our ephemeral addresses. - wallet::transparent::check_address_is_not_ephemeral( + wallet::transparent::ephemeral::check_address_is_not_ephemeral( wdb, &zcash_address.encode(), )?; @@ -1485,7 +1485,7 @@ impl WalletWrite for WalletDb n: i32, ) -> Result, Self::Error> { self.transactionally(|wdb| { - wallet::transparent::reserve_next_n_ephemeral_addresses(wdb, account_id, n) + wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses(wdb, account_id, n) }) } } diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 82e43cedeb..8f9e6ec593 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -1,5 +1,4 @@ //! Functions for transparent input support in the wallet. -use std::cmp::max; use std::collections::{HashMap, HashSet}; use rusqlite::OptionalExtension; @@ -9,31 +8,23 @@ use zip32::{DiversifierIndex, Scope}; use zcash_address::unified::{Encoding, Ivk, Uivk}; use zcash_client_backend::{ data_api::AccountBalance, - keys::AddressGenerationError, wallet::{TransparentAddressMetadata, WalletTransparentOutput}, }; -use zcash_keys::{ - address::Address, - encoding::{encode_transparent_address_p, AddressCodec}, -}; +use zcash_keys::{address::Address, encoding::AddressCodec}; use zcash_primitives::{ legacy::{ - keys::{EphemeralIvk, IncomingViewingKey, NonHardenedChildIndex}, + keys::{IncomingViewingKey, NonHardenedChildIndex}, Script, TransparentAddress, }, - transaction::{ - components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, - TxId, - }, + transaction::components::{amount::NonNegativeAmount, Amount, OutPoint, TxOut}, }; use zcash_protocol::consensus::{self, BlockHeight}; use crate::{error::SqliteClientError, AccountId, UtxoId}; -use crate::{SqlTransaction, WalletDb}; -use super::{chain_tip_height, get_account, get_account_ids}; +use super::{chain_tip_height, get_account_ids}; -mod ephemeral; +pub(crate) mod ephemeral; pub(crate) fn detect_spending_accounts<'a>( conn: &Connection, @@ -608,248 +599,6 @@ pub(crate) fn put_transparent_output( Ok(utxo_id) } -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in a transaction that we have just created. This has no effect if `address` is -/// not one of our ephemeral addresses. -/// -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -pub(crate) fn mark_ephemeral_address_as_used( - wdb: &mut WalletDb, P>, - ephemeral_address: &TransparentAddress, - tx_ref: i64, -) -> Result<(), SqliteClientError> { - let address_str = encode_transparent_address_p(&wdb.params, ephemeral_address); - ephemeral_address_check_internal(wdb, &address_str, false)?; - - wdb.conn.0.execute( - "UPDATE ephemeral_addresses SET used_in_tx = :used_in_tx WHERE address = :address", - named_params![":used_in_tx": &tx_ref, ":address": address_str], - )?; - Ok(()) -} - -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if `address` is -/// an ephemeral transparent address. -pub(crate) fn check_address_is_not_ephemeral( - wdb: &mut WalletDb, P>, - address_str: &str, -) -> Result<(), SqliteClientError> { - ephemeral_address_check_internal(wdb, address_str, true) -} - -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. If `reject_all_ephemeral` is set, return an error if the address -/// is ephemeral at all, regardless of reuse. -fn ephemeral_address_check_internal( - wdb: &mut WalletDb, P>, - address_str: &str, - reject_all_ephemeral: bool, -) -> Result<(), SqliteClientError> { - // It is intentional that we don't require `t.mined_height` to be non-null. - // That is, we conservatively treat an ephemeral address as potentially - // reused even if we think that the transaction where we had evidence of - // its use is at present unmined. This should never occur in supported - // situations where only a single correctly operating wallet instance is - // using a given seed, because such a wallet will not reuse an address that - // it ever reserved. - // - // `COALESCE(used_in_tx, mined_in_tx)` can only differ from `used_in_tx` - // if the address was reserved, an error occurred in transaction creation - // before calling `mark_ephemeral_address_as_used`, and then we observed - // the address to have been used in a mined transaction (presumably by - // another wallet instance, or due to a bug) anyway. - let res = wdb - .conn - .0 - .query_row( - "SELECT t.txid FROM ephemeral_addresses - LEFT OUTER JOIN transactions t - ON t.id_tx = COALESCE(used_in_tx, mined_in_tx) - WHERE address = :address", - named_params![":address": address_str], - |row| row.get::<_, Option>>(0), - ) - .optional()?; - - match res { - Some(Some(txid_bytes)) => { - let txid = TxId::from_bytes( - txid_bytes - .try_into() - .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, - ); - Err(SqliteClientError::EphemeralAddressReuse( - address_str.to_owned(), - Some(txid), - )) - } - Some(None) if reject_all_ephemeral => Err(SqliteClientError::EphemeralAddressReuse( - address_str.to_owned(), - None, - )), - _ => Ok(()), - } -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in the given mined transaction (which may or may not be a transaction we sent). -/// -/// `tx_ref` must be a valid transaction reference. This call has no effect if -/// `address` is not one of our ephemeral addresses. -pub(crate) fn mark_ephemeral_address_as_mined( - wdb: &mut WalletDb, P>, - address: &TransparentAddress, - tx_ref: i64, -) -> Result<(), SqliteClientError> { - let address_str = encode_transparent_address_p(&wdb.params, address); - - // Figure out which transaction was mined earlier: `tx_ref`, or any existing - // tx referenced by `mined_in_tx` for the given address. Prefer the existing - // reference in case of a tie or if both transactions are unmined. - // This slightly reduces the chance of unnecessarily reaching the gap limit - // too early in some corner cases (because the earlier transaction is less - // likely to be unmined). - // - // The query should always return a value if `tx_ref` is valid. - let earlier_ref = wdb.conn.0.query_row( - "SELECT id_tx FROM transactions - LEFT OUTER JOIN ephemeral_addresses e - ON id_tx = e.mined_in_tx - WHERE id_tx = :tx_ref OR e.address = :address - ORDER BY mined_height ASC NULLS LAST, - tx_index ASC NULLS LAST, - e.mined_in_tx ASC NULLS LAST - LIMIT 1", - named_params![":tx_ref": &tx_ref, ":address": address_str], - |row| row.get::<_, i64>(0), - )?; - - wdb.conn.0.execute( - "UPDATE ephemeral_addresses SET mined_in_tx = :mined_in_tx WHERE address = :address", - named_params![":mined_in_tx": &earlier_ref, ":address": address_str], - )?; - Ok(()) -} - -/// Returns the ephemeral transparent IVK for a given account ID. -pub(crate) fn get_ephemeral_ivk( - conn: &rusqlite::Connection, - params: &P, - account_id: AccountId, -) -> Result { - use zcash_client_backend::data_api::Account; - - Ok(get_account(conn, params, account_id)? - .ok_or(SqliteClientError::AccountUnknown)? - .ufvk() - .and_then(|ufvk| ufvk.transparent()) - .ok_or(SqliteClientError::UnknownZip32Derivation)? - .derive_ephemeral_ivk()?) -} - -/// Returns a vector with all ephemeral transparent addresses potentially belonging to this wallet. -/// If `for_detection` is true, this includes addresses for an additional GAP_LIMIT indices. -pub(crate) fn get_reserved_ephemeral_addresses( - conn: &rusqlite::Connection, - params: &P, - account_id: AccountId, - for_detection: bool, -) -> Result>, SqliteClientError> { - let mut stmt = conn.prepare( - "SELECT address, address_index FROM ephemeral_addresses WHERE account_id = :account ORDER BY address_index", - )?; - let mut rows = stmt.query(named_params! { ":account": account_id.0 })?; - - let mut result = HashMap::new(); - let mut first_unused_index: Option = Some(0); - - while let Some(row) = rows.next()? { - let addr_str: String = row.get(0)?; - let raw_index: u32 = row.get(1)?; - first_unused_index = i32::try_from(raw_index) - .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? - .checked_add(1); - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); - let address = TransparentAddress::decode(params, &addr_str)?; - result.insert(address, Some(ephemeral::metadata(address_index))); - } - - if for_detection { - if let Some(first) = first_unused_index { - let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; - - for raw_index in ephemeral::range_after(first, ephemeral::GAP_LIMIT) { - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); - let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; - result.insert(address, Some(ephemeral::metadata(address_index))); - } - } - } - Ok(result) -} - -/// Returns a vector with the next `n` previously unreserved ephemeral addresses for -/// the given account. -/// -/// Precondition: `n >= 0` -/// -/// # Errors -/// -/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. -/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and -/// it is not possible to derive new addresses for it. -/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses -/// within the gap limit after the last address in this account that is known to have an -/// output in a mined transaction. -/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, -/// if the limit on transparent address indices has been reached. -pub(crate) fn reserve_next_n_ephemeral_addresses( - wdb: &mut WalletDb, P>, - account_id: AccountId, - n: i32, -) -> Result, SqliteClientError> { - if n == 0 { - return Ok(vec![]); - } - assert!(n > 0); - - let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; - let last_reserved_index = ephemeral::last_reserved_index(wdb.conn.0, account_id)?; - let last_safe_index = ephemeral::last_safe_index(wdb.conn.0, account_id)?; - let allocation = ephemeral::range_after(last_reserved_index, n); - - if allocation.clone().count() < n.try_into().unwrap() { - return Err(SqliteClientError::AddressGeneration( - AddressGenerationError::DiversifierSpaceExhausted, - )); - } - if *allocation.end() > last_safe_index { - let unsafe_index = max(*allocation.start(), last_safe_index.saturating_add(1)); - return Err(SqliteClientError::ReachedGapLimit(account_id, unsafe_index)); - } - - // used_in_tx and mined_in_tx are initially NULL - let mut stmt_insert_ephemeral_address = wdb.conn.0.prepare_cached( - "INSERT INTO ephemeral_addresses (account_id, address_index, address) - VALUES (:account_id, :address_index, :address)", - )?; - - allocation - .map(|raw_index| { - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); - let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; - - stmt_insert_ephemeral_address.execute(named_params![ - ":account_id": account_id.0, - ":address_index": raw_index, - ":address": encode_transparent_address_p(&wdb.params, &address) - ])?; - Ok((address, ephemeral::metadata(address_index))) - }) - .collect() -} - #[cfg(test)] mod tests { use crate::testing::{AddressType, TestBuilder, TestState}; diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 7930d5bf3e..04f25f0237 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -1,12 +1,25 @@ //! Functions for wallet support of ephemeral transparent addresses. +use std::cmp::max; +use std::collections::HashMap; use std::ops::RangeInclusive; use rusqlite::{named_params, OptionalExtension}; -use zcash_client_backend::wallet::TransparentAddressMetadata; -use zcash_primitives::legacy::keys::{NonHardenedChildIndex, TransparentKeyScope}; +use zcash_client_backend::{data_api::Account, wallet::TransparentAddressMetadata}; +use zcash_keys::{ + encoding::{encode_transparent_address_p, AddressCodec}, + keys::AddressGenerationError, +}; +use zcash_primitives::{ + legacy::{ + keys::{EphemeralIvk, NonHardenedChildIndex, TransparentKeyScope}, + TransparentAddress, + }, + transaction::TxId, +}; +use zcash_protocol::consensus; -use crate::{error::SqliteClientError, AccountId}; +use crate::{error::SqliteClientError, wallet::get_account, AccountId, SqlTransaction, WalletDb}; /// The number of ephemeral addresses that can be safely reserved without observing any /// of them to be mined. This is the same as the gap limit in Bitcoin. @@ -16,14 +29,14 @@ pub(crate) const GAP_LIMIT: i32 = 20; // TODO: consider moving this to `zcash_primitives::legacy::keys`, or else // provide a way to derive `ivk`s for custom scopes in general there, so that // the constant isn't duplicated. -pub(crate) const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2) { +const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2) { Some(s) => s, None => unreachable!(), }; // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. -pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddressMetadata { +fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddressMetadata { TransparentAddressMetadata::new(EPHEMERAL_SCOPE, address_index) } @@ -96,3 +109,243 @@ pub(crate) fn range_after(i: i32, n: i32) -> RangeInclusive { let last = u32::try_from(i.saturating_add(n)).unwrap(); first..=last } + +/// Returns the ephemeral transparent IVK for a given account ID. +pub(crate) fn get_ephemeral_ivk( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, +) -> Result { + Ok(get_account(conn, params, account_id)? + .ok_or(SqliteClientError::AccountUnknown)? + .ufvk() + .and_then(|ufvk| ufvk.transparent()) + .ok_or(SqliteClientError::UnknownZip32Derivation)? + .derive_ephemeral_ivk()?) +} + +/// Returns a vector with all ephemeral transparent addresses potentially belonging to this wallet. +/// If `for_detection` is true, this includes addresses for an additional GAP_LIMIT indices. +pub(crate) fn get_reserved_ephemeral_addresses( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, + for_detection: bool, +) -> Result>, SqliteClientError> { + let mut stmt = conn.prepare( + "SELECT address, address_index FROM ephemeral_addresses WHERE account_id = :account ORDER BY address_index", + )?; + let mut rows = stmt.query(named_params! { ":account": account_id.0 })?; + + let mut result = HashMap::new(); + let mut first_unused_index: Option = Some(0); + + while let Some(row) = rows.next()? { + let addr_str: String = row.get(0)?; + let raw_index: u32 = row.get(1)?; + first_unused_index = i32::try_from(raw_index) + .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? + .checked_add(1); + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address = TransparentAddress::decode(params, &addr_str)?; + result.insert(address, Some(metadata(address_index))); + } + + if for_detection { + if let Some(first) = first_unused_index { + let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; + + for raw_index in range_after(first, GAP_LIMIT) { + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; + result.insert(address, Some(metadata(address_index))); + } + } + } + Ok(result) +} + +/// Returns a vector with the next `n` previously unreserved ephemeral addresses for +/// the given account. +/// +/// Precondition: `n >= 0` +/// +/// # Errors +/// +/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. +/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and +/// it is not possible to derive new addresses for it. +/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses +/// within the gap limit after the last address in this account that is known to have an +/// output in a mined transaction. +/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, +/// if the limit on transparent address indices has been reached. +pub(crate) fn reserve_next_n_ephemeral_addresses( + wdb: &mut WalletDb, P>, + account_id: AccountId, + n: i32, +) -> Result, SqliteClientError> { + if n == 0 { + return Ok(vec![]); + } + assert!(n > 0); + + let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; + let last_reserved_index = last_reserved_index(wdb.conn.0, account_id)?; + let last_safe_index = last_safe_index(wdb.conn.0, account_id)?; + let allocation = range_after(last_reserved_index, n); + + if allocation.clone().count() < n.try_into().unwrap() { + return Err(SqliteClientError::AddressGeneration( + AddressGenerationError::DiversifierSpaceExhausted, + )); + } + if *allocation.end() > last_safe_index { + let unsafe_index = max(*allocation.start(), last_safe_index.saturating_add(1)); + return Err(SqliteClientError::ReachedGapLimit(account_id, unsafe_index)); + } + + // used_in_tx and mined_in_tx are initially NULL + let mut stmt_insert_ephemeral_address = wdb.conn.0.prepare_cached( + "INSERT INTO ephemeral_addresses (account_id, address_index, address) + VALUES (:account_id, :address_index, :address)", + )?; + + allocation + .map(|raw_index| { + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; + + stmt_insert_ephemeral_address.execute(named_params![ + ":account_id": account_id.0, + ":address_index": raw_index, + ":address": encode_transparent_address_p(&wdb.params, &address) + ])?; + Ok((address, metadata(address_index))) + }) + .collect() +} + +/// Returns a `SqliteClientError::EphemeralAddressReuse` error if `address` is +/// an ephemeral transparent address. +pub(crate) fn check_address_is_not_ephemeral( + wdb: &mut WalletDb, P>, + address_str: &str, +) -> Result<(), SqliteClientError> { + ephemeral_address_check_internal(wdb, address_str, true) +} + +/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was +/// already used. If `reject_all_ephemeral` is set, return an error if the address +/// is ephemeral at all, regardless of reuse. +fn ephemeral_address_check_internal( + wdb: &mut WalletDb, P>, + address_str: &str, + reject_all_ephemeral: bool, +) -> Result<(), SqliteClientError> { + // It is intentional that we don't require `t.mined_height` to be non-null. + // That is, we conservatively treat an ephemeral address as potentially + // reused even if we think that the transaction where we had evidence of + // its use is at present unmined. This should never occur in supported + // situations where only a single correctly operating wallet instance is + // using a given seed, because such a wallet will not reuse an address that + // it ever reserved. + // + // `COALESCE(used_in_tx, mined_in_tx)` can only differ from `used_in_tx` + // if the address was reserved, an error occurred in transaction creation + // before calling `mark_ephemeral_address_as_used`, and then we observed + // the address to have been used in a mined transaction (presumably by + // another wallet instance, or due to a bug) anyway. + let res = wdb + .conn + .0 + .query_row( + "SELECT t.txid FROM ephemeral_addresses + LEFT OUTER JOIN transactions t + ON t.id_tx = COALESCE(used_in_tx, mined_in_tx) + WHERE address = :address", + named_params![":address": address_str], + |row| row.get::<_, Option>>(0), + ) + .optional()?; + + match res { + Some(Some(txid_bytes)) => { + let txid = TxId::from_bytes( + txid_bytes + .try_into() + .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, + ); + Err(SqliteClientError::EphemeralAddressReuse( + address_str.to_owned(), + Some(txid), + )) + } + Some(None) if reject_all_ephemeral => Err(SqliteClientError::EphemeralAddressReuse( + address_str.to_owned(), + None, + )), + _ => Ok(()), + } +} + +/// If `address` is one of our ephemeral addresses, mark it as having an output +/// in a transaction that we have just created. This has no effect if `address` is +/// not one of our ephemeral addresses. +/// +/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was +/// already used. +pub(crate) fn mark_ephemeral_address_as_used( + wdb: &mut WalletDb, P>, + ephemeral_address: &TransparentAddress, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let address_str = encode_transparent_address_p(&wdb.params, ephemeral_address); + ephemeral_address_check_internal(wdb, &address_str, false)?; + + wdb.conn.0.execute( + "UPDATE ephemeral_addresses SET used_in_tx = :used_in_tx WHERE address = :address", + named_params![":used_in_tx": &tx_ref, ":address": address_str], + )?; + Ok(()) +} + +/// If `address` is one of our ephemeral addresses, mark it as having an output +/// in the given mined transaction (which may or may not be a transaction we sent). +/// +/// `tx_ref` must be a valid transaction reference. This call has no effect if +/// `address` is not one of our ephemeral addresses. +pub(crate) fn mark_ephemeral_address_as_mined( + wdb: &mut WalletDb, P>, + address: &TransparentAddress, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let address_str = encode_transparent_address_p(&wdb.params, address); + + // Figure out which transaction was mined earlier: `tx_ref`, or any existing + // tx referenced by `mined_in_tx` for the given address. Prefer the existing + // reference in case of a tie or if both transactions are unmined. + // This slightly reduces the chance of unnecessarily reaching the gap limit + // too early in some corner cases (because the earlier transaction is less + // likely to be unmined). + // + // The query should always return a value if `tx_ref` is valid. + let earlier_ref = wdb.conn.0.query_row( + "SELECT id_tx FROM transactions + LEFT OUTER JOIN ephemeral_addresses e + ON id_tx = e.mined_in_tx + WHERE id_tx = :tx_ref OR e.address = :address + ORDER BY mined_height ASC NULLS LAST, + tx_index ASC NULLS LAST, + e.mined_in_tx ASC NULLS LAST + LIMIT 1", + named_params![":tx_ref": &tx_ref, ":address": address_str], + |row| row.get::<_, i64>(0), + )?; + + wdb.conn.0.execute( + "UPDATE ephemeral_addresses SET mined_in_tx = :mined_in_tx WHERE address = :address", + named_params![":mined_in_tx": &earlier_ref, ":address": address_str], + )?; + Ok(()) +} From 745054ba69aa7a6952f397fda024c5b475167a2a Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 24 Jun 2024 10:53:36 +0100 Subject: [PATCH 13/56] `find_account_for_transparent_output` now searches unreserved ephemeral addresses within the gap limit. This should make recording TXOs found at these addresses via `WalletWrite::put_received_transparent_utxo` work correctly. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/transparent.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 8f9e6ec593..dc8fe55566 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -504,19 +504,36 @@ pub(crate) fn find_account_for_transparent_output( return Ok(Some(account_id)); } + let account_ids = get_account_ids(conn)?; + // If the UTXO is received at the legacy transparent address (at BIP 44 address // index 0 within its particular account, which we specifically ensure is returned // from `get_transparent_receivers`), there may be no entry in the addresses table // that can be used to tie the address to a particular account. In this case, we // look up the legacy address for each account in the wallet, and check whether it // matches the address for the received UTXO. - for account_id in get_account_ids(conn)? { + for &account_id in account_ids.iter() { if let Some((legacy_taddr, _)) = get_legacy_transparent_address(params, conn, account_id)? { if &legacy_taddr == output.recipient_address() { return Ok(Some(account_id)); } } } + + // Finally we check for ephemeral addresses within the gap limit. + for account_id in account_ids { + let ephemeral_ivk = ephemeral::get_ephemeral_ivk(conn, params, account_id)?; + let last_reserved_index = ephemeral::last_reserved_index(conn, account_id)?; + + for raw_index in ephemeral::range_after(last_reserved_index, ephemeral::GAP_LIMIT) { + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + + if &ephemeral_ivk.derive_ephemeral_address(address_index)? == output.recipient_address() + { + return Ok(Some(account_id)); + } + } + } Ok(None) } From 6b465b702ebc92e642779501942f985cef345dad Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 24 Jun 2024 13:27:32 +0100 Subject: [PATCH 14/56] Document the mapping functions on `zcash_client_backend::wallet::Recipient`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/wallet.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index f6897fac08..8783206140 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -83,6 +83,7 @@ pub enum Recipient { } impl Recipient { + /// Return a copy of this `Recipient` with `f` applied to the note metadata, if any. pub fn map_internal_account_note B>( self, f: F, @@ -110,6 +111,7 @@ impl Recipient { } } + /// Return a copy of this `Recipient` with `f` applied to the output metadata, if any. pub fn map_ephemeral_transparent_outpoint B>( self, f: F, @@ -139,6 +141,8 @@ impl Recipient { } impl Recipient, O> { + /// Return a copy of this `Recipient` with optional note metadata transposed to + /// an optional result. pub fn internal_account_note_transpose_option(self) -> Option> { match self { Recipient::External(addr, pool) => Some(Recipient::External(addr, pool)), From 4f43a01f83c0bf823bf6033460a9f88f03f51f5a Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Tue, 25 Jun 2024 12:04:35 +0100 Subject: [PATCH 15/56] Refactor transparent address metadata lookups. This is correct as-is but will be simplified and made more efficient in subsequent commmits. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 5 +- zcash_client_backend/src/data_api.rs | 35 ++++++++++++++ zcash_client_backend/src/data_api/wallet.rs | 46 ++++++++----------- zcash_client_sqlite/src/wallet/transparent.rs | 9 +--- 4 files changed, 58 insertions(+), 37 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 49e2852859..ecab735ded 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -48,12 +48,13 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail `change_memo` is given, and defends against losing money by using `DustAction::AddDustToFee` with a too-high dust threshold. See [#1430](https://github.com/zcash/librustzcash/pull/1430) for details. -- `zcash_client_backend::zip321` has been extracted to, and is now a reexport +- `zcash_client_backend::zip321` has been extracted to, and is now a reexport of the root module of the `zip321` crate. Several of the APIs of this module have changed as a consequence of this extraction; please see the `zip321` CHANGELOG for details. - `zcash_client_backend::data_api`: - - `WalletRead` has a new `get_reserved_ephemeral_addresses` method. + - `WalletRead` has new `get_reserved_ephemeral_addresses` and + `get_transparent_address_metadata` methods. - `WalletWrite` has a new `reserve_next_n_ephemeral_addresses` method. - `error::Error` has a new `Address` variant. - `wallet::input_selection::InputSelectorError` has a new `Address` variant. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 4501ef87dc..d049d18467 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -921,6 +921,41 @@ pub trait WalletRead { Ok(HashMap::new()) } + /// Returns the metadata associated with a given transparent receiver in an account + /// controlled by this wallet. + /// + /// This is equivalent to (but may be implemented more efficiently than): + /// ```compile_fail + /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + /// return Ok(result.clone()); + /// } + /// if let Some(result) = self.get_reserved_ephemeral_addresses(account, false)?.get(address) { + /// return Ok(result.clone()); + /// } + /// Ok(None) + /// ``` + /// + /// Returns `Ok(None)` if the address is not recognized, or we do not have metadata for it. + /// Returns `Ok(Some(metadata))` if we have the metadata. + #[cfg(feature = "transparent-inputs")] + fn get_transparent_address_metadata( + &self, + account: Self::AccountId, + address: &TransparentAddress, + ) -> Result, Self::Error> { + // This should be overridden. + if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + return Ok(result.clone()); + } + if let Some(result) = self + .get_reserved_ephemeral_addresses(account, false)? + .get(address) + { + return Ok(result.clone()); + } + Ok(None) + } + /// Returns the set of reserved ephemeral transparent addresses associated with the /// given account controlled by this wallet. /// diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index dd36b369bf..da1bd173a4 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -81,7 +81,6 @@ use { }, input_selection::ShieldingSelector, std::collections::HashMap, - zcash_keys::encoding::AddressCodec, zcash_primitives::transaction::components::TxOut, }; @@ -829,41 +828,32 @@ where } #[cfg(feature = "transparent-inputs")] - let mut known_addrs = wallet_db - .get_transparent_receivers(account_id) - .map_err(Error::DataSource)?; - #[cfg(feature = "transparent-inputs")] - let mut ephemeral_added = false; + let mut cache = HashMap::::new(); #[cfg(feature = "transparent-inputs")] let mut metadata_from_address = |addr: TransparentAddress| -> Result< TransparentAddressMetadata, ErrorT, > { - match known_addrs.get(&addr) { - None if !ephemeral_added => { - // The ephemeral addresses are added lazily to avoid extra database operations - // in the common case. We don't need to include them in order to be able to - // construct ZIP 320 transactions, because in that case the ephemeral output - // is represented via a "change" reference to a previous step. However, we do - // need them in order to create a transaction from a proposal that explicitly - // spends an output from an ephemeral address. This need not set `for_detection` - // because we only need to be able to spend outputs already detected by this - // wallet instance. - ephemeral_added = true; - known_addrs.extend( - wallet_db - .get_reserved_ephemeral_addresses(account_id, false) - .map_err(Error::DataSource)? - .into_iter(), - ); - known_addrs.get(&addr) + match cache.get(&addr) { + Some(result) => Ok(result.clone()), + None => { + // `wallet_db.get_transparent_address_metadata` includes reserved ephemeral + // addresses in its lookup. We don't need to include these in order to be + // able to construct ZIP 320 transactions, because in that case the ephemeral + // output is represented via a "change" reference to a previous step. However, + // we do need them in order to create a transaction from a proposal that + // explicitly spends an output from an ephemeral address (only for outputs + // already detected by this wallet instance). + + let result = wallet_db + .get_transparent_address_metadata(account_id, &addr) + .map_err(InputSelectorError::DataSource)? + .ok_or(Error::AddressNotRecognized(addr))?; + cache.insert(addr, result.clone()); + Ok(result) } - result => result, } - .ok_or(Error::AddressNotRecognized(addr))? - .clone() - .ok_or_else(|| Error::NoSpendingKey(addr.encode(params))) }; #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index dc8fe55566..816662ea76 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -112,13 +112,8 @@ pub(crate) fn get_transparent_receivers( } if let Some((taddr, address_index)) = get_legacy_transparent_address(params, conn, account)? { - ret.insert( - taddr, - Some(TransparentAddressMetadata::new( - Scope::External.into(), - address_index, - )), - ); + let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + ret.insert(taddr, Some(metadata)); } Ok(ret) From 5a90fffed47b02ebef4742ccb4f3778bd8d15f87 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Tue, 25 Jun 2024 16:59:34 +0100 Subject: [PATCH 16/56] Factor out the conversion of the `diversifier_index_be` field in the `addresses` table to a `NonHardenedChildIndex`. (This moves where a `diversifier_index_be` field of the wrong length would be detected and so is not quite a no-op, but that shouldn't matter.) Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/transparent.rs | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 816662ea76..ee164071f8 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -54,6 +54,28 @@ pub(crate) fn detect_spending_accounts<'a>( Ok(acc) } +/// Returns the `NonHardenedChildIndex` corresponding to a diversifier index +/// given as bytes in big-endian order (the reverse of the usual order). +fn address_index_from_diversifier_index_be( + diversifier_index_be: &[u8], +) -> Result { + let mut di: [u8; 11] = diversifier_index_be.try_into().map_err(|_| { + SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) + })?; + di.reverse(); // BE -> LE conversion + + NonHardenedChildIndex::from_index(DiversifierIndex::from(di).try_into().map_err(|_| { + SqliteClientError::CorruptedData( + "Unable to get diversifier for transparent address.".to_string(), + ) + })?) + .ok_or_else(|| { + SqliteClientError::CorruptedData( + "Unexpected hardened index for transparent address.".to_string(), + ) + }) +} + pub(crate) fn get_transparent_receivers( conn: &rusqlite::Connection, params: &P, @@ -70,10 +92,6 @@ pub(crate) fn get_transparent_receivers( while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; let di_vec: Vec = row.get(1)?; - let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion let ua = Address::decode(params, &ua_str) .ok_or_else(|| { @@ -88,26 +106,9 @@ pub(crate) fn get_transparent_receivers( })?; if let Some(taddr) = ua.transparent() { - let index = NonHardenedChildIndex::from_index( - DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_string(), - ) - })?, - ) - .ok_or_else(|| { - SqliteClientError::CorruptedData( - "Unexpected hardened index for transparent address.".to_string(), - ) - })?; - - ret.insert( - *taddr, - Some(TransparentAddressMetadata::new( - Scope::External.into(), - index, - )), - ); + let address_index = address_index_from_diversifier_index_be(&di_vec)?; + let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + ret.insert(*taddr, Some(metadata)); } } From 7fb355739ed5b835efd97ffb0138934094f1936b Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Tue, 25 Jun 2024 17:05:36 +0100 Subject: [PATCH 17/56] Implement `WalletRead::get_transparent_address_metadata` for `zcash_client_sqlite` using direct database queries. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api.rs | 9 ++++ zcash_client_sqlite/src/lib.rs | 14 +++++ zcash_client_sqlite/src/wallet/transparent.rs | 52 +++++++++++++++++++ .../src/wallet/transparent/ephemeral.rs | 2 +- 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d049d18467..5faec524c5 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1971,6 +1971,15 @@ pub mod testing { Ok(HashMap::new()) } + #[cfg(feature = "transparent-inputs")] + fn get_transparent_address_metadata( + &self, + _account: Self::AccountId, + _address: &TransparentAddress, + ) -> Result, Self::Error> { + Ok(None) + } + #[cfg(feature = "transparent-inputs")] fn get_reserved_ephemeral_addresses( &self, diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 3e062b2bd3..35cadcd5e7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -534,6 +534,20 @@ impl, P: consensus::Parameters> WalletRead for W ) } + #[cfg(feature = "transparent-inputs")] + fn get_transparent_address_metadata( + &self, + account: Self::AccountId, + address: &TransparentAddress, + ) -> Result, Self::Error> { + wallet::transparent::get_transparent_address_metadata( + self.conn.borrow(), + &self.params, + account, + address, + ) + } + #[cfg(feature = "transparent-inputs")] fn get_reserved_ephemeral_addresses( &self, diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index ee164071f8..77fe21b0f0 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet}; use rusqlite::OptionalExtension; use rusqlite::{named_params, Connection, Row}; +use zcash_keys::encoding::encode_transparent_address_p; use zip32::{DiversifierIndex, Scope}; use zcash_address::unified::{Encoding, Ivk, Uivk}; @@ -461,6 +462,57 @@ pub(crate) fn put_received_transparent_utxo( } } +pub(crate) fn get_transparent_address_metadata( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, + address: &TransparentAddress, +) -> Result, SqliteClientError> { + let address_str = encode_transparent_address_p(params, address); + + if let Some(di_vec) = conn + .query_row( + "SELECT diversifier_index_be FROM addresses + WHERE account_id = :account_id AND cached_transparent_receiver_address = :address", + named_params![":account_id": account_id.0, ":address": &address_str], + |row| row.get::<_, Vec>(0), + ) + .optional()? + { + let address_index = address_index_from_diversifier_index_be(&di_vec)?; + let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + return Ok(Some(metadata)); + } + + if let Some((legacy_taddr, address_index)) = + get_legacy_transparent_address(params, conn, account_id)? + { + if &legacy_taddr == address { + let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + return Ok(Some(metadata)); + } + } + + // Search ephemeral addresses that have already been reserved. + if let Some(raw_index) = conn + .query_row( + "SELECT address_index FROM ephemeral_addresses + WHERE account_id = :account_id AND address = :address", + named_params![":account_id": account_id.0, ":address": &address_str], + |row| row.get::<_, u32>(0), + ) + .optional()? + { + let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + return Ok(Some(ephemeral::metadata(address_index))); + } + + // We intentionally don't check for unreserved ephemeral addresses within the gap + // limit here. It's unnecessary to look up metadata for addresses from which we + // can spend. + Ok(None) +} + /// Attempts to determine the account that received the given transparent output. /// /// The following three locations in the wallet's key tree are searched: diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 04f25f0237..6394d9be80 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -36,7 +36,7 @@ const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2 // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. -fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddressMetadata { +pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddressMetadata { TransparentAddressMetadata::new(EPHEMERAL_SCOPE, address_index) } From 7d8a96f827b697b14c0eeb80e11c6aaabbf49547 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Tue, 25 Jun 2024 21:15:22 +0100 Subject: [PATCH 18/56] Don't cache metadata between steps; it's not an important optimization. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 31 ++++++--------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index da1bd173a4..f1d9dc520e 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -33,7 +33,6 @@ to a wallet-internal shielded address, as described in [ZIP 316](https://zips.z. //! [`TransactionRequest`]: crate::zip321::TransactionRequest //! [`propose_transfer`]: crate::data_api::wallet::propose_transfer -use core::convert::Infallible; use nonempty::NonEmpty; use rand_core::OsRng; use sapling::{ @@ -79,6 +78,7 @@ use { proposal::{ProposalError, StepOutput}, wallet::TransparentAddressMetadata, }, + core::convert::Infallible, input_selection::ShieldingSelector, std::collections::HashMap, zcash_primitives::transaction::components::TxOut, @@ -663,11 +663,7 @@ fn create_proposed_transaction( proposal_step: &Step, #[cfg(feature = "transparent-inputs")] unused_transparent_outputs: &mut HashMap< StepOutput, - ( - TransparentAddress, - Option, - OutPoint, - ), + (TransparentAddress, OutPoint), >, ) -> Result> where @@ -884,14 +880,12 @@ where for input_ref in proposal_step.prior_step_inputs() { // A referenced transparent step output must exist and be referenced *at most* once. // (Exactly once in the case of ephemeral outputs.) - let (address, address_metadata_opt, outpoint) = unused_transparent_outputs + let (address, outpoint) = unused_transparent_outputs .remove(input_ref) .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))?; - let address_metadata = match address_metadata_opt { - Some(meta) => meta, - None => metadata_from_address(address)?, - }; + let address_metadata = metadata_from_address(address)?; + let txout = &prior_step_results[input_ref.step_index()] .1 .transaction() @@ -952,11 +946,6 @@ where Some(sapling_dfvk.to_ovk(Scope::Internal)) }; - #[cfg(feature = "transparent-inputs")] - type TransparentMetadataT = TransparentAddressMetadata; - #[cfg(not(feature = "transparent-inputs"))] - type TransparentMetadataT = Infallible; - #[cfg(feature = "orchard")] let mut orchard_output_meta: Vec<( Recipient<_, PoolType, _>, @@ -971,7 +960,6 @@ where let mut transparent_output_meta: Vec<( Recipient<_, _, ()>, TransparentAddress, - Option, NonNegativeAmount, StepOutputIndex, )> = vec![]; @@ -1030,7 +1018,6 @@ where transparent_output_meta.push(( Recipient::External(recipient_address.clone(), PoolType::TRANSPARENT), to, - None, payment.amount(), StepOutputIndex::Payment(payment_index), )); @@ -1154,7 +1141,8 @@ where .map_err(Error::DataSource)?; assert_eq!(addresses_and_metadata.len(), ephemeral_outputs.len()); - for ((change_index, change_value), (ephemeral_address, address_metadata)) in + // We don't need the TransparentAddressMetadata here; we can look it up from the data source later. + for ((change_index, change_value), (ephemeral_address, _)) in ephemeral_outputs.iter().zip(addresses_and_metadata) { // This is intended for an ephemeral transparent output, rather than a @@ -1168,7 +1156,6 @@ where outpoint_metadata: (), }, ephemeral_address, - Some(address_metadata), change_value.value(), StepOutputIndex::Change(*change_index), )) @@ -1253,13 +1240,13 @@ where #[allow(unused_variables)] let transparent_outputs = transparent_output_meta.into_iter().enumerate().map( - |(n, (recipient, ephemeral_address, address_metadata_opt, value, step_output_index))| { + |(n, (recipient, ephemeral_address, value, step_output_index))| { let outpoint = OutPoint::new(txid, n as u32); let recipient = recipient.map_ephemeral_transparent_outpoint(|()| outpoint.clone()); #[cfg(feature = "transparent-inputs")] unused_transparent_outputs.insert( StepOutput::new(step_index, step_output_index), - (ephemeral_address, address_metadata_opt, outpoint), + (ephemeral_address, outpoint), ); SentTransactionOutput::from_parts(n, recipient, value, None) }, From bd6c9f3599f7c2356790f08c2dcdc2aeeb4abcb3 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 26 Jun 2024 07:14:47 +0100 Subject: [PATCH 19/56] Apply documentation suggestions from code review. Co-authored-by: Kris Nuttycombe Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 3 +-- zcash_client_backend/src/data_api/wallet.rs | 4 +++- .../src/data_api/wallet/input_selection.rs | 9 +++++++-- .../src/wallet/transparent/ephemeral.rs | 16 ++++++++++------ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index ecab735ded..d227cca9e3 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -9,8 +9,7 @@ and this library adheres to Rust's notion of ### Notable changes `zcash_client_backend` now supports TEX (transparent-source-only) addresses as specified in ZIP 320. Sending to one or more TEX addresses will automatically create a multi-step -proposal that uses two transactions. This is intended to be used in conjunction with -`zcash_client_sqlite` 0.11 or later. +proposal that uses two transactions. In order to take advantage of this support, client wallets will need to be able to send multiple transactions created from `zcash_client_backend::data_api::wallet::create_proposed_transactions`. diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index f1d9dc520e..71ddf096fa 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -604,7 +604,9 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - // The set of transparent StepOutputs available and unused from prior steps. + // The set of transparent `StepOutput`s available and unused from prior steps. + // When a transparent `StepOutput` is created, it is added to the map. When it + // is consumed, it is removed from the map. #[cfg(feature = "transparent-inputs")] let mut unused_transparent_outputs = HashMap::new(); diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index bec5283ec1..9b1bf0e734 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -464,6 +464,9 @@ where // The ephemeral input going into transaction 1 must be able to pay that // transaction's fee, as well as the TEX address payments. + // First compute the required total without providing any input value, + // catching the `InsufficientFunds` error to obtain the required amount + // given the provided change strategy. let tr1_required_input_value = match self.change_strategy.compute_balance::<_, DbT::NoteRef>( params, @@ -475,7 +478,7 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - true, + true, // ignore change memo to avoid adding a change output #[cfg(feature = "transparent-inputs")] &[NonNegativeAmount::ZERO], #[cfg(feature = "transparent-inputs")] @@ -486,6 +489,8 @@ where Err(other) => return Err(other.into()), }; + // Now recompute to obtain the `TransactionBalance` and verify that it + // fully accounts for the required fees. let tr1_balance = self.change_strategy.compute_balance::<_, DbT::NoteRef>( params, target_height, @@ -496,7 +501,7 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - true, + true, // ignore change memo to avoid adding a change output #[cfg(feature = "transparent-inputs")] &[tr1_required_input_value], #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 6394d9be80..5327b487fc 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -26,6 +26,10 @@ use crate::{error::SqliteClientError, wallet::get_account, AccountId, SqlTransac pub(crate) const GAP_LIMIT: i32 = 20; // The custom scope used for derivation of ephemeral addresses. +// +// This must match the constant used in +// `zcash_primitives::legacy::keys::AccountPubKey::derive_ephemeral_ivk`. +// // TODO: consider moving this to `zcash_primitives::legacy::keys`, or else // provide a way to derive `ivk`s for custom scopes in general there, so that // the constant isn't duplicated. @@ -49,9 +53,9 @@ pub(crate) fn last_reserved_index( match conn .query_row( "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id - ORDER BY address_index DESC - LIMIT 1", + WHERE account_id = :account_id + ORDER BY address_index DESC + LIMIT 1", named_params![":account_id": account_id.0], |row| row.get::<_, i32>(0), ) @@ -261,9 +265,9 @@ fn ephemeral_address_check_internal( .0 .query_row( "SELECT t.txid FROM ephemeral_addresses - LEFT OUTER JOIN transactions t - ON t.id_tx = COALESCE(used_in_tx, mined_in_tx) - WHERE address = :address", + LEFT OUTER JOIN transactions t + ON t.id_tx = COALESCE(used_in_tx, mined_in_tx) + WHERE address = :address", named_params![":address": address_str], |row| row.get::<_, Option>>(0), ) From 0735390546c2b0b9ee20b0cdd9d98b5f4c3f5fad Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 26 Jun 2024 07:19:11 +0100 Subject: [PATCH 20/56] Rename `amount` to `transfer_amount` in `send_multi_step_proposed_transfer`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/testing/pool.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index ed6cb158ed..bbda91afdd 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -329,7 +329,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { }; let value = NonNegativeAmount::const_from_u64(100000); - let amount = NonNegativeAmount::const_from_u64(50000); + let transfer_amount = NonNegativeAmount::const_from_u64(50000); let run_test = |st: &mut TestState<_>, expected_index| { // Add funds to the wallet. @@ -337,7 +337,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { let expected_step0_fee = (zip317::MARGINAL_FEE * 3).unwrap(); let expected_step1_fee = zip317::MINIMUM_FEE; - let expected_ephemeral = (amount + expected_step1_fee).unwrap(); + let expected_ephemeral = (transfer_amount + expected_step1_fee).unwrap(); let expected_step0_change = (value - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); assert!(expected_step0_change.is_positive()); @@ -358,7 +358,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), &tex_addr, - amount, + transfer_amount, None, change_memo.clone(), T::SHIELDED_PROTOCOL, @@ -374,7 +374,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { steps[0].balance().proposed_change(), [ ChangeValue::shielded(T::SHIELDED_PROTOCOL, expected_step0_change, change_memo), - ChangeValue::ephemeral_transparent((amount + expected_step1_fee).unwrap()), + ChangeValue::ephemeral_transparent((transfer_amount + expected_step1_fee).unwrap()), ] ); assert_eq!(steps[1].balance().proposed_change(), []); @@ -437,7 +437,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { assert_matches!( confirmed_sent[1][0].clone(), (sent_v, sent_to_addr, None, None) - if sent_v == u64::try_from(amount).unwrap() && sent_to_addr == Some(tex_addr.encode(&st.wallet().params))); + if sent_v == u64::try_from(transfer_amount).unwrap() && sent_to_addr == Some(tex_addr.encode(&st.wallet().params))); (ephemeral_addr.unwrap(), txids.head) }; @@ -462,7 +462,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), &ephemeral_taddr, - amount, + transfer_amount, None, None, T::SHIELDED_PROTOCOL, From c926e7ce0aeacd4121924f1cbab5a69ff1cd4d0d Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 26 Jun 2024 07:20:50 +0100 Subject: [PATCH 21/56] Filter ephemeral transparent `ChangeValue`s by `is_ephemeral()` as well as pool type. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 71ddf096fa..a709bd2f03 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1133,7 +1133,9 @@ where .proposed_change() .iter() .enumerate() - .filter(|(_, change_value)| matches!(change_value.output_pool(), PoolType::Transparent)) + .filter(|(_, change_value)| { + change_value.is_ephemeral() && change_value.output_pool() == PoolType::Transparent + }) .collect(); let num_ephemeral_outputs = i32::try_from(ephemeral_outputs.len()).map_err(|_| Error::ProposalNotSupported)?; From ffb2ddf5943ddb5b5e62d430b12b484dbd048335 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 26 Jun 2024 08:06:26 +0100 Subject: [PATCH 22/56] `zcash_client_backend::fees::ChangeValue` is now an enum, allowing only valid combinations of output pool, memo presence, and ephemerality. Co-authored-by: Kris Nuttycombe Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 6 +- zcash_client_backend/src/fees.rs | 104 +++++++++++++++++++----------- 2 files changed, 70 insertions(+), 40 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index d227cca9e3..0319ca5679 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -64,9 +64,9 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - The return type of `ChangeValue::output_pool`, and the type of the `output_pool` argument to `ChangeValue::new`, have changed from `ShieldedProtocol` to `zcash_protocol::PoolType`. - - `ChangeValue::new` takes an additional `is_ephemeral` parameter indicating - whether the value is ephemeral (i.e. for use in a subsequent proposal step) - or change. + - When the "transparent-inputs" feature is enabled, `ChangeValue::new` + takes an additional `is_ephemeral` parameter indicating whether the value + is ephemeral (i.e. for use in a subsequent proposal step) or change. - The return type of `ChangeValue::new` is now optional; it returns `None` if a memo is given for the transparent pool. Use `ChangeValue::shielded` to avoid this error case when creating a `ChangeValue` known to be for a diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index a85acbdf28..893d53422c 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -21,51 +21,61 @@ pub mod sapling; pub mod standard; pub mod zip317; -/// A proposed change amount and output pool. +/// `ChangeValue` represents either a proposed change output to a shielded pool +/// (with an optional change memo), or if the "transparent-inputs" feature is +/// enabled, an ephemeral output to the transparent pool. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct ChangeValue { - output_pool: PoolType, - value: NonNegativeAmount, - memo: Option, - is_ephemeral: bool, +pub struct ChangeValue(ChangeValueInner); + +#[derive(Clone, Debug, PartialEq, Eq)] +enum ChangeValueInner { + Shielded { + protocol: ShieldedProtocol, + value: NonNegativeAmount, + memo: Option, + }, + #[cfg(feature = "transparent-inputs")] + EphemeralTransparent { value: NonNegativeAmount }, } impl ChangeValue { - /// Constructs a new change or ephemeral value from its constituent parts. - /// - /// Currently, the only supported combinations are change sent to a shielded - /// pool, or (if the "transparent-inputs" feature is enabled) an ephemeral - /// output to the transparent pool. + #[cfg_attr( + feature = "transparent-inputs", + doc = "Constructs a new change or ephemeral value from its constituent + parts. Currently, the only supported combinations are change sent + to a shielded pool, or an ephemeral output to the transparent pool." + )] + #[cfg_attr( + not(feature = "transparent-inputs"), + doc = "Constructs a new change value from its constituent parts. + Currently, `output_pool` must be a shielded pool (enable the + `transparent-inputs` feature to support ephemeral outputs to the + transparent pool). Use `ChangeValue::shielded` to avoid the + `Option` return." + )] pub fn new( output_pool: PoolType, value: NonNegativeAmount, memo: Option, - is_ephemeral: bool, + #[cfg(feature = "transparent-inputs")] is_ephemeral: bool, ) -> Option { - match output_pool { - PoolType::Shielded(_) => !is_ephemeral, + #[cfg(not(feature = "transparent-inputs"))] + let is_ephemeral = false; + + match (output_pool, is_ephemeral) { + (PoolType::Shielded(protocol), false) => Some(Self::shielded(protocol, value, memo)), #[cfg(feature = "transparent-inputs")] - PoolType::Transparent => is_ephemeral && memo.is_none(), - #[cfg(not(feature = "transparent-inputs"))] - PoolType::Transparent => false, + (PoolType::Transparent, true) if memo.is_none() => { + Some(Self::ephemeral_transparent(value)) + } + _ => None, } - .then_some(Self { - output_pool, - value, - memo, - is_ephemeral, - }) } /// Constructs a new ephemeral transparent output value. #[cfg(feature = "transparent-inputs")] pub fn ephemeral_transparent(value: NonNegativeAmount) -> Self { - Self { - output_pool: PoolType::TRANSPARENT, - value, - memo: None, - is_ephemeral: true, - } + Self(ChangeValueInner::EphemeralTransparent { value }) } /// Constructs a new change value that will be created as a shielded output. @@ -74,12 +84,11 @@ impl ChangeValue { value: NonNegativeAmount, memo: Option, ) -> Self { - Self { - output_pool: PoolType::Shielded(protocol), + Self(ChangeValueInner::Shielded { + protocol, value, memo, - is_ephemeral: false, - } + }) } /// Constructs a new change value that will be created as a Sapling output. @@ -95,22 +104,43 @@ impl ChangeValue { /// Returns the pool to which the change or ephemeral output should be sent. pub fn output_pool(&self) -> PoolType { - self.output_pool + match &self.0 { + ChangeValueInner::Shielded { protocol, .. } => PoolType::Shielded(*protocol), + #[cfg(feature = "transparent-inputs")] + ChangeValueInner::EphemeralTransparent { .. } => PoolType::Transparent, + } } /// Returns the value of the change or ephemeral output to be created, in zatoshis. pub fn value(&self) -> NonNegativeAmount { - self.value + match &self.0 { + ChangeValueInner::Shielded { value, .. } => *value, + #[cfg(feature = "transparent-inputs")] + ChangeValueInner::EphemeralTransparent { value } => *value, + } } /// Returns the memo to be associated with the output. pub fn memo(&self) -> Option<&MemoBytes> { - self.memo.as_ref() + match &self.0 { + ChangeValueInner::Shielded { memo, .. } => memo.as_ref(), + #[cfg(feature = "transparent-inputs")] + ChangeValueInner::EphemeralTransparent { .. } => None, + } } /// Whether this is to be an ephemeral output. + #[cfg_attr( + not(feature = "transparent-inputs"), + doc = "This is always false because the `transparent-inputs` feature is + not enabled." + )] pub fn is_ephemeral(&self) -> bool { - self.is_ephemeral + match &self.0 { + ChangeValueInner::Shielded { .. } => false, + #[cfg(feature = "transparent-inputs")] + ChangeValueInner::EphemeralTransparent { .. } => true, + } } } From b77813933ab8125371a6d84ac1b74cfb6830b1bb Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 26 Jun 2024 12:26:30 +0100 Subject: [PATCH 23/56] Refactor ephemeral output-related parameters to balance calculation. This now only supports a single ephemeral input and/or output when constructing a proposal (multiple ephemeral outputs are still supported when creating transactions). Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 11 +- .../src/data_api/wallet/input_selection.rs | 54 ++++------ zcash_client_backend/src/fees.rs | 54 +++++++++- zcash_client_backend/src/fees/common.rs | 101 ++++++++++-------- zcash_client_backend/src/fees/fixed.rs | 31 ++---- zcash_client_backend/src/fees/standard.rs | 25 ++--- zcash_client_backend/src/fees/zip317.rs | 81 ++++---------- .../src/transaction/fees/transparent.rs | 1 + 8 files changed, 162 insertions(+), 196 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 0319ca5679..c8a6f2cbb0 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -71,12 +71,11 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail if a memo is given for the transparent pool. Use `ChangeValue::shielded` to avoid this error case when creating a `ChangeValue` known to be for a shielded pool. - - `ChangeStrategy::compute_balance`: this trait method has three additional - parameters when the "transparent-inputs" feature is enabled. These are - used to specify whether the change memo should be ignored, and the amounts - of additional transparent P2PKH inputs and outputs. Passing `false` for - `ignore_change_memo` and empty slices for the other two arguments will - retain the previous behaviour. + - `ChangeStrategy::compute_balance`: this trait method has an additional + `&EphemeralParameters` parameter when the "transparent-inputs" feature is + enabled. This can be used to specify whether the change memo should be + ignored, and the amounts of additional transparent P2PKH inputs and + outputs. Passing `&Default::default()` will retain the previous behaviour. - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. - `zcash_client_backend::proto::ProposalDecodingError` has a new variant diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 9b1bf0e734..5758225197 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -33,7 +33,7 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { crate::{ - fees::ChangeValue, + fees::{ChangeValue, EphemeralParameters}, proposal::{Step, StepOutput, StepOutputIndex}, zip321::Payment, }, @@ -457,16 +457,17 @@ where } #[cfg(feature = "transparent-inputs")] - let (ephemeral_output_amounts, tr1_balance_opt) = { + let (ephemeral_parameters, tr1_balance_opt) = { if tr1_transparent_outputs.is_empty() { - (vec![], None) + (Default::default(), None) } else { // The ephemeral input going into transaction 1 must be able to pay that // transaction's fee, as well as the TEX address payments. - // First compute the required total without providing any input value, + // First compute the required total with an additional zero input, // catching the `InsufficientFunds` error to obtain the required amount - // given the provided change strategy. + // given the provided change strategy. Ignore the change memo in order + // to avoid adding a change output. let tr1_required_input_value = match self.change_strategy.compute_balance::<_, DbT::NoteRef>( params, @@ -477,12 +478,7 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - #[cfg(feature = "transparent-inputs")] - true, // ignore change memo to avoid adding a change output - #[cfg(feature = "transparent-inputs")] - &[NonNegativeAmount::ZERO], - #[cfg(feature = "transparent-inputs")] - &[], + &EphemeralParameters::new(true, Some(NonNegativeAmount::ZERO), None), ) { Err(ChangeError::InsufficientFunds { required, .. }) => required, Ok(_) => NonNegativeAmount::ZERO, // shouldn't happen @@ -500,16 +496,14 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - #[cfg(feature = "transparent-inputs")] - true, // ignore change memo to avoid adding a change output - #[cfg(feature = "transparent-inputs")] - &[tr1_required_input_value], - #[cfg(feature = "transparent-inputs")] - &[], + &EphemeralParameters::new(true, Some(tr1_required_input_value), None), )?; assert_eq!(tr1_balance.total(), tr1_balance.fee_required()); - (vec![tr1_required_input_value], Some(tr1_balance)) + ( + EphemeralParameters::new(false, None, Some(tr1_required_input_value)), + Some(tr1_balance), + ) } }; @@ -583,11 +577,7 @@ where ), &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &ephemeral_output_amounts, + &ephemeral_parameters, ); match balance { @@ -613,7 +603,11 @@ where // The ephemeral output should always be at the last change index. assert_eq!( *balance.proposed_change().last().expect("nonempty"), - ChangeValue::ephemeral_transparent(ephemeral_output_amounts[0]) + ChangeValue::ephemeral_transparent( + ephemeral_parameters + .ephemeral_output_amount() + .expect("ephemeral output is present") + ) ); let ephemeral_stepoutput = StepOutput::new( 0, @@ -775,11 +769,7 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); let balance = match trial_balance { @@ -798,11 +788,7 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), )? } Err(other) => { diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 893d53422c..6082b2e2a3 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -353,6 +353,51 @@ impl Default for DustOutputPolicy { } } +/// `EphemeralParameters` can be used to specify variations on how balance +/// and fees are computed that are relevant to transactions using ephemeral +/// transparent outputs. +#[cfg(feature = "transparent-inputs")] +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EphemeralParameters { + ignore_change_memo: bool, + ephemeral_input_amount: Option, + ephemeral_output_amount: Option, +} + +#[cfg(feature = "transparent-inputs")] +impl EphemeralParameters { + /// Returns an `EphemeralParameters` with the following parameters: + /// + /// * `ignore_change_memo`: `true` if the change memo should be + /// ignored for the purpose of deciding whether there will be a + /// change output. + /// * `ephemeral_input_amount`: specifies that there will be an + /// additional P2PKH input of the given amount. + /// * `ephemeral_output_amount`: specifies that there will be an + /// additional P2PKH output of the given amount. + pub fn new( + ignore_change_memo: bool, + ephemeral_input_amount: Option, + ephemeral_output_amount: Option, + ) -> Self { + Self { + ignore_change_memo, + ephemeral_input_amount, + ephemeral_output_amount, + } + } + + pub fn ignore_change_memo(&self) -> bool { + self.ignore_change_memo + } + pub fn ephemeral_input_amount(&self) -> Option { + self.ephemeral_input_amount + } + pub fn ephemeral_output_amount(&self) -> Option { + self.ephemeral_output_amount + } +} + /// A trait that represents the ability to compute the suggested change and fees that must be paid /// by a transaction having a specified set of inputs and outputs. pub trait ChangeStrategy { @@ -373,8 +418,9 @@ pub trait ChangeStrategy { /// #[cfg_attr( feature = "transparent-inputs", - doc = "The `ephemeral_input_amounts` and `ephemeral_output_amounts` parameters - specify the amounts of additional transparent P2PKH inputs and outputs." + doc = "`ephemeral_parameters` can be used to specify variations on how balance + and fees are computed that are relevant to transactions using ephemeral + transparent outputs; see [`EphemeralParameters::new`]." )] #[allow(clippy::too_many_arguments)] fn compute_balance( @@ -386,9 +432,7 @@ pub trait ChangeStrategy { sapling: &impl sapling::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, - #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], - #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result>; } diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 1d40d4966b..1739c283cc 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -5,11 +5,7 @@ use zcash_primitives::{ memo::MemoBytes, transaction::{ components::amount::{BalanceError, NonNegativeAmount}, - fees::{ - transparent::{self, InputSize}, - zip317::{MINIMUM_FEE, P2PKH_STANDARD_OUTPUT_SIZE}, - FeeRule, - }, + fees::{transparent, zip317::MINIMUM_FEE, FeeRule}, }, }; use zcash_protocol::ShieldedProtocol; @@ -19,6 +15,14 @@ use super::{ TransactionBalance, }; +#[cfg(feature = "transparent-inputs")] +use { + super::EphemeralParameters, + zcash_primitives::transaction::fees::{ + transparent::InputSize, zip317::P2PKH_STANDARD_OUTPUT_SIZE, + }, +}; + #[cfg(feature = "orchard")] use super::orchard as orchard_fees; @@ -53,24 +57,27 @@ pub(crate) fn calculate_net_flows( transparent_outputs: &[impl transparent::OutputView], sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, - ephemeral_input_amounts: &[NonNegativeAmount], - ephemeral_output_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_input_amount: Option, + #[cfg(feature = "transparent-inputs")] ephemeral_output_amount: Option, ) -> Result> where E: From + From, { let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); + #[cfg(not(feature = "transparent-inputs"))] + let (ephemeral_input_amount, ephemeral_output_amount) = (None, None); + let t_in = transparent_inputs .iter() .map(|t_in| t_in.coin().value) - .chain(ephemeral_input_amounts.iter().cloned()) + .chain(ephemeral_input_amount) .sum::>() .ok_or_else(overflow)?; let t_out = transparent_outputs .iter() .map(|t_out| t_out.value()) - .chain(ephemeral_output_amounts.iter().cloned()) + .chain(ephemeral_output_amount) .sum::>() .ok_or_else(overflow)?; let sapling_in = sapling @@ -162,29 +169,29 @@ pub(crate) fn single_change_output_balance< #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, default_dust_threshold: NonNegativeAmount, - change_memo: Option, + change_memo: Option<&MemoBytes>, fallback_change_pool: ShieldedProtocol, - #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], - #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> where E: From + From, { + #[cfg(feature = "transparent-inputs")] + let change_memo = change_memo.filter(|_| !ephemeral_parameters.ignore_change_memo()); + let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); let underflow = || ChangeError::StrategyError(E::from(BalanceError::Underflow)); - #[cfg(not(feature = "transparent-inputs"))] - let (ephemeral_input_amounts, ephemeral_output_amounts) = - (&[] as &[NonNegativeAmount], &[] as &[NonNegativeAmount]); - let net_flows = calculate_net_flows::( transparent_inputs, transparent_outputs, sapling, #[cfg(feature = "orchard")] orchard, - ephemeral_input_amounts, - ephemeral_output_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_parameters.ephemeral_input_amount(), + #[cfg(feature = "transparent-inputs")] + ephemeral_parameters.ephemeral_output_amount(), )?; let total_in = net_flows .total_in() @@ -251,27 +258,25 @@ where // and the sum of their outputs learns the sum of the inputs if no change // output is present); and // * we will then always have an shielded output in which to put change_memo, - // if one is given. + // if one is used. // // Note that using the `DustAction::AddDustToFee` policy inherently leaks // more information. - let transparent_input_sizes = transparent_inputs - .iter() - .map(|i| i.serialized_size()) - .chain( - ephemeral_input_amounts - .iter() - .map(|_| InputSize::STANDARD_P2PKH), - ); - let transparent_output_sizes = transparent_outputs - .iter() - .map(|i| i.serialized_size()) - .chain( - ephemeral_output_amounts - .iter() - .map(|_| P2PKH_STANDARD_OUTPUT_SIZE), - ); + let transparent_input_sizes = transparent_inputs.iter().map(|i| i.serialized_size()); + #[cfg(feature = "transparent-inputs")] + let transparent_input_sizes = transparent_input_sizes.chain( + ephemeral_parameters + .ephemeral_input_amount() + .map(|_| InputSize::STANDARD_P2PKH), + ); + let transparent_output_sizes = transparent_outputs.iter().map(|i| i.serialized_size()); + #[cfg(feature = "transparent-inputs")] + let transparent_output_sizes = transparent_output_sizes.chain( + ephemeral_parameters + .ephemeral_output_amount() + .map(|_| P2PKH_STANDARD_OUTPUT_SIZE), + ); let fee_without_change = fee_rule .fee_required( @@ -300,7 +305,7 @@ where .map_err(|fee_error| ChangeError::StrategyError(E::from(fee_error)))?, ); - // We don't create a fully-transparent transaction if a change memo is requested. + // We don't create a fully-transparent transaction if a change memo is used. let transparent = net_flows.is_transparent() && change_memo.is_none(); let total_out_plus_fee_without_change = @@ -328,9 +333,13 @@ where // Case 3b or 3c. let proposed_change = (total_in - total_out_plus_fee_with_change).expect("checked above"); - let simple_case = |memo| { + let simple_case = || { ( - vec![ChangeValue::shielded(change_pool, proposed_change, memo)], + vec![ChangeValue::shielded( + change_pool, + proposed_change, + change_memo.cloned(), + )], fee_with_change, ) }; @@ -349,7 +358,7 @@ where // * zero-valued notes do not require witness tracking; // * the effect on trial decryption overhead is small. if proposed_change.is_zero() { - simple_case(change_memo) + simple_case() } else { let shortfall = (dust_threshold - proposed_change).ok_or_else(underflow)?; @@ -360,7 +369,7 @@ where }); } } - DustAction::AllowDustChange => simple_case(change_memo), + DustAction::AllowDustChange => simple_case(), DustAction::AddDustToFee => { // Zero-valued change is also always allowed for this policy, but when // no change memo is given, we might omit the change output instead. @@ -376,13 +385,13 @@ where if fee_with_dust > reasonable_fee { // Defend against losing money by using AddDustToFee with a too-high // dust threshold. - simple_case(change_memo) + simple_case() } else if change_memo.is_some() { ( vec![ChangeValue::shielded( change_pool, NonNegativeAmount::ZERO, - change_memo, + change_memo.cloned(), )], fee_with_dust, ) @@ -392,15 +401,15 @@ where } } } else { - simple_case(change_memo) + simple_case() } } }; #[cfg(feature = "transparent-inputs")] change.extend( - ephemeral_output_amounts - .iter() - .map(|&amount| ChangeValue::ephemeral_transparent(amount)), + ephemeral_parameters + .ephemeral_output_amount() + .map(ChangeValue::ephemeral_transparent), ); TransactionBalance::new(change, fee).map_err(|_| overflow()) diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index f9ab9a9cae..de8e814fe5 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -20,7 +20,7 @@ use super::{ use super::orchard as orchard_fees; #[cfg(feature = "transparent-inputs")] -use super::NonNegativeAmount; +use super::EphemeralParameters; /// A change strategy that proposes change as a single output to the most current supported /// shielded pool and delegates fee calculation to the provided fee rule. @@ -66,13 +66,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, - #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], - #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> { - #[cfg(not(feature = "transparent-inputs"))] - let ignore_change_memo = false; - single_change_output_balance( params, &self.fee_rule, @@ -84,16 +79,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, self.fee_rule().fixed_fee(), - if ignore_change_memo { - None - } else { - self.change_memo.clone() - }, + self.change_memo.as_ref(), self.fallback_change_pool, #[cfg(feature = "transparent-inputs")] - ephemeral_input_amounts, - #[cfg(feature = "transparent-inputs")] - ephemeral_output_amounts, + ephemeral_parameters, ) } } @@ -150,11 +139,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -201,11 +186,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( diff --git a/zcash_client_backend/src/fees/standard.rs b/zcash_client_backend/src/fees/standard.rs index 8d53ecb66d..30d77d8fbc 100644 --- a/zcash_client_backend/src/fees/standard.rs +++ b/zcash_client_backend/src/fees/standard.rs @@ -21,6 +21,9 @@ use super::{ TransactionBalance, }; +#[cfg(feature = "transparent-inputs")] +use super::EphemeralParameters; + #[cfg(feature = "orchard")] use super::orchard as orchard_fees; @@ -68,9 +71,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, - #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], - #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> { #[allow(deprecated)] match self.fee_rule() { @@ -89,11 +90,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, #[cfg(feature = "transparent-inputs")] - ignore_change_memo, - #[cfg(feature = "transparent-inputs")] - ephemeral_input_amounts, - #[cfg(feature = "transparent-inputs")] - ephemeral_output_amounts, + ephemeral_parameters, ) .map_err(|e| e.map(Zip317FeeError::Balance)), StandardFeeRule::Zip313 => fixed::SingleOutputChangeStrategy::new( @@ -111,11 +108,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, #[cfg(feature = "transparent-inputs")] - ignore_change_memo, - #[cfg(feature = "transparent-inputs")] - ephemeral_input_amounts, - #[cfg(feature = "transparent-inputs")] - ephemeral_output_amounts, + ephemeral_parameters, ) .map_err(|e| e.map(Zip317FeeError::Balance)), StandardFeeRule::Zip317 => zip317::SingleOutputChangeStrategy::new( @@ -133,11 +126,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, #[cfg(feature = "transparent-inputs")] - ignore_change_memo, - #[cfg(feature = "transparent-inputs")] - ephemeral_input_amounts, - #[cfg(feature = "transparent-inputs")] - ephemeral_output_amounts, + ephemeral_parameters, ), } } diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index e6c8d7858e..4097d6bc21 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -17,10 +17,12 @@ use crate::ShieldedProtocol; use super::{ common::{calculate_net_flows, single_change_output_balance, single_change_output_policy}, - sapling as sapling_fees, ChangeError, ChangeStrategy, DustOutputPolicy, NonNegativeAmount, - TransactionBalance, + sapling as sapling_fees, ChangeError, ChangeStrategy, DustOutputPolicy, TransactionBalance, }; +#[cfg(feature = "transparent-inputs")] +use super::EphemeralParameters; + #[cfg(feature = "orchard")] use super::orchard as orchard_fees; @@ -68,9 +70,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ignore_change_memo: bool, - #[cfg(feature = "transparent-inputs")] ephemeral_input_amounts: &[NonNegativeAmount], - #[cfg(feature = "transparent-inputs")] ephemeral_output_amounts: &[NonNegativeAmount], + #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> { // We intentionally never count ephemeral inputs as dust. let mut transparent_dust: Vec<_> = transparent_inputs @@ -120,10 +120,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { let t_non_dust = transparent_inputs.len() - transparent_dust.len(); let t_allowed_dust = transparent_outputs.len().saturating_sub(t_non_dust); - #[cfg(not(feature = "transparent-inputs"))] - let (ephemeral_input_amounts, ephemeral_output_amounts) = - (&[] as &[NonNegativeAmount], &[] as &[NonNegativeAmount]); - // We add one to either the Sapling or Orchard outputs for the (single) // change output. Note that this means that wallet-internal shielding // transactions are an opportunity to spend a dust note. @@ -133,8 +129,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling, #[cfg(feature = "orchard")] orchard, - ephemeral_input_amounts, - ephemeral_output_amounts, + #[cfg(feature = "transparent-inputs")] + ephemeral_parameters.ephemeral_input_amount(), + #[cfg(feature = "transparent-inputs")] + ephemeral_parameters.ephemeral_output_amount(), )?; let (_, sapling_change, orchard_change) = single_change_output_policy(&net_flows, self.fallback_change_pool); @@ -204,9 +202,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { } } - #[cfg(not(feature = "transparent-inputs"))] - let ignore_change_memo = false; - single_change_output_balance( params, &self.fee_rule, @@ -218,16 +213,10 @@ impl ChangeStrategy for SingleOutputChangeStrategy { orchard, dust_output_policy, self.fee_rule.marginal_fee(), - if ignore_change_memo { - None - } else { - self.change_memo.clone() - }, + self.change_memo.as_ref(), self.fallback_change_pool, #[cfg(feature = "transparent-inputs")] - ephemeral_input_amounts, - #[cfg(feature = "transparent-inputs")] - ephemeral_output_amounts, + ephemeral_parameters, ) } } @@ -291,11 +280,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -340,11 +325,7 @@ mod tests { ), &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -398,11 +379,7 @@ mod tests { &orchard_fees::EmptyBundleView, dust_output_policy, #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -447,11 +424,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -496,11 +469,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -551,11 +520,7 @@ mod tests { Some(NonNegativeAmount::const_from_u64(1000)), ), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -617,11 +582,7 @@ mod tests { &orchard_fees::EmptyBundleView, dust_output_policy, #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); assert_matches!( @@ -675,11 +636,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - false, - #[cfg(feature = "transparent-inputs")] - &[], - #[cfg(feature = "transparent-inputs")] - &[], + &Default::default(), ); // We will get an error here, because the dust input now isn't free to add diff --git a/zcash_primitives/src/transaction/fees/transparent.rs b/zcash_primitives/src/transaction/fees/transparent.rs index 8beb394b54..a2fbfd4526 100644 --- a/zcash_primitives/src/transaction/fees/transparent.rs +++ b/zcash_primitives/src/transaction/fees/transparent.rs @@ -16,6 +16,7 @@ use crate::transaction::components::transparent::builder::TransparentInputInfo; /// The size of a transparent input, or the outpoint corresponding to the input /// if the size of the script required to spend that input is unknown. +#[derive(Clone, Debug, PartialEq, Eq)] pub enum InputSize { /// The txin size is known. Known(usize), From 7a05b44df9292aa2ce58e535efef12e98145c27a Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 26 Jun 2024 14:51:30 +0100 Subject: [PATCH 24/56] Make mutable inputs to closures in `create_proposed_transaction` explicit. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 46 +++++++++++++-------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index a709bd2f03..29619d50d8 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -857,9 +857,11 @@ where #[cfg(feature = "transparent-inputs")] let utxos_spent = { let mut utxos_spent: Vec = vec![]; - let mut add_transparent_input = |address_metadata: &TransparentAddressMetadata, - outpoint: OutPoint, - txout: TxOut| + let add_transparent_input = |builder: &mut Builder<_, _>, + utxos_spent: &mut Vec<_>, + address_metadata: &TransparentAddressMetadata, + outpoint: OutPoint, + txout: TxOut| -> Result<(), ErrorT> { let secret_key = usk .transparent() @@ -874,6 +876,8 @@ where for utxo in proposal_step.transparent_inputs() { add_transparent_input( + &mut builder, + &mut utxos_spent, &metadata_from_address(*utxo.recipient_address())?, utxo.outpoint().clone(), utxo.txout().clone(), @@ -895,7 +899,13 @@ where .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))? .vout[outpoint.n() as usize]; - add_transparent_input(&address_metadata, outpoint, txout.clone())?; + add_transparent_input( + &mut builder, + &mut utxos_spent, + &address_metadata, + outpoint, + txout.clone(), + )?; } utxos_spent }; @@ -976,8 +986,9 @@ where ); let recipient_address = payment.recipient_address(); - let mut add_sapling_output = |builder: &mut Builder<_, _>, - to: sapling::PaymentAddress| + let add_sapling_output = |builder: &mut Builder<_, _>, + sapling_output_meta: &mut Vec<_>, + to: sapling::PaymentAddress| -> Result<(), ErrorT> { let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); builder.add_sapling_output(sapling_external_ovk, to, payment.amount(), memo.clone())?; @@ -990,8 +1001,9 @@ where }; #[cfg(feature = "orchard")] - let mut add_orchard_output = |builder: &mut Builder<_, _>, - to: orchard::Address| + let add_orchard_output = |builder: &mut Builder<_, _>, + orchard_output_meta: &mut Vec<_>, + to: orchard::Address| -> Result<(), ErrorT> { let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); builder.add_orchard_output( @@ -1008,9 +1020,9 @@ where Ok(()) }; - #[allow(unused_mut)] - let mut add_transparent_output = |builder: &mut Builder<_, _>, - to: TransparentAddress| + let add_transparent_output = |builder: &mut Builder<_, _>, + transparent_output_meta: &mut Vec<_>, + to: TransparentAddress| -> Result<(), ErrorT> { if payment.memo().is_some() { return Err(Error::MemoForbidden); @@ -1038,22 +1050,22 @@ where #[cfg(feature = "orchard")] PoolType::Shielded(ShieldedProtocol::Orchard) => { let to = *ua.orchard().expect("The mapping between payment pool and receiver is checked in step construction"); - add_orchard_output(&mut builder, to)?; + add_orchard_output(&mut builder, &mut orchard_output_meta, to)?; } PoolType::Shielded(ShieldedProtocol::Sapling) => { let to = *ua.sapling().expect("The mapping between payment pool and receiver is checked in step construction"); - add_sapling_output(&mut builder, to)?; + add_sapling_output(&mut builder, &mut sapling_output_meta, to)?; } PoolType::Transparent => { let to = *ua.transparent().expect("The mapping between payment pool and receiver is checked in step construction"); - add_transparent_output(&mut builder, to)?; + add_transparent_output(&mut builder, &mut transparent_output_meta, to)?; } }, Address::Sapling(to) => { - add_sapling_output(&mut builder, to)?; + add_sapling_output(&mut builder, &mut sapling_output_meta, to)?; } Address::Transparent(to) => { - add_transparent_output(&mut builder, to)?; + add_transparent_output(&mut builder, &mut transparent_output_meta, to)?; } #[cfg(not(feature = "transparent-inputs"))] Address::Tex(_) => { @@ -1065,7 +1077,7 @@ where return Err(Error::ProposalNotSupported); } let to = TransparentAddress::PublicKeyHash(data); - add_transparent_output(&mut builder, to)?; + add_transparent_output(&mut builder, &mut transparent_output_meta, to)?; } } } From 6471d4c27afceb71053932764a24fd1d6a2c0401 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 27 Jun 2024 02:21:55 +0100 Subject: [PATCH 25/56] Don't assume that prior step outputs are ephemeral iff they are `StepOutputIndex::Change`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 17 ++--- zcash_client_sqlite/src/testing/pool.rs | 80 +++++++++++++++++++++ zcash_client_sqlite/src/wallet/orchard.rs | 6 ++ zcash_client_sqlite/src/wallet/sapling.rs | 6 ++ 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 29619d50d8..6b2f58a439 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -629,15 +629,16 @@ where step_results.push((step, step_result)); } - // Ephemeral outputs must be referenced exactly once. Currently this is all - // transparent outputs using `StepOutputIndex::Change`. - // TODO: if we support transparent change, this will need to be updated to - // not require it to be referenced by a later step. + // Ephemeral outputs must be referenced exactly once. #[cfg(feature = "transparent-inputs")] - if unused_transparent_outputs - .into_keys() - .any(|s: StepOutput| matches!(s.output_index(), StepOutputIndex::Change(_))) - { + if unused_transparent_outputs.into_keys().any(|s: StepOutput| { + if let StepOutputIndex::Change(i) = s.output_index() { + // indexing has already been checked + step_results[s.step_index()].0.balance().proposed_change()[i].is_ephemeral() + } else { + false + } + }) { return Err(Error::ProposalNotSupported); } diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index bbda91afdd..951015a544 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -479,6 +479,86 @@ pub(crate) fn send_multi_step_proposed_transfer() { Err(Error::DataSource(SqliteClientError::EphemeralAddressReuse(addr, Some(txid)))) if addr == &ephemeral0 && txid == &txid0); } +#[cfg(feature = "transparent-inputs")] +pub(crate) fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { + use nonempty::NonEmpty; + use zcash_client_backend::proposal::Proposal; + + let mut st = TestBuilder::new() + .with_block_cache() + .with_account_from_sapling_activation(BlockHash([0; 32])) + .build(); + + let account = st.test_account().cloned().unwrap(); + let dfvk = T::test_account_fvk(&st); + + let add_funds = |st: &mut TestState<_>, value| { + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); + + assert_eq!( + block_max_scanned(&st.wallet().conn, &st.wallet().params) + .unwrap() + .unwrap() + .block_height(), + h + ); + assert_eq!(st.get_spendable_balance(account.account_id(), 1), value); + }; + + let value = NonNegativeAmount::const_from_u64(100000); + let transfer_amount = NonNegativeAmount::const_from_u64(50000); + + // Add funds to the wallet. + add_funds(&mut st, value); + + // Generate a ZIP 320 proposal, sending to the wallet's default transparent address + // expressed as a TEX address. + let tex_addr = match account.usk().default_transparent_address().0 { + TransparentAddress::PublicKeyHash(data) => Address::Tex(data), + _ => unreachable!(), + }; + + let proposal = st + .propose_standard_transfer::( + account.account_id(), + StandardFeeRule::Zip317, + NonZeroU32::new(1).unwrap(), + &tex_addr, + transfer_amount, + None, + None, + T::SHIELDED_PROTOCOL, + ) + .unwrap(); + + // This is somewhat redundant with `send_multi_step_proposed_transfer`, + // but tests the case with no change memo and ensures we haven't messed + // up the test setup. + let create_proposed_result = st.create_proposed_transactions::( + account.usk(), + OvkPolicy::Sender, + &proposal, + ); + assert_matches!(create_proposed_result, Ok(_)); + + // Frobnicate the proposal to make it invalid because it does not consume + // the ephemeral output, by truncating it to the first step. + let frobbed_proposal = Proposal::multi_step( + *proposal.fee_rule(), + proposal.min_target_height(), + NonEmpty::singleton(proposal.steps().first().clone()), + ) + .unwrap(); + + let create_proposed_result = st.create_proposed_transactions::( + account.usk(), + OvkPolicy::Sender, + &frobbed_proposal, + ); + assert_matches!(create_proposed_result, Err(Error::ProposalNotSupported)); +} + #[allow(deprecated)] pub(crate) fn create_to_address_fails_on_incorrect_usk() { let mut st = TestBuilder::new() diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs index 6f28719d50..88e7f66f75 100644 --- a/zcash_client_sqlite/src/wallet/orchard.rs +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -575,6 +575,12 @@ pub(crate) mod tests { testing::pool::send_multi_step_proposed_transfer::() } + #[test] + #[cfg(feature = "transparent-inputs")] + fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { + testing::pool::proposal_fails_if_not_all_ephemeral_outputs_consumed::() + } + #[test] #[allow(deprecated)] fn create_to_address_fails_on_incorrect_usk() { diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 10834346c6..b43ce4a908 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -581,6 +581,12 @@ pub(crate) mod tests { testing::pool::send_multi_step_proposed_transfer::() } + #[test] + #[cfg(feature = "transparent-inputs")] + fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { + testing::pool::proposal_fails_if_not_all_ephemeral_outputs_consumed::() + } + #[test] #[allow(deprecated)] fn create_to_address_fails_on_incorrect_usk() { From ec4a6d0cad08f3d00455ea289f8b44782d08103b Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 27 Jun 2024 03:17:07 +0100 Subject: [PATCH 26/56] Documentation improvements. Co-authored-by: Jack Grigg Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/error.rs | 8 ++++++-- zcash_client_backend/src/data_api/wallet.rs | 3 ++- .../src/data_api/wallet/input_selection.rs | 7 +++++-- zcash_client_sqlite/src/wallet/transparent.rs | 3 ++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 562045a2af..c8c60efde2 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -36,8 +36,12 @@ pub enum Error { /// An error in transaction proposal construction Proposal(ProposalError), - /// The proposal was structurally valid, but spending shielded outputs of prior multi-step - /// transaction steps is not yet supported. + /// The proposal was structurally valid, but tries to do one of these unsupported things: + /// * spending a prior shielded output or non-ephemeral change output; + /// * leaving an ephemeral output unspent; + /// * paying to an output pool for which the corresponding feature is not enabled; + /// * paying to a TEX address if the "transparent-inputs" feature is not enabled; + /// * paying to a TEX address in a transaction that has shielded inputs. ProposalNotSupported, /// No account could be found corresponding to a provided spending key. diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 6b2f58a439..e819572613 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -677,7 +677,8 @@ where #[allow(unused_variables)] let step_index = prior_step_results.len(); - // We only support spending transparent payments or ephemeral outputs from a prior step. + // We only support spending transparent payments or transparent ephemeral outputs from a + // prior step. // // TODO: Maybe support spending prior shielded outputs at some point? Doing so would require // a higher-level approach in the wallet that waits for transactions with shielded outputs to diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 5758225197..24c521a35d 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -370,6 +370,9 @@ where let mut orchard_outputs = vec![]; let mut payment_pools = BTreeMap::new(); + // In a ZIP 320 pair, tr0 refers to the first transaction request that + // collects shielded value and sends it to an ephemeral address, and tr1 + // refers to the second transaction request that pays the TEX addresses. #[cfg(feature = "transparent-inputs")] let mut tr1_transparent_outputs = vec![]; #[cfg(feature = "transparent-inputs")] @@ -597,8 +600,8 @@ where if let Some(tr1_balance) = tr1_balance_opt { // Construct two new `TransactionRequest`s: // * `tr0` excludes the TEX outputs, and in their place includes - // a single additional "change" output to the transparent pool. - // * `tr1` spends from that change output to each TEX output. + // a single additional ephemeral output to the transparent pool. + // * `tr1` spends from that ephemeral output to each TEX output. // The ephemeral output should always be at the last change index. assert_eq!( diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 77fe21b0f0..c52d4ec986 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -517,7 +517,8 @@ pub(crate) fn get_transparent_address_metadata( /// /// The following three locations in the wallet's key tree are searched: /// - Transparent receivers that have been generated as part of a Unified Address. -/// - Transparent ephemeral addresses that have been reserved. +/// - Transparent ephemeral addresses that have been reserved or are within +/// the gap limit from the last reserved address. /// - "Legacy transparent addresses" (at BIP 44 address index 0 within an account). /// /// Returns `Ok(None)` if the transparent output's recipient address is not in any of the From 81a2846593c75542082d6ed489d101e731828684 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 27 Jun 2024 12:23:38 +0100 Subject: [PATCH 27/56] Simpler way to calculate `has_shielded_inputs`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index e819572613..71250a7dcb 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -807,24 +807,18 @@ where }, ); - #[cfg(feature = "transparent-inputs")] - let mut has_shielded_inputs = false; + #[cfg(all(feature = "transparent-inputs", not(feature = "orchard")))] + let has_shielded_inputs = !sapling_inputs.is_empty(); + #[cfg(all(feature = "transparent-inputs", feature = "orchard"))] + let has_shielded_inputs = !(sapling_inputs.is_empty() && orchard_inputs.is_empty()); for (sapling_key, sapling_note, merkle_path) in sapling_inputs.into_iter() { builder.add_sapling_spend(&sapling_key, sapling_note.clone(), merkle_path)?; - #[cfg(feature = "transparent-inputs")] - { - has_shielded_inputs = true; - } } #[cfg(feature = "orchard")] for (orchard_note, merkle_path) in orchard_inputs.into_iter() { builder.add_orchard_spend(usk.orchard(), *orchard_note, merkle_path.into())?; - #[cfg(feature = "transparent-inputs")] - { - has_shielded_inputs = true; - } } #[cfg(feature = "transparent-inputs")] From f0e5aab6924f1c36ddf6ec067809adfbb2c58e72 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 27 Jun 2024 12:22:30 +0100 Subject: [PATCH 28/56] Improve discrimination of proposal errors. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/error.rs | 23 +++++--- zcash_client_backend/src/data_api/wallet.rs | 61 +++++++++------------ zcash_client_backend/src/proposal.rs | 24 ++++++++ zcash_client_sqlite/src/testing/pool.rs | 8 ++- 4 files changed, 69 insertions(+), 47 deletions(-) diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index c8c60efde2..33081f6546 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -37,11 +37,10 @@ pub enum Error { Proposal(ProposalError), /// The proposal was structurally valid, but tries to do one of these unsupported things: - /// * spending a prior shielded output or non-ephemeral change output; - /// * leaving an ephemeral output unspent; + /// * spending a prior shielded output; /// * paying to an output pool for which the corresponding feature is not enabled; /// * paying to a TEX address if the "transparent-inputs" feature is not enabled; - /// * paying to a TEX address in a transaction that has shielded inputs. + /// * exceeding implementation limits. ProposalNotSupported, /// No account could be found corresponding to a provided spending key. @@ -120,12 +119,12 @@ where Error::Proposal(e) => { write!(f, "Input selection attempted to construct an invalid proposal: {}", e) } - Error::ProposalNotSupported => { - write!( - f, - "The proposal was valid, but spending shielded outputs of prior transaction steps is not yet supported." - ) - } + Error::ProposalNotSupported => write!( + f, + "The proposal was valid but tried to do something that is not supported \ + (spending shielded outputs of prior transaction steps, using a feature \ + that is not enabled, or exceeding an implementation limit).", + ), Error::KeyNotRecognized => { write!( f, @@ -191,6 +190,12 @@ impl From> for Error { } } +impl From for Error { + fn from(e: ProposalError) -> Self { + Error::Proposal(e) + } +} + impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 71250a7dcb..0d99238a33 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -51,7 +51,7 @@ use crate::{ decrypt_transaction, fees::{self, DustOutputPolicy}, keys::UnifiedSpendingKey, - proposal::{Proposal, Step, StepOutputIndex}, + proposal::{Proposal, ProposalError, Step, StepOutputIndex}, wallet::{Note, OvkPolicy, Recipient}, zip321::{self, Payment}, PoolType, ShieldedProtocol, @@ -73,11 +73,7 @@ use zip32::Scope; #[cfg(feature = "transparent-inputs")] use { - crate::{ - fees::ChangeValue, - proposal::{ProposalError, StepOutput}, - wallet::TransparentAddressMetadata, - }, + crate::{fees::ChangeValue, proposal::StepOutput, wallet::TransparentAddressMetadata}, core::convert::Infallible, input_selection::ShieldingSelector, std::collections::HashMap, @@ -631,15 +627,13 @@ where // Ephemeral outputs must be referenced exactly once. #[cfg(feature = "transparent-inputs")] - if unused_transparent_outputs.into_keys().any(|s: StepOutput| { - if let StepOutputIndex::Change(i) = s.output_index() { - // indexing has already been checked - step_results[s.step_index()].0.balance().proposed_change()[i].is_ephemeral() - } else { - false + for so in unused_transparent_outputs.into_keys() { + if let StepOutputIndex::Change(i) = so.output_index() { + // references have already been checked + if step_results[so.step_index()].0.balance().proposed_change()[i].is_ephemeral() { + return Err(ProposalError::EphemeralOutputLeftUnspent(so).into()); + } } - }) { - return Err(Error::ProposalNotSupported); } Ok(NonEmpty::from_vec( @@ -674,7 +668,7 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - #[allow(unused_variables)] + #[cfg(feature = "transparent-inputs")] let step_index = prior_step_results.len(); // We only support spending transparent payments or transparent ephemeral outputs from a @@ -683,32 +677,27 @@ where // TODO: Maybe support spending prior shielded outputs at some point? Doing so would require // a higher-level approach in the wallet that waits for transactions with shielded outputs to // be mined and only then attempts to perform the next step. - #[cfg(feature = "transparent-inputs")] for input_ref in proposal_step.prior_step_inputs() { - let supported = prior_step_results + let (prior_step, _) = prior_step_results .get(input_ref.step_index()) - .and_then(|(prior_step, _)| match input_ref.output_index() { - StepOutputIndex::Payment(i) => prior_step - .payment_pools() - .get(&i) - .map(|&pool| pool == PoolType::TRANSPARENT), - StepOutputIndex::Change(i) => { - prior_step.balance().proposed_change().get(i).map(|change| { - change.is_ephemeral() && change.output_pool() == PoolType::TRANSPARENT - }) + .ok_or(ProposalError::ReferenceError(*input_ref))?; + + let output_pool = match input_ref.output_index() { + StepOutputIndex::Payment(i) => prior_step.payment_pools().get(&i).cloned(), + StepOutputIndex::Change(i) => match prior_step.balance().proposed_change().get(i) { + Some(change) if !change.is_ephemeral() => { + return Err(ProposalError::SpendsChange(*input_ref).into()); } - }) - .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))?; + other => other.map(|change| change.output_pool()), + }, + } + .ok_or(ProposalError::ReferenceError(*input_ref))?; - // Return an error on trying to spend a prior shielded output or non-ephemeral change output. - if !supported { + // Return an error on trying to spend a prior shielded output. + if output_pool != PoolType::TRANSPARENT { return Err(Error::ProposalNotSupported); } } - #[cfg(not(feature = "transparent-inputs"))] - if !proposal_step.prior_step_inputs().is_empty() { - return Err(Error::ProposalNotSupported); - } let account_id = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) @@ -892,7 +881,7 @@ where .1 .transaction() .transparent_bundle() - .ok_or(Error::Proposal(ProposalError::ReferenceError(*input_ref)))? + .ok_or(ProposalError::ReferenceError(*input_ref))? .vout[outpoint.n() as usize]; add_transparent_input( @@ -1070,7 +1059,7 @@ where #[cfg(feature = "transparent-inputs")] Address::Tex(data) => { if has_shielded_inputs { - return Err(Error::ProposalNotSupported); + return Err(ProposalError::PaysTexFromShielded.into()); } let to = TransparentAddress::PublicKeyHash(data); add_transparent_output(&mut builder, &mut transparent_output_meta, to)?; diff --git a/zcash_client_backend/src/proposal.rs b/zcash_client_backend/src/proposal.rs index 212b8ba721..e67e3e676c 100644 --- a/zcash_client_backend/src/proposal.rs +++ b/zcash_client_backend/src/proposal.rs @@ -47,6 +47,14 @@ pub enum ProposalError { /// There was a mismatch between the payments in the proposal's transaction request /// and the payment pool selection values. PaymentPoolsMismatch, + /// The proposal tried to spend a change output. Mark the `ChangeValue` as ephemeral if this is intended. + SpendsChange(StepOutput), + /// A proposal step created an ephemeral output that was not spent in any later step. + #[cfg(feature = "transparent-inputs")] + EphemeralOutputLeftUnspent(StepOutput), + /// The proposal included a payment to a TEX address and a spend from a shielded input in the same step. + #[cfg(feature = "transparent-inputs")] + PaysTexFromShielded, } impl Display for ProposalError { @@ -90,6 +98,22 @@ impl Display for ProposalError { f, "The chosen payment pools did not match the payments of the transaction request." ), + ProposalError::SpendsChange(r) => write!( + f, + "The proposal attempts to spends the change output created at step {:?}.", + r, + ), + #[cfg(feature = "transparent-inputs")] + ProposalError::EphemeralOutputLeftUnspent(r) => write!( + f, + "The proposal created an ephemeral output at step {:?} that was not spent in any later step.", + r, + ), + #[cfg(feature = "transparent-inputs")] + ProposalError::PaysTexFromShielded => write!( + f, + "The proposal included a payment to a TEX address and a spend from a shielded input in the same step.", + ), } } } diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 951015a544..bf21663b95 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -482,7 +482,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { #[cfg(feature = "transparent-inputs")] pub(crate) fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { use nonempty::NonEmpty; - use zcash_client_backend::proposal::Proposal; + use zcash_client_backend::proposal::{Proposal, ProposalError, StepOutput, StepOutputIndex}; let mut st = TestBuilder::new() .with_block_cache() @@ -556,7 +556,11 @@ pub(crate) fn proposal_fails_if_not_all_ephemeral_outputs_consumed Date: Wed, 26 Jun 2024 17:00:09 +0100 Subject: [PATCH 29/56] Remove complicated code to calculate the number of dust spends. See #1316 for re-adding it. The previous code was essentially dead because it had no side effects other than potentially returning an error. That error is unnecessary and incorrect when we are not actually performing dust spends. When we re-enable them, we should not try to make dust spends in any transaction with non-default `ephemeral_parameters`, or if the `dust_output_policy` is `DustAction::AddDustToFee`. That would guarantee that we will always have a shielded change output, which makes the calculations more tractable and excludes some potentially complicated interactions. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/fees/zip317.rs | 135 +----------------------- 1 file changed, 3 insertions(+), 132 deletions(-) diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 4097d6bc21..81ec9667e1 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -16,8 +16,8 @@ use zcash_primitives::{ use crate::ShieldedProtocol; use super::{ - common::{calculate_net_flows, single_change_output_balance, single_change_output_policy}, - sapling as sapling_fees, ChangeError, ChangeStrategy, DustOutputPolicy, TransactionBalance, + common::single_change_output_balance, sapling as sapling_fees, ChangeError, ChangeStrategy, + DustOutputPolicy, TransactionBalance, }; #[cfg(feature = "transparent-inputs")] @@ -72,136 +72,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { dust_output_policy: &DustOutputPolicy, #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> { - // We intentionally never count ephemeral inputs as dust. - let mut transparent_dust: Vec<_> = transparent_inputs - .iter() - .filter_map(|i| { - // for now, we're just assuming p2pkh inputs, so we don't check the size of the input - // script - if i.coin().value < self.fee_rule.marginal_fee() { - Some(i.outpoint().clone()) - } else { - None - } - }) - .collect(); - - let mut sapling_dust: Vec<_> = sapling - .inputs() - .iter() - .filter_map(|i| { - if sapling_fees::InputView::::value(i) < self.fee_rule.marginal_fee() { - Some(sapling_fees::InputView::::note_id(i).clone()) - } else { - None - } - }) - .collect(); - - #[cfg(feature = "orchard")] - let mut orchard_dust: Vec = orchard - .inputs() - .iter() - .filter_map(|i| { - if orchard_fees::InputView::::value(i) < self.fee_rule.marginal_fee() { - Some(orchard_fees::InputView::::note_id(i).clone()) - } else { - None - } - }) - .collect(); - #[cfg(not(feature = "orchard"))] - let mut orchard_dust: Vec = vec![]; - - // Depending on the shape of the transaction, we may be able to spend up to - // `grace_actions - 1` dust inputs. If we don't have any dust inputs though, - // we don't need to worry about any of that. - if !(transparent_dust.is_empty() && sapling_dust.is_empty() && orchard_dust.is_empty()) { - let t_non_dust = transparent_inputs.len() - transparent_dust.len(); - let t_allowed_dust = transparent_outputs.len().saturating_sub(t_non_dust); - - // We add one to either the Sapling or Orchard outputs for the (single) - // change output. Note that this means that wallet-internal shielding - // transactions are an opportunity to spend a dust note. - let net_flows = calculate_net_flows::( - transparent_inputs, - transparent_outputs, - sapling, - #[cfg(feature = "orchard")] - orchard, - #[cfg(feature = "transparent-inputs")] - ephemeral_parameters.ephemeral_input_amount(), - #[cfg(feature = "transparent-inputs")] - ephemeral_parameters.ephemeral_output_amount(), - )?; - let (_, sapling_change, orchard_change) = - single_change_output_policy(&net_flows, self.fallback_change_pool); - - let s_non_dust = sapling.inputs().len() - sapling_dust.len(); - let s_allowed_dust = - (sapling.outputs().len() + sapling_change).saturating_sub(s_non_dust); - - #[cfg(feature = "orchard")] - let (orchard_inputs_len, orchard_outputs_len) = - (orchard.inputs().len(), orchard.outputs().len()); - #[cfg(not(feature = "orchard"))] - let (orchard_inputs_len, orchard_outputs_len) = (0, 0); - - let o_non_dust = orchard_inputs_len - orchard_dust.len(); - let o_allowed_dust = (orchard_outputs_len + orchard_change).saturating_sub(o_non_dust); - - let available_grace_inputs = self - .fee_rule - .grace_actions() - .saturating_sub(t_non_dust) - .saturating_sub(s_non_dust) - .saturating_sub(o_non_dust); - - let mut t_disallowed_dust = transparent_dust.len().saturating_sub(t_allowed_dust); - let mut s_disallowed_dust = sapling_dust.len().saturating_sub(s_allowed_dust); - let mut o_disallowed_dust = orchard_dust.len().saturating_sub(o_allowed_dust); - - if available_grace_inputs > 0 { - // If we have available grace inputs, allocate them first to transparent dust - // and then to Sapling dust followed by Orchard dust. The caller has provided - // inputs that it is willing to spend, so we don't need to consider privacy - // effects at this layer. - let t_grace_dust = available_grace_inputs.saturating_sub(t_disallowed_dust); - t_disallowed_dust = t_disallowed_dust.saturating_sub(t_grace_dust); - - let s_grace_dust = available_grace_inputs - .saturating_sub(t_grace_dust) - .saturating_sub(s_disallowed_dust); - s_disallowed_dust = s_disallowed_dust.saturating_sub(s_grace_dust); - - let o_grace_dust = available_grace_inputs - .saturating_sub(t_grace_dust) - .saturating_sub(s_grace_dust) - .saturating_sub(o_disallowed_dust); - o_disallowed_dust = o_disallowed_dust.saturating_sub(o_grace_dust); - } - - // Truncate the lists of inputs to be disregarded in input selection to just the - // disallowed lengths. This has the effect of prioritizing inputs for inclusion by the - // order of the original input slices, with the most preferred inputs first. - transparent_dust.reverse(); - transparent_dust.truncate(t_disallowed_dust); - sapling_dust.reverse(); - sapling_dust.truncate(s_disallowed_dust); - orchard_dust.reverse(); - orchard_dust.truncate(o_disallowed_dust); - - if !(transparent_dust.is_empty() && sapling_dust.is_empty() && orchard_dust.is_empty()) - { - return Err(ChangeError::DustInputs { - transparent: transparent_dust, - sapling: sapling_dust, - #[cfg(feature = "orchard")] - orchard: orchard_dust, - }); - } - } - + // TODO: consider opportunistic dust spends (#1316). single_change_output_balance( params, &self.fee_rule, From baccb4361b4317099a5bc5792744af2611e31063 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Fri, 28 Jun 2024 17:00:37 +0100 Subject: [PATCH 30/56] Restore the logic to determine whether we are spending inputs that are uneconomic to spend. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/fees.rs | 29 ++- zcash_client_backend/src/fees/common.rs | 247 ++++++++++++++++++++++-- zcash_client_backend/src/fees/fixed.rs | 6 +- zcash_client_backend/src/fees/zip317.rs | 13 +- 4 files changed, 269 insertions(+), 26 deletions(-) diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 6082b2e2a3..ae1a24da5e 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -206,15 +206,25 @@ pub enum ChangeError { /// including the required fees. required: NonNegativeAmount, }, - /// Some of the inputs provided to the transaction were determined to currently have no - /// economic value (i.e. their inclusion in a transaction causes fees to rise in an amount - /// greater than their value.) + /// Some of the inputs provided to the transaction have value less than the + /// marginal fee, and could not be determined to have any economic value in + /// the context of this input selection. + /// + /// This determination is potentially conservative in the sense that inputs + /// with value less than or equal to the marginal fee might be excluded, even + /// though in practice they would not cause the fee to increase. Inputs with + /// value greater than the marginal fee will never be excluded. + /// + /// The ordering of the inputs in each list is unspecified. DustInputs { - /// The outpoints corresponding to transparent inputs having no current economic value. + /// The outpoints for transparent inputs that could not be determined to + /// have economic value in the context of this input selection. transparent: Vec, - /// The identifiers for Sapling inputs having no current economic value + /// The identifiers for Sapling inputs that could not be determined to + /// have economic value in the context of this input selection. sapling: Vec, - /// The identifiers for Orchard inputs having no current economic value + /// The identifiers for Orchard inputs that could not be determined to + /// have economic value in the context of this input selection. #[cfg(feature = "orchard")] orchard: Vec, }, @@ -416,6 +426,13 @@ pub trait ChangeStrategy { /// supply the requested outputs and required fees, implementations should return /// [`ChangeError::InsufficientFunds`]. /// + /// If the inputs include notes or UTXOs that are not economic to spend in the context + /// of this input selection, a [`ChangeError::DustInputs`] error can be returned + /// indicating inputs that should be removed from the selection (all of which will + /// have value less than or equal to the marginal fee). The caller should order the + /// inputs from most to least preferred to spend within each pool, so that the most + /// preferred ones are less likely to be indicated to remove. + /// #[cfg_attr( feature = "transparent-inputs", doc = "`ephemeral_parameters` can be used to specify variations on how balance diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 1739c283cc..17e4f77b65 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -1,4 +1,4 @@ -use core::cmp::max; +use core::cmp::{max, min}; use zcash_primitives::{ consensus::{self, BlockHeight}, @@ -171,6 +171,8 @@ pub(crate) fn single_change_output_balance< default_dust_threshold: NonNegativeAmount, change_memo: Option<&MemoBytes>, fallback_change_pool: ShieldedProtocol, + marginal_fee: NonNegativeAmount, + grace_actions: usize, #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> where @@ -193,6 +195,43 @@ where #[cfg(feature = "transparent-inputs")] ephemeral_parameters.ephemeral_output_amount(), )?; + + #[allow(unused_variables)] + let (change_pool, sapling_change, orchard_change) = + single_change_output_policy(&net_flows, fallback_change_pool); + + // We don't create a fully-transparent transaction if a change memo is used. + let transparent = net_flows.is_transparent() && change_memo.is_none(); + + // If we have a non-zero marginal fee, we need to check for uneconomic inputs. + // This is basically assuming that fee rules with non-zero marginal fee are + // "ZIP 317-like", but we can generalize later if needed. + if marginal_fee.is_positive() { + // Is it certain that there will be a change output? If it is not certain, + // we should call `check_for_uneconomic_inputs` with `possible_change` + // including both possibilities. + let possible_change = + // These are the situations where we might not have a change output. + if transparent || (dust_output_policy.action() == DustAction::AddDustToFee && change_memo.is_none()) { + vec![(0, 0, 0), (0, sapling_change, orchard_change)] + } else { + vec![(0, sapling_change, orchard_change)] + }; + + check_for_uneconomic_inputs( + transparent_inputs, + transparent_outputs, + sapling, + #[cfg(feature = "orchard")] + orchard, + marginal_fee, + grace_actions, + &possible_change[..], + #[cfg(feature = "transparent-inputs")] + ephemeral_parameters, + )?; + } + let total_in = net_flows .total_in() .map_err(|e| ChangeError::StrategyError(E::from(e)))?; @@ -200,10 +239,6 @@ where .total_out() .map_err(|e| ChangeError::StrategyError(E::from(e)))?; - #[allow(unused_variables)] - let (change_pool, sapling_change, orchard_change) = - single_change_output_policy(&net_flows, fallback_change_pool); - let sapling_input_count = sapling .bundle_type() .num_spends(sapling.inputs().len()) @@ -305,9 +340,6 @@ where .map_err(|fee_error| ChangeError::StrategyError(E::from(fee_error)))?, ); - // We don't create a fully-transparent transaction if a change memo is used. - let transparent = net_flows.is_transparent() && change_memo.is_none(); - let total_out_plus_fee_without_change = (total_out + fee_without_change).ok_or_else(overflow)?; let total_out_plus_fee_with_change = (total_out + fee_with_change).ok_or_else(overflow)?; @@ -344,11 +376,11 @@ where ) }; - let dust_threshold = dust_output_policy + let change_dust_threshold = dust_output_policy .dust_threshold() .unwrap_or(default_dust_threshold); - if proposed_change < dust_threshold { + if proposed_change < change_dust_threshold { match dust_output_policy.action() { DustAction::Reject => { // Always allow zero-valued change even for the `Reject` policy: @@ -361,7 +393,7 @@ where simple_case() } else { let shortfall = - (dust_threshold - proposed_change).ok_or_else(underflow)?; + (change_dust_threshold - proposed_change).ok_or_else(underflow)?; return Err(ChangeError::InsufficientFunds { available: total_in, @@ -414,3 +446,196 @@ where TransactionBalance::new(change, fee).map_err(|_| overflow()) } + +/// Returns a `[ChangeStrategy::DustInputs]` error if some of the inputs provided +/// to the transaction have value less than the marginal fee, and could not be +/// determined to have any economic value in the context of this input selection. +/// +/// This determination is potentially conservative in the sense that outputs +/// with value less than the marginal fee might be excluded, even though in +/// practice they would not cause the fee to increase. Outputs with value +/// greater than the marginal fee will never be excluded. +#[allow(clippy::too_many_arguments)] +pub(crate) fn check_for_uneconomic_inputs( + transparent_inputs: &[impl transparent::InputView], + transparent_outputs: &[impl transparent::OutputView], + sapling: &impl sapling_fees::BundleView, + #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, + marginal_fee: NonNegativeAmount, + grace_actions: usize, + possible_change: &[(usize, usize, usize)], + #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, +) -> Result<(), ChangeError> { + let mut t_dust: Vec<_> = transparent_inputs + .iter() + .filter_map(|i| { + // For now, we're just assuming P2PKH inputs, so we don't check the + // size of the input script. + if i.coin().value <= marginal_fee { + Some(i.outpoint().clone()) + } else { + None + } + }) + .collect(); + + let mut s_dust: Vec<_> = sapling + .inputs() + .iter() + .filter_map(|i| { + if sapling_fees::InputView::::value(i) <= marginal_fee { + Some(sapling_fees::InputView::::note_id(i).clone()) + } else { + None + } + }) + .collect(); + + #[cfg(feature = "orchard")] + let mut o_dust: Vec = orchard + .inputs() + .iter() + .filter_map(|i| { + if orchard_fees::InputView::::value(i) <= marginal_fee { + Some(orchard_fees::InputView::::note_id(i).clone()) + } else { + None + } + }) + .collect(); + #[cfg(not(feature = "orchard"))] + let mut o_dust: Vec = vec![]; + + // If we don't have any dust inputs, there is nothing to check. + if t_dust.is_empty() && s_dust.is_empty() && o_dust.is_empty() { + return Ok(()); + } + + let (t_inputs_len, t_outputs_len) = (transparent_inputs.len(), transparent_outputs.len()); + #[cfg(feature = "transparent-inputs")] + let (t_inputs_len, t_outputs_len) = ( + t_inputs_len + usize::from(ephemeral_parameters.ephemeral_input_amount().is_some()), + t_outputs_len + usize::from(ephemeral_parameters.ephemeral_output_amount().is_some()), + ); + let (s_inputs_len, s_outputs_len) = (sapling.inputs().len(), sapling.outputs().len()); + #[cfg(feature = "orchard")] + let (o_inputs_len, o_outputs_len) = (orchard.inputs().len(), orchard.outputs().len()); + #[cfg(not(feature = "orchard"))] + let (o_inputs_len, o_outputs_len) = (0usize, 0usize); + + let t_non_dust = t_inputs_len.checked_sub(t_dust.len()).unwrap(); + let s_non_dust = s_inputs_len.checked_sub(s_dust.len()).unwrap(); + let o_non_dust = o_inputs_len.checked_sub(o_dust.len()).unwrap(); + + // Return the number of allowed dust inputs from each pool. + let allowed_dust = |(t_change, s_change, o_change): &(usize, usize, usize)| { + // What is the maximum number of dust inputs in each pool, out of the ones we + // actually have, that can be economically spent along with the non-dust inputs? + // Get an initial estimate by calculating the number of dust inputs in each pool + // that will be allowed regardless of padding or grace. + + let t_allowed = min( + t_dust.len(), + (t_outputs_len + t_change).saturating_sub(t_non_dust), + ); + let s_allowed = min( + s_dust.len(), + (s_outputs_len + s_change).saturating_sub(s_non_dust), + ); + let o_allowed = min( + o_dust.len(), + (o_outputs_len + o_change).saturating_sub(o_non_dust), + ); + + // We'll be spending the non-dust and allowed dust in each pool. + let t_req_inputs = t_non_dust + t_allowed; + let s_req_inputs = s_non_dust + s_allowed; + #[cfg(feature = "orchard")] + let o_req_inputs = o_non_dust + o_allowed; + + // This calculates the hypothetical number of actions with given extra inputs. + // The padding rules are subtle (they also depend on `bundle_required` for example). + // To reliably tell whether we can add an extra input of a given type, we will need + // to call them both with without that input; if the number of actions does not + // increase, then the input is free to add. + let hypothetical_actions = |t_extra, s_extra, _o_extra| { + let s_spend_count = sapling + .bundle_type() + .num_spends(s_req_inputs + s_extra) + .map_err(ChangeError::BundleError)?; + + let s_output_count = sapling + .bundle_type() + .num_outputs(s_req_inputs + s_extra, s_outputs_len + s_change) + .map_err(ChangeError::BundleError)?; + + #[cfg(feature = "orchard")] + let o_action_count = orchard + .bundle_type() + .num_actions(o_req_inputs + _o_extra, o_outputs_len + o_change) + .map_err(ChangeError::BundleError)?; + #[cfg(not(feature = "orchard"))] + let o_action_count = 0; + + // To calculate the number of unused actions, we assume that transparent inputs + // and outputs are P2PKH. + Ok(max(t_req_inputs + t_extra, t_outputs_len + t_change) + + max(s_spend_count, s_output_count) + + o_action_count) + }; + + // First calculate the baseline number of logical actions with only the definitely + // allowed inputs estimated above. If it is less than `grace_actions`, try to allocate + // a grace input first to transparent dust, then to Sapling dust, then to Orchard dust. + // If the number of actions increases, it was not possible to allocate that input for + // free. This approach is sufficient because at most one such input can be allocated, + // since `grace_actions` is at most 2 for ZIP 317 and there must be at least one + // logical action. (If `grace_actions` were greater than 2 then the code would still + // be correct, it would just not find all potential extra inputs.) + + let baseline = hypothetical_actions(0, 0, 0)?; + + let (t_extra, s_extra, o_extra) = if baseline >= grace_actions { + (0, 0, 0) + } else if t_dust.len() > t_allowed && hypothetical_actions(1, 0, 0)? <= baseline { + (1, 0, 0) + } else if s_dust.len() > s_allowed && hypothetical_actions(0, 1, 0)? <= baseline { + (0, 1, 0) + } else if o_dust.len() > o_allowed && hypothetical_actions(0, 0, 1)? <= baseline { + (0, 0, 1) + } else { + (0, 0, 0) + }; + Ok(( + t_allowed + t_extra, + s_allowed + s_extra, + o_allowed + o_extra, + )) + }; + + // Find the least number of allowed dust inputs for each pool for any `possible_change`. + let (t_allowed, s_allowed, o_allowed) = possible_change + .iter() + .map(allowed_dust) + .collect::, _>>()? + .into_iter() + .reduce(|(a, b, c), (x, y, z)| (min(a, x), min(b, y), min(c, z))) + .expect("possible_change is nonempty"); + + // The inputs in the tail of each list after the first `*_allowed` are returned as uneconomic. + // The caller should order the inputs from most to least preferred to spend. + let t_dust = t_dust.split_off(t_allowed); + let s_dust = s_dust.split_off(s_allowed); + let o_dust = o_dust.split_off(o_allowed); + + if t_dust.is_empty() && s_dust.is_empty() && o_dust.is_empty() { + Ok(()) + } else { + Err(ChangeError::DustInputs { + transparent: t_dust, + sapling: s_dust, + #[cfg(feature = "orchard")] + orchard: o_dust, + }) + } +} diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index de8e814fe5..a6a9961e72 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -4,7 +4,7 @@ use zcash_primitives::{ consensus::{self, BlockHeight}, memo::MemoBytes, transaction::{ - components::amount::BalanceError, + components::amount::{BalanceError, NonNegativeAmount}, fees::{fixed::FeeRule as FixedFeeRule, transparent}, }, }; @@ -78,9 +78,11 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - self.fee_rule().fixed_fee(), + self.fee_rule.fixed_fee(), self.change_memo.as_ref(), self.fallback_change_pool, + NonNegativeAmount::ZERO, + 0, #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ) diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 81ec9667e1..5a55153cf2 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -72,7 +72,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { dust_output_policy: &DustOutputPolicy, #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, ) -> Result> { - // TODO: consider opportunistic dust spends (#1316). single_change_output_balance( params, &self.fee_rule, @@ -86,6 +85,8 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fee_rule.marginal_fee(), self.change_memo.as_ref(), self.fallback_change_pool, + self.fee_rule.marginal_fee(), + self.fee_rule.grace_actions(), #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ) @@ -472,10 +473,8 @@ mod tests { ShieldedProtocol::Sapling, ); - // Attempt to spend three Sapling notes, one of them dust. Taking into account - // Sapling output padding, the dust note would be free to add to the transaction - // if there were only two notes (as in the `change_with_allowable_dust` test), but - // it is not free when there are three notes. + // Attempt to spend three Sapling notes, one of them dust. Adding the third + // note increases the number of actions, and so it is uneconomic to spend it. let result = change_strategy.compute_balance( &Network::TestNetwork, Network::TestNetwork @@ -500,7 +499,7 @@ mod tests { }, ][..], &[SaplingPayment::new(NonNegativeAmount::const_from_u64( - 40000, + 30000, ))][..], ), #[cfg(feature = "orchard")] @@ -510,7 +509,7 @@ mod tests { &Default::default(), ); - // We will get an error here, because the dust input now isn't free to add + // We will get an error here, because the dust input isn't free to add // to the transaction. assert_matches!( result, From 9c082dca3e163e41eb7c296e4a48dcd8c3f28a57 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Fri, 28 Jun 2024 17:55:15 +0100 Subject: [PATCH 31/56] `zcash_primitives::transaction::fees::zip317::FeeRule::non_standard` has been deprecated, because in general it can calculate fees that violate ZIP 317, which might cause transactions built with it to fail. Maintaining the generality of the current implementation imposes ongoing maintenance costs, and so it is likely to be removed in the near future. Use `transaction::fees::zip317::FeeRule::standard()` instead to comply with ZIP 317. Signed-off-by: Daira-Emma Hopwood --- zcash_primitives/CHANGELOG.md | 8 ++++++++ zcash_primitives/src/transaction/fees/zip317.rs | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 259d505529..6773d516e1 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -44,6 +44,14 @@ and this library adheres to Rust's notion of (expressed as iterators of `InputSize` for inputs and `usize` for outputs) rather than a slice of `InputView` or `OutputView`. +### Deprecated +- `zcash_primitives::transaction::fees::zip317::FeeRule::non_standard` has been + deprecated, because in general it can calculate fees that violate ZIP 317, which + might cause transactions built with it to fail. Maintaining the generality of the + current implementation imposes ongoing maintenance costs, and so it is likely to + be removed in the near future. Use `transaction::fees::zip317::FeeRule::standard()` + instead to comply with ZIP 317. + ### Removed - The `zcash_primitives::zip339` module, which reexported parts of the API of the `bip0039` crate, has been removed. Use the `bip0039` crate directly diff --git a/zcash_primitives/src/transaction/fees/zip317.rs b/zcash_primitives/src/transaction/fees/zip317.rs index 1d5d7513ac..48457b8007 100644 --- a/zcash_primitives/src/transaction/fees/zip317.rs +++ b/zcash_primitives/src/transaction/fees/zip317.rs @@ -78,6 +78,13 @@ impl FeeRule { /// /// Returns `None` if either `p2pkh_standard_input_size` or `p2pkh_standard_output_size` are /// zero. + #[deprecated( + note = "Using this fee rule with `marginal_fee < 5000 || grace_actions < 2 \ + || p2pkh_standard_input_size > P2PKH_STANDARD_INPUT_SIZE \ + || p2pkh_standard_output_size > P2PKH_STANDARD_OUTPUT_SIZE` \ + violates ZIP 317, and might cause transactions built with it to fail. \ + This API is likely to be removed. Use `[FeeRule::standard]` instead." + )] pub fn non_standard( marginal_fee: NonNegativeAmount, grace_actions: usize, From 38296634d9e5187d380a8ec7ad835e9db1384d1c Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Fri, 28 Jun 2024 18:03:43 +0100 Subject: [PATCH 32/56] Change note selection query to select notes > 5000 zats, not >= 5000 zats. Also the issue reference was wrong. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/wallet/common.rs b/zcash_client_sqlite/src/wallet/common.rs index 800d7f660d..3bcdbc97f5 100644 --- a/zcash_client_sqlite/src/wallet/common.rs +++ b/zcash_client_sqlite/src/wallet/common.rs @@ -158,7 +158,7 @@ where INNER JOIN transactions ON transactions.id_tx = {table_prefix}_received_notes.tx WHERE {table_prefix}_received_notes.account_id = :account - AND value >= 5000 -- FIXME #1016, allow selection of a dust inputs + AND value > 5000 -- FIXME #1316, allow selection of dust inputs AND accounts.ufvk IS NOT NULL AND recipient_key_scope IS NOT NULL AND nf IS NOT NULL From 25006ab0131b56f7452c6c5efe053602030b25dc Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Fri, 28 Jun 2024 20:32:30 +0100 Subject: [PATCH 33/56] Documentation improvement from code review. Co-authored-by: Kris Nuttycombe Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/error.rs | 14 +++++++------- zcash_client_backend/src/data_api/wallet.rs | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 33081f6546..371d57706e 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -36,11 +36,11 @@ pub enum Error { /// An error in transaction proposal construction Proposal(ProposalError), - /// The proposal was structurally valid, but tries to do one of these unsupported things: - /// * spending a prior shielded output; - /// * paying to an output pool for which the corresponding feature is not enabled; - /// * paying to a TEX address if the "transparent-inputs" feature is not enabled; - /// * exceeding implementation limits. + /// The proposal was structurally valid, but tried to do one of these unsupported things: + /// * spend a prior shielded output; + /// * pay to an output pool for which the corresponding feature is not enabled; + /// * pay to a TEX address if the "transparent-inputs" feature is not enabled; + /// or exceeded an implementation limit. ProposalNotSupported, /// No account could be found corresponding to a provided spending key. @@ -122,8 +122,8 @@ where Error::ProposalNotSupported => write!( f, "The proposal was valid but tried to do something that is not supported \ - (spending shielded outputs of prior transaction steps, using a feature \ - that is not enabled, or exceeding an implementation limit).", + (spend shielded outputs of prior transaction steps or use a feature that \ + is not enabled), or exceeded an implementation limit.", ), Error::KeyNotRecognized => { write!( diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 0d99238a33..be662dd3ca 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -645,6 +645,9 @@ where .expect("proposal.steps is NonEmpty")) } +// `unused_transparent_outputs` maps `StepOutput`s for transparent outputs +// that have not been consumed so far, to the corresponding pair of +// `TransparentAddress` and `Outpoint`. #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] fn create_proposed_transaction( From 3922d71ade387746ef15d70296f06064bdb613ce Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Fri, 28 Jun 2024 20:31:36 +0100 Subject: [PATCH 34/56] Change the type of `n` in `reserve_next_n_ephemeral_addresses` back to `u32`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api.rs | 8 ++++---- zcash_client_backend/src/data_api/wallet.rs | 11 +++++++---- zcash_client_sqlite/src/lib.rs | 2 +- .../src/wallet/transparent/ephemeral.rs | 5 +++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 5faec524c5..1edc2b0ffb 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1634,14 +1634,14 @@ pub trait WalletWrite: WalletRead { /// the given number of addresses, or if the account identifier does not correspond /// to a known account. /// - /// Precondition: `n >= 0` + /// Precondition: `n < 0x80000000` /// /// [BIP 44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#user-content-Address_gap_limit #[cfg(feature = "transparent-inputs")] fn reserve_next_n_ephemeral_addresses( &mut self, account_id: Self::AccountId, - n: i32, + n: u32, ) -> Result, Self::Error>; } @@ -2056,9 +2056,9 @@ pub mod testing { fn reserve_next_n_ephemeral_addresses( &mut self, _account_id: Self::AccountId, - n: i32, + n: u32, ) -> Result, Self::Error> { - assert!(n >= 0); + assert!(n < 0x80000000); Err(()) } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index be662dd3ca..c892952ec5 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1137,11 +1137,14 @@ where change_value.is_ephemeral() && change_value.output_pool() == PoolType::Transparent }) .collect(); - let num_ephemeral_outputs = - i32::try_from(ephemeral_outputs.len()).map_err(|_| Error::ProposalNotSupported)?; - + if ephemeral_outputs.len() > i32::MAX as usize { + return Err(Error::ProposalNotSupported); + } let addresses_and_metadata = wallet_db - .reserve_next_n_ephemeral_addresses(account_id, num_ephemeral_outputs) + .reserve_next_n_ephemeral_addresses( + account_id, + ephemeral_outputs.len().try_into().unwrap(), + ) .map_err(Error::DataSource)?; assert_eq!(addresses_and_metadata.len(), ephemeral_outputs.len()); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 35cadcd5e7..1a31ab31ad 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1496,7 +1496,7 @@ impl WalletWrite for WalletDb fn reserve_next_n_ephemeral_addresses( &mut self, account_id: Self::AccountId, - n: i32, + n: u32, ) -> Result, Self::Error> { self.transactionally(|wdb| { wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses(wdb, account_id, n) diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 5327b487fc..b79532eb1b 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -172,7 +172,7 @@ pub(crate) fn get_reserved_ephemeral_addresses( /// Returns a vector with the next `n` previously unreserved ephemeral addresses for /// the given account. /// -/// Precondition: `n >= 0` +/// Precondition: `n < 0x80000000` /// /// # Errors /// @@ -187,12 +187,13 @@ pub(crate) fn get_reserved_ephemeral_addresses( pub(crate) fn reserve_next_n_ephemeral_addresses( wdb: &mut WalletDb, P>, account_id: AccountId, - n: i32, + n: u32, ) -> Result, SqliteClientError> { if n == 0 { return Ok(vec![]); } assert!(n > 0); + let n = i32::try_from(n).expect("precondition violated"); let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; let last_reserved_index = last_reserved_index(wdb.conn.0, account_id)?; From d32b7dbd8f2a0675069ca28b53fbb9df988932bf Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Fri, 28 Jun 2024 21:49:51 +0100 Subject: [PATCH 35/56] Remove `ChangeValue::new`. Also document `ChangeValue::is_ephemeral` as an addition in the changelog. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 19 ++++++++---------- zcash_client_backend/src/fees.rs | 33 ------------------------------- 2 files changed, 8 insertions(+), 44 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c8a6f2cbb0..03d5147874 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -28,7 +28,8 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - `chain::BlockCache` trait, behind the `sync` feature flag. - `WalletRead::get_spendable_transparent_outputs`. - `zcash_client_backend::fees`: - - `ChangeValue::{ephemeral_transparent, shielded}` + - `ChangeValue::shielded, is_ephemeral` + - `ChangeValue::ephemeral_transparent` (when "transparent-inputs" is enabled) - `sapling::EmptyBundleView` - `orchard::EmptyBundleView` - `zcash_client_backend::proposal`: @@ -61,16 +62,10 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail try_into_standard_proposal}` each no longer require a `consensus::Parameters` argument. - `zcash_client_backend::data_api::fees` - - The return type of `ChangeValue::output_pool`, and the type of the - `output_pool` argument to `ChangeValue::new`, have changed from - `ShieldedProtocol` to `zcash_protocol::PoolType`. - - When the "transparent-inputs" feature is enabled, `ChangeValue::new` - takes an additional `is_ephemeral` parameter indicating whether the value - is ephemeral (i.e. for use in a subsequent proposal step) or change. - - The return type of `ChangeValue::new` is now optional; it returns `None` - if a memo is given for the transparent pool. Use `ChangeValue::shielded` - to avoid this error case when creating a `ChangeValue` known to be for a - shielded pool. + - When the "transparent-inputs" feature is enabled, `ChangeValue` can also + represent an ephemeral transparent output in a proposal. Accordingly, the + return type of `ChangeValue::output_pool` has (unconditionally) changed + from `ShieldedProtocol` to `zcash_protocol::PoolType`. - `ChangeStrategy::compute_balance`: this trait method has an additional `&EphemeralParameters` parameter when the "transparent-inputs" feature is enabled. This can be used to specify whether the change memo should be @@ -92,6 +87,8 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - `WalletRead::get_unspent_transparent_outputs` has been removed because its semantics were unclear and could not be clarified. Use `WalletRead::get_spendable_transparent_outputs` instead. +- `zcash_client_backend::fees::ChangeValue::new`. Use `ChangeValue::shielded` + or `ChangeValue::ephemeral_transparent` instead. ## [0.12.1] - 2024-03-27 diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index ae1a24da5e..1a300699ab 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -39,39 +39,6 @@ enum ChangeValueInner { } impl ChangeValue { - #[cfg_attr( - feature = "transparent-inputs", - doc = "Constructs a new change or ephemeral value from its constituent - parts. Currently, the only supported combinations are change sent - to a shielded pool, or an ephemeral output to the transparent pool." - )] - #[cfg_attr( - not(feature = "transparent-inputs"), - doc = "Constructs a new change value from its constituent parts. - Currently, `output_pool` must be a shielded pool (enable the - `transparent-inputs` feature to support ephemeral outputs to the - transparent pool). Use `ChangeValue::shielded` to avoid the - `Option` return." - )] - pub fn new( - output_pool: PoolType, - value: NonNegativeAmount, - memo: Option, - #[cfg(feature = "transparent-inputs")] is_ephemeral: bool, - ) -> Option { - #[cfg(not(feature = "transparent-inputs"))] - let is_ephemeral = false; - - match (output_pool, is_ephemeral) { - (PoolType::Shielded(protocol), false) => Some(Self::shielded(protocol, value, memo)), - #[cfg(feature = "transparent-inputs")] - (PoolType::Transparent, true) if memo.is_none() => { - Some(Self::ephemeral_transparent(value)) - } - _ => None, - } - } - /// Constructs a new ephemeral transparent output value. #[cfg(feature = "transparent-inputs")] pub fn ephemeral_transparent(value: NonNegativeAmount) -> Self { From 286439aa960701d5fb57345ae73579bf5422bbf5 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 29 Jun 2024 01:33:50 +0100 Subject: [PATCH 36/56] Define a constant `EphemeralParameters::NONE` instead of deriving `Default`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 3 ++- .../src/data_api/wallet/input_selection.rs | 6 +++--- zcash_client_backend/src/fees.rs | 12 ++++++++++-- zcash_client_backend/src/fees/fixed.rs | 7 +++++-- zcash_client_backend/src/fees/zip317.rs | 19 +++++++++++-------- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 03d5147874..901124bb4a 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -70,7 +70,8 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail `&EphemeralParameters` parameter when the "transparent-inputs" feature is enabled. This can be used to specify whether the change memo should be ignored, and the amounts of additional transparent P2PKH inputs and - outputs. Passing `&Default::default()` will retain the previous behaviour. + outputs. Passing `&EphemeralParameters::NONE` will retain the previous + behaviour. - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. - `zcash_client_backend::proto::ProposalDecodingError` has a new variant diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 24c521a35d..b0a64936f1 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -462,7 +462,7 @@ where #[cfg(feature = "transparent-inputs")] let (ephemeral_parameters, tr1_balance_opt) = { if tr1_transparent_outputs.is_empty() { - (Default::default(), None) + (EphemeralParameters::NONE, None) } else { // The ephemeral input going into transaction 1 must be able to pay that // transaction's fee, as well as the TEX address payments. @@ -772,7 +772,7 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); let balance = match trial_balance { @@ -791,7 +791,7 @@ where &orchard_fees::EmptyBundleView, &self.dust_output_policy, #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, )? } Err(other) => { diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 1a300699ab..ddf0ac241b 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -334,7 +334,7 @@ impl Default for DustOutputPolicy { /// and fees are computed that are relevant to transactions using ephemeral /// transparent outputs. #[cfg(feature = "transparent-inputs")] -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct EphemeralParameters { ignore_change_memo: bool, ephemeral_input_amount: Option, @@ -343,6 +343,14 @@ pub struct EphemeralParameters { #[cfg(feature = "transparent-inputs")] impl EphemeralParameters { + /// An `EphemeralParameters` indicating no use of ephemeral inputs + /// or outputs. It has: + /// + /// * `ignore_change_memo: false`, + /// * `ephemeral_input_amount: None`, + /// * `ephemeral_output_amount: None`. + pub const NONE: Self = Self::new(false, None, None); + /// Returns an `EphemeralParameters` with the following parameters: /// /// * `ignore_change_memo`: `true` if the change memo should be @@ -352,7 +360,7 @@ impl EphemeralParameters { /// additional P2PKH input of the given amount. /// * `ephemeral_output_amount`: specifies that there will be an /// additional P2PKH output of the given amount. - pub fn new( + pub const fn new( ignore_change_memo: bool, ephemeral_input_amount: Option, ephemeral_output_amount: Option, diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index a6a9961e72..8c27895b7f 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -109,6 +109,9 @@ mod tests { ShieldedProtocol, }; + #[cfg(feature = "transparent-inputs")] + use crate::fees::EphemeralParameters; + #[cfg(feature = "orchard")] use crate::fees::orchard as orchard_fees; @@ -141,7 +144,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -188,7 +191,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 5a55153cf2..3622b6850f 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -116,6 +116,9 @@ mod tests { ShieldedProtocol, }; + #[cfg(feature = "transparent-inputs")] + use crate::fees::EphemeralParameters; + #[cfg(feature = "orchard")] use { crate::data_api::wallet::input_selection::OrchardPayment, @@ -152,7 +155,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -197,7 +200,7 @@ mod tests { ), &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -251,7 +254,7 @@ mod tests { &orchard_fees::EmptyBundleView, dust_output_policy, #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -296,7 +299,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -341,7 +344,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -392,7 +395,7 @@ mod tests { Some(NonNegativeAmount::const_from_u64(1000)), ), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -454,7 +457,7 @@ mod tests { &orchard_fees::EmptyBundleView, dust_output_policy, #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); assert_matches!( @@ -506,7 +509,7 @@ mod tests { &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), #[cfg(feature = "transparent-inputs")] - &Default::default(), + &EphemeralParameters::NONE, ); // We will get an error here, because the dust input isn't free to add From bc38f2af807ebf7ce192c8265d5070cac734888a Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 29 Jun 2024 01:34:19 +0100 Subject: [PATCH 37/56] Document the `possible_change` parameter to `check_for_uneconomic_inputs`. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/fees/common.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 17e4f77b65..9c9b9f3109 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -455,6 +455,11 @@ where /// with value less than the marginal fee might be excluded, even though in /// practice they would not cause the fee to increase. Outputs with value /// greater than the marginal fee will never be excluded. +/// +/// `possible_change` is a slice of `(transparent_change, sapling_change, orchard_change)` +/// tuples indicating possible combinations of how many change outputs (0 or 1) +/// might be included in the transaction for each pool. The shape of the tuple +/// does not depend on which protocol features are enabled. #[allow(clippy::too_many_arguments)] pub(crate) fn check_for_uneconomic_inputs( transparent_inputs: &[impl transparent::InputView], From 7838c048a22201abc0d02cb2b97f2326450111bc Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 29 Jun 2024 12:17:05 +0100 Subject: [PATCH 38/56] Make `ephemeral_parameters` and `EphemeralParameters::NONE` available unconditionally. This removes a bunch of `#[cfg(feature = "transparent-inputs")]` conditionals, with negligible (if any, after compiler optimization) overhead. It also requires callers of functions with an `ephemeral_parameters` parameter to do the intended thing to work with "transparent-inputs" either enabled or disabled. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 7 +- .../src/data_api/wallet/input_selection.rs | 9 +-- zcash_client_backend/src/fees.rs | 19 ++++-- zcash_client_backend/src/fees/common.rs | 65 ++++++++----------- zcash_client_backend/src/fees/fixed.rs | 15 +---- zcash_client_backend/src/fees/standard.rs | 10 +-- zcash_client_backend/src/fees/zip317.rs | 20 +----- 7 files changed, 57 insertions(+), 88 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 901124bb4a..7c5b19843b 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -67,11 +67,12 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail return type of `ChangeValue::output_pool` has (unconditionally) changed from `ShieldedProtocol` to `zcash_protocol::PoolType`. - `ChangeStrategy::compute_balance`: this trait method has an additional - `&EphemeralParameters` parameter when the "transparent-inputs" feature is - enabled. This can be used to specify whether the change memo should be + `&EphemeralParameters` parameter. If the "transparent-inputs" feature is + enabled, this can be used to specify whether the change memo should be ignored, and the amounts of additional transparent P2PKH inputs and outputs. Passing `&EphemeralParameters::NONE` will retain the previous - behaviour. + behaviour (and is necessary when the "transparent-inputs" feature is + not enabled). - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. - `zcash_client_backend::proto::ProposalDecodingError` has a new variant diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index b0a64936f1..162290ef9e 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -23,7 +23,7 @@ use zcash_primitives::{ use crate::{ address::{Address, UnifiedAddress}, data_api::{InputSource, SimpleNoteRetention, SpendableNotes}, - fees::{sapling, ChangeError, ChangeStrategy, DustOutputPolicy}, + fees::{sapling, ChangeError, ChangeStrategy, DustOutputPolicy, EphemeralParameters}, proposal::{Proposal, ProposalError, ShieldedInputs}, wallet::WalletTransparentOutput, zip321::TransactionRequest, @@ -33,7 +33,7 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { crate::{ - fees::{ChangeValue, EphemeralParameters}, + fees::ChangeValue, proposal::{Step, StepOutput, StepOutputIndex}, zip321::Payment, }, @@ -459,6 +459,9 @@ where } } + #[cfg(not(feature = "transparent-inputs"))] + let ephemeral_parameters = EphemeralParameters::NONE; + #[cfg(feature = "transparent-inputs")] let (ephemeral_parameters, tr1_balance_opt) = { if tr1_transparent_outputs.is_empty() { @@ -579,7 +582,6 @@ where &orchard_outputs[..], ), &self.dust_output_policy, - #[cfg(feature = "transparent-inputs")] &ephemeral_parameters, ); @@ -771,7 +773,6 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index ddf0ac241b..51867a9f15 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -332,8 +332,9 @@ impl Default for DustOutputPolicy { /// `EphemeralParameters` can be used to specify variations on how balance /// and fees are computed that are relevant to transactions using ephemeral -/// transparent outputs. -#[cfg(feature = "transparent-inputs")] +/// transparent outputs. These are only relevant when the "transparent-inputs" +/// feature is enabled, but to reduce feature-dependent boilerplate, the type +/// and the `EphemeralParameters::NONE` constant are present unconditionally. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EphemeralParameters { ignore_change_memo: bool, @@ -341,7 +342,6 @@ pub struct EphemeralParameters { ephemeral_output_amount: Option, } -#[cfg(feature = "transparent-inputs")] impl EphemeralParameters { /// An `EphemeralParameters` indicating no use of ephemeral inputs /// or outputs. It has: @@ -349,7 +349,11 @@ impl EphemeralParameters { /// * `ignore_change_memo: false`, /// * `ephemeral_input_amount: None`, /// * `ephemeral_output_amount: None`. - pub const NONE: Self = Self::new(false, None, None); + pub const NONE: Self = Self { + ignore_change_memo: false, + ephemeral_input_amount: None, + ephemeral_output_amount: None, + }; /// Returns an `EphemeralParameters` with the following parameters: /// @@ -360,6 +364,7 @@ impl EphemeralParameters { /// additional P2PKH input of the given amount. /// * `ephemeral_output_amount`: specifies that there will be an /// additional P2PKH output of the given amount. + #[cfg(feature = "transparent-inputs")] pub const fn new( ignore_change_memo: bool, ephemeral_input_amount: Option, @@ -414,6 +419,10 @@ pub trait ChangeStrategy { and fees are computed that are relevant to transactions using ephemeral transparent outputs; see [`EphemeralParameters::new`]." )] + #[cfg_attr( + not(feature = "transparent-inputs"), + doc = "`ephemeral_parameters` should be set to `&EphemeralParameters::NONE`." + )] #[allow(clippy::too_many_arguments)] fn compute_balance( &self, @@ -424,7 +433,7 @@ pub trait ChangeStrategy { sapling: &impl sapling::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: &EphemeralParameters, ) -> Result>; } diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 9c9b9f3109..d0b3eebcac 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -5,22 +5,14 @@ use zcash_primitives::{ memo::MemoBytes, transaction::{ components::amount::{BalanceError, NonNegativeAmount}, - fees::{transparent, zip317::MINIMUM_FEE, FeeRule}, + fees::{transparent, zip317::MINIMUM_FEE, zip317::P2PKH_STANDARD_OUTPUT_SIZE, FeeRule}, }, }; use zcash_protocol::ShieldedProtocol; use super::{ sapling as sapling_fees, ChangeError, ChangeValue, DustAction, DustOutputPolicy, - TransactionBalance, -}; - -#[cfg(feature = "transparent-inputs")] -use { - super::EphemeralParameters, - zcash_primitives::transaction::fees::{ - transparent::InputSize, zip317::P2PKH_STANDARD_OUTPUT_SIZE, - }, + EphemeralParameters, TransactionBalance, }; #[cfg(feature = "orchard")] @@ -57,17 +49,14 @@ pub(crate) fn calculate_net_flows( transparent_outputs: &[impl transparent::OutputView], sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, - #[cfg(feature = "transparent-inputs")] ephemeral_input_amount: Option, - #[cfg(feature = "transparent-inputs")] ephemeral_output_amount: Option, + ephemeral_input_amount: Option, + ephemeral_output_amount: Option, ) -> Result> where E: From + From, { let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); - #[cfg(not(feature = "transparent-inputs"))] - let (ephemeral_input_amount, ephemeral_output_amount) = (None, None); - let t_in = transparent_inputs .iter() .map(|t_in| t_in.coin().value) @@ -173,12 +162,11 @@ pub(crate) fn single_change_output_balance< fallback_change_pool: ShieldedProtocol, marginal_fee: NonNegativeAmount, grace_actions: usize, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: &EphemeralParameters, ) -> Result> where E: From + From, { - #[cfg(feature = "transparent-inputs")] let change_memo = change_memo.filter(|_| !ephemeral_parameters.ignore_change_memo()); let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); @@ -190,9 +178,7 @@ where sapling, #[cfg(feature = "orchard")] orchard, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters.ephemeral_input_amount(), - #[cfg(feature = "transparent-inputs")] ephemeral_parameters.ephemeral_output_amount(), )?; @@ -227,7 +213,6 @@ where marginal_fee, grace_actions, &possible_change[..], - #[cfg(feature = "transparent-inputs")] ephemeral_parameters, )?; } @@ -298,20 +283,22 @@ where // Note that using the `DustAction::AddDustToFee` policy inherently leaks // more information. - let transparent_input_sizes = transparent_inputs.iter().map(|i| i.serialized_size()); - #[cfg(feature = "transparent-inputs")] - let transparent_input_sizes = transparent_input_sizes.chain( - ephemeral_parameters - .ephemeral_input_amount() - .map(|_| InputSize::STANDARD_P2PKH), - ); - let transparent_output_sizes = transparent_outputs.iter().map(|i| i.serialized_size()); - #[cfg(feature = "transparent-inputs")] - let transparent_output_sizes = transparent_output_sizes.chain( - ephemeral_parameters - .ephemeral_output_amount() - .map(|_| P2PKH_STANDARD_OUTPUT_SIZE), - ); + let transparent_input_sizes = transparent_inputs + .iter() + .map(|i| i.serialized_size()) + .chain( + ephemeral_parameters + .ephemeral_input_amount() + .map(|_| transparent::InputSize::STANDARD_P2PKH), + ); + let transparent_output_sizes = transparent_outputs + .iter() + .map(|i| i.serialized_size()) + .chain( + ephemeral_parameters + .ephemeral_output_amount() + .map(|_| P2PKH_STANDARD_OUTPUT_SIZE), + ); let fee_without_change = fee_rule .fee_required( @@ -469,7 +456,7 @@ pub(crate) fn check_for_uneconomic_inputs( marginal_fee: NonNegativeAmount, grace_actions: usize, possible_change: &[(usize, usize, usize)], - #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: &EphemeralParameters, ) -> Result<(), ChangeError> { let mut t_dust: Vec<_> = transparent_inputs .iter() @@ -516,11 +503,11 @@ pub(crate) fn check_for_uneconomic_inputs( return Ok(()); } - let (t_inputs_len, t_outputs_len) = (transparent_inputs.len(), transparent_outputs.len()); - #[cfg(feature = "transparent-inputs")] let (t_inputs_len, t_outputs_len) = ( - t_inputs_len + usize::from(ephemeral_parameters.ephemeral_input_amount().is_some()), - t_outputs_len + usize::from(ephemeral_parameters.ephemeral_output_amount().is_some()), + transparent_inputs.len() + + usize::from(ephemeral_parameters.ephemeral_input_amount().is_some()), + transparent_outputs.len() + + usize::from(ephemeral_parameters.ephemeral_output_amount().is_some()), ); let (s_inputs_len, s_outputs_len) = (sapling.inputs().len(), sapling.outputs().len()); #[cfg(feature = "orchard")] diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index 8c27895b7f..e42e4527db 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -13,15 +13,12 @@ use crate::ShieldedProtocol; use super::{ common::single_change_output_balance, sapling as sapling_fees, ChangeError, ChangeStrategy, - DustOutputPolicy, TransactionBalance, + DustOutputPolicy, EphemeralParameters, TransactionBalance, }; #[cfg(feature = "orchard")] use super::orchard as orchard_fees; -#[cfg(feature = "transparent-inputs")] -use super::EphemeralParameters; - /// A change strategy that proposes change as a single output to the most current supported /// shielded pool and delegates fee calculation to the provided fee rule. pub struct SingleOutputChangeStrategy { @@ -66,7 +63,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: &EphemeralParameters, ) -> Result> { single_change_output_balance( params, @@ -83,7 +80,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fallback_change_pool, NonNegativeAmount::ZERO, 0, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ) } @@ -104,14 +100,11 @@ mod tests { data_api::wallet::input_selection::SaplingPayment, fees::{ tests::{TestSaplingInput, TestTransparentInput}, - ChangeError, ChangeStrategy, ChangeValue, DustOutputPolicy, + ChangeError, ChangeStrategy, ChangeValue, DustOutputPolicy, EphemeralParameters, }, ShieldedProtocol, }; - #[cfg(feature = "transparent-inputs")] - use crate::fees::EphemeralParameters; - #[cfg(feature = "orchard")] use crate::fees::orchard as orchard_fees; @@ -143,7 +136,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -190,7 +182,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); diff --git a/zcash_client_backend/src/fees/standard.rs b/zcash_client_backend/src/fees/standard.rs index 30d77d8fbc..8c49e33b35 100644 --- a/zcash_client_backend/src/fees/standard.rs +++ b/zcash_client_backend/src/fees/standard.rs @@ -18,12 +18,9 @@ use crate::ShieldedProtocol; use super::{ fixed, sapling as sapling_fees, zip317, ChangeError, ChangeStrategy, DustOutputPolicy, - TransactionBalance, + EphemeralParameters, TransactionBalance, }; -#[cfg(feature = "transparent-inputs")] -use super::EphemeralParameters; - #[cfg(feature = "orchard")] use super::orchard as orchard_fees; @@ -71,7 +68,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: &EphemeralParameters, ) -> Result> { #[allow(deprecated)] match self.fee_rule() { @@ -89,7 +86,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ) .map_err(|e| e.map(Zip317FeeError::Balance)), @@ -107,7 +103,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ) .map_err(|e| e.map(Zip317FeeError::Balance)), @@ -125,7 +120,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ), } diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 3622b6850f..a054f9b41a 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -17,12 +17,9 @@ use crate::ShieldedProtocol; use super::{ common::single_change_output_balance, sapling as sapling_fees, ChangeError, ChangeStrategy, - DustOutputPolicy, TransactionBalance, + DustOutputPolicy, EphemeralParameters, TransactionBalance, }; -#[cfg(feature = "transparent-inputs")] -use super::EphemeralParameters; - #[cfg(feature = "orchard")] use super::orchard as orchard_fees; @@ -70,7 +67,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - #[cfg(feature = "transparent-inputs")] ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: &EphemeralParameters, ) -> Result> { single_change_output_balance( params, @@ -87,7 +84,6 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fallback_change_pool, self.fee_rule.marginal_fee(), self.fee_rule.grace_actions(), - #[cfg(feature = "transparent-inputs")] ephemeral_parameters, ) } @@ -112,13 +108,11 @@ mod tests { fees::{ tests::{TestSaplingInput, TestTransparentInput}, ChangeError, ChangeStrategy, ChangeValue, DustAction, DustOutputPolicy, + EphemeralParameters, }, ShieldedProtocol, }; - #[cfg(feature = "transparent-inputs")] - use crate::fees::EphemeralParameters; - #[cfg(feature = "orchard")] use { crate::data_api::wallet::input_selection::OrchardPayment, @@ -154,7 +148,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -199,7 +192,6 @@ mod tests { ))][..], ), &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -253,7 +245,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, dust_output_policy, - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -298,7 +289,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -343,7 +333,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -394,7 +383,6 @@ mod tests { DustAction::AllowDustChange, Some(NonNegativeAmount::const_from_u64(1000)), ), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -456,7 +444,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, dust_output_policy, - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); @@ -508,7 +495,6 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - #[cfg(feature = "transparent-inputs")] &EphemeralParameters::NONE, ); From 14bdcdeaaa559b031a9f4f50afd04e19d55cff7d Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 29 Jun 2024 12:06:10 +0100 Subject: [PATCH 39/56] We cannot spend prior outputs at all when "transparent-inputs" is not enabled. (I got this right in a previous commit but broke it when refactoring the proposal error handling.) Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api/wallet.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index c892952ec5..439e8a2b19 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -675,7 +675,7 @@ where let step_index = prior_step_results.len(); // We only support spending transparent payments or transparent ephemeral outputs from a - // prior step. + // prior step (when "transparent-inputs" is enabled). // // TODO: Maybe support spending prior shielded outputs at some point? Doing so would require // a higher-level approach in the wallet that waits for transactions with shielded outputs to @@ -685,6 +685,7 @@ where .get(input_ref.step_index()) .ok_or(ProposalError::ReferenceError(*input_ref))?; + #[allow(unused_variables)] let output_pool = match input_ref.output_index() { StepOutputIndex::Payment(i) => prior_step.payment_pools().get(&i).cloned(), StepOutputIndex::Change(i) => match prior_step.balance().proposed_change().get(i) { @@ -696,10 +697,13 @@ where } .ok_or(ProposalError::ReferenceError(*input_ref))?; - // Return an error on trying to spend a prior shielded output. + // Return an error on trying to spend a prior output that is not supported. + #[cfg(feature = "transparent-inputs")] if output_pool != PoolType::TRANSPARENT { return Err(Error::ProposalNotSupported); } + #[cfg(not(feature = "transparent-inputs"))] + return Err(Error::ProposalNotSupported); } let account_id = wallet_db From 8636daa4f3041a94e5f6d98402f69003d4fce352 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Sat, 29 Jun 2024 12:08:43 +0100 Subject: [PATCH 40/56] Tiny simplification. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/proto.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index f90967ae85..c832fb44f6 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -649,9 +649,7 @@ impl proposal::Proposal { }) .collect::, ProposalDecodingError>>()?; - #[cfg(not(feature = "transparent-inputs"))] - let transparent_inputs = vec![]; - #[cfg(feature = "transparent-inputs")] + #[allow(unused_mut)] let mut transparent_inputs = vec![]; let mut received_notes = vec![]; let mut prior_step_inputs = vec![]; From 01ff201ffb23316a5b90f3632d1b6d767bd6a2a5 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 3 Jul 2024 15:57:12 +0100 Subject: [PATCH 41/56] Minor changes responding to review comments. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 3 ++ zcash_client_backend/src/data_api.rs | 26 +++++++---- zcash_client_backend/src/data_api/wallet.rs | 10 ++++- zcash_client_backend/src/fees/common.rs | 25 +++++++---- zcash_client_backend/src/wallet.rs | 2 + .../src/wallet/transparent/ephemeral.rs | 45 ++++++++----------- zcash_primitives/CHANGELOG.md | 1 + zcash_primitives/src/legacy/keys.rs | 19 +++++++- 8 files changed, 84 insertions(+), 47 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 7c5b19843b..b29f63126f 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -75,6 +75,9 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail not enabled). - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. +- `zcash_client_backend::proposal::ProposalError` has new variants + `SpendsChange`, `EphemeralOutputLeftUnspent`, and `PaysTexFromShielded`. + (the last two are conditional on the "transparent-inputs" feature). - `zcash_client_backend::proto::ProposalDecodingError` has a new variant `InvalidEphemeralRecipient`. - `zcash_client_backend::wallet::Recipient` variants have changed. Instead of diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 1edc2b0ffb..bb3b2d589b 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -922,7 +922,7 @@ pub trait WalletRead { } /// Returns the metadata associated with a given transparent receiver in an account - /// controlled by this wallet. + /// controlled by this wallet, if available. /// /// This is equivalent to (but may be implemented more efficiently than): /// ```compile_fail @@ -956,13 +956,19 @@ pub trait WalletRead { Ok(None) } - /// Returns the set of reserved ephemeral transparent addresses associated with the - /// given account controlled by this wallet. + /// Returns a set of ephemeral transparent addresses associated with the given + /// account controlled by this wallet, along with their metadata. /// - /// The set contains all ephemeral transparent receivers that are known to have - /// been derived under this account. Wallets should scan the chain for UTXOs sent to - /// these receivers, but do not need to do so regularly. Under expected usage, outputs - /// would only be detected with these receivers in the following situations: + /// If `for_detection` is false, the set only includes addresses reserved by + /// `reserve_next_n_ephemeral_addresses`. If `for_detection` is true, it includes + /// those addresses and also the ones that will be reserved next, for an additional + /// `GAP_LIMIT` indices (up to and including the maximum index given by + /// `NonHardenedChildIndex::from_index(i32::MAX as u32)`). + /// + /// Wallets should scan the chain for UTXOs sent to the ephemeral transparent + /// receivers obtained with `for_detection` set to `true`, but do not need to do + /// so regularly. Under expected usage, outputs would only be detected with these + /// receivers in the following situations: /// /// - This wallet created a payment to a ZIP 320 (TEX) address, but the second /// transaction (that spent the output sent to the ephemeral address) did not get @@ -1542,6 +1548,8 @@ pub trait WalletWrite: WalletRead { /// funds have been received by the currently-available account (in order to enable automated /// account recovery). /// + /// # Panics + /// /// Panics if the length of the seed is not between 32 and 252 bytes inclusive. /// /// [ZIP 316]: https://zips.z.cash/zip-0316 @@ -1634,7 +1642,9 @@ pub trait WalletWrite: WalletRead { /// the given number of addresses, or if the account identifier does not correspond /// to a known account. /// - /// Precondition: `n < 0x80000000` + /// # Panics + /// + /// Panics if the precondition `n < 0x80000000` does not hold. /// /// [BIP 44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#user-content-Address_gap_limit #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 439e8a2b19..2712608a98 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -680,6 +680,7 @@ where // TODO: Maybe support spending prior shielded outputs at some point? Doing so would require // a higher-level approach in the wallet that waits for transactions with shielded outputs to // be mined and only then attempts to perform the next step. + #[allow(clippy::never_loop)] for input_ref in proposal_step.prior_step_inputs() { let (prior_step, _) = prior_step_results .get(input_ref.step_index()) @@ -1251,13 +1252,18 @@ where #[allow(unused_variables)] let transparent_outputs = transparent_output_meta.into_iter().enumerate().map( - |(n, (recipient, ephemeral_address, value, step_output_index))| { + |(n, (recipient, address, value, step_output_index))| { + // This assumes that transparent outputs are pushed onto `transparent_output_meta` + // with the same indices they have in the transaction's transparent outputs. + // We do not reorder transparent outputs; there is no reason to do so because it + // would not usefully improve privacy. let outpoint = OutPoint::new(txid, n as u32); + let recipient = recipient.map_ephemeral_transparent_outpoint(|()| outpoint.clone()); #[cfg(feature = "transparent-inputs")] unused_transparent_outputs.insert( StepOutput::new(step_index, step_output_index), - (ephemeral_address, outpoint), + (address, outpoint), ); SentTransactionOutput::from_parts(n, recipient, value, None) }, diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index d0b3eebcac..560d8b4e6d 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -521,10 +521,17 @@ pub(crate) fn check_for_uneconomic_inputs( // Return the number of allowed dust inputs from each pool. let allowed_dust = |(t_change, s_change, o_change): &(usize, usize, usize)| { - // What is the maximum number of dust inputs in each pool, out of the ones we - // actually have, that can be economically spent along with the non-dust inputs? - // Get an initial estimate by calculating the number of dust inputs in each pool - // that will be allowed regardless of padding or grace. + // Here we assume a "ZIP 317-like" fee model in which the existence of an output + // to a given pool implies that a corresponding input from that pool can be + // provided without increasing the fee. (This is also likely to be true for + // future fee models if we do not want to penalize use of Orchard relative to + // other pools.) + // + // Under that assumption, we want to calculate the maximum number of dust inputs + // from each pool, out of the ones we actually have, that can be economically + // spent along with the non-dust inputs. Get an initial estimate by calculating + // the number of dust inputs in each pool that will be allowed regardless of + // padding or grace. let t_allowed = min( t_dust.len(), @@ -545,10 +552,12 @@ pub(crate) fn check_for_uneconomic_inputs( #[cfg(feature = "orchard")] let o_req_inputs = o_non_dust + o_allowed; - // This calculates the hypothetical number of actions with given extra inputs. - // The padding rules are subtle (they also depend on `bundle_required` for example). - // To reliably tell whether we can add an extra input of a given type, we will need - // to call them both with without that input; if the number of actions does not + // This calculates the hypothetical number of actions with given extra inputs, + // for ZIP 317 and the padding rules in effect. The padding rules for each + // pool are subtle (they also depend on `bundle_required` for example), so we + // must actually call them rather than try to predict their effect. To tell + // whether we can freely add an extra input from a given pool, we need to call + // them both with and without that input; if the number of actions does not // increase, then the input is free to add. let hypothetical_actions = |t_extra, s_extra, _o_extra| { let s_spend_count = sapling diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index 8783206140..1bc4dae3fc 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -647,6 +647,8 @@ pub struct TransparentAddressMetadata { #[cfg(feature = "transparent-inputs")] impl TransparentAddressMetadata { + /// Returns a `TransparentAddressMetadata` in the given scope for the + /// given address index. pub fn new(scope: TransparentKeyScope, address_index: NonHardenedChildIndex) -> Self { Self { scope, diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index b79532eb1b..445275c762 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -25,23 +25,10 @@ use crate::{error::SqliteClientError, wallet::get_account, AccountId, SqlTransac /// of them to be mined. This is the same as the gap limit in Bitcoin. pub(crate) const GAP_LIMIT: i32 = 20; -// The custom scope used for derivation of ephemeral addresses. -// -// This must match the constant used in -// `zcash_primitives::legacy::keys::AccountPubKey::derive_ephemeral_ivk`. -// -// TODO: consider moving this to `zcash_primitives::legacy::keys`, or else -// provide a way to derive `ivk`s for custom scopes in general there, so that -// the constant isn't duplicated. -const EPHEMERAL_SCOPE: TransparentKeyScope = match TransparentKeyScope::custom(2) { - Some(s) => s, - None => unreachable!(), -}; - // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddressMetadata { - TransparentAddressMetadata::new(EPHEMERAL_SCOPE, address_index) + TransparentAddressMetadata::new(TransparentKeyScope::EPHEMERAL, address_index) } /// Returns the last reserved ephemeral address index in the given account, @@ -61,9 +48,7 @@ pub(crate) fn last_reserved_index( ) .optional()? { - Some(i) if i < 0 => Err(SqliteClientError::CorruptedData( - "negative index".to_owned(), - )), + Some(i) if i < 0 => unreachable!("violates constraint address_index_in_range"), Some(i) => Ok(i), None => Ok(-1), } @@ -92,12 +77,10 @@ pub(crate) fn last_safe_index( ) .optional()? { - Some(i) if i < 0 => Err(SqliteClientError::CorruptedData( - "negative index".to_owned(), - )), - Some(i) => Ok(i), - None => Ok(-1), - }?; + Some(i) if i < 0 => unreachable!("violates constraint address_index_in_range"), + Some(i) => i, + None => -1, + }; Ok(u32::try_from(last_mined_index.saturating_add(GAP_LIMIT)).unwrap()) } @@ -105,7 +88,9 @@ pub(crate) fn last_safe_index( /// and is of length up to `n`. The range is truncated if necessary to end at /// the maximum valid address index, `i32::MAX`. /// -/// Precondition: `i >= -1 and n > 0` +/// # Panics +/// +/// Panics if the precondition `i >= -1 and n > 0` does not hold. pub(crate) fn range_after(i: i32, n: i32) -> RangeInclusive { assert!(i >= -1); assert!(n > 0); @@ -128,8 +113,12 @@ pub(crate) fn get_ephemeral_ivk( .derive_ephemeral_ivk()?) } -/// Returns a vector with all ephemeral transparent addresses potentially belonging to this wallet. -/// If `for_detection` is true, this includes addresses for an additional GAP_LIMIT indices. +/// Returns a mapping of ephemeral transparent addresses potentially belonging +/// to this wallet to their metadata. +/// +/// If `for_detection` is false, the result only includes reserved addresses. +/// If `for_detection` is true, it includes addresses for an additional +/// `GAP_LIMIT` indices, up to the limit. pub(crate) fn get_reserved_ephemeral_addresses( conn: &rusqlite::Connection, params: &P, @@ -172,7 +161,9 @@ pub(crate) fn get_reserved_ephemeral_addresses( /// Returns a vector with the next `n` previously unreserved ephemeral addresses for /// the given account. /// -/// Precondition: `n < 0x80000000` +/// # Panics +/// +/// Panics if the precondition `n < 0x80000000` does not hold. /// /// # Errors /// diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 6773d516e1..ddd5850c71 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -14,6 +14,7 @@ and this library adheres to Rust's notion of - `EphemeralIvk` - `AccountPubKey::derive_ephemeral_ivk` - `TransparentKeyScope::custom` is now `const`. + - `TransparentKeyScope::{EXTERNAL, INTERNAL, EPHEMERAL}` - `zcash_primitives::legacy::Script::serialized_size` - `zcash_primitives::transaction::fees::transparent`: - `InputSize` diff --git a/zcash_primitives/src/legacy/keys.rs b/zcash_primitives/src/legacy/keys.rs index e969cd7330..60131671f6 100644 --- a/zcash_primitives/src/legacy/keys.rs +++ b/zcash_primitives/src/legacy/keys.rs @@ -21,6 +21,10 @@ use super::TransparentAddress; pub struct TransparentKeyScope(u32); impl TransparentKeyScope { + /// Returns an arbitrary custom `TransparentKeyScope`. This should be used + /// with care: funds associated with keys derived under a custom scope may + /// not be recoverable if the wallet seed is restored in another wallet. + /// It is usually preferable to use standardized key scopes. pub const fn custom(i: u32) -> Option { if i < (1 << 31) { Some(TransparentKeyScope(i)) @@ -28,13 +32,24 @@ impl TransparentKeyScope { None } } + + /// The scope used to derive keys for external transparent addresses, + /// intended to be used to send funds to this wallet. + pub const EXTERNAL: Self = TransparentKeyScope(0); + + /// The scope used to derive keys for internal wallet operations, e.g. + /// change or UTXO management. + pub const INTERNAL: Self = TransparentKeyScope(1); + + /// The scope used to derive keys for ephemeral transparent addresses. + pub const EPHEMERAL: Self = TransparentKeyScope(2); } impl From for TransparentKeyScope { fn from(value: zip32::Scope) -> Self { match value { - zip32::Scope::External => TransparentKeyScope(0), - zip32::Scope::Internal => TransparentKeyScope(1), + zip32::Scope::External => TransparentKeyScope::EXTERNAL, + zip32::Scope::Internal => TransparentKeyScope::INTERNAL, } } } From b63ff5bfcdae0471a7b07f1b19903db2abd50f30 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 3 Jul 2024 17:02:47 +0100 Subject: [PATCH 42/56] Rename `get_reserved_ephemeral_addresses` to `get_known_ephemeral_addresses` and change the `TransparentAddressMetadata` in its result map to not be optional. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 2 +- zcash_client_backend/src/data_api.rs | 19 +++++++++---------- zcash_client_sqlite/src/lib.rs | 6 +++--- .../src/wallet/transparent/ephemeral.rs | 8 ++++---- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index b29f63126f..2b4de630fa 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -53,7 +53,7 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail have changed as a consequence of this extraction; please see the `zip321` CHANGELOG for details. - `zcash_client_backend::data_api`: - - `WalletRead` has new `get_reserved_ephemeral_addresses` and + - `WalletRead` has new `get_known_ephemeral_addresses` and `get_transparent_address_metadata` methods. - `WalletWrite` has a new `reserve_next_n_ephemeral_addresses` method. - `error::Error` has a new `Address` variant. diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index bb3b2d589b..19ee52badc 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -897,7 +897,7 @@ pub trait WalletRead { /// been derived under this account. Wallets should scan the chain for UTXOs sent to /// these receivers. /// - /// Use [`Self::get_reserved_ephemeral_addresses`] to obtain the ephemeral transparent + /// Use [`Self::get_known_ephemeral_addresses`] to obtain the ephemeral transparent /// receivers. #[cfg(feature = "transparent-inputs")] fn get_transparent_receivers( @@ -929,8 +929,8 @@ pub trait WalletRead { /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { /// return Ok(result.clone()); /// } - /// if let Some(result) = self.get_reserved_ephemeral_addresses(account, false)?.get(address) { - /// return Ok(result.clone()); + /// if let Some(result) = self.get_known_ephemeral_addresses(account, false)?.get(address) { + /// return Ok(Some(result.clone())); /// } /// Ok(None) /// ``` @@ -948,10 +948,10 @@ pub trait WalletRead { return Ok(result.clone()); } if let Some(result) = self - .get_reserved_ephemeral_addresses(account, false)? + .get_known_ephemeral_addresses(account, false)? .get(address) { - return Ok(result.clone()); + return Ok(Some(result.clone())); } Ok(None) } @@ -991,11 +991,11 @@ pub trait WalletRead { /// In all cases, the wallet should re-shield the unspent outputs, in a separate /// transaction per ephemeral address, before re-spending the funds. #[cfg(feature = "transparent-inputs")] - fn get_reserved_ephemeral_addresses( + fn get_known_ephemeral_addresses( &self, _account: Self::AccountId, _for_detection: bool, - ) -> Result>, Self::Error> { + ) -> Result, Self::Error> { Ok(HashMap::new()) } } @@ -1991,12 +1991,11 @@ pub mod testing { } #[cfg(feature = "transparent-inputs")] - fn get_reserved_ephemeral_addresses( + fn get_known_ephemeral_addresses( &self, _account: Self::AccountId, _for_detection: bool, - ) -> Result>, Self::Error> - { + ) -> Result, Self::Error> { Ok(HashMap::new()) } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1a31ab31ad..ad0d03597a 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -549,12 +549,12 @@ impl, P: consensus::Parameters> WalletRead for W } #[cfg(feature = "transparent-inputs")] - fn get_reserved_ephemeral_addresses( + fn get_known_ephemeral_addresses( &self, account: Self::AccountId, for_detection: bool, - ) -> Result>, Self::Error> { - wallet::transparent::ephemeral::get_reserved_ephemeral_addresses( + ) -> Result, Self::Error> { + wallet::transparent::ephemeral::get_known_ephemeral_addresses( self.conn.borrow(), &self.params, account, diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 445275c762..83fa19ee42 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -119,12 +119,12 @@ pub(crate) fn get_ephemeral_ivk( /// If `for_detection` is false, the result only includes reserved addresses. /// If `for_detection` is true, it includes addresses for an additional /// `GAP_LIMIT` indices, up to the limit. -pub(crate) fn get_reserved_ephemeral_addresses( +pub(crate) fn get_known_ephemeral_addresses( conn: &rusqlite::Connection, params: &P, account_id: AccountId, for_detection: bool, -) -> Result>, SqliteClientError> { +) -> Result, SqliteClientError> { let mut stmt = conn.prepare( "SELECT address, address_index FROM ephemeral_addresses WHERE account_id = :account ORDER BY address_index", )?; @@ -141,7 +141,7 @@ pub(crate) fn get_reserved_ephemeral_addresses( .checked_add(1); let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); let address = TransparentAddress::decode(params, &addr_str)?; - result.insert(address, Some(metadata(address_index))); + result.insert(address, metadata(address_index)); } if for_detection { @@ -151,7 +151,7 @@ pub(crate) fn get_reserved_ephemeral_addresses( for raw_index in range_after(first, GAP_LIMIT) { let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; - result.insert(address, Some(metadata(address_index))); + result.insert(address, metadata(address_index)); } } } From e97da434094495ffc510a886d79bc0e59c9b4d3b Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Wed, 3 Jul 2024 17:05:25 +0100 Subject: [PATCH 43/56] Refactoring to address review comments. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 20 +++-- zcash_client_backend/src/data_api.rs | 38 ++++++++ zcash_client_backend/src/data_api/error.rs | 9 ++ zcash_client_backend/src/data_api/wallet.rs | 13 ++- zcash_client_sqlite/src/lib.rs | 20 +++-- zcash_client_sqlite/src/testing/pool.rs | 6 +- zcash_client_sqlite/src/wallet.rs | 3 +- zcash_client_sqlite/src/wallet/transparent.rs | 21 +---- .../src/wallet/transparent/ephemeral.rs | 89 +++++++++++-------- 9 files changed, 140 insertions(+), 79 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 2b4de630fa..c5cc60cd1c 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -53,14 +53,14 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail have changed as a consequence of this extraction; please see the `zip321` CHANGELOG for details. - `zcash_client_backend::data_api`: - - `WalletRead` has new `get_known_ephemeral_addresses` and - `get_transparent_address_metadata` methods. - - `WalletWrite` has a new `reserve_next_n_ephemeral_addresses` method. - - `error::Error` has a new `Address` variant. + - `WalletRead` has new `get_known_ephemeral_addresses`, + `find_account_for_ephemeral_address`, and `get_transparent_address_metadata` + methods when the "transparent-inputs" feature is enabled. + - `WalletWrite` has a new `reserve_next_n_ephemeral_addresses` method when + the "transparent-inputs" feature is enabled. + - `error::Error` has new `Address` and (when the "transparent-inputs" feature + is enabled) `PaysEphemeralTransparentAddress` variants. - `wallet::input_selection::InputSelectorError` has a new `Address` variant. -- `zcash_client_backend::proto::proposal::Proposal::{from_standard_proposal, - try_into_standard_proposal}` each no longer require a `consensus::Parameters` - argument. - `zcash_client_backend::data_api::fees` - When the "transparent-inputs" feature is enabled, `ChangeValue` can also represent an ephemeral transparent output in a proposal. Accordingly, the @@ -78,8 +78,10 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - `zcash_client_backend::proposal::ProposalError` has new variants `SpendsChange`, `EphemeralOutputLeftUnspent`, and `PaysTexFromShielded`. (the last two are conditional on the "transparent-inputs" feature). -- `zcash_client_backend::proto::ProposalDecodingError` has a new variant - `InvalidEphemeralRecipient`. +- `zcash_client_backend::proto`: + - `ProposalDecodingError` has a new variant `InvalidEphemeralRecipient`. + - `proposal::Proposal::{from_standard_proposal, try_into_standard_proposal}` + each no longer require a `consensus::Parameters` argument. - `zcash_client_backend::wallet::Recipient` variants have changed. Instead of wrapping protocol-address types, the `External` and `InternalAccount` variants now wrap a `zcash_address::ZcashAddress`. This simplifies the process of diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 19ee52badc..3140ad6483 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -998,6 +998,36 @@ pub trait WalletRead { ) -> Result, Self::Error> { Ok(HashMap::new()) } + + /// If a given transparent address has been reserved, i.e. would be included in + /// the map returned by `get_known_ephemeral_addresses(account_id, false)` for any + /// of the wallet's accounts, then return `Ok(Some(account_id))`. Otherwise return + /// `Ok(None)`. + /// + /// This is equivalent to (but may be implemented more efficiently than): + /// ```compile_fail + /// for account_id in self.get_account_ids()? { + /// if self.get_known_ephemeral_addresses(account_id, false)?.contains_key(address)? { + /// return Ok(Some(account_id)); + /// } + /// } + /// Ok(None) + /// ``` + #[cfg(feature = "transparent-inputs")] + fn find_account_for_ephemeral_address( + &self, + address: &TransparentAddress, + ) -> Result, Self::Error> { + for account_id in self.get_account_ids()? { + if self + .get_known_ephemeral_addresses(account_id, false)? + .contains_key(address) + { + return Ok(Some(account_id)); + } + } + Ok(None) + } } /// The relevance of a seed to a given wallet. @@ -1998,6 +2028,14 @@ pub mod testing { ) -> Result, Self::Error> { Ok(HashMap::new()) } + + #[cfg(feature = "transparent-inputs")] + fn find_account_for_ephemeral_address( + &self, + _address: &TransparentAddress, + ) -> Result, Self::Error> { + Ok(None) + } } impl WalletWrite for MockWalletDb { diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 371d57706e..acf451a3e4 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -92,6 +92,11 @@ pub enum Error { /// belonging to the wallet. #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), + + /// The wallet tried to pay to an ephemeral transparent address as a normal + /// output. + #[cfg(feature = "transparent-inputs")] + PaysEphemeralTransparentAddress(String), } impl fmt::Display for Error @@ -161,6 +166,10 @@ where Error::AddressNotRecognized(_) => { write!(f, "The specified transparent address was not recognized as belonging to the wallet.") } + #[cfg(feature = "transparent-inputs")] + Error::PaysEphemeralTransparentAddress(addr) => { + write!(f, "The wallet tried to pay to an ephemeral transparent address as a normal output: {}", addr) + } } } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 2712608a98..f2332405cb 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -77,6 +77,7 @@ use { core::convert::Infallible, input_selection::ShieldingSelector, std::collections::HashMap, + zcash_keys::encoding::AddressCodec, zcash_primitives::transaction::components::TxOut, }; @@ -1017,11 +1018,19 @@ where transparent_output_meta: &mut Vec<_>, to: TransparentAddress| -> Result<(), ErrorT> { + // Always reject sending to one of our known ephemeral addresses. + #[cfg(feature = "transparent-inputs")] + if wallet_db + .find_account_for_ephemeral_address(&to) + .map_err(Error::DataSource)? + .is_some() + { + return Err(Error::PaysEphemeralTransparentAddress(to.encode(params))); + } if payment.memo().is_some() { return Err(Error::MemoForbidden); - } else { - builder.add_transparent_output(&to, payment.amount())?; } + builder.add_transparent_output(&to, payment.amount())?; transparent_output_meta.push(( Recipient::External(recipient_address.clone(), PoolType::TRANSPARENT), to, diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index ad0d03597a..8b73b77810 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -88,6 +88,7 @@ use { #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::wallet::TransparentAddressMetadata, + zcash_keys::encoding::AddressCodec, zcash_primitives::{ legacy::TransparentAddress, transaction::components::{OutPoint, TxOut}, @@ -561,6 +562,17 @@ impl, P: consensus::Parameters> WalletRead for W for_detection, ) } + + #[cfg(feature = "transparent-inputs")] + fn find_account_for_ephemeral_address( + &self, + address: &TransparentAddress, + ) -> Result, Self::Error> { + wallet::transparent::ephemeral::find_account_for_ephemeral_address_str( + self.conn.borrow(), + &address.encode(&self.params), + ) + } } impl WalletWrite for WalletDb { @@ -1470,14 +1482,6 @@ impl WalletWrite for WalletDb tx_ref, )?; } - #[cfg(feature = "transparent-inputs")] - Recipient::External(zcash_address, PoolType::Transparent) => { - // Always reject sending to one of our ephemeral addresses. - wallet::transparent::ephemeral::check_address_is_not_ephemeral( - wdb, - &zcash_address.encode(), - )?; - } _ => {} } } diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index bf21663b95..ac4683ff5a 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -443,7 +443,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { }; // Each transfer should use a different ephemeral address. - let (ephemeral0, txid0) = run_test(&mut st, 0); + let (ephemeral0, _) = run_test(&mut st, 0); let (ephemeral1, _) = run_test(&mut st, 1); assert!(ephemeral0 != ephemeral1); @@ -455,7 +455,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { Address::Transparent(TransparentAddress::PublicKeyHash(_)) ); - // Attempting to use the same address again should cause an error. + // Attempting to pay to an ephemeral address should cause an error. let proposal = st .propose_standard_transfer::( account.account_id(), @@ -476,7 +476,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { ); assert_matches!( &create_proposed_result, - Err(Error::DataSource(SqliteClientError::EphemeralAddressReuse(addr, Some(txid)))) if addr == &ephemeral0 && txid == &txid0); + Err(Error::PaysEphemeralTransparentAddress(address_str)) if address_str == &ephemeral0); } #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index a1f9b15cfa..0559bb0866 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -68,7 +68,6 @@ use incrementalmerkletree::{Marking, Retention}; use rusqlite::{self, named_params, OptionalExtension}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; -use zcash_keys::encoding::encode_transparent_address_p; use zip32::fingerprint::SeedFingerprint; use std::collections::{HashMap, HashSet}; @@ -2148,7 +2147,7 @@ fn recipient_params( ephemeral_address, .. } => ( - Some(encode_transparent_address_p(params, ephemeral_address)), + Some(ephemeral_address.encode(params)), Some(*receiving_account), PoolType::TRANSPARENT, ), diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index c52d4ec986..d4476ff878 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -3,7 +3,6 @@ use std::collections::{HashMap, HashSet}; use rusqlite::OptionalExtension; use rusqlite::{named_params, Connection, Row}; -use zcash_keys::encoding::encode_transparent_address_p; use zip32::{DiversifierIndex, Scope}; use zcash_address::unified::{Encoding, Ivk, Uivk}; @@ -468,7 +467,7 @@ pub(crate) fn get_transparent_address_metadata( account_id: AccountId, address: &TransparentAddress, ) -> Result, SqliteClientError> { - let address_str = encode_transparent_address_p(params, address); + let address_str = address.encode(params); if let Some(di_vec) = conn .query_row( @@ -494,14 +493,8 @@ pub(crate) fn get_transparent_address_metadata( } // Search ephemeral addresses that have already been reserved. - if let Some(raw_index) = conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id AND address = :address", - named_params![":account_id": account_id.0, ":address": &address_str], - |row| row.get::<_, u32>(0), - ) - .optional()? + if let Some(raw_index) = + ephemeral::find_index_for_ephemeral_address_str(conn, account_id, &address_str)? { let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); return Ok(Some(ephemeral::metadata(address_index))); @@ -542,13 +535,7 @@ pub(crate) fn find_account_for_transparent_output( } // Search ephemeral addresses that have already been reserved. - if let Some(account_id) = conn - .query_row( - "SELECT account_id FROM ephemeral_addresses WHERE address = :address", - named_params![":address": &address_str], - |row| Ok(AccountId(row.get(0)?)), - ) - .optional()? + if let Some(account_id) = ephemeral::find_account_for_ephemeral_address_str(conn, &address_str)? { return Ok(Some(account_id)); } diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 83fa19ee42..6d5af23350 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -6,10 +6,7 @@ use std::ops::RangeInclusive; use rusqlite::{named_params, OptionalExtension}; use zcash_client_backend::{data_api::Account, wallet::TransparentAddressMetadata}; -use zcash_keys::{ - encoding::{encode_transparent_address_p, AddressCodec}, - keys::AddressGenerationError, -}; +use zcash_keys::{encoding::AddressCodec, keys::AddressGenerationError}; use zcash_primitives::{ legacy::{ keys::{EphemeralIvk, NonHardenedChildIndex, TransparentKeyScope}, @@ -158,6 +155,37 @@ pub(crate) fn get_known_ephemeral_addresses( Ok(result) } +/// If this is an ephemeral address in any account, return its account id. +pub(crate) fn find_account_for_ephemeral_address_str( + conn: &rusqlite::Connection, + address_str: &str, +) -> Result, SqliteClientError> { + // Search ephemeral addresses that have already been reserved. + Ok(conn + .query_row( + "SELECT account_id FROM ephemeral_addresses WHERE address = :address", + named_params![":address": &address_str], + |row| Ok(AccountId(row.get(0)?)), + ) + .optional()?) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn find_index_for_ephemeral_address_str( + conn: &rusqlite::Connection, + account_id: AccountId, + address_str: &str, +) -> Result, SqliteClientError> { + Ok(conn + .query_row( + "SELECT address_index FROM ephemeral_addresses + WHERE account_id = :account_id AND address = :address", + named_params![":account_id": account_id.0, ":address": &address_str], + |row| row.get::<_, u32>(0), + ) + .optional()?) +} + /// Returns a vector with the next `n` previously unreserved ephemeral addresses for /// the given account. /// @@ -215,29 +243,18 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( stmt_insert_ephemeral_address.execute(named_params![ ":account_id": account_id.0, ":address_index": raw_index, - ":address": encode_transparent_address_p(&wdb.params, &address) + ":address": address.encode(&wdb.params), ])?; Ok((address, metadata(address_index))) }) .collect() } -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if `address` is -/// an ephemeral transparent address. -pub(crate) fn check_address_is_not_ephemeral( - wdb: &mut WalletDb, P>, - address_str: &str, -) -> Result<(), SqliteClientError> { - ephemeral_address_check_internal(wdb, address_str, true) -} - /// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. If `reject_all_ephemeral` is set, return an error if the address -/// is ephemeral at all, regardless of reuse. -fn ephemeral_address_check_internal( +/// already used. +fn ephemeral_address_reuse_check( wdb: &mut WalletDb, P>, address_str: &str, - reject_all_ephemeral: bool, ) -> Result<(), SqliteClientError> { // It is intentional that we don't require `t.mined_height` to be non-null. // That is, we conservatively treat an ephemeral address as potentially @@ -263,25 +280,21 @@ fn ephemeral_address_check_internal( named_params![":address": address_str], |row| row.get::<_, Option>>(0), ) - .optional()?; + .optional()? + .flatten(); - match res { - Some(Some(txid_bytes)) => { - let txid = TxId::from_bytes( - txid_bytes - .try_into() - .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, - ); - Err(SqliteClientError::EphemeralAddressReuse( - address_str.to_owned(), - Some(txid), - )) - } - Some(None) if reject_all_ephemeral => Err(SqliteClientError::EphemeralAddressReuse( + if let Some(txid_bytes) = res { + let txid = TxId::from_bytes( + txid_bytes + .try_into() + .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, + ); + Err(SqliteClientError::EphemeralAddressReuse( address_str.to_owned(), - None, - )), - _ => Ok(()), + Some(txid), + )) + } else { + Ok(()) } } @@ -296,8 +309,8 @@ pub(crate) fn mark_ephemeral_address_as_used( ephemeral_address: &TransparentAddress, tx_ref: i64, ) -> Result<(), SqliteClientError> { - let address_str = encode_transparent_address_p(&wdb.params, ephemeral_address); - ephemeral_address_check_internal(wdb, &address_str, false)?; + let address_str = ephemeral_address.encode(&wdb.params); + ephemeral_address_reuse_check(wdb, &address_str)?; wdb.conn.0.execute( "UPDATE ephemeral_addresses SET used_in_tx = :used_in_tx WHERE address = :address", @@ -316,7 +329,7 @@ pub(crate) fn mark_ephemeral_address_as_mined( address: &TransparentAddress, tx_ref: i64, ) -> Result<(), SqliteClientError> { - let address_str = encode_transparent_address_p(&wdb.params, address); + let address_str = address.encode(&wdb.params); // Figure out which transaction was mined earlier: `tx_ref`, or any existing // tx referenced by `mined_in_tx` for the given address. Prefer the existing From a01588bc655fd0af9767e0e42818bbec0ac4ce0f Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 05:22:18 +0100 Subject: [PATCH 44/56] Ensure that `mark_ephemeral_address_as_mined` correctly handles indices within the gap. Also support paging for `get_known_ephemeral_addresses`. Co-authored-by: Jack Grigg Signed-off-by: Daira-Emma Hopwood --- Cargo.lock | 1 + Cargo.toml | 3 +- zcash_client_backend/src/data_api.rs | 81 +++--- zcash_client_backend/src/data_api/wallet.rs | 13 +- zcash_client_sqlite/Cargo.toml | 3 + zcash_client_sqlite/src/lib.rs | 8 +- zcash_client_sqlite/src/testing/pool.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 12 +- zcash_client_sqlite/src/wallet/db.rs | 38 ++- .../src/wallet/init/migrations.rs | 4 +- .../init/migrations/ephemeral_addresses.rs | 143 +++++++++- zcash_client_sqlite/src/wallet/transparent.rs | 21 +- .../src/wallet/transparent/ephemeral.rs | 253 +++++++++++------- 13 files changed, 395 insertions(+), 187 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d645f9b9a..bc53be191b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3121,6 +3121,7 @@ dependencies = [ "schemer-rusqlite", "secrecy", "shardtree", + "static_assertions", "subtle", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index fe77eb43a8..b3769877a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,8 +103,9 @@ tonic-build = { version = "0.11", default-features = false } secrecy = "0.8" subtle = "2.2.3" -# Static constants +# Static constants and assertions lazy_static = "1" +static_assertions = "1" # Tests and benchmarks assert_matches = "1.5" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 3140ad6483..1fdf3e633e 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -97,7 +97,10 @@ use zcash_primitives::{ }; #[cfg(feature = "transparent-inputs")] -use {crate::wallet::TransparentAddressMetadata, zcash_primitives::legacy::TransparentAddress}; +use { + crate::wallet::TransparentAddressMetadata, std::ops::Range, + zcash_primitives::legacy::TransparentAddress, +}; #[cfg(any(test, feature = "test-dependencies"))] use zcash_primitives::consensus::NetworkUpgrade; @@ -929,10 +932,11 @@ pub trait WalletRead { /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { /// return Ok(result.clone()); /// } - /// if let Some(result) = self.get_known_ephemeral_addresses(account, false)?.get(address) { - /// return Ok(Some(result.clone())); - /// } - /// Ok(None) + /// Ok(self + /// .get_known_ephemeral_addresses(account, None)? + /// .into_iter() + /// .find(|(known_addr, _)| known_addr == address) + /// .map(|(_, metadata)| metadata)) /// ``` /// /// Returns `Ok(None)` if the address is not recognized, or we do not have metadata for it. @@ -947,28 +951,24 @@ pub trait WalletRead { if let Some(result) = self.get_transparent_receivers(account)?.get(address) { return Ok(result.clone()); } - if let Some(result) = self - .get_known_ephemeral_addresses(account, false)? - .get(address) - { - return Ok(Some(result.clone())); - } - Ok(None) + Ok(self + .get_known_ephemeral_addresses(account, None)? + .into_iter() + .find(|(known_addr, _)| known_addr == address) + .map(|(_, metadata)| metadata)) } - /// Returns a set of ephemeral transparent addresses associated with the given - /// account controlled by this wallet, along with their metadata. + /// Returns a vector of ephemeral transparent addresses associated with the given + /// account controlled by this wallet, along with their metadata. The result includes + /// reserved addresses, and addresses for `GAP_LIMIT` additional indices (capped to + /// the maximum index). /// - /// If `for_detection` is false, the set only includes addresses reserved by - /// `reserve_next_n_ephemeral_addresses`. If `for_detection` is true, it includes - /// those addresses and also the ones that will be reserved next, for an additional - /// `GAP_LIMIT` indices (up to and including the maximum index given by - /// `NonHardenedChildIndex::from_index(i32::MAX as u32)`). + /// If `index_range` is some `Range`, it limits the result to addresses with indices + /// in that range. /// - /// Wallets should scan the chain for UTXOs sent to the ephemeral transparent - /// receivers obtained with `for_detection` set to `true`, but do not need to do - /// so regularly. Under expected usage, outputs would only be detected with these - /// receivers in the following situations: + /// Wallets should scan the chain for UTXOs sent to these ephemeral transparent + /// receivers, but do not need to do so regularly. Under expected usage, outputs + /// would only be detected with these receivers in the following situations: /// /// - This wallet created a payment to a ZIP 320 (TEX) address, but the second /// transaction (that spent the output sent to the ephemeral address) did not get @@ -994,9 +994,9 @@ pub trait WalletRead { fn get_known_ephemeral_addresses( &self, _account: Self::AccountId, - _for_detection: bool, - ) -> Result, Self::Error> { - Ok(HashMap::new()) + _index_range: Option>, + ) -> Result, Self::Error> { + Ok(vec![]) } /// If a given transparent address has been reserved, i.e. would be included in @@ -1007,7 +1007,11 @@ pub trait WalletRead { /// This is equivalent to (but may be implemented more efficiently than): /// ```compile_fail /// for account_id in self.get_account_ids()? { - /// if self.get_known_ephemeral_addresses(account_id, false)?.contains_key(address)? { + /// if self + /// .get_known_ephemeral_addresses(account_id, None)? + /// .into_iter() + /// .any(|(known_addr, _)| &known_addr == address) + /// { /// return Ok(Some(account_id)); /// } /// } @@ -1020,8 +1024,9 @@ pub trait WalletRead { ) -> Result, Self::Error> { for account_id in self.get_account_ids()? { if self - .get_known_ephemeral_addresses(account_id, false)? - .contains_key(address) + .get_known_ephemeral_addresses(account_id, None)? + .into_iter() + .any(|(known_addr, _)| &known_addr == address) { return Ok(Some(account_id)); } @@ -1672,10 +1677,6 @@ pub trait WalletWrite: WalletRead { /// the given number of addresses, or if the account identifier does not correspond /// to a known account. /// - /// # Panics - /// - /// Panics if the precondition `n < 0x80000000` does not hold. - /// /// [BIP 44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#user-content-Address_gap_limit #[cfg(feature = "transparent-inputs")] fn reserve_next_n_ephemeral_addresses( @@ -1787,7 +1788,10 @@ pub mod testing { }; #[cfg(feature = "transparent-inputs")] - use {crate::wallet::TransparentAddressMetadata, zcash_primitives::legacy::TransparentAddress}; + use { + crate::wallet::TransparentAddressMetadata, std::ops::Range, + zcash_primitives::legacy::TransparentAddress, + }; #[cfg(feature = "orchard")] use super::ORCHARD_SHARD_HEIGHT; @@ -2024,9 +2028,9 @@ pub mod testing { fn get_known_ephemeral_addresses( &self, _account: Self::AccountId, - _for_detection: bool, - ) -> Result, Self::Error> { - Ok(HashMap::new()) + _index_range: Option>, + ) -> Result, Self::Error> { + Ok(vec![]) } #[cfg(feature = "transparent-inputs")] @@ -2103,9 +2107,8 @@ pub mod testing { fn reserve_next_n_ephemeral_addresses( &mut self, _account_id: Self::AccountId, - n: u32, + _n: u32, ) -> Result, Self::Error> { - assert!(n < 0x80000000); Err(()) } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index f2332405cb..0b63c601da 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1151,14 +1151,13 @@ where change_value.is_ephemeral() && change_value.output_pool() == PoolType::Transparent }) .collect(); - if ephemeral_outputs.len() > i32::MAX as usize { - return Err(Error::ProposalNotSupported); - } + + let n = ephemeral_outputs + .len() + .try_into() + .map_err(|_| Error::ProposalNotSupported)?; let addresses_and_metadata = wallet_db - .reserve_next_n_ephemeral_addresses( - account_id, - ephemeral_outputs.len().try_into().unwrap(), - ) + .reserve_next_n_ephemeral_addresses(account_id, n) .map_err(Error::DataSource)?; assert_eq!(addresses_and_metadata.len(), ephemeral_outputs.len()); diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index a1903acb4a..1521d3a0e2 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -54,6 +54,9 @@ jubjub.workspace = true secrecy.workspace = true subtle.workspace = true +# - Static assertions +static_assertions.workspace = true + # - Shielded protocols orchard = { workspace = true, optional = true } sapling.workspace = true diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 8b73b77810..74563c3c67 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -302,7 +302,7 @@ impl, P: consensus::Parameters> WalletRead for W type Account = wallet::Account; fn get_account_ids(&self) -> Result, Self::Error> { - wallet::get_account_ids(self.conn.borrow()) + Ok(wallet::get_account_ids(self.conn.borrow())?) } fn get_account( @@ -553,13 +553,13 @@ impl, P: consensus::Parameters> WalletRead for W fn get_known_ephemeral_addresses( &self, account: Self::AccountId, - for_detection: bool, - ) -> Result, Self::Error> { + index_range: Option>, + ) -> Result, Self::Error> { wallet::transparent::ephemeral::get_known_ephemeral_addresses( self.conn.borrow(), &self.params, account, - for_detection, + index_range, ) } diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index ac4683ff5a..da6caf2722 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -445,7 +445,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { // Each transfer should use a different ephemeral address. let (ephemeral0, _) = run_test(&mut st, 0); let (ephemeral1, _) = run_test(&mut st, 1); - assert!(ephemeral0 != ephemeral1); + assert_ne!(ephemeral0, ephemeral1); add_funds(&mut st, value); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 0559bb0866..b7d0b9230f 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -133,6 +133,10 @@ pub(crate) mod transparent; pub(crate) const BLOCK_SAPLING_FRONTIER_ABSENT: &[u8] = &[0x0]; +/// The number of ephemeral addresses that can be safely reserved without observing any +/// of them to be mined. This is the same as the gap limit in Bitcoin. +pub(crate) const GAP_LIMIT: u32 = 20; + fn parse_account_source( account_kind: u32, hd_seed_fingerprint: Option<[u8; 32]>, @@ -509,6 +513,10 @@ pub(crate) fn add_account( let (address, d_idx) = account.default_address(DEFAULT_UA_REQUEST)?; insert_address(conn, params, account_id, d_idx, &address)?; + // Initialize the `ephemeral_addresses` table. + #[cfg(feature = "transparent-inputs")] + transparent::ephemeral::init_account(conn, params, account_id)?; + Ok(account_id) } @@ -1914,9 +1922,11 @@ pub(crate) fn truncate_to_height( } /// Returns a vector with the IDs of all accounts known to this wallet. +/// +/// Note that this is called from db migration code. pub(crate) fn get_account_ids( conn: &rusqlite::Connection, -) -> Result, SqliteClientError> { +) -> Result, rusqlite::Error> { let mut stmt = conn.prepare("SELECT id FROM accounts")?; let mut rows = stmt.query([])?; let mut result = Vec::new(); diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 9dc3b1a0bc..f4e9203c6f 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -13,10 +13,12 @@ // from showing up in `cargo doc --document-private-items`. #![allow(dead_code)] +use static_assertions::const_assert_eq; + use zcash_client_backend::data_api::scanning::ScanPriority; use zcash_protocol::consensus::{NetworkUpgrade, Parameters}; -use crate::wallet::scanning::priority_code; +use crate::wallet::{scanning::priority_code, GAP_LIMIT}; /// Stores information about the accounts that the wallet is tracking. pub(super) const TABLE_ACCOUNTS: &str = r#" @@ -76,12 +78,17 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( "account_id" ASC )"#; -/// Stores ephemeral transparent addresses used for ZIP 320. For each account, these addresses are -/// allocated sequentially by address index under custom scope 2 at the "change" level of the BIP 32 -/// address hierarchy. Only "reserved" ephemeral addresses, that is addresses that have been allocated -/// for use in a ZIP 320 transaction proposal, are stored in the table. Addresses are never removed. -/// New ones should only be reserved via the `WalletWrite::reserve_next_n_ephemeral_addresses` API. -/// All of the addresses in the table should be scanned for incoming funds. +/// Stores ephemeral transparent addresses used for ZIP 320. +/// +/// For each account, these addresses are allocated sequentially by address index under scope 2 +/// (`TransparentKeyScope::EPHEMERAL`) at the "change" level of the BIP 32 address hierarchy. +/// The ephemeral addresses stored in the table are exactly the "reserved" ephemeral addresses +/// (that is addresses that have been allocated for use in a ZIP 320 transaction proposal), plus +/// the addresses at the next `GAP_LIMIT` indices. +/// +/// Addresses are never removed. New ones should only be reserved via the +/// `WalletWrite::reserve_next_n_ephemeral_addresses` API. All of the addresses in the table +/// should be scanned for incoming funds. /// /// ### Columns /// - `address` contains the string (Base58Check) encoding of a transparent P2PKH address. @@ -96,22 +103,35 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( /// a debugging aid (although the latter allows us to account for whether the referenced transaction /// is unmined). We only really care which addresses have been used, and whether we can allocate a /// new address within the gap limit. +/// +/// It is an external invariant that within each account: +/// - the address indices are contiguous and start from 0; +/// - the last `GAP_LIMIT` addresses have `used_in_tx` and `mined_in_tx` both NULL. +/// +/// All but the last `GAP_LIMIT` addresses are defined to be "reserved" addresses. Since the next +/// index to reserve is determined by dead reckoning from the last stored address, we use dummy +/// entries after the maximum valid index in order to allow the last `GAP_LIMIT` addresses at the +/// end of the index range to be used. pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" CREATE TABLE ephemeral_addresses ( account_id INTEGER NOT NULL, address_index INTEGER NOT NULL, - address TEXT NOT NULL, + address TEXT, used_in_tx INTEGER, mined_in_tx INTEGER, FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), PRIMARY KEY (account_id, address_index), - CONSTRAINT address_index_in_range CHECK (address_index >= 0 AND address_index <= 0x7FFFFFFF) + CONSTRAINT index_range_and_address_nullity CHECK ( + (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR + (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND mined_in_tx IS NULL) + ) ) WITHOUT ROWID"#; // Hexadecimal integer literals were added in SQLite version 3.8.6 (2014-08-15). // libsqlite3-sys requires at least version 3.14.0. // "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. +const_assert_eq!(GAP_LIMIT, 20); pub(super) const INDEX_EPHEMERAL_ADDRESSES_ADDRESS: &str = r#" CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( address ASC diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index f8c2cf7b7f..3e6b056b9e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -119,7 +119,9 @@ pub(super) fn all_migrations( params: params.clone(), }), Box::new(utxos_to_txos::Migration), - Box::new(ephemeral_addresses::Migration), + Box::new(ephemeral_addresses::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 886f9bd871..25f9c26280 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -5,22 +5,31 @@ use rusqlite; use schemer; use schemer_rusqlite::RusqliteMigration; use uuid::Uuid; +use zcash_protocol::consensus; use crate::wallet::init::WalletMigrationError; +#[cfg(feature = "transparent-inputs")] +use crate::wallet::{self, init, transparent::ephemeral}; + use super::utxos_to_txos; pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0e1d4274_1f8e_44e2_909d_689a4bc2967b); -pub(super) struct Migration; +const DEPENDENCIES: [Uuid; 1] = [utxos_to_txos::MIGRATION_ID]; + +#[allow(dead_code)] +pub(super) struct Migration

{ + pub(super) params: P, +} -impl schemer::Migration for Migration { +impl

schemer::Migration for Migration

{ fn id(&self) -> Uuid { MIGRATION_ID } fn dependencies(&self) -> HashSet { - [utxos_to_txos::MIGRATION_ID].into_iter().collect() + DEPENDENCIES.into_iter().collect() } fn description(&self) -> &'static str { @@ -28,7 +37,7 @@ impl schemer::Migration for Migration { } } -impl RusqliteMigration for Migration { +impl RusqliteMigration for Migration

{ type Error = WalletMigrationError; fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { @@ -36,19 +45,30 @@ impl RusqliteMigration for Migration { "CREATE TABLE ephemeral_addresses ( account_id INTEGER NOT NULL, address_index INTEGER NOT NULL, - address TEXT NOT NULL, + address TEXT, used_in_tx INTEGER, mined_in_tx INTEGER, FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), PRIMARY KEY (account_id, address_index), - CONSTRAINT address_index_in_range CHECK (address_index >= 0 AND address_index <= 0x7FFFFFFF) + CONSTRAINT index_range_and_address_nullity CHECK ( + (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR + (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND mined_in_tx IS NULL) + ) ) WITHOUT ROWID; CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( address ASC );", )?; + + // Make sure that at least `GAP_LIMIT` ephemeral transparent addresses are + // stored in each account. + #[cfg(feature = "transparent-inputs")] + for account_id in wallet::get_account_ids(transaction)? { + ephemeral::init_account(transaction, &self.params, account_id) + .map_err(init::sqlite_client_error_to_wallet_migration_error)?; + } Ok(()) } @@ -65,4 +85,115 @@ mod tests { fn migrate() { test_migrate(&[super::MIGRATION_ID]); } + + #[test] + #[cfg(feature = "transparent-inputs")] + fn initialize_table() { + use rusqlite::named_params; + use secrecy::Secret; + use tempfile::NamedTempFile; + use zcash_client_backend::{ + data_api::{AccountBirthday, AccountSource, WalletWrite}, + wallet::TransparentAddressMetadata, + }; + use zcash_keys::keys::UnifiedSpendingKey; + use zcash_primitives::{block::BlockHash, legacy::keys::NonHardenedChildIndex}; + use zcash_protocol::consensus::Network; + use zip32::{fingerprint::SeedFingerprint, AccountId as Zip32AccountId}; + + use crate::{ + error::SqliteClientError, + wallet::{ + account_kind_code, init::init_wallet_db_internal, transparent::ephemeral, GAP_LIMIT, + }, + WalletDb, + }; + + let network = Network::TestNetwork; + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), network).unwrap(); + + let seed0 = vec![0x00; 32]; + init_wallet_db_internal( + &mut db_data, + Some(Secret::new(seed0.clone())), + &super::DEPENDENCIES, + false, + ) + .unwrap(); + + let birthday = AccountBirthday::from_sapling_activation(&network, BlockHash([0; 32])); + + // Simulate creating an account prior to this migration. + let account0_index = Zip32AccountId::ZERO; + let account0_seed_fp = [0u8; 32]; + let account0_kind = account_kind_code(AccountSource::Derived { + seed_fingerprint: SeedFingerprint::from_seed(&account0_seed_fp).unwrap(), + account_index: account0_index, + }); + assert_eq!(u32::from(account0_index), 0); + let account0_id = crate::AccountId(0); + + let usk0 = UnifiedSpendingKey::from_seed(&network, &seed0, account0_index).unwrap(); + let ufvk0 = usk0.to_unified_full_viewing_key(); + let uivk0 = ufvk0.to_unified_incoming_viewing_key(); + + db_data + .conn + .execute( + "INSERT INTO accounts (id, account_kind, hd_seed_fingerprint, hd_account_index, ufvk, uivk, birthday_height) + VALUES (:id, :account_kind, :hd_seed_fingerprint, :hd_account_index, :ufvk, :uivk, :birthday_height)", + named_params![ + ":id": account0_id.0, + ":account_kind": account0_kind, + ":hd_seed_fingerprint": account0_seed_fp, + ":hd_account_index": u32::from(account0_index), + ":ufvk": ufvk0.encode(&network), + ":uivk": uivk0.encode(&network), + ":birthday_height": u32::from(birthday.height()), + ], + ) + .unwrap(); + + // The `ephemeral_addresses` table is expected not to exist before migration. + assert_matches!( + ephemeral::first_unstored_index(&db_data.conn, account0_id), + Err(SqliteClientError::DbError(_)) + ); + + let check = |db: &WalletDb<_, _>, account_id| { + eprintln!("checking {account_id:?}"); + assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); + assert_matches!(ephemeral::first_unreserved_index(&db.conn, account_id), Ok(addr_index) if addr_index == 0); + + let known_addrs = + ephemeral::get_known_ephemeral_addresses(&db.conn, &db.params, account_id, None) + .unwrap(); + + let expected_metadata: Vec = (0..GAP_LIMIT) + .map(|i| ephemeral::metadata(NonHardenedChildIndex::from_index(i).unwrap())) + .collect(); + let actual_metadata: Vec = + known_addrs.into_iter().map(|(_, meta)| meta).collect(); + assert_eq!(actual_metadata, expected_metadata); + }; + + // The migration should initialize `ephemeral_addresses`. + init_wallet_db_internal( + &mut db_data, + Some(Secret::new(seed0)), + &[super::MIGRATION_ID], + false, + ) + .unwrap(); + check(&db_data, account0_id); + + // Creating a new account should initialize `ephemeral_addresses` for that account. + let seed1 = vec![0x01; 32]; + let (account1_id, _usk) = db_data + .create_account(&Secret::new(seed1), &birthday) + .unwrap(); + assert_ne!(account0_id, account1_id); + check(&db_data, account1_id); + } } diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index d4476ff878..d45e32f853 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -492,7 +492,7 @@ pub(crate) fn get_transparent_address_metadata( } } - // Search ephemeral addresses that have already been reserved. + // Search known ephemeral addresses. if let Some(raw_index) = ephemeral::find_index_for_ephemeral_address_str(conn, account_id, &address_str)? { @@ -500,9 +500,6 @@ pub(crate) fn get_transparent_address_metadata( return Ok(Some(ephemeral::metadata(address_index))); } - // We intentionally don't check for unreserved ephemeral addresses within the gap - // limit here. It's unnecessary to look up metadata for addresses from which we - // can spend. Ok(None) } @@ -534,7 +531,7 @@ pub(crate) fn find_account_for_transparent_output( return Ok(Some(account_id)); } - // Search ephemeral addresses that have already been reserved. + // Search known ephemeral addresses. if let Some(account_id) = ephemeral::find_account_for_ephemeral_address_str(conn, &address_str)? { return Ok(Some(account_id)); @@ -556,20 +553,6 @@ pub(crate) fn find_account_for_transparent_output( } } - // Finally we check for ephemeral addresses within the gap limit. - for account_id in account_ids { - let ephemeral_ivk = ephemeral::get_ephemeral_ivk(conn, params, account_id)?; - let last_reserved_index = ephemeral::last_reserved_index(conn, account_id)?; - - for raw_index in ephemeral::range_after(last_reserved_index, ephemeral::GAP_LIMIT) { - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); - - if &ephemeral_ivk.derive_ephemeral_address(address_index)? == output.recipient_address() - { - return Ok(Some(account_id)); - } - } - } Ok(None) } diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 6d5af23350..06d5f314b4 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -1,7 +1,6 @@ //! Functions for wallet support of ephemeral transparent addresses. -use std::cmp::max; -use std::collections::HashMap; -use std::ops::RangeInclusive; +use std::cmp::{max, min}; +use std::ops::Range; use rusqlite::{named_params, OptionalExtension}; @@ -16,11 +15,11 @@ use zcash_primitives::{ }; use zcash_protocol::consensus; -use crate::{error::SqliteClientError, wallet::get_account, AccountId, SqlTransaction, WalletDb}; - -/// The number of ephemeral addresses that can be safely reserved without observing any -/// of them to be mined. This is the same as the gap limit in Bitcoin. -pub(crate) const GAP_LIMIT: i32 = 20; +use crate::{ + error::SqliteClientError, + wallet::{get_account, GAP_LIMIT}, + AccountId, SqlTransaction, WalletDb, +}; // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. @@ -28,12 +27,11 @@ pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddre TransparentAddressMetadata::new(TransparentKeyScope::EPHEMERAL, address_index) } -/// Returns the last reserved ephemeral address index in the given account, -/// or -1 if the account has no reserved ephemeral addresses. -pub(crate) fn last_reserved_index( +/// Returns the first unstored ephemeral address index in the given account. +pub(crate) fn first_unstored_index( conn: &rusqlite::Connection, account_id: AccountId, -) -> Result { +) -> Result { match conn .query_row( "SELECT address_index FROM ephemeral_addresses @@ -41,19 +39,33 @@ pub(crate) fn last_reserved_index( ORDER BY address_index DESC LIMIT 1", named_params![":account_id": account_id.0], - |row| row.get::<_, i32>(0), + |row| row.get::<_, u32>(0), ) .optional()? { - Some(i) if i < 0 => unreachable!("violates constraint address_index_in_range"), - Some(i) => Ok(i), - None => Ok(-1), + Some(i) if i >= (1 << 31) + GAP_LIMIT => { + unreachable!("violates constraint index_range_and_address_nullity") + } + Some(i) => Ok(i.checked_add(1).unwrap()), + None => Ok(0), } } -/// Returns the last ephemeral address index in the given account that -/// would not violate the gap invariant if used. -pub(crate) fn last_safe_index( +/// Returns the first unreserved ephemeral address index in the given account. +pub(crate) fn first_unreserved_index( + conn: &rusqlite::Connection, + account_id: AccountId, +) -> Result { + first_unstored_index(conn, account_id)? + .checked_sub(GAP_LIMIT) + .ok_or(SqliteClientError::CorruptedData( + "ephemeral_addresses table has not been initialized".to_owned(), + )) +} + +/// Returns the first ephemeral address index in the given account that +/// would violate the gap invariant if used. +pub(crate) fn first_unsafe_index( conn: &rusqlite::Connection, account_id: AccountId, ) -> Result { @@ -62,7 +74,7 @@ pub(crate) fn last_safe_index( // to have been mined in a transaction that we currently see as unmined. // This is conservative in terms of avoiding violation of the gap // invariant: it can only cause us to get to the end of the gap sooner. - let last_mined_index: i32 = match conn + let first_unmined_index: u32 = match conn .query_row( "SELECT address_index FROM ephemeral_addresses JOIN transactions t ON t.id_tx = mined_in_tx @@ -70,30 +82,30 @@ pub(crate) fn last_safe_index( ORDER BY address_index DESC LIMIT 1", named_params![":account_id": account_id.0], - |row| row.get::<_, i32>(0), + |row| row.get::<_, u32>(0), ) .optional()? { - Some(i) if i < 0 => unreachable!("violates constraint address_index_in_range"), - Some(i) => i, - None => -1, + Some(i) if i >= 1 << 31 => { + unreachable!("violates constraint index_range_and_address_nullity") + } + Some(i) => i.checked_add(1).unwrap(), + None => 0, }; - Ok(u32::try_from(last_mined_index.saturating_add(GAP_LIMIT)).unwrap()) + Ok(min( + 1 << 31, + first_unmined_index.checked_add(GAP_LIMIT).unwrap(), + )) } -/// Utility function to return an `InclusiveRange` that starts at `i + 1` -/// and is of length up to `n`. The range is truncated if necessary to end at -/// the maximum valid address index, `i32::MAX`. -/// -/// # Panics -/// -/// Panics if the precondition `i >= -1 and n > 0` does not hold. -pub(crate) fn range_after(i: i32, n: i32) -> RangeInclusive { - assert!(i >= -1); - assert!(n > 0); - let first = u32::try_from(i64::from(i) + 1).unwrap(); - let last = u32::try_from(i.saturating_add(n)).unwrap(); - first..=last +/// Utility function to return an `Range` that starts at `i` +/// and is of length up to `n`. The range is truncated if necessary +/// so that it contains no elements beyond the maximum valid address +/// index, `(1 << 31) - 1`. +pub(crate) fn range_from(i: u32, n: u32) -> Range { + let first = min(1 << 31, i); + let last = min(1 << 31, i.saturating_add(n)); + first..last } /// Returns the ephemeral transparent IVK for a given account ID. @@ -110,47 +122,40 @@ pub(crate) fn get_ephemeral_ivk( .derive_ephemeral_ivk()?) } -/// Returns a mapping of ephemeral transparent addresses potentially belonging -/// to this wallet to their metadata. +/// Returns a vector of ephemeral transparent addresses associated with the given +/// account controlled by this wallet, along with their metadata. The result includes +/// reserved addresses, and addresses for `GAP_LIMIT` additional indices (capped to +/// the maximum index). /// -/// If `for_detection` is false, the result only includes reserved addresses. -/// If `for_detection` is true, it includes addresses for an additional -/// `GAP_LIMIT` indices, up to the limit. +/// If `index_range` is some `Range`, it limits the result to addresses with indices +/// in that range. pub(crate) fn get_known_ephemeral_addresses( conn: &rusqlite::Connection, params: &P, account_id: AccountId, - for_detection: bool, -) -> Result, SqliteClientError> { + index_range: Option>, +) -> Result, SqliteClientError> { + let index_range = index_range.unwrap_or(0..(1 << 31)); + let mut stmt = conn.prepare( - "SELECT address, address_index FROM ephemeral_addresses WHERE account_id = :account ORDER BY address_index", + "SELECT address, address_index FROM ephemeral_addresses + WHERE account_id = :account AND address_index >= :start AND address_index < :end + ORDER BY address_index", )?; - let mut rows = stmt.query(named_params! { ":account": account_id.0 })?; + let mut rows = stmt.query(named_params![ + ":account": account_id.0, + ":start": index_range.start, + ":end": min(1 << 31, index_range.end), + ])?; - let mut result = HashMap::new(); - let mut first_unused_index: Option = Some(0); + let mut result = vec![]; while let Some(row) = rows.next()? { let addr_str: String = row.get(0)?; let raw_index: u32 = row.get(1)?; - first_unused_index = i32::try_from(raw_index) - .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? - .checked_add(1); let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); let address = TransparentAddress::decode(params, &addr_str)?; - result.insert(address, metadata(address_index)); - } - - if for_detection { - if let Some(first) = first_unused_index { - let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; - - for raw_index in range_after(first, GAP_LIMIT) { - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); - let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; - result.insert(address, metadata(address_index)); - } - } + result.push((address, metadata(address_index))); } Ok(result) } @@ -189,10 +194,6 @@ pub(crate) fn find_index_for_ephemeral_address_str( /// Returns a vector with the next `n` previously unreserved ephemeral addresses for /// the given account. /// -/// # Panics -/// -/// Panics if the precondition `n < 0x80000000` does not hold. -/// /// # Errors /// /// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. @@ -211,43 +212,81 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( if n == 0 { return Ok(vec![]); } - assert!(n > 0); - let n = i32::try_from(n).expect("precondition violated"); - let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; - let last_reserved_index = last_reserved_index(wdb.conn.0, account_id)?; - let last_safe_index = last_safe_index(wdb.conn.0, account_id)?; - let allocation = range_after(last_reserved_index, n); + let first_unreserved = first_unreserved_index(wdb.conn.0, account_id)?; + let first_unsafe = first_unsafe_index(wdb.conn.0, account_id)?; + let allocation = range_from(first_unreserved, n); - if allocation.clone().count() < n.try_into().unwrap() { - return Err(SqliteClientError::AddressGeneration( - AddressGenerationError::DiversifierSpaceExhausted, + if allocation.len() < n.try_into().unwrap() { + return Err(AddressGenerationError::DiversifierSpaceExhausted.into()); + } + if allocation.end > first_unsafe { + return Err(SqliteClientError::ReachedGapLimit( + account_id, + max(first_unreserved, first_unsafe), )); } - if *allocation.end() > last_safe_index { - let unsafe_index = max(*allocation.start(), last_safe_index.saturating_add(1)); - return Err(SqliteClientError::ReachedGapLimit(account_id, unsafe_index)); + reserve_until(wdb.conn.0, &wdb.params, account_id, allocation.end)?; + get_known_ephemeral_addresses(wdb.conn.0, &wdb.params, account_id, Some(allocation)) +} + +/// Initialize the `ephemeral_addresses` table. This must be called when +/// creating or migrating an account. +pub(crate) fn init_account( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountId, +) -> Result<(), SqliteClientError> { + reserve_until(conn, params, account_id, 0) +} + +/// Extend the range of stored addresses in an account if necessary so that the +/// index of the next address to reserve will be *at least* `next_to_reserve`. +/// If it would already have been at least `next_to_reserve`, then do nothing. +/// +/// Note that this is called from db migration code. +/// +/// # Panics +/// +/// Panics if `next_to_reserve > (1 << 31)`. +fn reserve_until( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountId, + next_to_reserve: u32, +) -> Result<(), SqliteClientError> { + assert!(next_to_reserve <= 1 << 31); + + let first_unstored = first_unstored_index(conn, account_id)?; + let range_to_store = first_unstored..(next_to_reserve.checked_add(GAP_LIMIT).unwrap()); + if range_to_store.is_empty() { + return Ok(()); } + let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; + // used_in_tx and mined_in_tx are initially NULL - let mut stmt_insert_ephemeral_address = wdb.conn.0.prepare_cached( + let mut stmt_insert_ephemeral_address = conn.prepare_cached( "INSERT INTO ephemeral_addresses (account_id, address_index, address) VALUES (:account_id, :address_index, :address)", )?; - allocation - .map(|raw_index| { - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); - let address = ephemeral_ivk.derive_ephemeral_address(address_index)?; - - stmt_insert_ephemeral_address.execute(named_params![ - ":account_id": account_id.0, - ":address_index": raw_index, - ":address": address.encode(&wdb.params), - ])?; - Ok((address, metadata(address_index))) - }) - .collect() + for raw_index in range_to_store { + let address_str_opt = match NonHardenedChildIndex::from_index(raw_index) { + Some(address_index) => Some( + ephemeral_ivk + .derive_ephemeral_address(address_index)? + .encode(params), + ), + None => None, + }; + stmt_insert_ephemeral_address.execute(named_params![ + ":account_id": account_id.0, + ":address_index": raw_index, + ":address": address_str_opt, + ])?; + } + Ok(()) } /// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was @@ -352,9 +391,25 @@ pub(crate) fn mark_ephemeral_address_as_mined( |row| row.get::<_, i64>(0), )?; - wdb.conn.0.execute( - "UPDATE ephemeral_addresses SET mined_in_tx = :mined_in_tx WHERE address = :address", - named_params![":mined_in_tx": &earlier_ref, ":address": address_str], - )?; + let mined_ephemeral = wdb + .conn + .0 + .query_row( + "UPDATE ephemeral_addresses + SET mined_in_tx = :mined_in_tx + WHERE address = :address + RETURNING (account_id, address_index)", + named_params![":mined_in_tx": &earlier_ref, ":address": address_str], + |row| Ok((AccountId(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), + ) + .optional()?; + + // If this is a known ephemeral address for an account in this wallet, we might need + // to extend the indices stored for that account to maintain the invariant that the + // last `GAP_LIMIT` addresses are unused and unmined. + if let Some((account_id, address_index)) = mined_ephemeral { + let next_to_reserve = min(1 << 31, address_index.saturating_add(1)); + reserve_until(wdb.conn.0, &wdb.params, account_id, next_to_reserve)?; + } Ok(()) } From 86428c4afe825b04adcf2c554ba47af61d96bb9d Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 19:43:02 +0100 Subject: [PATCH 45/56] Refactor `find_account_for_transparent_output` (now called `find_account_for_transparent_address`) to take a `TransparentAddress` rather than a `WalletTransparentOutput`. Co-authored-by: Kris Nuttycombe Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/transparent.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index d45e32f853..8839c353bf 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -443,21 +443,20 @@ pub(crate) fn put_received_transparent_utxo( params: &P, output: &WalletTransparentOutput, ) -> Result { - if let Some(receiving_account) = find_account_for_transparent_output(conn, params, output)? { + let address = output.recipient_address(); + if let Some(receiving_account) = find_account_for_transparent_address(conn, params, address)? { put_transparent_output( conn, params, output.outpoint(), output.txout(), Some(output.height()), - output.recipient_address(), + address, receiving_account, ) } else { // The UTXO was not for any of our transparent addresses. - Err(SqliteClientError::AddressNotRecognized( - *output.recipient_address(), - )) + Err(SqliteClientError::AddressNotRecognized(*address)) } } @@ -513,12 +512,12 @@ pub(crate) fn get_transparent_address_metadata( /// /// Returns `Ok(None)` if the transparent output's recipient address is not in any of the /// above locations. This means the wallet considers the output "not interesting". -pub(crate) fn find_account_for_transparent_output( +pub(crate) fn find_account_for_transparent_address( conn: &rusqlite::Connection, params: &P, - output: &WalletTransparentOutput, + address: &TransparentAddress, ) -> Result, SqliteClientError> { - let address_str = output.recipient_address().encode(params); + let address_str = address.encode(params); if let Some(account_id) = conn .query_row( @@ -547,7 +546,7 @@ pub(crate) fn find_account_for_transparent_output( // matches the address for the received UTXO. for &account_id in account_ids.iter() { if let Some((legacy_taddr, _)) = get_legacy_transparent_address(params, conn, account_id)? { - if &legacy_taddr == output.recipient_address() { + if &legacy_taddr == address { return Ok(Some(account_id)); } } From b48f6272f0daf14ec9db7fc9120d2ad395d63954 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 19:44:20 +0100 Subject: [PATCH 46/56] Minor simplification in a test. Co-authored-by: Jack Grigg Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/testing/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index da6caf2722..c48102f730 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -374,7 +374,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { steps[0].balance().proposed_change(), [ ChangeValue::shielded(T::SHIELDED_PROTOCOL, expected_step0_change, change_memo), - ChangeValue::ephemeral_transparent((transfer_amount + expected_step1_fee).unwrap()), + ChangeValue::ephemeral_transparent(expected_ephemeral), ] ); assert_eq!(steps[1].balance().proposed_change(), []); From 6bc22f411ea07d67e92f33e0b94dd5d9c6015bea Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 19:46:56 +0100 Subject: [PATCH 47/56] Documentation fixes and improvements. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api.rs | 14 +++++++------ zcash_client_backend/src/data_api/wallet.rs | 5 ++--- zcash_client_sqlite/src/error.rs | 9 ++++----- zcash_client_sqlite/src/lib.rs | 3 --- zcash_client_sqlite/src/wallet/transparent.rs | 20 ++++++++++++------- .../src/wallet/transparent/ephemeral.rs | 3 +++ zcash_primitives/src/transaction/builder.rs | 2 -- 7 files changed, 30 insertions(+), 26 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 1fdf3e633e..7561fa4586 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -680,12 +680,14 @@ pub trait InputSource { Ok(None) } - /// Returns the list of transparent outputs received at `address` such that: - /// * The transaction that produced these outputs is mined or mineable as of `max_height`. - /// * Each returned output is unspent as of the current chain tip. - /// - /// The caller should filter these outputs to ensure they respect the desired number of - /// confirmations before attempting to spend them. + /// Returns the list of spendable transparent outputs received by this wallet at `address` + /// such that, at height `target_height`: + /// * the transaction that produced the output had or will have at least `min_confirmations` + /// confirmations; and + /// * the output is unspent as of the current chain tip. + /// + /// An output that is potentially spent by an unmined transaction in the mempool is excluded + /// iff the spending transaction will not be expired at `target_height`. #[cfg(feature = "transparent-inputs")] fn get_spendable_transparent_outputs( &self, diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 0b63c601da..6c1e500404 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1165,9 +1165,8 @@ where for ((change_index, change_value), (ephemeral_address, _)) in ephemeral_outputs.iter().zip(addresses_and_metadata) { - // This is intended for an ephemeral transparent output, rather than a - // non-ephemeral transparent change output. We will report an error in - // `create_proposed_transactions` if a later step does not consume this output. + // This output is ephemeral; we will report an error in `create_proposed_transactions` + // if a later step does not consume it. builder.add_transparent_output(&ephemeral_address, change_value.value())?; transparent_output_meta.push(( Recipient::EphemeralTransparent { diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 5e4fc1c976..445c7ed78e 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -120,10 +120,9 @@ pub enum SqliteClientError { #[cfg(feature = "transparent-inputs")] ReachedGapLimit(AccountId, u32), - /// An ephemeral address would be reused, or incorrectly used as an external address. - /// The parameters are the address in string form, and if it is known to have been - /// used in one or more previous transactions, the txid of the earliest of those - /// transactions. + /// An ephemeral address would be reused. The parameters are the address in string + /// form, and if it is known to have been used in one or more previous transactions, + /// the txid of the earliest of those transactions. #[cfg(feature = "transparent-inputs")] EphemeralAddressReuse(String, Option), } @@ -184,7 +183,7 @@ impl fmt::Display for SqliteClientError { #[cfg(feature = "transparent-inputs")] SqliteClientError::EphemeralAddressReuse(address_str, Some(txid)) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(address_str, None) => write!(f, "The ephemeral address {address_str} would be reused, or incorrectly used as an external address."), + SqliteClientError::EphemeralAddressReuse(address_str, None) => write!(f, "The ephemeral address {address_str} would be reused."), } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 74563c3c67..e62fcca4bf 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1316,9 +1316,6 @@ impl WalletWrite for WalletDb .enumerate() { if let Some(address) = txout.recipient_address() { - // TODO: we really want to only mark outputs when a transaction has been - // *reliably* mined, because that is strictly more conservative in avoiding - // going over the gap limit. #[cfg(feature = "transparent-inputs")] wallet::transparent::ephemeral::mark_ephemeral_address_as_mined(wdb, &address, tx_ref)?; diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 8839c353bf..9554343258 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -238,9 +238,19 @@ pub(crate) fn get_unspent_transparent_output( result } -/// Returns spendable transparent outputs that have been received by this wallet at the given -/// transparent address, as outputs of transactions in blocks mined at a height less than or -/// equal to the provided `max_height`. +/// Returns the list of spendable transparent outputs received by this wallet at `address` +/// such that, at height `target_height`: +/// * the transaction that produced the output had or will have at least `min_confirmations` +/// confirmations; and +/// * the output is unspent as of the current chain tip. +/// +/// An output that is potentially spent by an unmined transaction in the mempool is excluded +/// iff the spending transaction will not be expired at `target_height`. +/// +/// This could, in very rare circumstances, return as unspent outputs that are actually not +/// spendable, if they are the outputs of deshielding transactions where the spend anchors have +/// been invalidated by a rewind. There isn't a way to detect this circumstance at present, but +/// it should be vanishingly rare as the vast majority of rewinds are of a single block. pub(crate) fn get_spendable_transparent_outputs( conn: &rusqlite::Connection, params: &P, @@ -250,10 +260,6 @@ pub(crate) fn get_spendable_transparent_outputs( ) -> Result, SqliteClientError> { let confirmed_height = target_height - min_confirmations; - // This could, in very rare circumstances, return as unspent outputs that are actually not - // spendable, if they are the outputs of deshielding transactions where the spend anchors have - // been invalidated by a rewind. There isn't a way to detect this circumstance at present, but - // it should be vanishingly rare as the vast majority of rewinds are of a single block. let mut stmt_utxos = conn.prepare( "SELECT t.txid, u.output_index, u.script, u.value_zat, t.mined_height AS received_height diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 06d5f314b4..1cd6001280 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -74,6 +74,9 @@ pub(crate) fn first_unsafe_index( // to have been mined in a transaction that we currently see as unmined. // This is conservative in terms of avoiding violation of the gap // invariant: it can only cause us to get to the end of the gap sooner. + // + // TODO: do we want to only consider transactions with a minimum number + // of confirmations here? let first_unmined_index: u32 = match conn .query_row( "SELECT address_index FROM ephemeral_addresses diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index b4f0c69507..47c0995028 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -538,8 +538,6 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< /// /// This fee is a function of the spends and outputs that have been added to the builder, /// pursuant to the specified [`FeeRule`]. - /// - /// Any ephemeral inputs or outputs are *not* taken into account. pub fn get_fee( &self, fee_rule: &FR, From 27ca6e44d63c42afc1fae813eb176563a92a5c92 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 20:10:38 +0100 Subject: [PATCH 48/56] Remove unneeded `impl SealedChangeLevelKey for EphemeralIvk`. Signed-off-by: Daira-Emma Hopwood --- zcash_primitives/src/legacy/keys.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/zcash_primitives/src/legacy/keys.rs b/zcash_primitives/src/legacy/keys.rs index 60131671f6..c7b7c610a2 100644 --- a/zcash_primitives/src/legacy/keys.rs +++ b/zcash_primitives/src/legacy/keys.rs @@ -435,18 +435,6 @@ impl IncomingViewingKey for InternalIvk {} #[derive(Clone, Debug)] pub struct EphemeralIvk(ExtendedPublicKey); -impl private::SealedChangeLevelKey for EphemeralIvk { - const SCOPE: TransparentKeyScope = TransparentKeyScope(2); - - fn extended_pubkey(&self) -> &ExtendedPublicKey { - &self.0 - } - - fn from_extended_pubkey(key: ExtendedPublicKey) -> Self { - EphemeralIvk(key) - } -} - #[cfg(feature = "transparent-inputs")] impl EphemeralIvk { /// Derives a transparent address at the provided child index. @@ -454,8 +442,7 @@ impl EphemeralIvk { &self, address_index: NonHardenedChildIndex, ) -> Result { - use private::SealedChangeLevelKey; - let child_key = self.extended_pubkey().derive_child(address_index.into())?; + let child_key = self.0.derive_child(address_index.into())?; #[allow(deprecated)] Ok(pubkey_to_address(child_key.public_key())) } From bda6451273eae4d12aabcddda1cd125eb01d8926 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 20:13:03 +0100 Subject: [PATCH 49/56] Change `unwrap`s to `expect`s when constructing `NonHardenedChildIndex`. Also change the return type of `find_index_for_ephemeral_address_str` to `Result, SqliteClientError>` so that the `expect` is in the right place. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/wallet/transparent.rs | 3 +-- .../src/wallet/transparent/ephemeral.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/transparent.rs b/zcash_client_sqlite/src/wallet/transparent.rs index 9554343258..000c515e2b 100644 --- a/zcash_client_sqlite/src/wallet/transparent.rs +++ b/zcash_client_sqlite/src/wallet/transparent.rs @@ -498,10 +498,9 @@ pub(crate) fn get_transparent_address_metadata( } // Search known ephemeral addresses. - if let Some(raw_index) = + if let Some(address_index) = ephemeral::find_index_for_ephemeral_address_str(conn, account_id, &address_str)? { - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); return Ok(Some(ephemeral::metadata(address_index))); } diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 1cd6001280..024158c018 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -156,7 +156,8 @@ pub(crate) fn get_known_ephemeral_addresses( while let Some(row) = rows.next()? { let addr_str: String = row.get(0)?; let raw_index: u32 = row.get(1)?; - let address_index = NonHardenedChildIndex::from_index(raw_index).unwrap(); + let address_index = NonHardenedChildIndex::from_index(raw_index) + .expect("where clause ensures this is in range"); let address = TransparentAddress::decode(params, &addr_str)?; result.push((address, metadata(address_index))); } @@ -183,7 +184,7 @@ pub(crate) fn find_index_for_ephemeral_address_str( conn: &rusqlite::Connection, account_id: AccountId, address_str: &str, -) -> Result, SqliteClientError> { +) -> Result, SqliteClientError> { Ok(conn .query_row( "SELECT address_index FROM ephemeral_addresses @@ -191,7 +192,11 @@ pub(crate) fn find_index_for_ephemeral_address_str( named_params![":account_id": account_id.0, ":address": &address_str], |row| row.get::<_, u32>(0), ) - .optional()?) + .optional()? + .map(|index| { + NonHardenedChildIndex::from_index(index) + .expect("valid by constraint index_range_and_address_nullity") + })) } /// Returns a vector with the next `n` previously unreserved ephemeral addresses for From 22b8cff6d1469106ec7ec90dd5f47e3af93d3890 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 20:37:07 +0100 Subject: [PATCH 50/56] The `TxId` argument to `EphemeralAddressReuse` does not need to be optional. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/error.rs | 10 ++++------ .../src/wallet/transparent/ephemeral.rs | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 445c7ed78e..3e9ce57211 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -121,10 +121,10 @@ pub enum SqliteClientError { ReachedGapLimit(AccountId, u32), /// An ephemeral address would be reused. The parameters are the address in string - /// form, and if it is known to have been used in one or more previous transactions, - /// the txid of the earliest of those transactions. + /// form, and the txid of the earliest transaction in which it is known to have been + /// used. #[cfg(feature = "transparent-inputs")] - EphemeralAddressReuse(String, Option), + EphemeralAddressReuse(String, TxId), } impl error::Error for SqliteClientError { @@ -181,9 +181,7 @@ impl fmt::Display for SqliteClientError { The ephemeral address in account {account_id:?} at index {bad_index} could not be safely reserved.", ), #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(address_str, Some(txid)) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(address_str, None) => write!(f, "The ephemeral address {address_str} would be reused."), + SqliteClientError::EphemeralAddressReuse(address_str, txid) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), } } } diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 024158c018..521143cc4e 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -338,7 +338,7 @@ fn ephemeral_address_reuse_check( ); Err(SqliteClientError::EphemeralAddressReuse( address_str.to_owned(), - Some(txid), + txid, )) } else { Ok(()) From 9856a708402ce7f8ce6d17a71e26b3e072c465bb Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 20:57:51 +0100 Subject: [PATCH 51/56] Simpler handling of a potential overflow. Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/src/data_api.rs | 4 ++-- zcash_client_backend/src/data_api/error.rs | 5 ++--- zcash_client_backend/src/data_api/wallet.rs | 6 +----- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet/transparent/ephemeral.rs | 9 ++++++--- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 7561fa4586..caa07e8ace 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1684,7 +1684,7 @@ pub trait WalletWrite: WalletRead { fn reserve_next_n_ephemeral_addresses( &mut self, account_id: Self::AccountId, - n: u32, + n: usize, ) -> Result, Self::Error>; } @@ -2109,7 +2109,7 @@ pub mod testing { fn reserve_next_n_ephemeral_addresses( &mut self, _account_id: Self::AccountId, - _n: u32, + _n: usize, ) -> Result, Self::Error> { Err(()) } diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index acf451a3e4..fd1d35aaae 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -39,8 +39,7 @@ pub enum Error { /// The proposal was structurally valid, but tried to do one of these unsupported things: /// * spend a prior shielded output; /// * pay to an output pool for which the corresponding feature is not enabled; - /// * pay to a TEX address if the "transparent-inputs" feature is not enabled; - /// or exceeded an implementation limit. + /// * pay to a TEX address if the "transparent-inputs" feature is not enabled. ProposalNotSupported, /// No account could be found corresponding to a provided spending key. @@ -128,7 +127,7 @@ where f, "The proposal was valid but tried to do something that is not supported \ (spend shielded outputs of prior transaction steps or use a feature that \ - is not enabled), or exceeded an implementation limit.", + is not enabled).", ), Error::KeyNotRecognized => { write!( diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 6c1e500404..29486b54b4 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1152,12 +1152,8 @@ where }) .collect(); - let n = ephemeral_outputs - .len() - .try_into() - .map_err(|_| Error::ProposalNotSupported)?; let addresses_and_metadata = wallet_db - .reserve_next_n_ephemeral_addresses(account_id, n) + .reserve_next_n_ephemeral_addresses(account_id, ephemeral_outputs.len()) .map_err(Error::DataSource)?; assert_eq!(addresses_and_metadata.len(), ephemeral_outputs.len()); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index e62fcca4bf..d6ad1acca4 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1497,7 +1497,7 @@ impl WalletWrite for WalletDb fn reserve_next_n_ephemeral_addresses( &mut self, account_id: Self::AccountId, - n: u32, + n: usize, ) -> Result, Self::Error> { self.transactionally(|wdb| { wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses(wdb, account_id, n) diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 521143cc4e..08b95f98f9 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -215,7 +215,7 @@ pub(crate) fn find_index_for_ephemeral_address_str( pub(crate) fn reserve_next_n_ephemeral_addresses( wdb: &mut WalletDb, P>, account_id: AccountId, - n: u32, + n: usize, ) -> Result, SqliteClientError> { if n == 0 { return Ok(vec![]); @@ -223,9 +223,12 @@ pub(crate) fn reserve_next_n_ephemeral_addresses( let first_unreserved = first_unreserved_index(wdb.conn.0, account_id)?; let first_unsafe = first_unsafe_index(wdb.conn.0, account_id)?; - let allocation = range_from(first_unreserved, n); + let allocation = range_from( + first_unreserved, + u32::try_from(n).map_err(|_| AddressGenerationError::DiversifierSpaceExhausted)?, + ); - if allocation.len() < n.try_into().unwrap() { + if allocation.len() < n { return Err(AddressGenerationError::DiversifierSpaceExhausted.into()); } if allocation.end > first_unsafe { From 56aa348a41f8eff4427e89d9e1826f0781911949 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Thu, 4 Jul 2024 23:00:15 +0100 Subject: [PATCH 52/56] Extend the `send_multi_step_proposed_transfer` test to check the behaviour when another wallet creates a transaction with an output to one of our ephemeral addresses, and repair the implementation to pass this test. Signed-off-by: Daira-Emma Hopwood --- zcash_client_sqlite/src/lib.rs | 55 ++--- zcash_client_sqlite/src/testing/pool.rs | 216 +++++++++++++++++- zcash_client_sqlite/src/wallet/db.rs | 29 +-- .../init/migrations/ephemeral_addresses.rs | 9 +- .../src/wallet/transparent/ephemeral.rs | 75 +++--- 5 files changed, 294 insertions(+), 90 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index d6ad1acca4..028124f2c7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1099,6 +1099,8 @@ impl WalletWrite for WalletDb self.transactionally(|wdb| { let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx(), None, None)?; let funding_accounts = wallet::get_funding_accounts(wdb.conn.0, d_tx.tx())?; + + // TODO(#1305): Correctly track accounts that fund each transaction output. let funding_account = funding_accounts.iter().next().copied(); if funding_accounts.len() > 1 { warn!( @@ -1287,38 +1289,27 @@ impl WalletWrite for WalletDb wallet::transparent::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, &txin.prevout)?; } - // If we have some transparent outputs: - if d_tx - .tx() - .transparent_bundle() - .iter() - .any(|b| !b.vout.is_empty()) - { - // If the transaction contains spends from our wallet, we will store z->t - // transactions we observe in the same way they would be stored by - // create_spend_to_address. - let funding_accounts = wallet::get_funding_accounts(wdb.conn.0, d_tx.tx())?; - let funding_account = funding_accounts.iter().next().copied(); - if let Some(account_id) = funding_account { - if funding_accounts.len() > 1 { - warn!( - "More than one wallet account detected as funding transaction {:?}, selecting {:?}", - d_tx.tx().txid(), - account_id - ) - } - - for (output_index, txout) in d_tx - .tx() - .transparent_bundle() - .iter() - .flat_map(|b| b.vout.iter()) - .enumerate() - { - if let Some(address) = txout.recipient_address() { - #[cfg(feature = "transparent-inputs")] - wallet::transparent::ephemeral::mark_ephemeral_address_as_mined(wdb, &address, tx_ref)?; - + // This `if` is just an optimization for cases where we would do nothing in the loop. + if funding_account.is_some() || cfg!(feature = "transparent-inputs") { + for (output_index, txout) in d_tx + .tx() + .transparent_bundle() + .iter() + .flat_map(|b| b.vout.iter()) + .enumerate() + { + if let Some(address) = txout.recipient_address() { + // The transaction is not necessarily mined yet, but we want to record + // that an output to the address was seen in this tx anyway. This will + // advance the gap regardless of whether it is mined, but an output in + // an unmined transaction won't advance the range of safe indices. + #[cfg(feature = "transparent-inputs")] + wallet::transparent::ephemeral::mark_ephemeral_address_as_seen(wdb, &address, tx_ref)?; + + // If a transaction we observe contains spends from our wallet, we will + // store its transparent outputs in the same way they would be stored by + // create_spend_to_address. + if let Some(account_id) = funding_account { let receiver = Receiver::Transparent(address); #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index c48102f730..5e64a06eb2 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -302,9 +302,19 @@ pub(crate) fn send_single_step_proposed_transfer() { #[cfg(feature = "transparent-inputs")] pub(crate) fn send_multi_step_proposed_transfer() { - use std::str::FromStr; + use std::{collections::HashSet, str::FromStr}; - use zcash_client_backend::fees::ChangeValue; + use rand_core::OsRng; + use zcash_client_backend::{ + fees::ChangeValue, + wallet::{TransparentAddressMetadata, WalletTx}, + }; + use zcash_primitives::{ + legacy::keys::{NonHardenedChildIndex, TransparentKeyScope}, + transaction::builder::{BuildConfig, Builder}, + }; + + use crate::wallet::{sapling::tests::test_prover, GAP_LIMIT}; let mut st = TestBuilder::new() .with_block_cache() @@ -312,6 +322,8 @@ pub(crate) fn send_multi_step_proposed_transfer() { .build(); let account = st.test_account().cloned().unwrap(); + let account_id = account.account_id(); + let (default_addr, default_index) = account.usk().default_transparent_address(); let dfvk = T::test_account_fvk(&st); let add_funds = |st: &mut TestState<_>, value| { @@ -325,7 +337,8 @@ pub(crate) fn send_multi_step_proposed_transfer() { .block_height(), h ); - assert_eq!(st.get_spendable_balance(account.account_id(), 1), value); + assert_eq!(st.get_spendable_balance(account_id, 1), value); + h }; let value = NonNegativeAmount::const_from_u64(100000); @@ -344,7 +357,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { // Generate a ZIP 320 proposal, sending to the wallet's default transparent address // expressed as a TEX address. - let tex_addr = match account.usk().default_transparent_address().0 { + let tex_addr = match default_addr { TransparentAddress::PublicKeyHash(data) => Address::Tex(data), _ => unreachable!(), }; @@ -354,7 +367,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { // serialization of the proposal. let proposal = st .propose_standard_transfer::( - account.account_id(), + account_id, StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), &tex_addr, @@ -439,15 +452,15 @@ pub(crate) fn send_multi_step_proposed_transfer() { (sent_v, sent_to_addr, None, None) if sent_v == u64::try_from(transfer_amount).unwrap() && sent_to_addr == Some(tex_addr.encode(&st.wallet().params))); - (ephemeral_addr.unwrap(), txids.head) + (ephemeral_addr.unwrap(), txids) }; // Each transfer should use a different ephemeral address. - let (ephemeral0, _) = run_test(&mut st, 0); - let (ephemeral1, _) = run_test(&mut st, 1); + let (ephemeral0, txids0) = run_test(&mut st, 0); + let (ephemeral1, txids1) = run_test(&mut st, 1); assert_ne!(ephemeral0, ephemeral1); - add_funds(&mut st, value); + let height = add_funds(&mut st, value); let ephemeral_taddr = Address::decode(&st.wallet().params, &ephemeral0).expect("valid address"); assert_matches!( @@ -458,7 +471,7 @@ pub(crate) fn send_multi_step_proposed_transfer() { // Attempting to pay to an ephemeral address should cause an error. let proposal = st .propose_standard_transfer::( - account.account_id(), + account_id, StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), &ephemeral_taddr, @@ -477,6 +490,184 @@ pub(crate) fn send_multi_step_proposed_transfer() { assert_matches!( &create_proposed_result, Err(Error::PaysEphemeralTransparentAddress(address_str)) if address_str == &ephemeral0); + + // Simulate another wallet sending to an ephemeral address with an index + // within the current gap limit. The `PaysEphemeralTransparentAddress` error + // prevents us from doing so straightforwardly, so we'll do it by building + // a transaction and calling `store_decrypted_tx` with it. + let known_addrs = st + .wallet() + .get_known_ephemeral_addresses(account_id, None) + .unwrap(); + assert_eq!(known_addrs.len(), (GAP_LIMIT as usize) + 2); + + // Check that the addresses are all distinct. + let known_set: HashSet<_> = known_addrs.iter().map(|(addr, _)| addr).collect(); + assert_eq!(known_set.len(), known_addrs.len()); + // Check that the metadata is as expected. + for (i, (_, meta)) in known_addrs.iter().enumerate() { + assert_eq!( + meta, + &TransparentAddressMetadata::new( + TransparentKeyScope::EPHEMERAL, + NonHardenedChildIndex::from_index(i.try_into().unwrap()).unwrap() + ) + ); + } + + let mut builder = Builder::new( + st.wallet().params, + height + 1, + BuildConfig::Standard { + sapling_anchor: None, + orchard_anchor: None, + }, + ); + let (colliding_addr, _) = &known_addrs[10]; + assert_matches!( + builder.add_transparent_output(colliding_addr, (value - zip317::MINIMUM_FEE).unwrap()), + Ok(_) + ); + let sk = account + .usk() + .transparent() + .derive_secret_key(Scope::External.into(), default_index) + .unwrap(); + let outpoint = OutPoint::fake(); + let txout = TxOut { + script_pubkey: default_addr.script(), + value, + }; + assert_matches!(builder.add_transparent_input(sk, outpoint, txout), Ok(_)); + let test_prover = test_prover(); + let build_result = builder + .build( + OsRng, + &test_prover, + &test_prover, + &zip317::FeeRule::standard(), + ) + .unwrap(); + let decrypted_tx = DecryptedTransaction::::new( + build_result.transaction(), + vec![], + #[cfg(feature = "orchard")] + vec![], + ); + st.wallet_mut().store_decrypted_tx(decrypted_tx).unwrap(); + + // That should have advanced the start of the gap to index 11. + let new_known_addrs = st + .wallet() + .get_known_ephemeral_addresses(account_id, None) + .unwrap(); + assert_eq!(new_known_addrs.len(), (GAP_LIMIT as usize) + 11); + assert!(new_known_addrs.starts_with(&known_addrs)); + + let reservation_should_succeed = |st: &mut TestState<_>, n| { + let reserved = st + .wallet_mut() + .reserve_next_n_ephemeral_addresses(account_id, n) + .unwrap(); + assert_eq!(reserved.len(), n); + reserved + }; + let reservation_should_fail = |st: &mut TestState<_>, n, expected_bad_index| { + assert_matches!(st + .wallet_mut() + .reserve_next_n_ephemeral_addresses(account_id, n), + Err(SqliteClientError::ReachedGapLimit(acct, bad_index)) + if acct == account_id && bad_index == expected_bad_index); + }; + + let next_reserved = reservation_should_succeed(&mut st, 1); + assert_eq!(next_reserved[0], known_addrs[11]); + + // Calling `reserve_next_n_ephemeral_addresses(account_id, 1)` will have advanced + // the start of the gap to index 12. This also tests the `index_range` parameter. + let newer_known_addrs = st + .wallet() + .get_known_ephemeral_addresses(account_id, Some(5..100)) + .unwrap(); + assert_eq!(newer_known_addrs.len(), (GAP_LIMIT as usize) + 12 - 5); + assert!(newer_known_addrs.starts_with(&new_known_addrs[5..])); + + // None of the five transactions created above (two from each proposal and the + // one built manually) have been mined yet. So, the range of address indices + // that are safe to reserve is still 0..20, and we have already reserved 12 + // addresses, so trying to reserve another 9 should fail. + reservation_should_fail(&mut st, 9, 20); + reservation_should_succeed(&mut st, 8); + reservation_should_fail(&mut st, 1, 20); + + // Now mine the transaction with the ephemeral output at index 1. + // We already reserved 20 addresses, so this should allow 2 more (..22). + // It does not matter that the transaction with ephemeral output at index 0 + // remains unmined. + let (h, _) = st.generate_next_block_including(txids1.head); + st.scan_cached_blocks(h, 1); + reservation_should_succeed(&mut st, 2); + reservation_should_fail(&mut st, 1, 22); + + // Mining the transaction with the ephemeral output at index 0 at this point + // should make no difference. + let (h, _) = st.generate_next_block_including(txids0.head); + st.scan_cached_blocks(h, 1); + reservation_should_fail(&mut st, 1, 22); + + // Now mine the transaction with the ephemeral output at index 10. + let tx = build_result.transaction(); + let tx_index = 1; + let (h, _) = st.generate_next_block_from_tx(tx_index, tx); + st.scan_cached_blocks(h, 1); + + // The rest of this test would currently fail without the explicit call to + // `put_tx_meta` below. Ideally the above `scan_cached_blocks` would be + // sufficient, but it does not detect the transaction as interesting to the + // wallet. If a transaction is in the database with a null `mined_height`, + // as in this case, its `mined_height` will remain null unless `put_tx_meta` + // is called on it. Normally `put_tx_meta` would be called via `put_blocks` + // as a result of scanning, but that won't happen for any fully transparent + // transaction, and currently it also will not happen for a partially shielded + // transaction unless it is interesting to the wallet for another reason. + // Therefore we will not currently detect either collisions with uses of + // ephemeral outputs by other wallets, or refunds of funds sent to TEX + // addresses. (#1354, #1379) + + // Check that what we say in the above paragraph remains true, so that we + // don't accidentally fix it without updating this test. + reservation_should_fail(&mut st, 1, 22); + + // For now, we demonstrate that this problem is the only obstacle to the rest + // of the ZIP 320 code doing the right thing, by manually calling `put_tx_meta`: + crate::wallet::put_tx_meta( + &st.wallet_mut().conn, + &WalletTx::new( + tx.txid(), + tx_index, + vec![], + vec![], + #[cfg(feature = "orchard")] + vec![], + #[cfg(feature = "orchard")] + vec![], + ), + h, + ) + .unwrap(); + + // We already reserved 22 addresses, so mining the transaction with the + // ephemeral output at index 10 should allow 9 more (..31). + reservation_should_succeed(&mut st, 9); + reservation_should_fail(&mut st, 1, 31); + + let newest_known_addrs = st + .wallet() + .get_known_ephemeral_addresses(account_id, None) + .unwrap(); + assert_eq!(newest_known_addrs.len(), (GAP_LIMIT as usize) + 31); + assert!(newest_known_addrs.starts_with(&known_addrs)); + assert!(newest_known_addrs[5..].starts_with(&newer_known_addrs)); } #[cfg(feature = "transparent-inputs")] @@ -490,6 +681,7 @@ pub(crate) fn proposal_fails_if_not_all_ephemeral_outputs_consumed, value| { @@ -503,7 +695,7 @@ pub(crate) fn proposal_fails_if_not_all_ephemeral_outputs_consumed( - account.account_id(), + account_id, StandardFeeRule::Zip317, NonZeroU32::new(1).unwrap(), &tex_addr, diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index f4e9203c6f..64b45acc0f 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -94,38 +94,41 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( /// - `address` contains the string (Base58Check) encoding of a transparent P2PKH address. /// - `used_in_tx` indicates that the address has been used by this wallet in a transaction (which /// has not necessarily been mined yet). This should only be set once, when the txid is known. -/// - `mined_in_tx` is non-null iff the address has been observed in a mined transaction (which may -/// have been sent by this wallet or another one using the same seed, or by a TEX address recipient -/// sending back the funds). This is used to advance the "gap limit", as well as to heuristically -/// reduce the chance of address reuse collisions with another wallet using the same seed. -/// -/// Note that the fact that `used_in_tx` and `mined_in_tx` reference specific transactions is primarily -/// a debugging aid (although the latter allows us to account for whether the referenced transaction -/// is unmined). We only really care which addresses have been used, and whether we can allocate a -/// new address within the gap limit. +/// - `seen_in_tx` is non-null iff an output to the address has been seed in a transaction observed +/// on the network and passed to `store_decrypted_tx`. The transaction may have been sent by this +// wallet or another one using the same seed, or by a TEX address recipient sending back the +/// funds. This is used to advance the "gap", as well as to heuristically reduce the chance of +/// address reuse collisions with another wallet using the same seed. /// /// It is an external invariant that within each account: /// - the address indices are contiguous and start from 0; -/// - the last `GAP_LIMIT` addresses have `used_in_tx` and `mined_in_tx` both NULL. +/// - the last `GAP_LIMIT` addresses have `used_in_tx` and `seen_in_tx` both NULL. /// /// All but the last `GAP_LIMIT` addresses are defined to be "reserved" addresses. Since the next /// index to reserve is determined by dead reckoning from the last stored address, we use dummy /// entries after the maximum valid index in order to allow the last `GAP_LIMIT` addresses at the /// end of the index range to be used. +/// +/// Note that the fact that `used_in_tx` references a specific transaction is just a debugging aid. +/// The same is mostly true of `seen_in_tx`, but we also take into account whether the referenced +/// transaction is unmined in order to determine the last index that is safe to reserve. pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" CREATE TABLE ephemeral_addresses ( account_id INTEGER NOT NULL, address_index INTEGER NOT NULL, address TEXT, used_in_tx INTEGER, - mined_in_tx INTEGER, + seen_in_tx INTEGER, FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), - FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), + FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), PRIMARY KEY (account_id, address_index), + CONSTRAINT used_implies_seen CHECK ( + used_in_tx IS NULL OR seen_in_tx IS NOT NULL + ), CONSTRAINT index_range_and_address_nullity CHECK ( (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR - (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND mined_in_tx IS NULL) + (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND seen_in_tx IS NULL) ) ) WITHOUT ROWID"#; // Hexadecimal integer literals were added in SQLite version 3.8.6 (2014-08-15). diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 25f9c26280..7f68792b33 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -47,14 +47,17 @@ impl RusqliteMigration for Migration

{ address_index INTEGER NOT NULL, address TEXT, used_in_tx INTEGER, - mined_in_tx INTEGER, + seen_in_tx INTEGER, FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), - FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), + FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), PRIMARY KEY (account_id, address_index), + CONSTRAINT used_implies_seen CHECK ( + used_in_tx IS NULL OR seen_in_tx IS NOT NULL + ), CONSTRAINT index_range_and_address_nullity CHECK ( (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR - (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND mined_in_tx IS NULL) + (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND seen_in_tx IS NULL) ) ) WITHOUT ROWID; CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 08b95f98f9..a700168857 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -70,7 +70,7 @@ pub(crate) fn first_unsafe_index( account_id: AccountId, ) -> Result { // The inner join with `transactions` excludes addresses for which - // `mined_in_tx` is NULL. The query also excludes addresses observed + // `seen_in_tx` is NULL. The query also excludes addresses observed // to have been mined in a transaction that we currently see as unmined. // This is conservative in terms of avoiding violation of the gap // invariant: it can only cause us to get to the end of the gap sooner. @@ -80,7 +80,7 @@ pub(crate) fn first_unsafe_index( let first_unmined_index: u32 = match conn .query_row( "SELECT address_index FROM ephemeral_addresses - JOIN transactions t ON t.id_tx = mined_in_tx + JOIN transactions t ON t.id_tx = seen_in_tx WHERE account_id = :account_id AND t.mined_height IS NOT NULL ORDER BY address_index DESC LIMIT 1", @@ -164,12 +164,11 @@ pub(crate) fn get_known_ephemeral_addresses( Ok(result) } -/// If this is an ephemeral address in any account, return its account id. +/// If this is a known ephemeral address in any account, return its account id. pub(crate) fn find_account_for_ephemeral_address_str( conn: &rusqlite::Connection, address_str: &str, ) -> Result, SqliteClientError> { - // Search ephemeral addresses that have already been reserved. Ok(conn .query_row( "SELECT account_id FROM ephemeral_addresses WHERE address = :address", @@ -179,7 +178,7 @@ pub(crate) fn find_account_for_ephemeral_address_str( .optional()?) } -#[cfg(feature = "transparent-inputs")] +/// If this is a known ephemeral address in the given account, return its index. pub(crate) fn find_index_for_ephemeral_address_str( conn: &rusqlite::Connection, account_id: AccountId, @@ -259,7 +258,7 @@ pub(crate) fn init_account( /// /// # Panics /// -/// Panics if `next_to_reserve > (1 << 31)`. +/// Panics if the precondition `next_to_reserve <= (1 << 31)` does not hold. fn reserve_until( conn: &rusqlite::Transaction, params: &P, @@ -276,7 +275,7 @@ fn reserve_until( let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; - // used_in_tx and mined_in_tx are initially NULL + // used_in_tx and seen_in_tx are initially NULL let mut stmt_insert_ephemeral_address = conn.prepare_cached( "INSERT INTO ephemeral_addresses (account_id, address_index, address) VALUES (:account_id, :address_index, :address)", @@ -314,18 +313,18 @@ fn ephemeral_address_reuse_check( // using a given seed, because such a wallet will not reuse an address that // it ever reserved. // - // `COALESCE(used_in_tx, mined_in_tx)` can only differ from `used_in_tx` + // `COALESCE(used_in_tx, seen_in_tx)` can only differ from `used_in_tx` // if the address was reserved, an error occurred in transaction creation - // before calling `mark_ephemeral_address_as_used`, and then we observed - // the address to have been used in a mined transaction (presumably by - // another wallet instance, or due to a bug) anyway. + // before calling `mark_ephemeral_address_as_used`, and then we saw the + // address in another transaction (presumably created by another wallet + // instance, or as a result of a bug) anyway. let res = wdb .conn .0 .query_row( "SELECT t.txid FROM ephemeral_addresses LEFT OUTER JOIN transactions t - ON t.id_tx = COALESCE(used_in_tx, mined_in_tx) + ON t.id_tx = COALESCE(used_in_tx, seen_in_tx) WHERE address = :address", named_params![":address": address_str], |row| row.get::<_, Option>>(0), @@ -362,10 +361,28 @@ pub(crate) fn mark_ephemeral_address_as_used( let address_str = ephemeral_address.encode(&wdb.params); ephemeral_address_reuse_check(wdb, &address_str)?; - wdb.conn.0.execute( - "UPDATE ephemeral_addresses SET used_in_tx = :used_in_tx WHERE address = :address", - named_params![":used_in_tx": &tx_ref, ":address": address_str], - )?; + // We update both `used_in_tx` and `seen_in_tx` here, because a used address has + // necessarily been seen in a transaction. We will not treat this as extending the + // range of addresses that are safe to reserve unless and until the transaction is + // observed as mined. + let update_result = wdb + .conn + .0 + .query_row( + "UPDATE ephemeral_addresses + SET used_in_tx = :tx_ref, seen_in_tx = :tx_ref + WHERE address = :address + RETURNING account_id, address_index", + named_params![":tx_ref": &tx_ref, ":address": address_str], + |row| Ok((AccountId(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), + ) + .optional()?; + + // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. + if let Some((account_id, address_index)) = update_result { + let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); + reserve_until(wdb.conn.0, &wdb.params, account_id, next_to_reserve)?; + } Ok(()) } @@ -374,7 +391,7 @@ pub(crate) fn mark_ephemeral_address_as_used( /// /// `tx_ref` must be a valid transaction reference. This call has no effect if /// `address` is not one of our ephemeral addresses. -pub(crate) fn mark_ephemeral_address_as_mined( +pub(crate) fn mark_ephemeral_address_as_seen( wdb: &mut WalletDb, P>, address: &TransparentAddress, tx_ref: i64, @@ -382,7 +399,7 @@ pub(crate) fn mark_ephemeral_address_as_mined( let address_str = address.encode(&wdb.params); // Figure out which transaction was mined earlier: `tx_ref`, or any existing - // tx referenced by `mined_in_tx` for the given address. Prefer the existing + // tx referenced by `seen_in_tx` for the given address. Prefer the existing // reference in case of a tie or if both transactions are unmined. // This slightly reduces the chance of unnecessarily reaching the gap limit // too early in some corner cases (because the earlier transaction is less @@ -392,34 +409,32 @@ pub(crate) fn mark_ephemeral_address_as_mined( let earlier_ref = wdb.conn.0.query_row( "SELECT id_tx FROM transactions LEFT OUTER JOIN ephemeral_addresses e - ON id_tx = e.mined_in_tx + ON id_tx = e.seen_in_tx WHERE id_tx = :tx_ref OR e.address = :address ORDER BY mined_height ASC NULLS LAST, tx_index ASC NULLS LAST, - e.mined_in_tx ASC NULLS LAST + e.seen_in_tx ASC NULLS LAST LIMIT 1", named_params![":tx_ref": &tx_ref, ":address": address_str], |row| row.get::<_, i64>(0), )?; - let mined_ephemeral = wdb + let update_result = wdb .conn .0 .query_row( "UPDATE ephemeral_addresses - SET mined_in_tx = :mined_in_tx - WHERE address = :address - RETURNING (account_id, address_index)", - named_params![":mined_in_tx": &earlier_ref, ":address": address_str], + SET seen_in_tx = :seen_in_tx + WHERE address = :address + RETURNING account_id, address_index", + named_params![":seen_in_tx": &earlier_ref, ":address": address_str], |row| Ok((AccountId(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), ) .optional()?; - // If this is a known ephemeral address for an account in this wallet, we might need - // to extend the indices stored for that account to maintain the invariant that the - // last `GAP_LIMIT` addresses are unused and unmined. - if let Some((account_id, address_index)) = mined_ephemeral { - let next_to_reserve = min(1 << 31, address_index.saturating_add(1)); + // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. + if let Some((account_id, address_index)) = update_result { + let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); reserve_until(wdb.conn.0, &wdb.params, account_id, next_to_reserve)?; } Ok(()) From aa4312326d8106e2287e60915ea8acc2f8b1ea1d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 16 Jul 2024 13:53:31 -0600 Subject: [PATCH 53/56] Use `EphemeralBalance` instead of `EphemeralParameters` The `EphemeralParameters` type makes too many states representable, in particular, it represents that it is possible for a proposal step to have both ephemeral inputs and ephemeral outputs. This change reduces the representable state space to make it so that only one or the other is true, and also makes clear that the change memo must be attached to the change output of the intermediate step, and not to the ultimate output to the final recipient (where we expect there to be no shielded change, and therefore it is not possible to attach a memo.) --- zcash_client_backend/CHANGELOG.md | 5 +- .../src/data_api/wallet/input_selection.rs | 27 ++++---- zcash_client_backend/src/fees.rs | 69 ++++++------------- zcash_client_backend/src/fees/common.rs | 41 ++++++----- zcash_client_backend/src/fees/fixed.rs | 12 ++-- zcash_client_backend/src/fees/standard.rs | 10 +-- zcash_client_backend/src/fees/zip317.rs | 21 +++--- 7 files changed, 79 insertions(+), 106 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c5cc60cd1c..e0d0a8cfbb 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -28,6 +28,7 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail - `chain::BlockCache` trait, behind the `sync` feature flag. - `WalletRead::get_spendable_transparent_outputs`. - `zcash_client_backend::fees`: + - `EphemeralBalance` - `ChangeValue::shielded, is_ephemeral` - `ChangeValue::ephemeral_transparent` (when "transparent-inputs" is enabled) - `sapling::EmptyBundleView` @@ -67,10 +68,10 @@ funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for detail return type of `ChangeValue::output_pool` has (unconditionally) changed from `ShieldedProtocol` to `zcash_protocol::PoolType`. - `ChangeStrategy::compute_balance`: this trait method has an additional - `&EphemeralParameters` parameter. If the "transparent-inputs" feature is + `Option<&EphemeralBalance>` parameter. If the "transparent-inputs" feature is enabled, this can be used to specify whether the change memo should be ignored, and the amounts of additional transparent P2PKH inputs and - outputs. Passing `&EphemeralParameters::NONE` will retain the previous + outputs. Passing `None` will retain the previous behaviour (and is necessary when the "transparent-inputs" feature is not enabled). - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 162290ef9e..a3518795c5 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -23,7 +23,7 @@ use zcash_primitives::{ use crate::{ address::{Address, UnifiedAddress}, data_api::{InputSource, SimpleNoteRetention, SpendableNotes}, - fees::{sapling, ChangeError, ChangeStrategy, DustOutputPolicy, EphemeralParameters}, + fees::{sapling, ChangeError, ChangeStrategy, DustOutputPolicy}, proposal::{Proposal, ProposalError, ShieldedInputs}, wallet::WalletTransparentOutput, zip321::TransactionRequest, @@ -33,7 +33,7 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { crate::{ - fees::ChangeValue, + fees::{ChangeValue, EphemeralBalance}, proposal::{Step, StepOutput, StepOutputIndex}, zip321::Payment, }, @@ -460,12 +460,12 @@ where } #[cfg(not(feature = "transparent-inputs"))] - let ephemeral_parameters = EphemeralParameters::NONE; + let ephemeral_balance = None; #[cfg(feature = "transparent-inputs")] - let (ephemeral_parameters, tr1_balance_opt) = { + let (ephemeral_balance, tr1_balance_opt) = { if tr1_transparent_outputs.is_empty() { - (EphemeralParameters::NONE, None) + (None, None) } else { // The ephemeral input going into transaction 1 must be able to pay that // transaction's fee, as well as the TEX address payments. @@ -484,7 +484,7 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - &EphemeralParameters::new(true, Some(NonNegativeAmount::ZERO), None), + Some(&EphemeralBalance::Input(NonNegativeAmount::ZERO)), ) { Err(ChangeError::InsufficientFunds { required, .. }) => required, Ok(_) => NonNegativeAmount::ZERO, // shouldn't happen @@ -502,12 +502,12 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - &EphemeralParameters::new(true, Some(tr1_required_input_value), None), + Some(&EphemeralBalance::Input(tr1_required_input_value)), )?; assert_eq!(tr1_balance.total(), tr1_balance.fee_required()); ( - EphemeralParameters::new(false, None, Some(tr1_required_input_value)), + Some(EphemeralBalance::Output(tr1_required_input_value)), Some(tr1_balance), ) } @@ -582,7 +582,7 @@ where &orchard_outputs[..], ), &self.dust_output_policy, - &ephemeral_parameters, + ephemeral_balance.as_ref(), ); match balance { @@ -609,8 +609,8 @@ where assert_eq!( *balance.proposed_change().last().expect("nonempty"), ChangeValue::ephemeral_transparent( - ephemeral_parameters - .ephemeral_output_amount() + ephemeral_balance + .and_then(|b| b.ephemeral_output_amount()) .expect("ephemeral output is present") ) ); @@ -773,7 +773,7 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - &EphemeralParameters::NONE, + None, ); let balance = match trial_balance { @@ -791,8 +791,7 @@ where #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &self.dust_output_policy, - #[cfg(feature = "transparent-inputs")] - &EphemeralParameters::NONE, + None, )? } Err(other) => { diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 51867a9f15..218e9cc76c 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -330,61 +330,36 @@ impl Default for DustOutputPolicy { } } -/// `EphemeralParameters` can be used to specify variations on how balance -/// and fees are computed that are relevant to transactions using ephemeral -/// transparent outputs. These are only relevant when the "transparent-inputs" -/// feature is enabled, but to reduce feature-dependent boilerplate, the type -/// and the `EphemeralParameters::NONE` constant are present unconditionally. +/// `EphemeralBalance` describes the ephemeral input or output value for +/// a transaction. It is use in the computation of fees are relevant to transactions using +/// ephemeral transparent outputs. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct EphemeralParameters { - ignore_change_memo: bool, - ephemeral_input_amount: Option, - ephemeral_output_amount: Option, +pub enum EphemeralBalance { + Input(NonNegativeAmount), + Output(NonNegativeAmount), } -impl EphemeralParameters { - /// An `EphemeralParameters` indicating no use of ephemeral inputs - /// or outputs. It has: - /// - /// * `ignore_change_memo: false`, - /// * `ephemeral_input_amount: None`, - /// * `ephemeral_output_amount: None`. - pub const NONE: Self = Self { - ignore_change_memo: false, - ephemeral_input_amount: None, - ephemeral_output_amount: None, - }; - - /// Returns an `EphemeralParameters` with the following parameters: - /// - /// * `ignore_change_memo`: `true` if the change memo should be - /// ignored for the purpose of deciding whether there will be a - /// change output. - /// * `ephemeral_input_amount`: specifies that there will be an - /// additional P2PKH input of the given amount. - /// * `ephemeral_output_amount`: specifies that there will be an - /// additional P2PKH output of the given amount. - #[cfg(feature = "transparent-inputs")] - pub const fn new( - ignore_change_memo: bool, - ephemeral_input_amount: Option, - ephemeral_output_amount: Option, - ) -> Self { - Self { - ignore_change_memo, - ephemeral_input_amount, - ephemeral_output_amount, - } +impl EphemeralBalance { + pub fn is_input(&self) -> bool { + matches!(self, EphemeralBalance::Input(_)) } - pub fn ignore_change_memo(&self) -> bool { - self.ignore_change_memo + pub fn is_output(&self) -> bool { + matches!(self, EphemeralBalance::Output(_)) } + pub fn ephemeral_input_amount(&self) -> Option { - self.ephemeral_input_amount + match self { + EphemeralBalance::Input(v) => Some(*v), + EphemeralBalance::Output(_) => None, + } } + pub fn ephemeral_output_amount(&self) -> Option { - self.ephemeral_output_amount + match self { + EphemeralBalance::Input(_) => None, + EphemeralBalance::Output(v) => Some(*v), + } } } @@ -433,7 +408,7 @@ pub trait ChangeStrategy { sapling: &impl sapling::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard::BundleView, dust_output_policy: &DustOutputPolicy, - ephemeral_parameters: &EphemeralParameters, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result>; } diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 560d8b4e6d..f1b0b7319e 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -12,7 +12,7 @@ use zcash_protocol::ShieldedProtocol; use super::{ sapling as sapling_fees, ChangeError, ChangeValue, DustAction, DustOutputPolicy, - EphemeralParameters, TransactionBalance, + EphemeralBalance, TransactionBalance, }; #[cfg(feature = "orchard")] @@ -49,8 +49,7 @@ pub(crate) fn calculate_net_flows( transparent_outputs: &[impl transparent::OutputView], sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, - ephemeral_input_amount: Option, - ephemeral_output_amount: Option, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result> where E: From + From, @@ -60,13 +59,13 @@ where let t_in = transparent_inputs .iter() .map(|t_in| t_in.coin().value) - .chain(ephemeral_input_amount) + .chain(ephemeral_balance.and_then(|b| b.ephemeral_input_amount())) .sum::>() .ok_or_else(overflow)?; let t_out = transparent_outputs .iter() .map(|t_out| t_out.value()) - .chain(ephemeral_output_amount) + .chain(ephemeral_balance.and_then(|b| b.ephemeral_output_amount())) .sum::>() .ok_or_else(overflow)?; let sapling_in = sapling @@ -162,12 +161,15 @@ pub(crate) fn single_change_output_balance< fallback_change_pool: ShieldedProtocol, marginal_fee: NonNegativeAmount, grace_actions: usize, - ephemeral_parameters: &EphemeralParameters, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result> where E: From + From, { - let change_memo = change_memo.filter(|_| !ephemeral_parameters.ignore_change_memo()); + // The change memo, if any, must be attached to the change in the intermediate step that + // produces the ephemeral output, and so it should be discarded in the ultimate step; this is + // distinguished by identifying that this transaction has ephemeral inputs. + let change_memo = change_memo.filter(|_| ephemeral_balance.map_or(true, |b| !b.is_input())); let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); let underflow = || ChangeError::StrategyError(E::from(BalanceError::Underflow)); @@ -178,8 +180,7 @@ where sapling, #[cfg(feature = "orchard")] orchard, - ephemeral_parameters.ephemeral_input_amount(), - ephemeral_parameters.ephemeral_output_amount(), + ephemeral_balance, )?; #[allow(unused_variables)] @@ -213,7 +214,7 @@ where marginal_fee, grace_actions, &possible_change[..], - ephemeral_parameters, + ephemeral_balance, )?; } @@ -287,16 +288,16 @@ where .iter() .map(|i| i.serialized_size()) .chain( - ephemeral_parameters - .ephemeral_input_amount() + ephemeral_balance + .and_then(|b| b.ephemeral_input_amount()) .map(|_| transparent::InputSize::STANDARD_P2PKH), ); let transparent_output_sizes = transparent_outputs .iter() .map(|i| i.serialized_size()) .chain( - ephemeral_parameters - .ephemeral_output_amount() + ephemeral_balance + .and_then(|b| b.ephemeral_output_amount()) .map(|_| P2PKH_STANDARD_OUTPUT_SIZE), ); @@ -426,8 +427,8 @@ where }; #[cfg(feature = "transparent-inputs")] change.extend( - ephemeral_parameters - .ephemeral_output_amount() + ephemeral_balance + .and_then(|b| b.ephemeral_output_amount()) .map(ChangeValue::ephemeral_transparent), ); @@ -456,7 +457,7 @@ pub(crate) fn check_for_uneconomic_inputs( marginal_fee: NonNegativeAmount, grace_actions: usize, possible_change: &[(usize, usize, usize)], - ephemeral_parameters: &EphemeralParameters, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result<(), ChangeError> { let mut t_dust: Vec<_> = transparent_inputs .iter() @@ -504,10 +505,8 @@ pub(crate) fn check_for_uneconomic_inputs( } let (t_inputs_len, t_outputs_len) = ( - transparent_inputs.len() - + usize::from(ephemeral_parameters.ephemeral_input_amount().is_some()), - transparent_outputs.len() - + usize::from(ephemeral_parameters.ephemeral_output_amount().is_some()), + transparent_inputs.len() + usize::from(ephemeral_balance.map_or(false, |b| b.is_input())), + transparent_outputs.len() + usize::from(ephemeral_balance.map_or(false, |b| b.is_output())), ); let (s_inputs_len, s_outputs_len) = (sapling.inputs().len(), sapling.outputs().len()); #[cfg(feature = "orchard")] diff --git a/zcash_client_backend/src/fees/fixed.rs b/zcash_client_backend/src/fees/fixed.rs index e42e4527db..933b008207 100644 --- a/zcash_client_backend/src/fees/fixed.rs +++ b/zcash_client_backend/src/fees/fixed.rs @@ -13,7 +13,7 @@ use crate::ShieldedProtocol; use super::{ common::single_change_output_balance, sapling as sapling_fees, ChangeError, ChangeStrategy, - DustOutputPolicy, EphemeralParameters, TransactionBalance, + DustOutputPolicy, EphemeralBalance, TransactionBalance, }; #[cfg(feature = "orchard")] @@ -63,7 +63,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - ephemeral_parameters: &EphemeralParameters, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result> { single_change_output_balance( params, @@ -80,7 +80,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fallback_change_pool, NonNegativeAmount::ZERO, 0, - ephemeral_parameters, + ephemeral_balance, ) } } @@ -100,7 +100,7 @@ mod tests { data_api::wallet::input_selection::SaplingPayment, fees::{ tests::{TestSaplingInput, TestTransparentInput}, - ChangeError, ChangeStrategy, ChangeValue, DustOutputPolicy, EphemeralParameters, + ChangeError, ChangeStrategy, ChangeValue, DustOutputPolicy, }, ShieldedProtocol, }; @@ -136,7 +136,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -182,7 +182,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); assert_matches!( diff --git a/zcash_client_backend/src/fees/standard.rs b/zcash_client_backend/src/fees/standard.rs index 8c49e33b35..5c08c7b1af 100644 --- a/zcash_client_backend/src/fees/standard.rs +++ b/zcash_client_backend/src/fees/standard.rs @@ -18,7 +18,7 @@ use crate::ShieldedProtocol; use super::{ fixed, sapling as sapling_fees, zip317, ChangeError, ChangeStrategy, DustOutputPolicy, - EphemeralParameters, TransactionBalance, + EphemeralBalance, TransactionBalance, }; #[cfg(feature = "orchard")] @@ -68,7 +68,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - ephemeral_parameters: &EphemeralParameters, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result> { #[allow(deprecated)] match self.fee_rule() { @@ -86,7 +86,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - ephemeral_parameters, + ephemeral_balance, ) .map_err(|e| e.map(Zip317FeeError::Balance)), StandardFeeRule::Zip313 => fixed::SingleOutputChangeStrategy::new( @@ -103,7 +103,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - ephemeral_parameters, + ephemeral_balance, ) .map_err(|e| e.map(Zip317FeeError::Balance)), StandardFeeRule::Zip317 => zip317::SingleOutputChangeStrategy::new( @@ -120,7 +120,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { #[cfg(feature = "orchard")] orchard, dust_output_policy, - ephemeral_parameters, + ephemeral_balance, ), } } diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index a054f9b41a..e2c6152c8a 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -17,7 +17,7 @@ use crate::ShieldedProtocol; use super::{ common::single_change_output_balance, sapling as sapling_fees, ChangeError, ChangeStrategy, - DustOutputPolicy, EphemeralParameters, TransactionBalance, + DustOutputPolicy, EphemeralBalance, TransactionBalance, }; #[cfg(feature = "orchard")] @@ -67,7 +67,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - ephemeral_parameters: &EphemeralParameters, + ephemeral_parameters: Option<&EphemeralBalance>, ) -> Result> { single_change_output_balance( params, @@ -108,7 +108,6 @@ mod tests { fees::{ tests::{TestSaplingInput, TestTransparentInput}, ChangeError, ChangeStrategy, ChangeValue, DustAction, DustOutputPolicy, - EphemeralParameters, }, ShieldedProtocol, }; @@ -148,7 +147,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -192,7 +191,7 @@ mod tests { ))][..], ), &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -245,7 +244,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, dust_output_policy, - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -289,7 +288,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -333,7 +332,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -383,7 +382,7 @@ mod tests { DustAction::AllowDustChange, Some(NonNegativeAmount::const_from_u64(1000)), ), - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -444,7 +443,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, dust_output_policy, - &EphemeralParameters::NONE, + None, ); assert_matches!( @@ -495,7 +494,7 @@ mod tests { #[cfg(feature = "orchard")] &orchard_fees::EmptyBundleView, &DustOutputPolicy::default(), - &EphemeralParameters::NONE, + None, ); // We will get an error here, because the dust input isn't free to add From dbb5eeb704474b4f41a078ba8a5aca049af65e66 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 16 Jul 2024 15:43:21 -0600 Subject: [PATCH 54/56] Fix a potential crash related to varying behavior between change strategies. It was possible for `GreedyInputSelector` to crash if the change strategy being used for input selection were to place a ZIP 320 ephemeral output anywhere but as the last element in the returned change values. This has been replaced by an error; the error will also be returned if the change strategy returns more than one ephemeral output in the change values. --- .../src/data_api/wallet/input_selection.rs | 43 +++++++++++++------ zcash_client_backend/src/proposal.rs | 9 ++++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index a3518795c5..7ffc873829 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -33,7 +33,7 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { crate::{ - fees::{ChangeValue, EphemeralBalance}, + fees::EphemeralBalance, proposal::{Step, StepOutput, StepOutputIndex}, zip321::Payment, }, @@ -328,6 +328,11 @@ pub struct GreedyInputSelector { impl GreedyInputSelector { /// Constructs a new greedy input selector that uses the provided change strategy to determine /// change values and fee amounts. + /// + /// The [`ChangeStrategy`] provided must produce exactly one ephemeral change value when + /// computing a transaction balance if an [`EphemeralBalance::Output`] value is provided for + /// its ephemeral balance, or the resulting [`GreedyInputSelector`] will return an error when + /// attempting to construct a transaction proposal that requires such an output. pub fn new(change_strategy: ChangeT, dust_output_policy: DustOutputPolicy) -> Self { GreedyInputSelector { change_strategy, @@ -605,19 +610,29 @@ where // a single additional ephemeral output to the transparent pool. // * `tr1` spends from that ephemeral output to each TEX output. - // The ephemeral output should always be at the last change index. - assert_eq!( - *balance.proposed_change().last().expect("nonempty"), - ChangeValue::ephemeral_transparent( - ephemeral_balance - .and_then(|b| b.ephemeral_output_amount()) - .expect("ephemeral output is present") - ) - ); - let ephemeral_stepoutput = StepOutput::new( - 0, - StepOutputIndex::Change(balance.proposed_change().len() - 1), - ); + // Find exactly one ephemeral change output. + let ephemeral_outputs = balance + .proposed_change() + .iter() + .enumerate() + .filter(|(_, c)| c.is_ephemeral()) + .collect::>(); + + let ephemeral_value = ephemeral_balance + .and_then(|b| b.ephemeral_output_amount()) + .expect("ephemeral output balance exists (constructed above)"); + + let ephemeral_output_index = match &ephemeral_outputs[..] { + [(i, change_value)] if change_value.value() == ephemeral_value => { + Ok(*i) + } + _ => Err(InputSelectorError::Proposal( + ProposalError::EphemeralOutputsInvalid, + )), + }?; + + let ephemeral_stepoutput = + StepOutput::new(0, StepOutputIndex::Change(ephemeral_output_index)); let tr0 = TransactionRequest::from_indexed( transaction_request diff --git a/zcash_client_backend/src/proposal.rs b/zcash_client_backend/src/proposal.rs index e67e3e676c..e88dbcf8d5 100644 --- a/zcash_client_backend/src/proposal.rs +++ b/zcash_client_backend/src/proposal.rs @@ -55,6 +55,10 @@ pub enum ProposalError { /// The proposal included a payment to a TEX address and a spend from a shielded input in the same step. #[cfg(feature = "transparent-inputs")] PaysTexFromShielded, + /// The change strategy provided to input selection failed to correctly generate an ephemeral + /// change output when needed for sending to a TEX address. + #[cfg(feature = "transparent-inputs")] + EphemeralOutputsInvalid, } impl Display for ProposalError { @@ -114,6 +118,11 @@ impl Display for ProposalError { f, "The proposal included a payment to a TEX address and a spend from a shielded input in the same step.", ), + #[cfg(feature = "transparent-inputs")] + ProposalError::EphemeralOutputsInvalid => write!( + f, + "The change strategy provided to input selection failed to correctly generate an ephemeral change output when needed for sending to a TEX address." + ), } } } From 24b6d50d778df619d393c9b411ab440984f6a0da Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 16 Jul 2024 17:24:27 -0600 Subject: [PATCH 55/56] Apply suggestions from code review Co-authored-by: Jack Grigg --- zcash_client_backend/src/data_api.rs | 38 ++++++++++--------- .../src/data_api/wallet/input_selection.rs | 3 ++ zcash_client_backend/src/fees.rs | 24 ++++++------ zcash_client_backend/src/fees/zip317.rs | 4 +- .../src/wallet/transparent/ephemeral.rs | 2 +- zcash_primitives/src/legacy/keys.rs | 9 +++-- 6 files changed, 43 insertions(+), 37 deletions(-) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index caa07e8ace..9c8e5a9326 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -931,14 +931,16 @@ pub trait WalletRead { /// /// This is equivalent to (but may be implemented more efficiently than): /// ```compile_fail - /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { - /// return Ok(result.clone()); - /// } - /// Ok(self - /// .get_known_ephemeral_addresses(account, None)? - /// .into_iter() - /// .find(|(known_addr, _)| known_addr == address) - /// .map(|(_, metadata)| metadata)) + /// Ok( + /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + /// result.clone() + /// } else { + /// self.get_known_ephemeral_addresses(account, None)? + /// .into_iter() + /// .find(|(known_addr, _)| known_addr == address) + /// .map(|(_, metadata)| metadata) + /// }, + /// ) /// ``` /// /// Returns `Ok(None)` if the address is not recognized, or we do not have metadata for it. @@ -950,14 +952,16 @@ pub trait WalletRead { address: &TransparentAddress, ) -> Result, Self::Error> { // This should be overridden. - if let Some(result) = self.get_transparent_receivers(account)?.get(address) { - return Ok(result.clone()); - } - Ok(self - .get_known_ephemeral_addresses(account, None)? - .into_iter() - .find(|(known_addr, _)| known_addr == address) - .map(|(_, metadata)| metadata)) + Ok( + if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + result.clone() + } else { + self.get_known_ephemeral_addresses(account, None)? + .into_iter() + .find(|(known_addr, _)| known_addr == address) + .map(|(_, metadata)| metadata) + }, + ) } /// Returns a vector of ephemeral transparent addresses associated with the given @@ -1001,7 +1005,7 @@ pub trait WalletRead { Ok(vec![]) } - /// If a given transparent address has been reserved, i.e. would be included in + /// If a given ephemeral address might have been reserved, i.e. would be included in /// the map returned by `get_known_ephemeral_addresses(account_id, false)` for any /// of the wallet's accounts, then return `Ok(Some(account_id))`. Otherwise return /// `Ok(None)`. diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 7ffc873829..ad9b56a03a 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -384,6 +384,9 @@ where let mut tr1_payments = vec![]; #[cfg(feature = "transparent-inputs")] let mut tr1_payment_pools = BTreeMap::new(); + // This balance value is just used for overflow checking; the actual value of ephemeral + // outputs will be computed from the constructed `tr1_transparent_outputs` value + // constructed below. #[cfg(feature = "transparent-inputs")] let mut total_ephemeral = NonNegativeAmount::ZERO; diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 218e9cc76c..fea9424243 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -330,9 +330,11 @@ impl Default for DustOutputPolicy { } } -/// `EphemeralBalance` describes the ephemeral input or output value for -/// a transaction. It is use in the computation of fees are relevant to transactions using -/// ephemeral transparent outputs. +/// `EphemeralBalance` describes the ephemeral input or output value for a transaction. It is used +/// in fee computation for series of transactions that use an ephemeral transparent output in an +/// intermediate step, such as when sending from a shielded pool to a [ZIP 320] "TEX" address. +/// +/// [ZIP 320]: https://zips.z.cash/zip-0320 #[derive(Clone, Debug, PartialEq, Eq)] pub enum EphemeralBalance { Input(NonNegativeAmount), @@ -388,16 +390,12 @@ pub trait ChangeStrategy { /// inputs from most to least preferred to spend within each pool, so that the most /// preferred ones are less likely to be indicated to remove. /// - #[cfg_attr( - feature = "transparent-inputs", - doc = "`ephemeral_parameters` can be used to specify variations on how balance - and fees are computed that are relevant to transactions using ephemeral - transparent outputs; see [`EphemeralParameters::new`]." - )] - #[cfg_attr( - not(feature = "transparent-inputs"), - doc = "`ephemeral_parameters` should be set to `&EphemeralParameters::NONE`." - )] + /// - `ephemeral_balance`: if the transaction is to be constructed with either an + /// ephemeral transparent input or an ephemeral transparent output this argument + /// may be used to provide the value of that input or output. The value of this + /// output should be `None` in the case that there are no such items. + /// + /// [ZIP 320]: https://zips.z.cash/zip-0320 #[allow(clippy::too_many_arguments)] fn compute_balance( &self, diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index e2c6152c8a..c5cd7d499a 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -67,7 +67,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, dust_output_policy: &DustOutputPolicy, - ephemeral_parameters: Option<&EphemeralBalance>, + ephemeral_balance: Option<&EphemeralBalance>, ) -> Result> { single_change_output_balance( params, @@ -84,7 +84,7 @@ impl ChangeStrategy for SingleOutputChangeStrategy { self.fallback_change_pool, self.fee_rule.marginal_fee(), self.fee_rule.grace_actions(), - ephemeral_parameters, + ephemeral_balance, ) } } diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index a700168857..461261de6e 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -187,7 +187,7 @@ pub(crate) fn find_index_for_ephemeral_address_str( Ok(conn .query_row( "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id AND address = :address", + WHERE account_id = :account_id AND address = :address", named_params![":account_id": account_id.0, ":address": &address_str], |row| row.get::<_, u32>(0), ) diff --git a/zcash_primitives/src/legacy/keys.rs b/zcash_primitives/src/legacy/keys.rs index c7b7c610a2..25a20eed1a 100644 --- a/zcash_primitives/src/legacy/keys.rs +++ b/zcash_primitives/src/legacy/keys.rs @@ -21,10 +21,11 @@ use super::TransparentAddress; pub struct TransparentKeyScope(u32); impl TransparentKeyScope { - /// Returns an arbitrary custom `TransparentKeyScope`. This should be used - /// with care: funds associated with keys derived under a custom scope may - /// not be recoverable if the wallet seed is restored in another wallet. - /// It is usually preferable to use standardized key scopes. + /// Returns an arbitrary custom `TransparentKeyScope`. + /// + /// This should be used with care: funds associated with keys derived under a custom + /// scope may not be recoverable if the wallet seed is restored in another wallet. It + /// is usually preferable to use standardized key scopes. pub const fn custom(i: u32) -> Option { if i < (1 << 31) { Some(TransparentKeyScope(i)) From f8bedd89e73718888b4ef86c5ab53f406085b72a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 17 Jul 2024 08:09:04 -0600 Subject: [PATCH 56/56] Make ephemeral_addresses.address unique This also provides additional documentation for why it's necessary to store ephemeral_addresses table entries at indicies that do not correspond to valid addresses. --- zcash_client_sqlite/src/wallet/db.rs | 10 ++++------ zcash_client_sqlite/src/wallet/init.rs | 1 - .../init/migrations/ephemeral_addresses.rs | 7 +++---- .../src/wallet/transparent/ephemeral.rs | 18 +++++++++++------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 64b45acc0f..65ce3b148e 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -106,8 +106,8 @@ CREATE INDEX "addresses_accounts" ON "addresses" ( /// /// All but the last `GAP_LIMIT` addresses are defined to be "reserved" addresses. Since the next /// index to reserve is determined by dead reckoning from the last stored address, we use dummy -/// entries after the maximum valid index in order to allow the last `GAP_LIMIT` addresses at the -/// end of the index range to be used. +/// entries having `NULL` for the value of the `address` column after the maximum valid index in +/// order to allow the last `GAP_LIMIT` addresses at the end of the index range to be used. /// /// Note that the fact that `used_in_tx` references a specific transaction is just a debugging aid. /// The same is mostly true of `seen_in_tx`, but we also take into account whether the referenced @@ -116,6 +116,7 @@ pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" CREATE TABLE ephemeral_addresses ( account_id INTEGER NOT NULL, address_index INTEGER NOT NULL, + -- nullability of this column is controlled by the index_range_and_address_nullity check address TEXT, used_in_tx INTEGER, seen_in_tx INTEGER, @@ -123,6 +124,7 @@ CREATE TABLE ephemeral_addresses ( FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), PRIMARY KEY (account_id, address_index), + CONSTRAINT ephemeral_addr_uniq UNIQUE (address), CONSTRAINT used_implies_seen CHECK ( used_in_tx IS NULL OR seen_in_tx IS NOT NULL ), @@ -135,10 +137,6 @@ CREATE TABLE ephemeral_addresses ( // libsqlite3-sys requires at least version 3.14.0. // "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. const_assert_eq!(GAP_LIMIT, 20); -pub(super) const INDEX_EPHEMERAL_ADDRESSES_ADDRESS: &str = r#" -CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( - address ASC -)"#; /// Stores information about every block that the wallet has scanned. /// diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index b14ecd9e5b..7c5a27f73e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -422,7 +422,6 @@ mod tests { db::INDEX_ACCOUNTS_UIVK, db::INDEX_HD_ACCOUNT, db::INDEX_ADDRESSES_ACCOUNTS, - db::INDEX_EPHEMERAL_ADDRESSES_ADDRESS, db::INDEX_NF_MAP_LOCATOR_IDX, db::INDEX_ORCHARD_RECEIVED_NOTES_ACCOUNT, db::INDEX_ORCHARD_RECEIVED_NOTES_TX, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 7f68792b33..d9dffaf89b 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -45,6 +45,7 @@ impl RusqliteMigration for Migration

{ "CREATE TABLE ephemeral_addresses ( account_id INTEGER NOT NULL, address_index INTEGER NOT NULL, + -- nullability of this column is controlled by the index_range_and_address_nullity check address TEXT, used_in_tx INTEGER, seen_in_tx INTEGER, @@ -52,6 +53,7 @@ impl RusqliteMigration for Migration

{ FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), PRIMARY KEY (account_id, address_index), + CONSTRAINT ephemeral_addr_uniq UNIQUE (address), CONSTRAINT used_implies_seen CHECK ( used_in_tx IS NULL OR seen_in_tx IS NOT NULL ), @@ -59,10 +61,7 @@ impl RusqliteMigration for Migration

{ (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND seen_in_tx IS NULL) ) - ) WITHOUT ROWID; - CREATE INDEX ephemeral_addresses_address ON ephemeral_addresses ( - address ASC - );", + ) WITHOUT ROWID;" )?; // Make sure that at least `GAP_LIMIT` ephemeral transparent addresses are diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 461261de6e..017ad002f9 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -282,14 +282,18 @@ fn reserve_until( )?; for raw_index in range_to_store { - let address_str_opt = match NonHardenedChildIndex::from_index(raw_index) { - Some(address_index) => Some( + // The range to store may contain indicies that are out of the valid range of non hardened + // child indices; we still store explicit rows in the ephemeral_addresses table for these + // so that it's possible to find the first unused address using dead reckoning with the gap + // limit. + let address_str_opt = NonHardenedChildIndex::from_index(raw_index) + .map(|address_index| { ephemeral_ivk - .derive_ephemeral_address(address_index)? - .encode(params), - ), - None => None, - }; + .derive_ephemeral_address(address_index) + .map(|addr| addr.encode(params)) + }) + .transpose()?; + stmt_insert_ephemeral_address.execute(named_params![ ":account_id": account_id.0, ":address_index": raw_index,