From 9b1193520b4018c6908c4d390917aba18f7664e5 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Tue, 15 Oct 2024 10:58:40 +0200 Subject: [PATCH 1/8] refactor(evmlib): break up `pay_for_quotes` into smaller functions --- evmlib/src/common.rs | 1 + evmlib/src/contract/data_payments/mod.rs | 49 ++++++++++++++++-------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/evmlib/src/common.rs b/evmlib/src/common.rs index af210f9285..88d8855245 100644 --- a/evmlib/src/common.rs +++ b/evmlib/src/common.rs @@ -16,3 +16,4 @@ pub type QuoteHash = Hash; pub type Amount = U256; pub type QuotePayment = (QuoteHash, Address, Amount); pub type EthereumWallet = alloy::network::EthereumWallet; +pub type Calldata = alloy::primitives::Bytes; diff --git a/evmlib/src/contract/data_payments/mod.rs b/evmlib/src/contract/data_payments/mod.rs index 352f294581..79f90f9b04 100644 --- a/evmlib/src/contract/data_payments/mod.rs +++ b/evmlib/src/contract/data_payments/mod.rs @@ -9,9 +9,10 @@ pub mod error; use crate::common; -use crate::common::{Address, TxHash}; +use crate::common::{Address, Calldata, TxHash}; use crate::contract::data_payments::error::Error; use crate::contract::data_payments::DataPaymentsContract::DataPaymentsContractInstance; +use alloy::network::TransactionBuilder; use alloy::providers::{Network, Provider}; use alloy::sol; use alloy::transports::Transport; @@ -64,6 +65,33 @@ where &self, data_payments: I, ) -> Result { + let (calldata, to) = self.pay_for_quotes_calldata(data_payments)?; + + let transaction_request = self + .contract + .provider() + .transaction_request() + .with_to(to) + .with_input(calldata); + + let tx_hash = self + .contract + .provider() + .send_transaction(transaction_request) + .await? + .watch() + .await?; + + Ok(tx_hash) + } + + /// Pay for quotes. + /// Input: (quote_id, reward_address, amount). + /// Returns the transaction calldata. + pub fn pay_for_quotes_calldata>( + &self, + data_payments: I, + ) -> Result<(Calldata, Address), Error> { let data_payments: Vec = data_payments .into_iter() .map(|(hash, addr, amount)| DataPayments::DataPayment { @@ -74,26 +102,15 @@ where .collect(); if data_payments.len() > MAX_TRANSFERS_PER_TRANSACTION { - error!( - "Data payments limit exceeded: {} > {}", - data_payments.len(), - MAX_TRANSFERS_PER_TRANSACTION - ); return Err(Error::TransferLimitExceeded); } - let tx_hash = self + let calldata = self .contract .submitDataPayments(data_payments) - .send() - .await - .inspect_err(|e| error!("Failed to submit data payments during pay_for_quotes: {e:?}"))? - .watch() - .await - .inspect_err(|e| { - error!("Failed to watch data payments during pay_for_quotes: {e:?}") - })?; + .calldata() + .to_owned(); - Ok(tx_hash) + Ok((calldata, *self.contract.address())) } } From 325410a2db66ab44dca1dd902e543c1fd4a952d5 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 16 Oct 2024 09:35:30 +0200 Subject: [PATCH 2/8] refactor(evmlib): add `approve_calldata` and `transfer_calldata` --- evmlib/src/contract/network_token.rs | 65 ++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/evmlib/src/contract/network_token.rs b/evmlib/src/contract/network_token.rs index 4c8112e869..a6fad9243d 100644 --- a/evmlib/src/contract/network_token.rs +++ b/evmlib/src/contract/network_token.rs @@ -6,8 +6,9 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::common::{Address, TxHash, U256}; +use crate::common::{Address, Calldata, TxHash, U256}; use crate::contract::network_token::NetworkTokenContract::NetworkTokenContractInstance; +use alloy::network::TransactionBuilder; use alloy::providers::{Network, Provider}; use alloy::sol; use alloy::transports::{RpcError, Transport, TransportErrorKind}; @@ -77,15 +78,30 @@ where /// Approve spender to spend a raw amount of tokens. pub async fn approve(&self, spender: Address, value: U256) -> Result { debug!("Approving spender to spend raw amt of tokens: {value}"); - let call = self.contract.approve(spender, value); - let pending_tx_builder = call.send().await.inspect_err(|err| { - error!( + let (calldata, to) = self.approve_calldata(spender, value); + + let transaction_request = self + .contract + .provider() + .transaction_request() + .with_to(to) + .with_input(calldata); + + let pending_tx_builder = self + .contract + .provider() + .send_transaction(transaction_request) + .await + .inspect_err(|err| { + error!( "Error approving spender {spender:?} to spend raw amt of tokens {value}: {err:?}" ) - })?; + })?; let pending_tx_hash = *pending_tx_builder.tx_hash(); + debug!("The approval from sender {spender:?} is pending with tx_hash: {pending_tx_hash:?}",); + let tx_hash = pending_tx_builder.watch().await.inspect_err(|err| { error!("Error watching approve tx with hash {pending_tx_hash:?}: {err:?}") })?; @@ -95,13 +111,33 @@ where Ok(tx_hash) } + /// Approve spender to spend a raw amount of tokens. + /// Returns the transaction calldata. + pub fn approve_calldata(&self, spender: Address, value: U256) -> (Calldata, Address) { + let calldata = self.contract.approve(spender, value).calldata().to_owned(); + (calldata, *self.contract.address()) + } + /// Transfer a raw amount of tokens. pub async fn transfer(&self, receiver: Address, amount: U256) -> Result { debug!("Transferring raw amt of tokens: {amount} to {receiver:?}"); - let call = self.contract.transfer(receiver, amount); - let pending_tx_builder = call.send().await.inspect_err(|err| { - error!("Error transferring raw amt of tokens to {receiver:?}: {err:?}") - })?; + let (calldata, to) = self.transfer_calldata(receiver, amount); + + let transaction_request = self + .contract + .provider() + .transaction_request() + .with_to(to) + .with_input(calldata); + + let pending_tx_builder = self + .contract + .provider() + .send_transaction(transaction_request) + .await + .inspect_err(|err| { + error!("Error transferring raw amt of tokens to {receiver:?}: {err:?}") + })?; let pending_tx_hash = *pending_tx_builder.tx_hash(); debug!( @@ -115,4 +151,15 @@ where Ok(tx_hash) } + + /// Transfer a raw amount of tokens. + /// Returns the transaction calldata. + pub fn transfer_calldata(&self, receiver: Address, amount: U256) -> (Calldata, Address) { + let calldata = self + .contract + .transfer(receiver, amount) + .calldata() + .to_owned(); + (calldata, *self.contract.address()) + } } From 7be689484df65deff6084d7c4f8e12f400a10a03 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 16 Oct 2024 10:27:35 +0200 Subject: [PATCH 3/8] feat(evmlib): add raw transaction functions to support external signers --- evmlib/Cargo.toml | 1 + evmlib/src/external_signer.rs | 98 +++++++++++++++++++++++++++++++++++ evmlib/src/lib.rs | 2 + evmlib/src/utils.rs | 23 ++++++++ evmlib/src/wallet.rs | 18 +------ 5 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 evmlib/src/external_signer.rs diff --git a/evmlib/Cargo.toml b/evmlib/Cargo.toml index 9a26778c36..75bf4dc7b1 100644 --- a/evmlib/Cargo.toml +++ b/evmlib/Cargo.toml @@ -11,6 +11,7 @@ version = "0.1.0" [features] wasm-bindgen = ["alloy/wasm-bindgen"] local = [] +external_signer = [] [dependencies] alloy = { version = "0.4.2", default-features = false, features = ["std", "reqwest-rustls-tls", "provider-anvil-node", "sol-types", "json", "signers", "contract", "signer-local", "network"] } diff --git a/evmlib/src/external_signer.rs b/evmlib/src/external_signer.rs new file mode 100644 index 0000000000..7dfef2a480 --- /dev/null +++ b/evmlib/src/external_signer.rs @@ -0,0 +1,98 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::common::{Address, Amount, Calldata, QuoteHash, QuotePayment, U256}; +use crate::contract::data_payments::{DataPaymentsHandler, MAX_TRANSFERS_PER_TRANSACTION}; +use crate::contract::network_token::NetworkToken; +use crate::contract::{data_payments, network_token}; +use crate::utils::http_provider; +use crate::Network; +use std::collections::HashMap; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Network token contract error: {0}")] + NetworkTokenContract(#[from] network_token::Error), + #[error("Chunk payments contract error: {0}")] + ChunkPaymentsContract(#[from] data_payments::error::Error), +} + +#[allow(dead_code)] +/// Approve an address / smart contract to spend this wallet's payment tokens. +/// +/// Returns the transaction calldata (input, to). +pub fn approve_to_spend_tokens_calldata( + network: &Network, + spender: Address, + value: U256, +) -> (Calldata, Address) { + let provider = http_provider(network.rpc_url().clone()); + let network_token = NetworkToken::new(*network.payment_token_address(), provider); + network_token.approve_calldata(spender, value) +} + +#[allow(dead_code)] +/// Transfer payment tokens from the supplied wallet to an address. +/// +/// Returns the transaction calldata (input, to). +pub fn transfer_tokens_calldata( + network: &Network, + receiver: Address, + amount: U256, +) -> (Calldata, Address) { + let provider = http_provider(network.rpc_url().clone()); + let network_token = NetworkToken::new(*network.payment_token_address(), provider); + network_token.transfer_calldata(receiver, amount) +} + +#[allow(dead_code)] +pub struct PayForQuotesCalldataReturnType { + batched_calldata_map: HashMap>, + to: Address, + approve_spender: Address, + approve_amount: Amount, +} + +#[allow(dead_code)] +/// Use this wallet to pay for chunks in batched transfer transactions. +/// If the amount of transfers is more than one transaction can contain, the transfers will be split up over multiple transactions. +/// +/// Returns PayForQuotesCalldataReturnType, containing calldata of the transaction batches along with the approval details for the spender. +pub fn pay_for_quotes_calldata>( + network: &Network, + payments: T, +) -> Result { + let payments: Vec<_> = payments.into_iter().collect(); + + let total_amount = payments.iter().map(|(_, _, amount)| amount).sum(); + + let approve_spender = *network.data_payments_address(); + let approve_amount = total_amount; + + let provider = http_provider(network.rpc_url().clone()); + let data_payments = DataPaymentsHandler::new(*network.data_payments_address(), provider); + + // Divide transfers over multiple transactions if they exceed the max per transaction. + let chunks = payments.chunks(MAX_TRANSFERS_PER_TRANSACTION); + + let mut calldata_map: HashMap> = HashMap::new(); + + for batch in chunks { + let quote_payments = batch.to_vec(); + let (calldata, _) = data_payments.pay_for_quotes_calldata(quote_payments.clone())?; + let quote_hashes = quote_payments.into_iter().map(|(qh, _, _)| qh).collect(); + calldata_map.insert(calldata, quote_hashes); + } + + Ok(PayForQuotesCalldataReturnType { + batched_calldata_map: calldata_map, + to: *data_payments.contract.address(), + approve_spender, + approve_amount, + }) +} diff --git a/evmlib/src/lib.rs b/evmlib/src/lib.rs index 0093aeac0e..3b200c7389 100644 --- a/evmlib/src/lib.rs +++ b/evmlib/src/lib.rs @@ -22,6 +22,8 @@ pub mod common; pub mod contract; pub mod cryptography; pub(crate) mod event; +#[cfg(feature = "external_signer")] +pub mod external_signer; pub mod testnet; pub mod transaction; pub mod utils; diff --git a/evmlib/src/utils.rs b/evmlib/src/utils.rs index e6f657938b..13c9b675c5 100644 --- a/evmlib/src/utils.rs +++ b/evmlib/src/utils.rs @@ -10,6 +10,12 @@ use crate::common::{Address, Hash}; use crate::{CustomNetwork, Network}; +use alloy::network::Ethereum; +use alloy::providers::fillers::{ + BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, +}; +use alloy::providers::{Identity, ProviderBuilder, ReqwestProvider}; +use alloy::transports::http::{reqwest, Client, Http}; use dirs_next::data_dir; use rand::Rng; use std::env; @@ -143,3 +149,20 @@ fn local_evm_network_from_csv() -> Result { } } } + +#[allow(clippy::type_complexity)] +pub(crate) fn http_provider( + rpc_url: reqwest::Url, +) -> FillProvider< + JoinFill< + Identity, + JoinFill>>, + >, + ReqwestProvider, + Http, + Ethereum, +> { + ProviderBuilder::new() + .with_recommended_fillers() + .on_http(rpc_url) +} diff --git a/evmlib/src/wallet.rs b/evmlib/src/wallet.rs index e758e58eee..8d79908efc 100644 --- a/evmlib/src/wallet.rs +++ b/evmlib/src/wallet.rs @@ -10,6 +10,7 @@ use crate::common::{Address, QuoteHash, QuotePayment, TxHash, U256}; use crate::contract::data_payments::{DataPaymentsHandler, MAX_TRANSFERS_PER_TRANSACTION}; use crate::contract::network_token::NetworkToken; use crate::contract::{data_payments, network_token}; +use crate::utils::http_provider; use crate::Network; use alloy::network::{Ethereum, EthereumWallet, NetworkWallet, TransactionBuilder}; use alloy::providers::fillers::{ @@ -130,23 +131,6 @@ fn from_private_key(private_key: &str) -> Result { // TODO(optimization): Find a way to reuse/persist contracts and/or a provider without the wallet nonce going out of sync -#[allow(clippy::type_complexity)] -fn http_provider( - rpc_url: reqwest::Url, -) -> FillProvider< - JoinFill< - Identity, - JoinFill>>, - >, - ReqwestProvider, - Http, - Ethereum, -> { - ProviderBuilder::new() - .with_recommended_fillers() - .on_http(rpc_url) -} - #[allow(clippy::type_complexity)] fn http_provider_with_wallet( rpc_url: reqwest::Url, From 3c88ef769a752acb9ea8ccaed01862f6d084e689 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 16 Oct 2024 19:24:25 +0200 Subject: [PATCH 4/8] feat(autonomi): working external signer put --- Cargo.lock | 1 + autonomi/Cargo.toml | 2 + autonomi/src/client/external_signer.rs | 97 +++++++++++++++++++++++ autonomi/src/client/mod.rs | 2 + autonomi/src/client/utils.rs | 2 +- autonomi/tests/put.rs | 103 +++++++++++++++++++++++++ evmlib/Cargo.toml | 2 +- evmlib/src/external_signer.rs | 12 +-- evmlib/src/lib.rs | 2 +- evmlib/src/wallet.rs | 20 +++-- sn_evm/Cargo.toml | 1 + sn_evm/src/lib.rs | 2 + 12 files changed, 231 insertions(+), 15 deletions(-) create mode 100644 autonomi/src/client/external_signer.rs diff --git a/Cargo.lock b/Cargo.lock index 5d397e2a98..e76033a306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1071,6 +1071,7 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" name = "autonomi" version = "0.1.2" dependencies = [ + "alloy", "bip39", "blsttc", "bytes", diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml index c7ecf07338..f035137930 100644 --- a/autonomi/Cargo.toml +++ b/autonomi/Cargo.toml @@ -21,6 +21,7 @@ fs = ["tokio/fs", "data"] local = ["sn_networking/local", "test_utils/local", "sn_evm/local"] registers = ["data"] loud = [] +external-signer = ["sn_evm/external-signer", "data"] [dependencies] bip39 = "2.0.0" @@ -53,6 +54,7 @@ wasm-bindgen-futures = "0.4.43" serde-wasm-bindgen = "0.6.5" [dev-dependencies] +alloy = { version = "0.4.2", default-features = false, features = ["std", "reqwest-rustls-tls", "provider-anvil-node", "sol-types", "json", "signers", "contract", "signer-local", "network"] } eyre = "0.6.5" sha2 = "0.10.6" sn_logging = { path = "../sn_logging", version = "0.2.33" } diff --git a/autonomi/src/client/external_signer.rs b/autonomi/src/client/external_signer.rs new file mode 100644 index 0000000000..fde41c6f4d --- /dev/null +++ b/autonomi/src/client/external_signer.rs @@ -0,0 +1,97 @@ +use crate::client::data::{DataAddr, PutError}; +use crate::client::utils::extract_quote_payments; +use crate::self_encryption::encrypt; +use crate::Client; +use bytes::Bytes; +use sn_evm::{ProofOfPayment, QuotePayment}; +use sn_protocol::storage::Chunk; +use std::collections::HashMap; +use xor_name::XorName; + +#[allow(unused_imports)] +pub use sn_evm::external_signer::*; +use sn_networking::PayeeQuote; + +impl Client { + /// Upload a piece of data to the network. This data will be self-encrypted. + /// Payment will not be done automatically as opposed to the regular `data_put`, so the proof of payment has to be provided. + /// Returns the Data Address at which the data was stored. + pub async fn data_put_with_proof_of_payment( + &self, + data: Bytes, + proof: HashMap, + ) -> Result { + let (data_map_chunk, chunks, _) = encrypt_data(data)?; + self.upload_data_map(&proof, &data_map_chunk).await?; + self.upload_chunks(&chunks, &proof).await?; + Ok(*data_map_chunk.address().xorname()) + } + + /// Get quotes for certain content addresses. + /// Returns a cost map, data payments to be executed and a list of free chunks. + pub async fn get_quotes_for_content_addrs( + &self, + content_addrs: impl Iterator, + ) -> Result< + ( + HashMap, + Vec, + Vec, + ), + PutError, + > { + let cost_map = self.get_store_quotes(content_addrs).await?; + let (quote_payments, free_chunks) = extract_quote_payments(&cost_map); + Ok((cost_map, quote_payments, free_chunks)) + } + + async fn upload_data_map( + &self, + payment_proofs: &HashMap, + data_map_chunk: &Chunk, + ) -> Result<(), PutError> { + let map_xor_name = data_map_chunk.name(); + + if let Some(proof) = payment_proofs.get(map_xor_name) { + debug!("Uploading data map chunk: {map_xor_name:?}"); + self.chunk_upload_with_payment(data_map_chunk.clone(), proof.clone()) + .await + .inspect_err(|err| error!("Error uploading data map chunk: {err:?}")) + } else { + Ok(()) + } + } + + async fn upload_chunks( + &self, + chunks: &[Chunk], + payment_proofs: &HashMap, + ) -> Result<(), PutError> { + debug!("Uploading {} chunks", chunks.len()); + for chunk in chunks { + if let Some(proof) = payment_proofs.get(chunk.name()) { + let address = *chunk.address(); + self.chunk_upload_with_payment(chunk.clone(), proof.clone()) + .await + .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}"))?; + } + } + Ok(()) + } +} + +pub fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec, Vec), PutError> { + let now = sn_networking::target_arch::Instant::now(); + let result = encrypt(data)?; + + debug!("Encryption took: {:.2?}", now.elapsed()); + + let map_xor_name = *result.0.address().xorname(); + let mut xor_names = vec![map_xor_name]; + + for chunk in &result.1 { + xor_names.push(*chunk.name()); + } + + Ok((result.0, result.1, xor_names)) +} diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index df5dab4ec0..4771d19e2a 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -12,6 +12,8 @@ pub mod address; pub mod archive; #[cfg(feature = "data")] pub mod data; +#[cfg(feature = "external-signer")] +pub mod external_signer; #[cfg(feature = "fs")] pub mod fs; #[cfg(feature = "registers")] diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index 7d24fbad77..fc752cd59b 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -229,7 +229,7 @@ async fn fetch_store_quote( } /// Form to be executed payments and already executed payments from a cost map. -fn extract_quote_payments( +pub(crate) fn extract_quote_payments( cost_map: &HashMap, ) -> (Vec, Vec) { let mut to_be_paid = vec![]; diff --git a/autonomi/tests/put.rs b/autonomi/tests/put.rs index dbced37d00..bc856dccc6 100644 --- a/autonomi/tests/put.rs +++ b/autonomi/tests/put.rs @@ -8,12 +8,17 @@ #![cfg(feature = "data")] +use alloy::network::TransactionBuilder; +use alloy::providers::Provider; use autonomi::Client; use eyre::Result; +use sn_evm::{ProofOfPayment, QuoteHash, TxHash}; use sn_logging::LogBuilder; +use std::collections::HashMap; use std::time::Duration; use test_utils::{evm::get_funded_wallet, gen_random_data, peers_from_env}; use tokio::time::sleep; +use xor_name::XorName; #[tokio::test] async fn put() -> Result<()> { @@ -32,3 +37,101 @@ async fn put() -> Result<()> { Ok(()) } + +// Example of how put would be done using external signers. +#[cfg(feature = "external-signer")] +#[tokio::test] +async fn external_signer_put() -> Result<()> { + let _log_appender_guard = + LogBuilder::init_single_threaded_tokio_test("external_signer_put", false); + + let client = Client::connect(&peers_from_env()?).await?; + let wallet = get_funded_wallet(); + let data = gen_random_data(1024 * 1024 * 10); + + // Encrypt the data as chunks + let (_data_map_chunk, _chunks, xor_names) = + autonomi::client::external_signer::encrypt_data(data.clone())?; + + let (quotes, quote_payments, _skipped_chunks) = client + .get_quotes_for_content_addrs(xor_names.into_iter()) + .await?; + + // Form quotes payment transaction data + let pay_for_quotes_calldata = autonomi::client::external_signer::pay_for_quotes_calldata( + wallet.network(), + quote_payments.into_iter(), + )?; + + let provider = wallet.to_provider(); + + // Form approve to spend tokens transaction data + let approve_calldata = autonomi::client::external_signer::approve_to_spend_tokens_calldata( + wallet.network(), + pay_for_quotes_calldata.approve_spender, + pay_for_quotes_calldata.approve_amount, + ); + + // Prepare approve to spend tokens transaction + let transaction_request = provider + .transaction_request() + .with_to(approve_calldata.1) + .with_input(approve_calldata.0); + + // Send approve to spend tokens transaction + let _tx_hash = provider + .send_transaction(transaction_request) + .await? + .watch() + .await?; + + let mut payments: HashMap = HashMap::new(); + + // Execute all quote payment transactions in batches + for (calldata, quote_hashes) in pay_for_quotes_calldata.batched_calldata_map { + // Prepare batched quote payments transaction + let transaction_request = provider + .transaction_request() + .with_to(pay_for_quotes_calldata.to) + .with_input(calldata); + + // Send batched quote payments transaction + let tx_hash = provider + .send_transaction(transaction_request) + .await? + .watch() + .await?; + + // Add to payments to be later use to construct the proofs + for quote_hash in quote_hashes { + payments.insert(quote_hash, tx_hash); + } + } + + // Payment proofs + let proofs: HashMap = quotes + .iter() + .filter_map(|(xor_name, (_, _, quote))| { + payments.get("e.hash()).map(|tx_hash| { + ( + *xor_name, + ProofOfPayment { + quote: quote.clone(), + tx_hash: *tx_hash, + }, + ) + }) + }) + .collect(); + + let addr = client + .data_put_with_proof_of_payment(data.clone(), proofs) + .await?; + + sleep(Duration::from_secs(10)).await; + + let data_fetched = client.data_get(addr).await?; + assert_eq!(data, data_fetched, "data fetched should match data put"); + + Ok(()) +} diff --git a/evmlib/Cargo.toml b/evmlib/Cargo.toml index 75bf4dc7b1..c3cfbdf432 100644 --- a/evmlib/Cargo.toml +++ b/evmlib/Cargo.toml @@ -11,7 +11,7 @@ version = "0.1.0" [features] wasm-bindgen = ["alloy/wasm-bindgen"] local = [] -external_signer = [] +external-signer = [] [dependencies] alloy = { version = "0.4.2", default-features = false, features = ["std", "reqwest-rustls-tls", "provider-anvil-node", "sol-types", "json", "signers", "contract", "signer-local", "network"] } diff --git a/evmlib/src/external_signer.rs b/evmlib/src/external_signer.rs index 7dfef2a480..a182b88cd5 100644 --- a/evmlib/src/external_signer.rs +++ b/evmlib/src/external_signer.rs @@ -52,10 +52,10 @@ pub fn transfer_tokens_calldata( #[allow(dead_code)] pub struct PayForQuotesCalldataReturnType { - batched_calldata_map: HashMap>, - to: Address, - approve_spender: Address, - approve_amount: Amount, + pub batched_calldata_map: HashMap>, + pub to: Address, + pub approve_spender: Address, + pub approve_amount: Amount, } #[allow(dead_code)] @@ -71,7 +71,7 @@ pub fn pay_for_quotes_calldata>( let total_amount = payments.iter().map(|(_, _, amount)| amount).sum(); - let approve_spender = *network.data_payments_address(); + let approve_to = *network.data_payments_address(); let approve_amount = total_amount; let provider = http_provider(network.rpc_url().clone()); @@ -92,7 +92,7 @@ pub fn pay_for_quotes_calldata>( Ok(PayForQuotesCalldataReturnType { batched_calldata_map: calldata_map, to: *data_payments.contract.address(), - approve_spender, + approve_spender: approve_to, approve_amount, }) } diff --git a/evmlib/src/lib.rs b/evmlib/src/lib.rs index 3b200c7389..f051e3c81d 100644 --- a/evmlib/src/lib.rs +++ b/evmlib/src/lib.rs @@ -22,7 +22,7 @@ pub mod common; pub mod contract; pub mod cryptography; pub(crate) mod event; -#[cfg(feature = "external_signer")] +#[cfg(feature = "external-signer")] pub mod external_signer; pub mod testnet; pub mod transaction; diff --git a/evmlib/src/wallet.rs b/evmlib/src/wallet.rs index 8d79908efc..8c4ec78298 100644 --- a/evmlib/src/wallet.rs +++ b/evmlib/src/wallet.rs @@ -63,6 +63,11 @@ impl Wallet { wallet_address(&self.wallet) } + /// Returns the `Network` of this wallet. + pub fn network(&self) -> &Network { + &self.network + } + /// Returns the raw balance of payment tokens for this wallet. pub async fn balance_of_tokens(&self) -> Result { balance_of_tokens(wallet_address(&self.wallet), &self.network).await @@ -112,6 +117,11 @@ impl Wallet { ) -> Result, PayForQuotesError> { pay_for_quotes(self.wallet.clone(), &self.network, data_payments).await } + + /// Build a provider using this wallet. + pub fn to_provider(&self) -> ProviderWithWallet { + http_provider_with_wallet(self.network.rpc_url().clone(), self.wallet.clone()) + } } /// Generate an EthereumWallet with a random private key. @@ -131,11 +141,7 @@ fn from_private_key(private_key: &str) -> Result { // TODO(optimization): Find a way to reuse/persist contracts and/or a provider without the wallet nonce going out of sync -#[allow(clippy::type_complexity)] -fn http_provider_with_wallet( - rpc_url: reqwest::Url, - wallet: EthereumWallet, -) -> FillProvider< +pub type ProviderWithWallet = FillProvider< JoinFill< JoinFill< Identity, @@ -146,7 +152,9 @@ fn http_provider_with_wallet( ReqwestProvider, Http, Ethereum, -> { +>; + +fn http_provider_with_wallet(rpc_url: reqwest::Url, wallet: EthereumWallet) -> ProviderWithWallet { ProviderBuilder::new() .with_recommended_fillers() .wallet(wallet) diff --git a/sn_evm/Cargo.toml b/sn_evm/Cargo.toml index 87da8453e4..8e5ed4923a 100644 --- a/sn_evm/Cargo.toml +++ b/sn_evm/Cargo.toml @@ -13,6 +13,7 @@ version = "0.1.0" [features] test-utils = [] local = ["evmlib/local"] +external-signer = ["evmlib/external-signer"] [dependencies] custom_debug = "~0.6.1" diff --git a/sn_evm/src/lib.rs b/sn_evm/src/lib.rs index a62fa5c0fd..222e4da326 100644 --- a/sn_evm/src/lib.rs +++ b/sn_evm/src/lib.rs @@ -12,6 +12,8 @@ extern crate tracing; pub use evmlib::common::Address as RewardsAddress; pub use evmlib::common::QuotePayment; pub use evmlib::common::{QuoteHash, TxHash}; +#[cfg(feature = "external-signer")] +pub use evmlib::external_signer; pub use evmlib::utils; pub use evmlib::utils::get_evm_network_from_env; pub use evmlib::utils::{DATA_PAYMENTS_ADDRESS, PAYMENT_TOKEN_ADDRESS, RPC_URL}; From 58e485ad2b6202c814ca3da69ad5b6e05976a537 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 17 Oct 2024 09:13:53 +0200 Subject: [PATCH 5/8] docs(autonomi): update encrypt_data docs --- autonomi/src/client/external_signer.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/autonomi/src/client/external_signer.rs b/autonomi/src/client/external_signer.rs index fde41c6f4d..f7cc5b9cd3 100644 --- a/autonomi/src/client/external_signer.rs +++ b/autonomi/src/client/external_signer.rs @@ -80,6 +80,9 @@ impl Client { } } +/// Encrypts data as chunks. +/// +/// Returns the data map chunk, file chunks and a list of all content addresses including the data map. pub fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec, Vec), PutError> { let now = sn_networking::target_arch::Instant::now(); let result = encrypt(data)?; From 72bb5a3dd0d29feb93aeefde5dbbec150fba1a15 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 17 Oct 2024 11:19:47 +0200 Subject: [PATCH 6/8] refactor(autonomi): simplify external signer put --- autonomi/src/client/external_signer.rs | 16 ++-- autonomi/src/client/utils.rs | 34 ++------ autonomi/src/lib.rs | 3 + autonomi/src/utils.rs | 24 ++++++ autonomi/tests/external_signer.rs | 91 ++++++++++++++++++++++ autonomi/tests/put.rs | 103 ------------------------- 6 files changed, 132 insertions(+), 139 deletions(-) create mode 100644 autonomi/src/utils.rs create mode 100644 autonomi/tests/external_signer.rs diff --git a/autonomi/src/client/external_signer.rs b/autonomi/src/client/external_signer.rs index f7cc5b9cd3..143e8340eb 100644 --- a/autonomi/src/client/external_signer.rs +++ b/autonomi/src/client/external_signer.rs @@ -4,13 +4,13 @@ use crate::self_encryption::encrypt; use crate::Client; use bytes::Bytes; use sn_evm::{ProofOfPayment, QuotePayment}; +use sn_networking::PayeeQuote; use sn_protocol::storage::Chunk; use std::collections::HashMap; use xor_name::XorName; #[allow(unused_imports)] pub use sn_evm::external_signer::*; -use sn_networking::PayeeQuote; impl Client { /// Upload a piece of data to the network. This data will be self-encrypted. @@ -27,11 +27,11 @@ impl Client { Ok(*data_map_chunk.address().xorname()) } - /// Get quotes for certain content addresses. - /// Returns a cost map, data payments to be executed and a list of free chunks. - pub async fn get_quotes_for_content_addrs( + /// Get quotes for data. + /// Returns a cost map, data payments to be executed and a list of free (already paid for) chunks. + pub async fn get_quotes_for_data( &self, - content_addrs: impl Iterator, + data: Bytes, ) -> Result< ( HashMap, @@ -40,7 +40,9 @@ impl Client { ), PutError, > { - let cost_map = self.get_store_quotes(content_addrs).await?; + // Encrypt the data as chunks + let (_data_map_chunk, _chunks, xor_names) = encrypt_data(data)?; + let cost_map = self.get_store_quotes(xor_names.into_iter()).await?; let (quote_payments, free_chunks) = extract_quote_payments(&cost_map); Ok((cost_map, quote_payments, free_chunks)) } @@ -83,7 +85,7 @@ impl Client { /// Encrypts data as chunks. /// /// Returns the data map chunk, file chunks and a list of all content addresses including the data map. -pub fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec, Vec), PutError> { +fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec, Vec), PutError> { let now = sn_networking::target_arch::Instant::now(); let result = encrypt(data)?; diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index fc752cd59b..33dde8d7b9 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -6,16 +6,13 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use std::{ - collections::{BTreeMap, HashMap}, - num::NonZero, -}; +use std::{collections::HashMap, num::NonZero}; use bytes::Bytes; use libp2p::kad::{Quorum, Record}; use rand::{thread_rng, Rng}; use self_encryption::{decrypt_full_set, DataMap, EncryptedChunk}; -use sn_evm::{EvmWallet, ProofOfPayment, QuoteHash, QuotePayment, TxHash}; +use sn_evm::{EvmWallet, ProofOfPayment, QuotePayment}; use sn_networking::{ GetRecordCfg, Network, NetworkError, PayeeQuote, PutRecordCfg, VerificationKind, }; @@ -26,12 +23,12 @@ use sn_protocol::{ }; use xor_name::XorName; -use crate::self_encryption::DataMapLevel; - use super::{ data::{CostError, GetError, PayError, PutError}, Client, }; +use crate::self_encryption::DataMapLevel; +use crate::utils::payment_proof_from_quotes_and_payments; impl Client { /// Fetch and decrypt all chunks in the data map. @@ -163,7 +160,7 @@ impl Client { .await .map_err(|err| PayError::from(err.0))?; - let proofs = construct_proofs(&cost_map, &payments); + let proofs = payment_proof_from_quotes_and_payments(&cost_map, &payments); trace!( "Chunk payments of {} chunks completed. {} chunks were free / already paid for", @@ -249,24 +246,3 @@ pub(crate) fn extract_quote_payments( (to_be_paid, already_paid) } - -/// Construct payment proofs from cost map and payments map. -fn construct_proofs( - cost_map: &HashMap, - payments: &BTreeMap, -) -> HashMap { - cost_map - .iter() - .filter_map(|(xor_name, (_, _, quote))| { - payments.get("e.hash()).map(|tx_hash| { - ( - *xor_name, - ProofOfPayment { - quote: quote.clone(), - tx_hash: *tx_hash, - }, - ) - }) - }) - .collect() -} diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index abfbd7563a..c73bef1378 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -35,11 +35,14 @@ extern crate tracing; pub mod client; #[cfg(feature = "data")] mod self_encryption; +mod utils; pub use sn_evm::get_evm_network_from_env; pub use sn_evm::EvmNetwork; pub use sn_evm::EvmWallet as Wallet; pub use sn_evm::RewardsAddress; +#[cfg(feature = "external-signer")] +pub use utils::payment_proof_from_quotes_and_payments; #[doc(no_inline)] // Place this under 'Re-exports' in the docs. pub use bytes::Bytes; diff --git a/autonomi/src/utils.rs b/autonomi/src/utils.rs new file mode 100644 index 0000000000..a7273f9bae --- /dev/null +++ b/autonomi/src/utils.rs @@ -0,0 +1,24 @@ +use sn_evm::{ProofOfPayment, QuoteHash, TxHash}; +use sn_networking::PayeeQuote; +use std::collections::{BTreeMap, HashMap}; +use xor_name::XorName; + +pub fn payment_proof_from_quotes_and_payments( + quotes: &HashMap, + payments: &BTreeMap, +) -> HashMap { + quotes + .iter() + .filter_map(|(xor_name, (_, _, quote))| { + payments.get("e.hash()).map(|tx_hash| { + ( + *xor_name, + ProofOfPayment { + quote: quote.clone(), + tx_hash: *tx_hash, + }, + ) + }) + }) + .collect() +} diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs new file mode 100644 index 0000000000..d97107cb39 --- /dev/null +++ b/autonomi/tests/external_signer.rs @@ -0,0 +1,91 @@ +#![cfg(feature = "external-signer")] + +use alloy::network::TransactionBuilder; +use alloy::providers::Provider; +use autonomi::Client; +use sn_evm::{QuoteHash, TxHash}; +use sn_logging::LogBuilder; +use std::collections::BTreeMap; +use std::time::Duration; +use test_utils::evm::get_funded_wallet; +use test_utils::{gen_random_data, peers_from_env}; +use tokio::time::sleep; + +// Example of how put would be done using external signers. +#[tokio::test] +async fn external_signer_put() -> eyre::Result<()> { + let _log_appender_guard = + LogBuilder::init_single_threaded_tokio_test("external_signer_put", false); + + let client = Client::connect(&peers_from_env()?).await?; + let wallet = get_funded_wallet(); + let data = gen_random_data(1024 * 1024 * 10); + + let (quotes, quote_payments, _free_chunks) = client.get_quotes_for_data(data.clone()).await?; + + // Form quotes payment transaction data + let pay_for_quotes_calldata = autonomi::client::external_signer::pay_for_quotes_calldata( + wallet.network(), + quote_payments.into_iter(), + )?; + + // Init an external wallet provider. In the webapp, this would be MetaMask for example + let provider = wallet.to_provider(); + + // Form approve to spend tokens transaction data + let approve_calldata = autonomi::client::external_signer::approve_to_spend_tokens_calldata( + wallet.network(), + pay_for_quotes_calldata.approve_spender, + pay_for_quotes_calldata.approve_amount, + ); + + // Prepare approve to spend tokens transaction + let transaction_request = provider + .transaction_request() + .with_to(approve_calldata.1) + .with_input(approve_calldata.0); + + // Send approve to spend tokens transaction + let _tx_hash = provider + .send_transaction(transaction_request) + .await? + .watch() + .await?; + + let mut payments: BTreeMap = Default::default(); + + // Execute all quote payment transactions in batches + for (calldata, quote_hashes) in pay_for_quotes_calldata.batched_calldata_map { + // Prepare batched quote payments transaction + let transaction_request = provider + .transaction_request() + .with_to(pay_for_quotes_calldata.to) + .with_input(calldata); + + // Send batched quote payments transaction + let tx_hash = provider + .send_transaction(transaction_request) + .await? + .watch() + .await?; + + // Add to payments to be later use to construct the proofs + for quote_hash in quote_hashes { + payments.insert(quote_hash, tx_hash); + } + } + + // Payment proofs + let proofs = autonomi::payment_proof_from_quotes_and_payments("es, &payments); + + let addr = client + .data_put_with_proof_of_payment(data.clone(), proofs) + .await?; + + sleep(Duration::from_secs(10)).await; + + let data_fetched = client.data_get(addr).await?; + assert_eq!(data, data_fetched, "data fetched should match data put"); + + Ok(()) +} diff --git a/autonomi/tests/put.rs b/autonomi/tests/put.rs index bc856dccc6..dbced37d00 100644 --- a/autonomi/tests/put.rs +++ b/autonomi/tests/put.rs @@ -8,17 +8,12 @@ #![cfg(feature = "data")] -use alloy::network::TransactionBuilder; -use alloy::providers::Provider; use autonomi::Client; use eyre::Result; -use sn_evm::{ProofOfPayment, QuoteHash, TxHash}; use sn_logging::LogBuilder; -use std::collections::HashMap; use std::time::Duration; use test_utils::{evm::get_funded_wallet, gen_random_data, peers_from_env}; use tokio::time::sleep; -use xor_name::XorName; #[tokio::test] async fn put() -> Result<()> { @@ -37,101 +32,3 @@ async fn put() -> Result<()> { Ok(()) } - -// Example of how put would be done using external signers. -#[cfg(feature = "external-signer")] -#[tokio::test] -async fn external_signer_put() -> Result<()> { - let _log_appender_guard = - LogBuilder::init_single_threaded_tokio_test("external_signer_put", false); - - let client = Client::connect(&peers_from_env()?).await?; - let wallet = get_funded_wallet(); - let data = gen_random_data(1024 * 1024 * 10); - - // Encrypt the data as chunks - let (_data_map_chunk, _chunks, xor_names) = - autonomi::client::external_signer::encrypt_data(data.clone())?; - - let (quotes, quote_payments, _skipped_chunks) = client - .get_quotes_for_content_addrs(xor_names.into_iter()) - .await?; - - // Form quotes payment transaction data - let pay_for_quotes_calldata = autonomi::client::external_signer::pay_for_quotes_calldata( - wallet.network(), - quote_payments.into_iter(), - )?; - - let provider = wallet.to_provider(); - - // Form approve to spend tokens transaction data - let approve_calldata = autonomi::client::external_signer::approve_to_spend_tokens_calldata( - wallet.network(), - pay_for_quotes_calldata.approve_spender, - pay_for_quotes_calldata.approve_amount, - ); - - // Prepare approve to spend tokens transaction - let transaction_request = provider - .transaction_request() - .with_to(approve_calldata.1) - .with_input(approve_calldata.0); - - // Send approve to spend tokens transaction - let _tx_hash = provider - .send_transaction(transaction_request) - .await? - .watch() - .await?; - - let mut payments: HashMap = HashMap::new(); - - // Execute all quote payment transactions in batches - for (calldata, quote_hashes) in pay_for_quotes_calldata.batched_calldata_map { - // Prepare batched quote payments transaction - let transaction_request = provider - .transaction_request() - .with_to(pay_for_quotes_calldata.to) - .with_input(calldata); - - // Send batched quote payments transaction - let tx_hash = provider - .send_transaction(transaction_request) - .await? - .watch() - .await?; - - // Add to payments to be later use to construct the proofs - for quote_hash in quote_hashes { - payments.insert(quote_hash, tx_hash); - } - } - - // Payment proofs - let proofs: HashMap = quotes - .iter() - .filter_map(|(xor_name, (_, _, quote))| { - payments.get("e.hash()).map(|tx_hash| { - ( - *xor_name, - ProofOfPayment { - quote: quote.clone(), - tx_hash: *tx_hash, - }, - ) - }) - }) - .collect(); - - let addr = client - .data_put_with_proof_of_payment(data.clone(), proofs) - .await?; - - sleep(Duration::from_secs(10)).await; - - let data_fetched = client.data_get(addr).await?; - assert_eq!(data, data_fetched, "data fetched should match data put"); - - Ok(()) -} From cbb6669063740e1525cdf1c0f39b27c242b0252f Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 17 Oct 2024 11:53:59 +0200 Subject: [PATCH 7/8] fix(autonomi): `PutError` from `CostError` --- autonomi/src/client/data.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data.rs index 366ad643be..2d9673103a 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data.rs @@ -37,6 +37,8 @@ pub enum PutError { VaultXorName, #[error("A network error occurred.")] Network(#[from] NetworkError), + #[error("Error occurred during cost estimation.")] + CostError(#[from] CostError), #[error("Error occurred during payment.")] PayError(#[from] PayError), #[error("Failed to serialize {0}")] From 37d9022c0feb107f7cef289a2b3da4fce1e16167 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Fri, 18 Oct 2024 11:08:51 +0200 Subject: [PATCH 8/8] refactor(evmlib): rename error variant and remove dead code allowances --- evmlib/src/external_signer.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/evmlib/src/external_signer.rs b/evmlib/src/external_signer.rs index a182b88cd5..83b43695d0 100644 --- a/evmlib/src/external_signer.rs +++ b/evmlib/src/external_signer.rs @@ -18,11 +18,10 @@ use std::collections::HashMap; pub enum Error { #[error("Network token contract error: {0}")] NetworkTokenContract(#[from] network_token::Error), - #[error("Chunk payments contract error: {0}")] - ChunkPaymentsContract(#[from] data_payments::error::Error), + #[error("Data payments contract error: {0}")] + DataPaymentsContract(#[from] data_payments::error::Error), } -#[allow(dead_code)] /// Approve an address / smart contract to spend this wallet's payment tokens. /// /// Returns the transaction calldata (input, to). @@ -36,7 +35,6 @@ pub fn approve_to_spend_tokens_calldata( network_token.approve_calldata(spender, value) } -#[allow(dead_code)] /// Transfer payment tokens from the supplied wallet to an address. /// /// Returns the transaction calldata (input, to). @@ -50,7 +48,6 @@ pub fn transfer_tokens_calldata( network_token.transfer_calldata(receiver, amount) } -#[allow(dead_code)] pub struct PayForQuotesCalldataReturnType { pub batched_calldata_map: HashMap>, pub to: Address, @@ -58,7 +55,6 @@ pub struct PayForQuotesCalldataReturnType { pub approve_amount: Amount, } -#[allow(dead_code)] /// Use this wallet to pay for chunks in batched transfer transactions. /// If the amount of transfers is more than one transaction can contain, the transfers will be split up over multiple transactions. ///