diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 95e456fb67..1d59de2431 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -125,11 +125,11 @@ jobs: - name: Run autonomi tests timeout-minutes: 25 - run: cargo test --release --package autonomi --features full --lib + run: cargo test --release --package autonomi --features full,local --lib - name: Run autonomi doc tests timeout-minutes: 25 - run: cargo test --release --package autonomi --features full --doc + run: cargo test --release --package autonomi --features full,local --doc - name: Run bootstrap tests timeout-minutes: 25 diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml index d7c424d822..32692f8ca2 100644 --- a/autonomi/Cargo.toml +++ b/autonomi/Cargo.toml @@ -13,9 +13,13 @@ repository = "https://github.com/maidsafe/autonomi" name = "autonomi" crate-type = ["cdylib", "rlib"] +[[example]] +name = "data_and_archive" +required-features = ["full"] + [[example]] name = "put_and_dir_upload" -features = ["full"] +required-features = ["full"] [features] default = ["vault"] diff --git a/autonomi/README.md b/autonomi/README.md index da18e98aa8..d77c38a81b 100644 --- a/autonomi/README.md +++ b/autonomi/README.md @@ -134,3 +134,11 @@ Chunk payments address: 0x8464135c8F25Da09e49BC8782676a84730C318bC Deployer wallet private key: 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 Genesis wallet balance: (tokens: 20000000000000000000000000, gas: 9998998011366954730202) ``` + +# WASM + +For documentation on WASM, see [./README_WASM.md]. + +# Python + +For documentation on the Python bindings, see [./README_PYTHON.md]. diff --git a/autonomi/examples/data_and_archive.rs b/autonomi/examples/data_and_archive.rs new file mode 100644 index 0000000000..07fddd560f --- /dev/null +++ b/autonomi/examples/data_and_archive.rs @@ -0,0 +1,37 @@ +use autonomi::{Bytes, Client, Metadata, PrivateArchive}; +use test_utils::evm::get_funded_wallet; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_env("RUST_LOG")) + .init(); + + let client = Client::init().await?; + let wallet = get_funded_wallet(); + + // Upload 10MiB of random data and verify it by fetching it back. + let data = Bytes::from("Hello, World!"); + let data_map = client.data_put(data.clone(), (&wallet).into()).await?; + let data_fetched = client.data_get(data_map.clone()).await?; + assert_eq!(data, data_fetched); + + // Upload the data as part of an archive, giving it the name `test.txt`. + let mut archive = PrivateArchive::new(); + archive.add_file( + "test.txt".into(), + data_map, + Metadata::new_with_size(data.len() as u64), + ); + + // Upload the archive to the network. + let archive_data_map = client.archive_put(&archive, (&wallet).into()).await?; + let archive_fetched = client.archive_get(archive_data_map).await?; + assert_eq!(archive, archive_fetched); + + println!("Archive uploaded successfully"); + + Ok(()) +} diff --git a/autonomi/examples/put_and_dir_upload.rs b/autonomi/examples/put_and_dir_upload.rs index 9b6d7a6a47..4af5e20b11 100644 --- a/autonomi/examples/put_and_dir_upload.rs +++ b/autonomi/examples/put_and_dir_upload.rs @@ -1,12 +1,16 @@ -use autonomi::{Bytes, Client, Wallet}; +use autonomi::{Bytes, Client}; +use test_utils::evm::get_funded_wallet; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; #[tokio::main] async fn main() -> Result<(), Box> { - // Default wallet of testnet. - let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_env("RUST_LOG")) + .init(); - let client = Client::init_local().await?; - let wallet = Wallet::new_from_private_key(Default::default(), key)?; + let client = Client::init().await?; + let wallet = get_funded_wallet(); // Put and fetch data. let data_addr = client diff --git a/autonomi/examples/autonomi_advanced.py b/autonomi/python/examples/autonomi_advanced.py similarity index 100% rename from autonomi/examples/autonomi_advanced.py rename to autonomi/python/examples/autonomi_advanced.py diff --git a/autonomi/examples/autonomi_data_registers.py b/autonomi/python/examples/autonomi_data_registers.py similarity index 100% rename from autonomi/examples/autonomi_data_registers.py rename to autonomi/python/examples/autonomi_data_registers.py diff --git a/autonomi/examples/autonomi_example.py b/autonomi/python/examples/autonomi_example.py similarity index 100% rename from autonomi/examples/autonomi_example.py rename to autonomi/python/examples/autonomi_example.py diff --git a/autonomi/examples/autonomi_private_data.py b/autonomi/python/examples/autonomi_private_data.py similarity index 100% rename from autonomi/examples/autonomi_private_data.py rename to autonomi/python/examples/autonomi_private_data.py diff --git a/autonomi/examples/autonomi_private_encryption.py b/autonomi/python/examples/autonomi_private_encryption.py similarity index 100% rename from autonomi/examples/autonomi_private_encryption.py rename to autonomi/python/examples/autonomi_private_encryption.py diff --git a/autonomi/examples/autonomi_vault.py b/autonomi/python/examples/autonomi_vault.py similarity index 100% rename from autonomi/examples/autonomi_vault.py rename to autonomi/python/examples/autonomi_vault.py diff --git a/autonomi/examples/basic.py b/autonomi/python/examples/basic.py similarity index 100% rename from autonomi/examples/basic.py rename to autonomi/python/examples/basic.py diff --git a/autonomi/src/client/data/mod.rs b/autonomi/src/client/data/mod.rs index e1967f0c95..e64c6872e4 100644 --- a/autonomi/src/client/data/mod.rs +++ b/autonomi/src/client/data/mod.rs @@ -160,6 +160,19 @@ fn hash_to_short_string(input: &str) -> String { impl Client { /// Fetch a blob of (private) data from the network + /// + /// # Example + /// + /// ```no_run + /// use autonomi::{Client, Bytes}; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// # let client = Client::init().await?; + /// # let data_map = todo!(); + /// let data_fetched = client.data_get(data_map).await?; + /// # Ok(()) + /// # } + /// ``` pub async fn data_get(&self, data_map: DataMapChunk) -> Result { info!( "Fetching private data from Data Map {:?}", @@ -175,6 +188,22 @@ impl Client { /// The [`DataMapChunk`] is not uploaded to the network, keeping the data private. /// /// Returns the [`DataMapChunk`] containing the map to the encrypted chunks. + /// + /// # Example + /// + /// ```no_run + /// use autonomi::{Client, Bytes}; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// # let client = Client::init().await?; + /// # let wallet = todo!(); + /// let data = Bytes::from("Hello, World"); + /// let data_map = client.data_put(data, wallet).await?; + /// let data_fetched = client.data_get(data_map).await?; + /// assert_eq!(data, data_fetched); + /// # Ok(()) + /// # } + /// ``` pub async fn data_put( &self, data: Bytes, diff --git a/autonomi/src/client/data_private.rs b/autonomi/src/client/data_private.rs deleted file mode 100644 index d1288bb193..0000000000 --- a/autonomi/src/client/data_private.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use std::hash::{DefaultHasher, Hash, Hasher}; - -use ant_evm::Amount; -use ant_protocol::storage::Chunk; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; - -use super::data::{GetError, PutError}; -use crate::client::payment::PaymentOption; -use crate::client::{ClientEvent, UploadSummary}; -use crate::{self_encryption::encrypt, Client}; - -/// Private data on the network can be accessed with this -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PrivateDataAccess(Chunk); - -impl PrivateDataAccess { - pub fn to_hex(&self) -> String { - hex::encode(self.0.value()) - } - - pub fn from_hex(hex: &str) -> Result { - let data = hex::decode(hex)?; - Ok(Self(Chunk::new(Bytes::from(data)))) - } - - /// Get a private address for [`PrivateDataAccess`]. Note that this is not a network address, it is only used for refering to private data client side. - pub fn address(&self) -> String { - hash_to_short_string(&self.to_hex()) - } -} - -fn hash_to_short_string(input: &str) -> String { - let mut hasher = DefaultHasher::new(); - input.hash(&mut hasher); - let hash_value = hasher.finish(); - hash_value.to_string() -} - -impl Client { - /// Fetch a blob of private data from the network - pub async fn private_data_get(&self, data_map: PrivateDataAccess) -> Result { - info!( - "Fetching private data from Data Map {:?}", - data_map.0.address() - ); - let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?; - - Ok(data) - } - - /// Upload a piece of private data to the network. This data will be self-encrypted. - /// Returns the [`PrivateDataAccess`] containing the map to the encrypted chunks. - /// This data is private and only accessible with the [`PrivateDataAccess`]. - pub async fn private_data_put( - &self, - data: Bytes, - payment_option: PaymentOption, - ) -> Result { - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, chunks) = encrypt(data)?; - debug!("Encryption took: {:.2?}", now.elapsed()); - - // Pay for all chunks - let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect(); - info!("Paying for {} addresses", xor_names.len()); - let receipt = self - .pay_for_content_addrs(xor_names.into_iter(), payment_option) - .await - .inspect_err(|err| error!("Error paying for data: {err:?}"))?; - - // Upload the chunks with the payments - debug!("Uploading {} chunks", chunks.len()); - - let mut failed_uploads = self - .upload_chunks_with_retries(chunks.iter().collect(), &receipt) - .await; - - // Return the last chunk upload error - if let Some(last_chunk_fail) = failed_uploads.pop() { - tracing::error!( - "Error uploading chunk ({:?}): {:?}", - last_chunk_fail.0.address(), - last_chunk_fail.1 - ); - return Err(last_chunk_fail.1); - } - - let record_count = chunks.len(); - - // Reporting - if let Some(channel) = self.client_event_sender.as_ref() { - let tokens_spent = receipt - .values() - .map(|(_proof, price)| price.as_atto()) - .sum::(); - - let summary = UploadSummary { - record_count, - tokens_spent, - }; - if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { - error!("Failed to send client event: {err:?}"); - } - } - - Ok(PrivateDataAccess(data_map_chunk)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hex() { - let data_map = PrivateDataAccess(Chunk::new(Bytes::from_static(b"hello"))); - let hex = data_map.to_hex(); - let data_map2 = PrivateDataAccess::from_hex(&hex).expect("Failed to decode hex"); - assert_eq!(data_map, data_map2); - } -} diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/files/archive.rs index 58f0788059..8aebc1df85 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -142,7 +142,7 @@ impl PrivateArchive { } /// Serialize to bytes. - pub fn into_bytes(&self) -> Result { + pub fn to_bytes(&self) -> Result { let root_serialized = rmp_serde::to_vec(&self)?; let root_serialized = Bytes::from(root_serialized); @@ -163,11 +163,11 @@ impl Client { /// Upload a [`PrivateArchive`] to the network pub async fn archive_put( &self, - archive: PrivateArchive, + archive: &PrivateArchive, payment_option: PaymentOption, ) -> Result { let bytes = archive - .into_bytes() + .to_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; let result = self.data_put(bytes, payment_option).await; debug!("Uploaded private archive {archive:?} to the network and address is {result:?}"); diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/files/archive_public.rs index 54121ae919..f4b487747f 100644 --- a/autonomi/src/client/files/archive_public.rs +++ b/autonomi/src/client/files/archive_public.rs @@ -104,7 +104,7 @@ impl PublicArchive { } /// Serialize to bytes. - pub fn into_bytes(&self) -> Result { + pub fn to_bytes(&self) -> Result { let root_serialized = rmp_serde::to_vec(&self)?; let root_serialized = Bytes::from(root_serialized); @@ -146,17 +146,17 @@ impl Client { /// # let wallet = todo!(); /// let mut archive = PublicArchive::new(); /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); - /// let address = client.archive_put_public(archive, &wallet).await?; + /// let address = client.archive_put_public(&archive, &wallet).await?; /// # Ok(()) /// # } /// ``` pub async fn archive_put_public( &self, - archive: PublicArchive, + archive: &PublicArchive, wallet: &EvmWallet, ) -> Result { let bytes = archive - .into_bytes() + .to_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; let result = self.data_put_public(bytes, wallet.into()).await; debug!("Uploaded archive {archive:?} to the network and the address is {result:?}"); @@ -166,7 +166,7 @@ impl Client { /// Get the cost to upload an archive pub async fn archive_cost(&self, archive: PublicArchive) -> Result { let bytes = archive - .into_bytes() + .to_bytes() .map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?; let result = self.data_cost(bytes).await; debug!("Calculated the cost to upload archive {archive:?} is {result:?}"); diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs index 0d41f0744d..2428f2d344 100644 --- a/autonomi/src/client/files/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -173,7 +173,7 @@ impl Client { wallet: &EvmWallet, ) -> Result { let archive = self.dir_upload(dir_path, wallet).await?; - let archive_addr = self.archive_put(archive, wallet.into()).await?; + let archive_addr = self.archive_put(&archive, wallet.into()).await?; Ok(archive_addr) } diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs index 52e79c300a..a35cce82f2 100644 --- a/autonomi/src/client/files/fs_public.rs +++ b/autonomi/src/client/files/fs_public.rs @@ -118,7 +118,7 @@ impl Client { wallet: &EvmWallet, ) -> Result { let archive = self.dir_upload_public(dir_path, wallet).await?; - let archive_addr = self.archive_put_public(archive, wallet).await?; + let archive_addr = self.archive_put_public(&archive, wallet).await?; Ok(archive_addr) } diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 88c181f02d..f245833b91 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -36,7 +36,6 @@ mod utils; use ant_bootstrap::{BootstrapCacheConfig, BootstrapCacheStore, PeersArgs}; pub use ant_evm::Amount; - use ant_evm::EvmNetwork; use ant_networking::{interval, multiaddr_is_global, Network, NetworkBuilder, NetworkEvent}; use ant_protocol::version::IDENTIFY_PROTOCOL_STR; @@ -74,9 +73,11 @@ pub struct Client { } /// Configuration for [`Client::init_with_config`]. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct ClientConfig { /// Whether we're expected to connect to a local network. + /// + /// If `local` feature is enabled, [`ClientConfig::default()`] will set this to `true`. pub local: bool, /// List of peers to connect to. @@ -85,7 +86,19 @@ pub struct ClientConfig { pub peers: Option>, } -/// Error returned by [`Client::connect`]. +impl Default for ClientConfig { + fn default() -> Self { + Self { + #[cfg(feature = "local")] + local: true, + #[cfg(not(feature = "local"))] + local: false, + peers: None, + } + } +} + +/// Error returned by [`Client::init`]. #[derive(Debug, thiserror::Error)] pub enum ConnectError { /// Did not manage to populate the routing table with enough peers. diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 5203c11c05..ce49ba83d2 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -263,10 +263,7 @@ mod archive { archive: &JsArchive, wallet: &JsWallet, ) -> Result { - let addr = self - .0 - .archive_put_public(archive.0.clone(), &wallet.0) - .await?; + let addr = self.0.archive_put_public(&archive.0, &wallet.0).await?; Ok(addr_to_str(addr)) } @@ -348,10 +345,7 @@ mod archive_private { archive: &JsPrivateArchive, wallet: &JsWallet, ) -> Result { - let private_archive_access = self - .0 - .archive_put(archive.0.clone(), (&wallet.0).into()) - .await?; + let private_archive_access = self.0.archive_put(&archive.0, (&wallet.0).into()).await?; let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; @@ -370,10 +364,7 @@ mod archive_private { ) -> Result { let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let private_archive_access = self - .0 - .archive_put(archive.0.clone(), receipt.into()) - .await?; + let private_archive_access = self.0.archive_put(&archive.0, receipt.into()).await?; let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index aa95c6f648..81ff866006 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -76,7 +76,7 @@ pub use bytes::Bytes; pub use libp2p::Multiaddr; #[doc(inline)] -pub use client::{files::archive::PrivateArchive, Client, ClientConfig}; +pub use client::{files::archive::Metadata, files::archive::PrivateArchive, Client, ClientConfig}; #[cfg(feature = "extension-module")] mod python; diff --git a/autonomi/tests/evm/file.rs b/autonomi/tests/evm/file.rs index 228efa1ed1..0c2aff9fe6 100644 --- a/autonomi/tests/evm/file.rs +++ b/autonomi/tests/evm/file.rs @@ -47,7 +47,7 @@ mod test { async fn file_into_vault() -> eyre::Result<()> { common::enable_logging(); - let mut client = Client::connect(&[]) + let mut client = Client::init() .await? .with_vault_entropy(Bytes::from("at least 32 bytes of entropy here"))?; @@ -66,7 +66,7 @@ mod test { ); // now assert over the stored account packet - let new_client = Client::connect(&[]) + let new_client = Client::init() .await? .with_vault_entropy(Bytes::from("at least 32 bytes of entropy here"))?; diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index 9cc15c0a69..755a1cac8f 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -120,13 +120,13 @@ async fn external_signer_put() -> eyre::Result<()> { Metadata::new_with_size(data.len() as u64), ); - let archive_serialized = private_archive.into_bytes()?; + let archive_serialized = private_archive.to_bytes()?; let receipt = pay_for_data(&client, &wallet, archive_serialized.clone()).await?; sleep(Duration::from_secs(5)).await; - let private_archive_access = client.archive_put(private_archive, receipt.into()).await?; + let private_archive_access = client.archive_put(&private_archive, receipt.into()).await?; let vault_key = VaultSecretKey::random(); diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 941d49cb84..926baeb4fd 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -93,12 +93,7 @@ async fn file_into_vault() -> Result<()> { let archive = client.archive_get_public(addr).await?; let set_version = 0; client - .write_bytes_to_vault( - archive.into_bytes()?, - wallet.into(), - &client_sk, - set_version, - ) + .write_bytes_to_vault(archive.to_bytes()?, wallet.into(), &client_sk, set_version) .await?; // now assert over the stored account packet