From 64118c53ec972fb39fa9e22863c893e7a640ae9c Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 09:53:20 +0100 Subject: [PATCH 01/13] refactor(autonomi): suffix API calls with _public --- ant-cli/src/actions/download.rs | 4 ++-- ant-cli/src/commands/file.rs | 2 +- ant-node/tests/data_with_churn.rs | 4 ++-- ant-node/tests/storage_payments.rs | 4 ++-- ant-node/tests/verify_data_location.rs | 2 +- autonomi/README.md | 8 +++---- autonomi/README_PYTHON.md | 14 ++++++------- autonomi/examples/autonomi_advanced.py | 4 ++-- autonomi/examples/autonomi_data_registers.py | 8 +++---- autonomi/examples/autonomi_example.py | 6 +++--- autonomi/examples/basic.py | 6 +++--- autonomi/examples/put_and_dir_upload.rs | 10 +++++---- autonomi/src/client/archive.rs | 14 ++++++------- autonomi/src/client/data.rs | 4 ++-- autonomi/src/client/fs.rs | 22 +++++++++++--------- autonomi/src/client/wasm.rs | 13 +++++++----- autonomi/src/lib.rs | 8 +++---- autonomi/src/python.rs | 8 +++---- autonomi/tests/fs.rs | 8 +++---- autonomi/tests/put.rs | 4 ++-- autonomi/tests/wasm.rs | 4 ++-- 21 files changed, 82 insertions(+), 75 deletions(-) diff --git a/ant-cli/src/actions/download.rs b/ant-cli/src/actions/download.rs index ff737ac2c1..b75d29c152 100644 --- a/ant-cli/src/actions/download.rs +++ b/ant-cli/src/actions/download.rs @@ -86,7 +86,7 @@ async fn download_public( client: &mut Client, ) -> Result<()> { let archive = client - .archive_get(address) + .archive_get_public(address) .await .wrap_err("Failed to fetch data from address")?; @@ -94,7 +94,7 @@ async fn download_public( let mut all_errs = vec![]; for (path, addr, _meta) in archive.iter() { progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.data_get(*addr).await { + let bytes = match client.data_get_public(*addr).await { Ok(bytes) => bytes, Err(e) => { let err = format!("Failed to fetch file {path:?}: {e}"); diff --git a/ant-cli/src/commands/file.rs b/ant-cli/src/commands/file.rs index 6d3f051015..8ac2e284c5 100644 --- a/ant-cli/src/commands/file.rs +++ b/ant-cli/src/commands/file.rs @@ -53,7 +53,7 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( let local_addr; let archive = if public { let xor_name = client - .dir_upload(dir_path, &wallet) + .dir_upload_public(dir_path, &wallet) .await .wrap_err("Failed to upload file")?; local_addr = addr_to_str(xor_name); diff --git a/ant-node/tests/data_with_churn.rs b/ant-node/tests/data_with_churn.rs index ffe2a879ab..64b3064350 100644 --- a/ant-node/tests/data_with_churn.rs +++ b/ant-node/tests/data_with_churn.rs @@ -338,7 +338,7 @@ fn store_chunks_task( let mut retries = 1; loop { match client - .data_put(random_data.clone(), (&wallet).into()) + .data_put_public(random_data.clone(), (&wallet).into()) .await .inspect_err(|err| { println!("Error to put chunk: {err:?}"); @@ -537,7 +537,7 @@ async fn query_content(client: &Client, net_addr: &NetworkAddress) -> Result<()> Ok(()) } NetworkAddress::ChunkAddress(addr) => { - client.data_get(*addr.xorname()).await?; + client.data_get_public(*addr.xorname()).await?; Ok(()) } _other => Ok(()), // we don't create/store any other type of content in this test yet diff --git a/ant-node/tests/storage_payments.rs b/ant-node/tests/storage_payments.rs index d2aabead94..bfb6d4ae75 100644 --- a/ant-node/tests/storage_payments.rs +++ b/ant-node/tests/storage_payments.rs @@ -205,7 +205,7 @@ // let _upload_stats = uploader.start_upload().await?; // let mut files_download = FilesDownload::new(files_api); -// let _ = files_download.download_file(file_addr, None).await?; +// let _ = files_download.file_download_public(file_addr, None).await?; // Ok(()) // } @@ -252,7 +252,7 @@ // let mut files_download = FilesDownload::new(files_api); // assert!( // matches!( -// files_download.download_file(content_addr, None).await, +// files_download.file_download_public(content_addr, None).await, // Err(ClientError::Network(NetworkError::GetRecordError( // GetRecordError::RecordNotFound // ))) diff --git a/ant-node/tests/verify_data_location.rs b/ant-node/tests/verify_data_location.rs index efdd848df8..0a82634ffe 100644 --- a/ant-node/tests/verify_data_location.rs +++ b/ant-node/tests/verify_data_location.rs @@ -351,7 +351,7 @@ async fn store_chunks( let random_bytes = Bytes::from(random_bytes); - client.data_put(random_bytes, wallet.into()).await?; + client.data_put_public(random_bytes, wallet.into()).await?; uploaded_chunks_count += 1; diff --git a/autonomi/README.md b/autonomi/README.md index c781c46bf9..7c759bd315 100644 --- a/autonomi/README.md +++ b/autonomi/README.md @@ -28,14 +28,14 @@ async fn main() -> Result<(), Box> { // Put and fetch data. let data_addr = client - .data_put(Bytes::from("Hello, World"), (&wallet).into()) + .data_put_public(Bytes::from("Hello, World"), (&wallet).into()) .await?; - let _data_fetched = client.data_get(data_addr).await?; + let _data_fetched = client.data_get_public(data_addr).await?; // Put and fetch directory from local file system. - let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?; + let dir_addr = client.dir_upload_public("files/to/upload".into(), &wallet).await?; client - .dir_download(dir_addr, "files/downloaded".into()) + .dir_download_public(dir_addr, "files/downloaded".into()) .await?; Ok(()) diff --git a/autonomi/README_PYTHON.md b/autonomi/README_PYTHON.md index 9bbb5a79b8..16c0fce428 100644 --- a/autonomi/README_PYTHON.md +++ b/autonomi/README_PYTHON.md @@ -26,11 +26,11 @@ payment = PaymentOption.wallet(wallet) # Upload data data = b"Hello, Safe Network!" -addr = client.data_put(data, payment) +addr = client.data_put_public(data, payment) print(f"Data uploaded to: {addr}") # Download data -retrieved = client.data_get(addr) +retrieved = client.data_get_public(addr) print(f"Retrieved: {retrieved.decode()}") ``` @@ -40,8 +40,8 @@ print(f"Retrieved: {retrieved.decode()}") - `Client`: Main interface to the Autonomi network - `connect(peers: List[str])`: Connect to network nodes - - `data_put(data: bytes, payment: PaymentOption)`: Upload data - - `data_get(addr: str)`: Download data + - `data_put_public(data: bytes, payment: PaymentOption)`: Upload data + - `data_get_public(addr: str)`: Download data - `private_data_put(data: bytes, payment: PaymentOption)`: Store private data - `private_data_get(access: PrivateDataAccess)`: Retrieve private data - `register_generate_key()`: Generate register key @@ -117,15 +117,15 @@ data, content_type = client.fetch_and_decrypt_vault(vault_key) def handle_data_operations(client, payment): # Upload text text_data = b"Hello, Safe Network!" - text_addr = client.data_put(text_data, payment) + text_addr = client.data_put_public(text_data, payment) # Upload binary data with open("image.jpg", "rb") as f: image_data = f.read() - image_addr = client.data_put(image_data, payment) + image_addr = client.data_put_public(image_data, payment) # Download and verify - downloaded = client.data_get(text_addr) + downloaded = client.data_get_public(text_addr) assert downloaded == text_data ``` diff --git a/autonomi/examples/autonomi_advanced.py b/autonomi/examples/autonomi_advanced.py index 310766192e..25e6333cb3 100644 --- a/autonomi/examples/autonomi_advanced.py +++ b/autonomi/examples/autonomi_advanced.py @@ -25,7 +25,7 @@ def connect_to_network(peers: list[str]) -> Client: def upload_data(client: Client, data: bytes, payment: PaymentOption) -> str: try: - addr = client.data_put(data, payment) + addr = client.data_put_public(data, payment) print(f"Successfully uploaded data to: {addr}") return addr except Exception as e: @@ -34,7 +34,7 @@ def upload_data(client: Client, data: bytes, payment: PaymentOption) -> str: def download_data(client: Client, addr: str) -> bytes: try: - data = client.data_get(addr) + data = client.data_get_public(addr) print(f"Successfully downloaded {len(data)} bytes") return data except Exception as e: diff --git a/autonomi/examples/autonomi_data_registers.py b/autonomi/examples/autonomi_data_registers.py index a7b8ba42ff..4d258fefa1 100644 --- a/autonomi/examples/autonomi_data_registers.py +++ b/autonomi/examples/autonomi_data_registers.py @@ -7,22 +7,22 @@ def handle_data_operations(client: Client, payment: PaymentOption): # Upload some text data text_data = b"Hello, Safe Network!" - text_addr = client.data_put(text_data, payment) + text_addr = client.data_put_public(text_data, payment) print(f"Text data uploaded to: {text_addr}") # Upload binary data (like an image) with open("example.jpg", "rb") as f: image_data = f.read() - image_addr = client.data_put(image_data, payment) + image_addr = client.data_put_public(image_data, payment) print(f"Image uploaded to: {image_addr}") # Download and verify data - downloaded_text = client.data_get(text_addr) + downloaded_text = client.data_get_public(text_addr) assert downloaded_text == text_data, "Text data verification failed!" print("Text data verified successfully") # Download and save image - downloaded_image = client.data_get(image_addr) + downloaded_image = client.data_get_public(image_addr) with open("downloaded_example.jpg", "wb") as f: f.write(downloaded_image) print("Image downloaded successfully") diff --git a/autonomi/examples/autonomi_example.py b/autonomi/examples/autonomi_example.py index 496446173c..14d6bbfc0e 100644 --- a/autonomi/examples/autonomi_example.py +++ b/autonomi/examples/autonomi_example.py @@ -21,17 +21,17 @@ def main(): # Upload some data data = b"Hello, Safe Network!" - addr = client.data_put(data, payment) + addr = client.data_put_public(data, payment) print(f"Data uploaded to address: {addr}") # Download the data back - downloaded = client.data_get(addr) + downloaded = client.data_get_public(addr) print(f"Downloaded data: {downloaded.decode()}") # You can also upload files with open("example.txt", "rb") as f: file_data = f.read() - file_addr = client.data_put(file_data, payment) + file_addr = client.data_put_public(file_data, payment) print(f"File uploaded to address: {file_addr}") if __name__ == "__main__": diff --git a/autonomi/examples/basic.py b/autonomi/examples/basic.py index b7d8f21619..e619df24d3 100644 --- a/autonomi/examples/basic.py +++ b/autonomi/examples/basic.py @@ -22,9 +22,9 @@ def main(): # Upload public data data = b"Hello World!" - addr = client.data_put(data, wallet) + addr = client.data_put_public(data, wallet) print(f"Uploaded public data to: {addr}") - retrieved = client.data_get(addr) + retrieved = client.data_get_public(addr) print(f"Retrieved public data: {retrieved}") # Upload private data @@ -40,7 +40,7 @@ def main(): print(f"Register values: {reg_values}") # Upload file/directory - file_addr = client.file_upload("./test_data", wallet) + file_addr = client.file_upload_public("./test_data", wallet) print(f"Uploaded files to: {file_addr}") client.file_download(file_addr, "./downloaded_data") print("Downloaded files") diff --git a/autonomi/examples/put_and_dir_upload.rs b/autonomi/examples/put_and_dir_upload.rs index f90480d101..45ebc96627 100644 --- a/autonomi/examples/put_and_dir_upload.rs +++ b/autonomi/examples/put_and_dir_upload.rs @@ -10,14 +10,16 @@ async fn main() -> Result<(), Box> { // Put and fetch data. let data_addr = client - .data_put(Bytes::from("Hello, World"), (&wallet).into()) + .data_put_public(Bytes::from("Hello, World"), (&wallet).into()) .await?; - let _data_fetched = client.data_get(data_addr).await?; + let _data_fetched = client.data_get_public(data_addr).await?; // Put and fetch directory from local file system. - let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?; + let dir_addr = client + .dir_upload_public("files/to/upload".into(), &wallet) + .await?; client - .dir_download(dir_addr, "files/downloaded".into()) + .dir_download_public(dir_addr, "files/downloaded".into()) .await?; Ok(()) diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/archive.rs index bed341c450..62794454d2 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/archive.rs @@ -157,12 +157,12 @@ impl Client { /// # async fn main() -> Result<(), Box> { /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; /// let client = Client::connect(&peers).await?; - /// let archive = client.archive_get(ArchiveAddr::random(&mut rand::thread_rng())).await?; + /// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?; /// # Ok(()) /// # } - /// ``` - pub async fn archive_get(&self, addr: ArchiveAddr) -> Result { - let data = self.data_get(addr).await?; + /// ```data_get_public + pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { + let data = self.data_get_public(addr).await?; Ok(Archive::from_bytes(data)?) } @@ -182,11 +182,11 @@ impl Client { /// # let wallet = todo!(); /// let mut archive = Archive::new(); /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); - /// let address = client.archive_put(archive, &wallet).await?; + /// let address = client.archive_put_public(archive, &wallet).await?; /// # Ok(()) /// # } /// ``` - pub async fn archive_put( + pub async fn archive_put_public( &self, archive: Archive, wallet: &EvmWallet, @@ -194,7 +194,7 @@ impl Client { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.data_put(bytes, wallet.into()).await; + let result = self.data_put_public(bytes, wallet.into()).await; debug!("Uploaded archive {archive:?} to the network and the address is {result:?}"); result } diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data.rs index e7f5d80a8e..c1018c871e 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data.rs @@ -127,7 +127,7 @@ pub enum CostError { impl Client { /// Fetch a blob of data from the network - pub async fn data_get(&self, addr: DataAddr) -> Result { + pub async fn data_get_public(&self, addr: DataAddr) -> Result { info!("Fetching data from Data Address: {addr:?}"); let data_map_chunk = self.chunk_get(addr).await?; let data = self @@ -141,7 +141,7 @@ impl Client { /// Upload a piece of data to the network. /// Returns the Data Address at which the data was stored. /// This data is publicly accessible. - pub async fn data_put( + pub async fn data_put_public( &self, data: Bytes, payment_option: PaymentOption, diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/fs.rs index 3eaf49b212..549f0808d2 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/fs.rs @@ -78,12 +78,12 @@ pub enum FileCostError { impl Client { /// Download file from network to local file system - pub async fn file_download( + pub async fn file_download_public( &self, data_addr: DataAddr, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let data = self.data_get(data_addr).await?; + let data = self.data_get_public(data_addr).await?; if let Some(parent) = to_dest.parent() { tokio::fs::create_dir_all(parent).await?; debug!("Created parent directories {parent:?} for {to_dest:?}"); @@ -94,15 +94,15 @@ impl Client { } /// Download directory from network to local file system - pub async fn dir_download( + pub async fn dir_download_public( &self, archive_addr: ArchiveAddr, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let archive = self.archive_get(archive_addr).await?; + let archive = self.archive_get_public(archive_addr).await?; debug!("Downloaded archive for the directory from the network at {archive_addr:?}"); for (path, addr, _meta) in archive.iter() { - self.file_download(*addr, to_dest.join(path)).await?; + self.file_download_public(*addr, to_dest.join(path)).await?; } debug!( "All files in the directory downloaded to {:?} from the network address {:?}", @@ -114,7 +114,7 @@ impl Client { /// Upload a directory to the network. The directory is recursively walked. /// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive) - pub async fn dir_upload( + pub async fn dir_upload_public( &self, dir_path: PathBuf, wallet: &EvmWallet, @@ -133,7 +133,7 @@ impl Client { let metadata = metadata_from_entry(&entry); let path = entry.path().to_path_buf(); upload_tasks.push(async move { - let file = self.file_upload(path.clone(), wallet).await; + let file = self.file_upload_public(path.clone(), wallet).await; (path, metadata, file) }); } @@ -159,7 +159,9 @@ impl Client { // upload archive let archive_serialized = archive.into_bytes()?; - let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; + let arch_addr = self + .data_put_public(archive_serialized, wallet.into()) + .await?; info!("Complete archive upload completed in {:?}", start.elapsed()); #[cfg(feature = "loud")] @@ -170,7 +172,7 @@ impl Client { /// Upload a file to the network. /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) - async fn file_upload( + async fn file_upload_public( &self, path: PathBuf, wallet: &EvmWallet, @@ -181,7 +183,7 @@ impl Client { let data = tokio::fs::read(path.clone()).await?; let data = Bytes::from(data); - let addr = self.data_put(data, wallet.into()).await?; + let addr = self.data_put_public(data, wallet.into()).await?; debug!("File {path:?} uploaded to the network at {addr:?}"); Ok(addr) } diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index fac5ec6343..e1ff6f027b 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -100,7 +100,7 @@ impl JsClient { #[wasm_bindgen(js_name = putData)] pub async fn put_data(&self, data: Vec, wallet: &JsWallet) -> Result { let data = crate::Bytes::from(data); - let xorname = self.0.data_put(data, (&wallet.0).into()).await?; + let xorname = self.0.data_put_public(data, (&wallet.0).into()).await?; Ok(addr_to_str(xorname)) } @@ -143,7 +143,7 @@ impl JsClient { #[wasm_bindgen(js_name = getData)] pub async fn get_data(&self, addr: String) -> Result, JsError> { let addr = str_to_addr(&addr)?; - let data = self.0.data_get(addr).await?; + let data = self.0.data_get_public(addr).await?; Ok(data.to_vec()) } @@ -249,7 +249,7 @@ mod archive { #[wasm_bindgen(js_name = getArchive)] pub async fn get_archive(&self, addr: String) -> Result { let addr = str_to_addr(&addr)?; - let archive = self.0.archive_get(addr).await?; + let archive = self.0.archive_get_public(addr).await?; let archive = JsArchive(archive); Ok(archive) @@ -264,7 +264,10 @@ mod archive { archive: &JsArchive, wallet: &JsWallet, ) -> Result { - let addr = self.0.archive_put(archive.0.clone(), &wallet.0).await?; + let addr = self + .0 + .archive_put_public(archive.0.clone(), &wallet.0) + .await?; Ok(addr_to_str(addr)) } @@ -691,7 +694,7 @@ mod external_signer { ) -> Result { let data = crate::Bytes::from(data); let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let xorname = self.0.data_put(data, receipt.into()).await?; + let xorname = self.0.data_put_public(data, receipt.into()).await?; Ok(addr_to_str(xorname)) } } diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index 4f219ea116..97fe148095 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -22,12 +22,12 @@ //! let wallet = Wallet::new_from_private_key(Default::default(), key)?; //! //! // Put and fetch data. -//! let data_addr = client.data_put(Bytes::from("Hello, World"), (&wallet).into()).await?; -//! let _data_fetched = client.data_get(data_addr).await?; +//! let data_addr = client.data_put_public(Bytes::from("Hello, World"), (&wallet).into()).await?; +//! let _data_fetched = client.data_get_public(data_addr).await?; //! //! // Put and fetch directory from local file system. -//! let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?; -//! client.dir_download(dir_addr, "files/downloaded".into()).await?; +//! let dir_addr = client.dir_upload_public("files/to/upload".into(), &wallet).await?; +//! client.dir_download_public(dir_addr, "files/downloaded".into()).await?; //! //! Ok(()) //! } diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index 2106327347..ac88ee43b0 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -67,12 +67,12 @@ impl PyClient { Ok(data.to_vec()) } - fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { + fn data_put_public(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let addr = rt .block_on( self.inner - .data_put(bytes::Bytes::from(data), payment.inner.clone()), + .data_put_public(bytes::Bytes::from(data), payment.inner.clone()), ) .map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to put data: {e}")) @@ -81,13 +81,13 @@ impl PyClient { Ok(crate::client::address::addr_to_str(addr)) } - fn data_get(&self, addr: &str) -> PyResult> { + fn data_get_public(&self, addr: &str) -> PyResult> { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let addr = crate::client::address::str_to_addr(addr).map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {e}")) })?; - let data = rt.block_on(self.inner.data_get(addr)).map_err(|e| { + let data = rt.block_on(self.inner.data_get_public(addr)).map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to get data: {e}")) })?; diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 274fc447f2..28aa62e55a 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -30,13 +30,13 @@ async fn dir_upload_download() -> Result<()> { let wallet = get_funded_wallet(); let addr = client - .dir_upload("tests/file/test_dir".into(), &wallet) + .dir_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(10)).await; client - .dir_download(addr, "tests/file/test_dir_fetched".into()) + .dir_download_public(addr, "tests/file/test_dir_fetched".into()) .await?; // compare the two directories @@ -86,11 +86,11 @@ async fn file_into_vault() -> Result<()> { let client_sk = bls::SecretKey::random(); let addr = client - .dir_upload("tests/file/test_dir".into(), &wallet) + .dir_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(2)).await; - let archive = client.archive_get(addr).await?; + let archive = client.archive_get_public(addr).await?; let set_version = 0; client .write_bytes_to_vault( diff --git a/autonomi/tests/put.rs b/autonomi/tests/put.rs index 401b5d3356..f5d411e691 100644 --- a/autonomi/tests/put.rs +++ b/autonomi/tests/put.rs @@ -21,11 +21,11 @@ async fn put() -> Result<()> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put(data.clone(), wallet.into()).await?; + let addr = client.data_put_public(data.clone(), wallet.into()).await?; sleep(Duration::from_secs(10)).await; - let data_fetched = client.data_get(addr).await?; + let data_fetched = client.data_get_public(addr).await?; assert_eq!(data, data_fetched, "data fetched should match data put"); Ok(()) diff --git a/autonomi/tests/wasm.rs b/autonomi/tests/wasm.rs index 980682765c..efdc8d179e 100644 --- a/autonomi/tests/wasm.rs +++ b/autonomi/tests/wasm.rs @@ -25,11 +25,11 @@ async fn put() -> Result<(), Box> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put(data.clone(), wallet.into()).await?; + let addr = client.data_put_public(data.clone(), wallet.into()).await?; sleep(Duration::from_secs(10)).await; - let data_fetched = client.data_get(addr).await?; + let data_fetched = client.data_get_public(addr).await?; assert_eq!(data, data_fetched, "data fetched should match data put"); Ok(()) From c8c36bfc459a32249a187adb757dd4caf57d456c Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 10:08:45 +0100 Subject: [PATCH 02/13] refactor(autonomi): remove _private suffixes --- ant-cli/src/actions/download.rs | 4 ++-- ant-cli/src/commands/file.rs | 2 +- autonomi/README_PYTHON.md | 12 +++++----- autonomi/examples/autonomi_private_data.py | 4 ++-- .../examples/autonomi_private_encryption.py | 4 ++-- autonomi/examples/basic.py | 8 +++---- autonomi/src/client/archive_private.rs | 8 +++---- autonomi/src/client/data_private.rs | 4 ++-- autonomi/src/client/fs_private.rs | 23 ++++++++----------- autonomi/src/client/wasm.rs | 12 +++++----- autonomi/src/python.rs | 12 ++++------ autonomi/tests/external_signer.rs | 14 ++++------- 12 files changed, 47 insertions(+), 60 deletions(-) diff --git a/ant-cli/src/actions/download.rs b/ant-cli/src/actions/download.rs index b75d29c152..f4edf8da8e 100644 --- a/ant-cli/src/actions/download.rs +++ b/ant-cli/src/actions/download.rs @@ -40,7 +40,7 @@ async fn download_private( client: &mut Client, ) -> Result<()> { let archive = client - .private_archive_get(private_address) + .archive_get(private_address) .await .wrap_err("Failed to fetch data from address")?; @@ -48,7 +48,7 @@ async fn download_private( let mut all_errs = vec![]; for (path, access, _meta) in archive.iter() { progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.private_data_get(access.clone()).await { + let bytes = match client.data_get(access.clone()).await { Ok(bytes) => bytes, Err(e) => { let err = format!("Failed to fetch file {path:?}: {e}"); diff --git a/ant-cli/src/commands/file.rs b/ant-cli/src/commands/file.rs index 8ac2e284c5..fde2e9e1d0 100644 --- a/ant-cli/src/commands/file.rs +++ b/ant-cli/src/commands/file.rs @@ -60,7 +60,7 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( local_addr.clone() } else { let private_data_access = client - .private_dir_upload(dir_path, &wallet) + .dir_upload(dir_path, &wallet) .await .wrap_err("Failed to upload file")?; local_addr = private_data_access.address(); diff --git a/autonomi/README_PYTHON.md b/autonomi/README_PYTHON.md index 16c0fce428..43e6ceaf04 100644 --- a/autonomi/README_PYTHON.md +++ b/autonomi/README_PYTHON.md @@ -42,8 +42,8 @@ print(f"Retrieved: {retrieved.decode()}") - `connect(peers: List[str])`: Connect to network nodes - `data_put_public(data: bytes, payment: PaymentOption)`: Upload data - `data_get_public(addr: str)`: Download data - - `private_data_put(data: bytes, payment: PaymentOption)`: Store private data - - `private_data_get(access: PrivateDataAccess)`: Retrieve private data + - `data_put(data: bytes, payment: PaymentOption)`: Store private data + - `data_get(access: PrivateDataAccess)`: Retrieve private data - `register_generate_key()`: Generate register key - `Wallet`: Ethereum wallet management @@ -63,9 +63,9 @@ print(f"Retrieved: {retrieved.decode()}") ```python # Private data example -access = client.private_data_put(secret_data, payment) +access = client.data_put(secret_data, payment) print(f"Private data stored at: {access.to_hex()}") -retrieved = client.private_data_get(access) +retrieved = client.data_get(access) ``` #### Registers @@ -138,11 +138,11 @@ def handle_private_data(client, payment): data = json.dumps(secret).encode() # Store privately - access = client.private_data_put(data, payment) + access = client.data_put(data, payment) print(f"Access token: {access.to_hex()}") # Retrieve - retrieved = client.private_data_get(access) + retrieved = client.data_get(access) secret = json.loads(retrieved.decode()) ``` diff --git a/autonomi/examples/autonomi_private_data.py b/autonomi/examples/autonomi_private_data.py index 3b0d9327e4..4d68acd3ea 100644 --- a/autonomi/examples/autonomi_private_data.py +++ b/autonomi/examples/autonomi_private_data.py @@ -10,12 +10,12 @@ def __init__(self, client: Client, wallet: Wallet): def store_private_data(self, data: bytes) -> str: """Store data privately and return its address""" - addr = self.client.private_data_put(data, self.payment) + addr = self.client.data_put(data, self.payment) return addr def retrieve_private_data(self, addr: str) -> bytes: """Retrieve privately stored data""" - return self.client.private_data_get(addr) + return self.client.data_get(addr) def create_shared_register(self, name: str, initial_value: bytes, allowed_writers: List[str]) -> str: diff --git a/autonomi/examples/autonomi_private_encryption.py b/autonomi/examples/autonomi_private_encryption.py index 7f71a6b8d6..3cfdfe54a1 100644 --- a/autonomi/examples/autonomi_private_encryption.py +++ b/autonomi/examples/autonomi_private_encryption.py @@ -16,12 +16,12 @@ def demonstrate_private_data(client: Client, payment: PaymentOption): data_bytes = json.dumps(secret_data).encode() # Store it privately - access = client.private_data_put(data_bytes, payment) + access = client.data_put(data_bytes, payment) print(f"Stored private data, access token: {access.to_hex()}") print(f"Short reference: {access.address()}") # Retrieve it - retrieved_bytes = client.private_data_get(access) + retrieved_bytes = client.data_get(access) retrieved_data = json.loads(retrieved_bytes.decode()) print(f"Retrieved private data: {retrieved_data}") diff --git a/autonomi/examples/basic.py b/autonomi/examples/basic.py index e619df24d3..4ddaee182c 100644 --- a/autonomi/examples/basic.py +++ b/autonomi/examples/basic.py @@ -28,9 +28,9 @@ def main(): print(f"Retrieved public data: {retrieved}") # Upload private data - private_access = client.private_data_put(b"Secret message", wallet) + private_access = client.data_put(b"Secret message", wallet) print(f"Private data access: {private_access}") - private_data = client.private_data_get(private_access) + private_data = client.data_get(private_access) print(f"Retrieved private data: {private_data}") # Create register @@ -58,9 +58,9 @@ def main(): print(f"Retrieved user data: {retrieved_data}") # Private directory operations - private_dir_access = client.private_dir_upload("./test_data", wallet) + private_dir_access = client.dir_upload("./test_data", wallet) print(f"Uploaded private directory, access: {private_dir_access}") - client.private_dir_download(private_dir_access, "./downloaded_private") + client.dir_download(private_dir_access, "./downloaded_private") print("Downloaded private directory") # External signer example diff --git a/autonomi/src/client/archive_private.rs b/autonomi/src/client/archive_private.rs index ee8705be2a..7734f92c79 100644 --- a/autonomi/src/client/archive_private.rs +++ b/autonomi/src/client/archive_private.rs @@ -114,16 +114,16 @@ impl PrivateArchive { impl Client { /// Fetch a private archive from the network - pub async fn private_archive_get( + pub async fn archive_get( &self, addr: PrivateArchiveAccess, ) -> Result { - let data = self.private_data_get(addr).await?; + let data = self.data_get(addr).await?; Ok(PrivateArchive::from_bytes(data)?) } /// Upload a private archive to the network - pub async fn private_archive_put( + pub async fn archive_put( &self, archive: PrivateArchive, payment_option: PaymentOption, @@ -131,7 +131,7 @@ impl Client { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.private_data_put(bytes, payment_option).await; + let result = self.data_put(bytes, payment_option).await; debug!("Uploaded private archive {archive:?} to the network and address is {result:?}"); result } diff --git a/autonomi/src/client/data_private.rs b/autonomi/src/client/data_private.rs index d31a13f437..1dab896d3b 100644 --- a/autonomi/src/client/data_private.rs +++ b/autonomi/src/client/data_private.rs @@ -47,7 +47,7 @@ fn hash_to_short_string(input: &str) -> String { impl Client { /// Fetch a blob of private data from the network - pub async fn private_data_get(&self, data_map: PrivateDataAccess) -> Result { + pub async fn data_get(&self, data_map: PrivateDataAccess) -> Result { info!( "Fetching private data from Data Map {:?}", data_map.0.address() @@ -61,7 +61,7 @@ impl Client { /// Upload a piece of private data to the network. This data will be self-encrypted. /// Returns the [`PrivateDataAccess`] containing the map to the encrypted chunks. /// This data is private and only accessible with the [`PrivateDataAccess`]. - pub async fn private_data_put( + pub async fn data_put( &self, data: Bytes, payment_option: PaymentOption, diff --git a/autonomi/src/client/fs_private.rs b/autonomi/src/client/fs_private.rs index 654fd4cef3..e9257209e3 100644 --- a/autonomi/src/client/fs_private.rs +++ b/autonomi/src/client/fs_private.rs @@ -28,12 +28,12 @@ use super::fs::FILE_UPLOAD_BATCH_SIZE; impl Client { /// Download a private file from network to local file system - pub async fn private_file_download( + pub async fn file_download( &self, data_access: PrivateDataAccess, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let data = self.private_data_get(data_access).await?; + let data = self.data_get(data_access).await?; if let Some(parent) = to_dest.parent() { tokio::fs::create_dir_all(parent).await?; debug!("Created parent directories for {to_dest:?}"); @@ -44,15 +44,14 @@ impl Client { } /// Download a private directory from network to local file system - pub async fn private_dir_download( + pub async fn dir_download( &self, archive_access: PrivateArchiveAccess, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let archive = self.private_archive_get(archive_access).await?; + let archive = self.archive_get(archive_access).await?; for (path, addr, _meta) in archive.iter() { - self.private_file_download(addr.clone(), to_dest.join(path)) - .await?; + self.file_download(addr.clone(), to_dest.join(path)).await?; } debug!("Downloaded directory to {to_dest:?}"); Ok(()) @@ -60,7 +59,7 @@ impl Client { /// Upload a private directory to the network. The directory is recursively walked. /// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive) - pub async fn private_dir_upload( + pub async fn dir_upload( &self, dir_path: PathBuf, wallet: &EvmWallet, @@ -79,7 +78,7 @@ impl Client { let metadata = super::fs::metadata_from_entry(&entry); let path = entry.path().to_path_buf(); upload_tasks.push(async move { - let file = self.private_file_upload(path.clone(), wallet).await; + let file = self.file_upload(path.clone(), wallet).await; (path, metadata, file) }); } @@ -105,9 +104,7 @@ impl Client { // upload archive let archive_serialized = archive.into_bytes()?; - let arch_addr = self - .private_data_put(archive_serialized, wallet.into()) - .await?; + let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; info!( "Complete private archive upload completed in {:?}", @@ -120,7 +117,7 @@ impl Client { /// Upload a private file to the network. /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap) - async fn private_file_upload( + async fn file_upload( &self, path: PathBuf, wallet: &EvmWallet, @@ -131,7 +128,7 @@ impl Client { let data = tokio::fs::read(path).await?; let data = Bytes::from(data); - let addr = self.private_data_put(data, wallet.into()).await?; + let addr = self.data_put(data, wallet.into()).await?; debug!("Uploaded file successfully in the privateAchive: {addr:?}"); Ok(addr) } diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index e1ff6f027b..8353e55ab9 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -115,7 +115,7 @@ impl JsClient { wallet: &JsWallet, ) -> Result { let data = crate::Bytes::from(data); - let private_data_access = self.0.private_data_put(data, (&wallet.0).into()).await?; + let private_data_access = self.0.data_put(data, (&wallet.0).into()).await?; let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; Ok(js_value) @@ -133,7 +133,7 @@ impl JsClient { ) -> Result { let data = crate::Bytes::from(data); let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let private_data_access = self.0.private_data_put(data, receipt.into()).await?; + let private_data_access = self.0.data_put(data, receipt.into()).await?; let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; Ok(js_value) @@ -153,7 +153,7 @@ impl JsClient { pub async fn get_private_data(&self, private_data_access: JsValue) -> Result, JsError> { let private_data_access: PrivateDataAccess = serde_wasm_bindgen::from_value(private_data_access)?; - let data = self.0.private_data_get(private_data_access).await?; + let data = self.0.data_get(private_data_access).await?; Ok(data.to_vec()) } @@ -335,7 +335,7 @@ mod archive_private { ) -> Result { let private_archive_access: PrivateArchiveAccess = serde_wasm_bindgen::from_value(private_archive_access)?; - let archive = self.0.private_archive_get(private_archive_access).await?; + let archive = self.0.archive_get(private_archive_access).await?; let archive = JsPrivateArchive(archive); Ok(archive) @@ -352,7 +352,7 @@ mod archive_private { ) -> Result { let private_archive_access = self .0 - .private_archive_put(archive.0.clone(), (&wallet.0).into()) + .archive_put(archive.0.clone(), (&wallet.0).into()) .await?; let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; @@ -374,7 +374,7 @@ mod archive_private { let private_archive_access = self .0 - .private_archive_put(archive.0.clone(), receipt.into()) + .archive_put(archive.0.clone(), receipt.into()) .await?; let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index ac88ee43b0..e8dafb1f42 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -39,16 +39,12 @@ impl PyClient { Ok(Self { inner: client }) } - fn private_data_put( - &self, - data: Vec, - payment: &PyPaymentOption, - ) -> PyResult { + fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let access = rt .block_on( self.inner - .private_data_put(Bytes::from(data), payment.inner.clone()), + .data_put(Bytes::from(data), payment.inner.clone()), ) .map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}")) @@ -57,10 +53,10 @@ impl PyClient { Ok(PyPrivateDataAccess { inner: access }) } - fn private_data_get(&self, access: &PyPrivateDataAccess) -> PyResult> { + fn data_get(&self, access: &PyPrivateDataAccess) -> PyResult> { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let data = rt - .block_on(self.inner.private_data_get(access.inner.clone())) + .block_on(self.inner.data_get(access.inner.clone())) .map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to get private data: {e}")) })?; diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index a9755400a4..997b651348 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -112,7 +112,7 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; let private_data_access = client - .private_data_put(data.clone(), receipt.into()) + .data_put(data.clone(), receipt.into()) .await?; let mut private_archive = PrivateArchive::new(); @@ -128,9 +128,7 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; - let private_archive_access = client - .private_archive_put(private_archive, receipt.into()) - .await?; + let private_archive_access = client.archive_put(private_archive, receipt.into()).await?; let vault_key = VaultSecretKey::random(); @@ -174,9 +172,7 @@ async fn external_signer_put() -> eyre::Result<()> { .expect("No private archive present in the UserData") .clone(); - let fetched_private_archive = client - .private_archive_get(fetched_private_archive_access) - .await?; + let fetched_private_archive = client.archive_get(fetched_private_archive_access).await?; let (_, (fetched_private_file_access, _)) = fetched_private_archive .map() @@ -184,9 +180,7 @@ async fn external_signer_put() -> eyre::Result<()> { .next() .expect("No file present in private archive"); - let fetched_private_file = client - .private_data_get(fetched_private_file_access.clone()) - .await?; + let fetched_private_file = client.data_get(fetched_private_file_access.clone()).await?; assert_eq!( fetched_private_file, data, From b6571f38a0b6f8147b70e9dd7224fa80ccc28b29 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 10:13:54 +0100 Subject: [PATCH 03/13] docs(autonomi): fix doc code example --- autonomi/src/client/archive.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/archive.rs index 62794454d2..c6b12f171c 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/archive.rs @@ -160,7 +160,7 @@ impl Client { /// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?; /// # Ok(()) /// # } - /// ```data_get_public + /// ``` pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { let data = self.data_get_public(addr).await?; Ok(Archive::from_bytes(data)?) From f8c4251fe826411b28de38018586bc00a4eac17a Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 10:43:16 +0100 Subject: [PATCH 04/13] refactor(autonomi): move data/archive/fs modules --- .../client/{data_private.rs => data/mod.rs} | 109 +++++++++++++++++- .../src/client/{data.rs => data/public.rs} | 103 +---------------- autonomi/src/client/{ => files}/archive.rs | 9 +- .../src/client/{ => files}/archive_private.rs | 11 +- autonomi/src/client/{ => files}/fs.rs | 6 +- autonomi/src/client/{ => files}/fs_private.rs | 2 +- autonomi/src/client/files/mod.rs | 8 ++ autonomi/src/client/mod.rs | 10 +- autonomi/src/client/vault/user_data.rs | 4 +- autonomi/src/python.rs | 5 +- autonomi/tests/external_signer.rs | 8 +- autonomi/tests/fs.rs | 2 +- 12 files changed, 140 insertions(+), 137 deletions(-) rename autonomi/src/client/{data_private.rs => data/mod.rs} (55%) rename autonomi/src/client/{data.rs => data/public.rs} (70%) rename autonomi/src/client/{ => files}/archive.rs (99%) rename autonomi/src/client/{ => files}/archive_private.rs (96%) rename autonomi/src/client/{ => files}/fs.rs (98%) rename autonomi/src/client/{ => files}/fs_private.rs (99%) create mode 100644 autonomi/src/client/files/mod.rs diff --git a/autonomi/src/client/data_private.rs b/autonomi/src/client/data/mod.rs similarity index 55% rename from autonomi/src/client/data_private.rs rename to autonomi/src/client/data/mod.rs index 1dab896d3b..f333616d67 100644 --- a/autonomi/src/client/data_private.rs +++ b/autonomi/src/client/data/mod.rs @@ -7,17 +7,122 @@ // permissions and limitations relating to use of the SAFE Network Software. use std::hash::{DefaultHasher, Hash, Hasher}; +use std::sync::LazyLock; -use ant_evm::Amount; +use ant_evm::{Amount, EvmWalletError}; +use ant_networking::NetworkError; use ant_protocol::storage::Chunk; +use ant_protocol::NetworkAddress; use bytes::Bytes; use serde::{Deserialize, Serialize}; +use xor_name::XorName; -use super::data::{GetError, PutError}; use crate::client::payment::PaymentOption; use crate::client::{ClientEvent, UploadSummary}; use crate::{self_encryption::encrypt, Client}; +pub mod public; + +/// Number of chunks to upload in parallel. +/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. +pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk upload batch size: {}", batch_size); + batch_size +}); + +/// Number of retries to upload chunks. +pub const RETRY_ATTEMPTS: usize = 3; + +/// Number of chunks to download in parallel. +/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. +pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk download batch size: {}", batch_size); + batch_size +}); + +/// Raw Data Address (points to a DataMap) +pub type DataAddr = XorName; +/// Raw Chunk Address (points to a [`Chunk`]) +pub type ChunkAddr = XorName; + +/// Errors that can occur during the put operation. +#[derive(Debug, thiserror::Error)] +pub enum PutError { + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("A network error occurred.")] + Network(#[from] NetworkError), + #[error("Error occurred during cost estimation.")] + CostError(#[from] CostError), + #[error("Error occurred during payment.")] + PayError(#[from] PayError), + #[error("Serialization error: {0}")] + Serialization(String), + #[error("A wallet error occurred.")] + Wallet(#[from] ant_evm::EvmError), + #[error("The vault owner key does not match the client's public key")] + VaultBadOwner, + #[error("Payment unexpectedly invalid for {0:?}")] + PaymentUnexpectedlyInvalid(NetworkAddress), +} + +/// Errors that can occur during the pay operation. +#[derive(Debug, thiserror::Error)] +pub enum PayError { + #[error("Wallet error: {0:?}")] + EvmWalletError(#[from] EvmWalletError), + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Cost error: {0:?}")] + Cost(#[from] CostError), +} + +/// Errors that can occur during the get operation. +#[derive(Debug, thiserror::Error)] +pub enum GetError { + #[error("Could not deserialize data map.")] + InvalidDataMap(rmp_serde::decode::Error), + #[error("Failed to decrypt data.")] + Decryption(crate::self_encryption::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), + #[error("General networking error: {0:?}")] + Network(#[from] NetworkError), + #[error("General protocol error: {0:?}")] + Protocol(#[from] ant_protocol::Error), +} + +/// Errors that can occur during the cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum CostError { + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Could not get store quote for: {0:?} after several retries")] + CouldNotGetStoreQuote(XorName), + #[error("Could not get store costs: {0:?}")] + CouldNotGetStoreCosts(NetworkError), + #[error("Failed to serialize {0}")] + Serialization(String), +} + /// Private data on the network can be accessed with this #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct PrivateDataAccess(Chunk); diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data/public.rs similarity index 70% rename from autonomi/src/client/data.rs rename to autonomi/src/client/data/public.rs index c1018c871e..a4ff4e1a40 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data/public.rs @@ -10,120 +10,21 @@ use bytes::Bytes; use libp2p::kad::Quorum; use std::collections::{HashMap, HashSet}; -use std::sync::LazyLock; use xor_name::XorName; use crate::client::payment::PaymentOption; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::{ClientEvent, UploadSummary}; use crate::{self_encryption::encrypt, Client}; +use ant_evm::ProofOfPayment; use ant_evm::{Amount, AttoTokens}; -use ant_evm::{EvmWalletError, ProofOfPayment}; use ant_networking::{GetRecordCfg, NetworkError}; use ant_protocol::{ storage::{try_deserialize_record, Chunk, ChunkAddress, RecordHeader, RecordKind}, NetworkAddress, }; -/// Number of chunks to upload in parallel. -/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("Chunk upload batch size: {}", batch_size); - batch_size -}); - -/// Number of retries to upload chunks. -pub const RETRY_ATTEMPTS: usize = 3; - -/// Number of chunks to download in parallel. -/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("Chunk download batch size: {}", batch_size); - batch_size -}); - -/// Raw Data Address (points to a DataMap) -pub type DataAddr = XorName; -/// Raw Chunk Address (points to a [`Chunk`]) -pub type ChunkAddr = XorName; - -/// Errors that can occur during the put operation. -#[derive(Debug, thiserror::Error)] -pub enum PutError { - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("A network error occurred.")] - Network(#[from] NetworkError), - #[error("Error occurred during cost estimation.")] - CostError(#[from] CostError), - #[error("Error occurred during payment.")] - PayError(#[from] PayError), - #[error("Serialization error: {0}")] - Serialization(String), - #[error("A wallet error occurred.")] - Wallet(#[from] ant_evm::EvmError), - #[error("The vault owner key does not match the client's public key")] - VaultBadOwner, - #[error("Payment unexpectedly invalid for {0:?}")] - PaymentUnexpectedlyInvalid(NetworkAddress), -} - -/// Errors that can occur during the pay operation. -#[derive(Debug, thiserror::Error)] -pub enum PayError { - #[error("Wallet error: {0:?}")] - EvmWalletError(#[from] EvmWalletError), - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Cost error: {0:?}")] - Cost(#[from] CostError), -} - -/// Errors that can occur during the get operation. -#[derive(Debug, thiserror::Error)] -pub enum GetError { - #[error("Could not deserialize data map.")] - InvalidDataMap(rmp_serde::decode::Error), - #[error("Failed to decrypt data.")] - Decryption(crate::self_encryption::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), - #[error("General networking error: {0:?}")] - Network(#[from] NetworkError), - #[error("General protocol error: {0:?}")] - Protocol(#[from] ant_protocol::Error), -} - -/// Errors that can occur during the cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum CostError { - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Could not get store quote for: {0:?} after several retries")] - CouldNotGetStoreQuote(XorName), - #[error("Could not get store costs: {0:?}")] - CouldNotGetStoreCosts(NetworkError), - #[error("Failed to serialize {0}")] - Serialization(String), -} +use super::*; impl Client { /// Fetch a blob of data from the network diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/files/archive.rs similarity index 99% rename from autonomi/src/client/archive.rs rename to autonomi/src/client/files/archive.rs index c6b12f171c..7b8323032a 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -13,10 +13,6 @@ use std::{ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; -use super::{ - data::{CostError, DataAddr, GetError, PutError}, - Client, -}; use ant_evm::{AttoTokens, EvmWallet}; use bytes::Bytes; use serde::{Deserialize, Serialize}; @@ -27,6 +23,11 @@ pub type ArchiveAddr = XorName; use thiserror::Error; +use crate::{ + client::data::{CostError, DataAddr, GetError, PutError}, + Client, +}; + #[derive(Error, Debug, PartialEq, Eq)] pub enum RenameError { #[error("File not found in archive: {0}")] diff --git a/autonomi/src/client/archive_private.rs b/autonomi/src/client/files/archive_private.rs similarity index 96% rename from autonomi/src/client/archive_private.rs rename to autonomi/src/client/files/archive_private.rs index 7734f92c79..d71b40f915 100644 --- a/autonomi/src/client/archive_private.rs +++ b/autonomi/src/client/files/archive_private.rs @@ -13,13 +13,14 @@ use std::{ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; -use super::{ - archive::{Metadata, RenameError}, - data::{GetError, PutError}, - data_private::PrivateDataAccess, +use super::archive::{Metadata, RenameError}; +use crate::{ + client::{ + data::{GetError, PrivateDataAccess, PutError}, + payment::PaymentOption, + }, Client, }; -use crate::client::payment::PaymentOption; use bytes::Bytes; use serde::{Deserialize, Serialize}; diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/files/fs.rs similarity index 98% rename from autonomi/src/client/fs.rs rename to autonomi/src/client/files/fs.rs index 549f0808d2..30fd73f1a0 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -6,8 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::archive::Metadata; -use crate::client::data::CostError; +use crate::client::data::{CostError, DataAddr, GetError, PutError}; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; @@ -16,8 +15,7 @@ use bytes::Bytes; use std::path::PathBuf; use std::sync::LazyLock; -use super::archive::{Archive, ArchiveAddr}; -use super::data::{DataAddr, GetError, PutError}; +use super::archive::{Archive, ArchiveAddr, Metadata}; /// Number of files to upload in parallel. /// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. diff --git a/autonomi/src/client/fs_private.rs b/autonomi/src/client/files/fs_private.rs similarity index 99% rename from autonomi/src/client/fs_private.rs rename to autonomi/src/client/files/fs_private.rs index e9257209e3..3fef1264b3 100644 --- a/autonomi/src/client/fs_private.rs +++ b/autonomi/src/client/files/fs_private.rs @@ -14,6 +14,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::client::data::PrivateDataAccess; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; @@ -21,7 +22,6 @@ use bytes::Bytes; use std::path::PathBuf; use super::archive_private::{PrivateArchive, PrivateArchiveAccess}; -use super::data_private::PrivateDataAccess; use super::fs::{DownloadError, UploadError}; use super::fs::FILE_UPLOAD_BATCH_SIZE; diff --git a/autonomi/src/client/files/mod.rs b/autonomi/src/client/files/mod.rs new file mode 100644 index 0000000000..0f76d26d28 --- /dev/null +++ b/autonomi/src/client/files/mod.rs @@ -0,0 +1,8 @@ +pub mod archive; +pub mod archive_private; +#[cfg(feature = "fs")] +#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] +pub mod fs; +#[cfg(feature = "fs")] +#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] +pub mod fs_private; diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 05003d1b19..914b01478b 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -12,19 +12,11 @@ pub mod address; pub mod payment; -pub mod archive; -pub mod archive_private; pub mod data; -pub mod data_private; #[cfg(feature = "external-signer")] #[cfg_attr(docsrs, doc(cfg(feature = "external-signer")))] pub mod external_signer; -#[cfg(feature = "fs")] -#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs; -#[cfg(feature = "fs")] -#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs_private; +pub mod files; #[cfg(feature = "registers")] #[cfg_attr(docsrs, doc(cfg(feature = "registers")))] pub mod registers; diff --git a/autonomi/src/client/vault/user_data.rs b/autonomi/src/client/vault/user_data.rs index d9bff46f6f..c30ba5f574 100644 --- a/autonomi/src/client/vault/user_data.rs +++ b/autonomi/src/client/vault/user_data.rs @@ -8,10 +8,10 @@ use std::collections::HashMap; -use crate::client::archive::ArchiveAddr; -use crate::client::archive_private::PrivateArchiveAccess; use crate::client::data::GetError; use crate::client::data::PutError; +use crate::client::files::archive::ArchiveAddr; +use crate::client::files::archive_private::PrivateArchiveAccess; use crate::client::payment::PaymentOption; use crate::client::registers::RegisterAddress; use crate::client::vault::VaultError; diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index e8dafb1f42..da11ebe6c3 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -2,9 +2,8 @@ #![allow(non_local_definitions)] use crate::client::{ - archive::ArchiveAddr, - archive_private::PrivateArchiveAccess, - data_private::PrivateDataAccess, + data::PrivateDataAccess, + files::{archive::ArchiveAddr, archive_private::PrivateArchiveAccess}, payment::PaymentOption as RustPaymentOption, vault::{UserData, VaultSecretKey}, Client as RustClient, diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index 997b651348..59573a6c44 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -4,9 +4,9 @@ use alloy::network::TransactionBuilder; use alloy::providers::Provider; use ant_evm::{QuoteHash, TxHash}; use ant_logging::LogBuilder; -use autonomi::client::archive::Metadata; -use autonomi::client::archive_private::PrivateArchive; use autonomi::client::external_signer::encrypt_data; +use autonomi::client::files::archive::Metadata; +use autonomi::client::files::archive_private::PrivateArchive; use autonomi::client::payment::Receipt; use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; use autonomi::client::vault::VaultSecretKey; @@ -111,9 +111,7 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; - let private_data_access = client - .data_put(data.clone(), receipt.into()) - .await?; + let private_data_access = client.data_put(data.clone(), receipt.into()).await?; let mut private_archive = PrivateArchive::new(); private_archive.add_file( diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 28aa62e55a..9eb0604681 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -106,7 +106,7 @@ async fn file_into_vault() -> Result<()> { let (ap, got_version) = new_client.fetch_and_decrypt_vault(&client_sk).await?; assert_eq!(set_version, got_version); - let ap_archive_fetched = autonomi::client::archive::Archive::from_bytes(ap)?; + let ap_archive_fetched = autonomi::client::files::archive::Archive::from_bytes(ap)?; assert_eq!( archive, ap_archive_fetched, From dd3a71e3d65a01687c43a116f49725cea50c60b6 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 10:44:26 +0100 Subject: [PATCH 05/13] refactor(autonomi): use archive_public instead of _private --- autonomi/src/client/files/archive.rs | 157 +++-------- autonomi/src/client/files/archive_private.rs | 139 --------- autonomi/src/client/files/archive_public.rs | 212 ++++++++++++++ autonomi/src/client/files/fs.rs | 236 +++------------- autonomi/src/client/files/fs_private.rs | 135 --------- autonomi/src/client/files/fs_public.rs | 281 +++++++++++++++++++ autonomi/src/client/files/mod.rs | 4 +- autonomi/src/client/vault/user_data.rs | 4 +- autonomi/src/python.rs | 2 +- autonomi/tests/external_signer.rs | 4 +- autonomi/tests/fs.rs | 2 +- 11 files changed, 588 insertions(+), 588 deletions(-) delete mode 100644 autonomi/src/client/files/archive_private.rs create mode 100644 autonomi/src/client/files/archive_public.rs delete mode 100644 autonomi/src/client/files/fs_private.rs create mode 100644 autonomi/src/client/files/fs_public.rs diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/files/archive.rs index 7b8323032a..c188f04043 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -13,66 +13,29 @@ use std::{ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; -use ant_evm::{AttoTokens, EvmWallet}; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; -use xor_name::XorName; - -/// The address of an archive on the network. Points to an [`Archive`]. -pub type ArchiveAddr = XorName; - -use thiserror::Error; - +use super::archive_public::{Metadata, RenameError}; use crate::{ - client::data::{CostError, DataAddr, GetError, PutError}, + client::{ + data::{GetError, PrivateDataAccess, PutError}, + payment::PaymentOption, + }, Client, }; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; -#[derive(Error, Debug, PartialEq, Eq)] -pub enum RenameError { - #[error("File not found in archive: {0}")] - FileNotFound(PathBuf), -} +/// The address of a private archive +/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data +pub type PrivateArchiveAccess = PrivateDataAccess; -/// An archive of files that containing file paths, their metadata and the files data addresses +/// A private archive of files that containing file paths, their metadata and the files data maps /// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. -/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::archive_private::PrivateArchive`]. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] -pub struct Archive { - map: HashMap, -} - -/// Metadata for a file in an archive. Time values are UNIX timestamps. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Metadata { - /// When the file was (last) uploaded to the network. - pub uploaded: u64, - /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. - pub created: u64, - /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. - pub modified: u64, - /// File size in bytes - pub size: u64, -} - -impl Metadata { - /// Create a new metadata struct with the current time as uploaded, created and modified. - pub fn new_with_size(size: u64) -> Self { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(); - - Self { - uploaded: now, - created: now, - modified: now, - size, - } - } +pub struct PrivateArchive { + map: HashMap, } -impl Archive { +impl PrivateArchive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network pub fn new() -> Self { @@ -94,14 +57,14 @@ impl Archive { .as_secs(); meta.modified = now; self.map.insert(new_path.to_path_buf(), (data_addr, meta)); - debug!("Renamed file successfully in the archive, old path: {old_path:?} new_path: {new_path:?}"); + debug!("Renamed file successfully in the private archive, old path: {old_path:?} new_path: {new_path:?}"); Ok(()) } /// Add a file to a local archive /// Note that this does not upload the archive to the network - pub fn add_file(&mut self, path: PathBuf, data_addr: DataAddr, meta: Metadata) { - self.map.insert(path.clone(), (data_addr, meta)); + pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) { + self.map.insert(path.clone(), (data_map, meta)); debug!("Added a new file to the archive, path: {:?}", path); } @@ -114,26 +77,29 @@ impl Archive { } /// List all data addresses of the files in the archive - pub fn addresses(&self) -> Vec { - self.map.values().map(|(addr, _)| *addr).collect() + pub fn addresses(&self) -> Vec { + self.map + .values() + .map(|(data_map, _)| data_map.clone()) + .collect() } /// Iterate over the archive items - /// Returns an iterator over (PathBuf, DataAddr, Metadata) - pub fn iter(&self) -> impl Iterator { + /// Returns an iterator over (PathBuf, SecretDataMap, Metadata) + pub fn iter(&self) -> impl Iterator { self.map .iter() - .map(|(path, (addr, meta))| (path, addr, meta)) + .map(|(path, (data_map, meta))| (path, data_map, meta)) } /// Get the underlying map - pub fn map(&self) -> &HashMap { + pub fn map(&self) -> &HashMap { &self.map } /// Deserialize from bytes. - pub fn from_bytes(data: Bytes) -> Result { - let root: Archive = rmp_serde::from_slice(&data[..])?; + pub fn from_bytes(data: Bytes) -> Result { + let root: PrivateArchive = rmp_serde::from_slice(&data[..])?; Ok(root) } @@ -148,65 +114,26 @@ impl Archive { } impl Client { - /// Fetch an archive from the network - /// - /// # Example - /// - /// ```no_run - /// # use autonomi::client::{Client, archive::ArchiveAddr}; - /// # #[tokio::main] - /// # async fn main() -> Result<(), Box> { - /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; - /// let client = Client::connect(&peers).await?; - /// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { - let data = self.data_get_public(addr).await?; - Ok(Archive::from_bytes(data)?) + /// Fetch a private archive from the network + pub async fn archive_get( + &self, + addr: PrivateArchiveAccess, + ) -> Result { + let data = self.data_get(addr).await?; + Ok(PrivateArchive::from_bytes(data)?) } - /// Upload an archive to the network - /// - /// # Example - /// - /// Create simple archive containing `file.txt` pointing to random XOR name. - /// - /// ```no_run - /// # use autonomi::client::{Client, data::DataAddr, archive::{Archive, ArchiveAddr, Metadata}}; - /// # use std::path::PathBuf; - /// # #[tokio::main] - /// # async fn main() -> Result<(), Box> { - /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; - /// # let client = Client::connect(&peers).await?; - /// # let wallet = todo!(); - /// let mut archive = Archive::new(); - /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); - /// let address = client.archive_put_public(archive, &wallet).await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn archive_put_public( + /// Upload a private archive to the network + pub async fn archive_put( &self, - archive: Archive, - wallet: &EvmWallet, - ) -> Result { + archive: PrivateArchive, + payment_option: PaymentOption, + ) -> Result { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.data_put_public(bytes, wallet.into()).await; - debug!("Uploaded archive {archive:?} to the network and the address is {result:?}"); - result - } - - /// Get the cost to upload an archive - pub async fn archive_cost(&self, archive: Archive) -> Result { - let bytes = archive - .into_bytes() - .map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.data_cost(bytes).await; - debug!("Calculated the cost to upload archive {archive:?} is {result:?}"); + let result = self.data_put(bytes, payment_option).await; + debug!("Uploaded private archive {archive:?} to the network and address is {result:?}"); result } } diff --git a/autonomi/src/client/files/archive_private.rs b/autonomi/src/client/files/archive_private.rs deleted file mode 100644 index d71b40f915..0000000000 --- a/autonomi/src/client/files/archive_private.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; - -use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; - -use super::archive::{Metadata, RenameError}; -use crate::{ - client::{ - data::{GetError, PrivateDataAccess, PutError}, - payment::PaymentOption, - }, - Client, -}; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; - -/// The address of a private archive -/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data -pub type PrivateArchiveAccess = PrivateDataAccess; - -/// A private archive of files that containing file paths, their metadata and the files data maps -/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] -pub struct PrivateArchive { - map: HashMap, -} - -impl PrivateArchive { - /// Create a new emtpy local archive - /// Note that this does not upload the archive to the network - pub fn new() -> Self { - Self { - map: HashMap::new(), - } - } - - /// Rename a file in an archive - /// Note that this does not upload the archive to the network - pub fn rename_file(&mut self, old_path: &Path, new_path: &Path) -> Result<(), RenameError> { - let (data_addr, mut meta) = self - .map - .remove(old_path) - .ok_or(RenameError::FileNotFound(old_path.to_path_buf()))?; - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(); - meta.modified = now; - self.map.insert(new_path.to_path_buf(), (data_addr, meta)); - debug!("Renamed file successfully in the private archive, old path: {old_path:?} new_path: {new_path:?}"); - Ok(()) - } - - /// Add a file to a local archive - /// Note that this does not upload the archive to the network - pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) { - self.map.insert(path.clone(), (data_map, meta)); - debug!("Added a new file to the archive, path: {:?}", path); - } - - /// List all files in the archive - pub fn files(&self) -> Vec<(PathBuf, Metadata)> { - self.map - .iter() - .map(|(path, (_, meta))| (path.clone(), meta.clone())) - .collect() - } - - /// List all data addresses of the files in the archive - pub fn addresses(&self) -> Vec { - self.map - .values() - .map(|(data_map, _)| data_map.clone()) - .collect() - } - - /// Iterate over the archive items - /// Returns an iterator over (PathBuf, SecretDataMap, Metadata) - pub fn iter(&self) -> impl Iterator { - self.map - .iter() - .map(|(path, (data_map, meta))| (path, data_map, meta)) - } - - /// Get the underlying map - pub fn map(&self) -> &HashMap { - &self.map - } - - /// Deserialize from bytes. - pub fn from_bytes(data: Bytes) -> Result { - let root: PrivateArchive = rmp_serde::from_slice(&data[..])?; - - Ok(root) - } - - /// Serialize to bytes. - pub fn into_bytes(&self) -> Result { - let root_serialized = rmp_serde::to_vec(&self)?; - let root_serialized = Bytes::from(root_serialized); - - Ok(root_serialized) - } -} - -impl Client { - /// Fetch a private archive from the network - pub async fn archive_get( - &self, - addr: PrivateArchiveAccess, - ) -> Result { - let data = self.data_get(addr).await?; - Ok(PrivateArchive::from_bytes(data)?) - } - - /// Upload a private archive to the network - pub async fn archive_put( - &self, - archive: PrivateArchive, - payment_option: PaymentOption, - ) -> Result { - let bytes = archive - .into_bytes() - .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.data_put(bytes, payment_option).await; - debug!("Uploaded private archive {archive:?} to the network and address is {result:?}"); - result - } -} diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/files/archive_public.rs new file mode 100644 index 0000000000..7b8323032a --- /dev/null +++ b/autonomi/src/client/files/archive_public.rs @@ -0,0 +1,212 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; + +use ant_evm::{AttoTokens, EvmWallet}; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use xor_name::XorName; + +/// The address of an archive on the network. Points to an [`Archive`]. +pub type ArchiveAddr = XorName; + +use thiserror::Error; + +use crate::{ + client::data::{CostError, DataAddr, GetError, PutError}, + Client, +}; + +#[derive(Error, Debug, PartialEq, Eq)] +pub enum RenameError { + #[error("File not found in archive: {0}")] + FileNotFound(PathBuf), +} + +/// An archive of files that containing file paths, their metadata and the files data addresses +/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. +/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::archive_private::PrivateArchive`]. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct Archive { + map: HashMap, +} + +/// Metadata for a file in an archive. Time values are UNIX timestamps. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Metadata { + /// When the file was (last) uploaded to the network. + pub uploaded: u64, + /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. + pub created: u64, + /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. + pub modified: u64, + /// File size in bytes + pub size: u64, +} + +impl Metadata { + /// Create a new metadata struct with the current time as uploaded, created and modified. + pub fn new_with_size(size: u64) -> Self { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(); + + Self { + uploaded: now, + created: now, + modified: now, + size, + } + } +} + +impl Archive { + /// Create a new emtpy local archive + /// Note that this does not upload the archive to the network + pub fn new() -> Self { + Self { + map: HashMap::new(), + } + } + + /// Rename a file in an archive + /// Note that this does not upload the archive to the network + pub fn rename_file(&mut self, old_path: &Path, new_path: &Path) -> Result<(), RenameError> { + let (data_addr, mut meta) = self + .map + .remove(old_path) + .ok_or(RenameError::FileNotFound(old_path.to_path_buf()))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(); + meta.modified = now; + self.map.insert(new_path.to_path_buf(), (data_addr, meta)); + debug!("Renamed file successfully in the archive, old path: {old_path:?} new_path: {new_path:?}"); + Ok(()) + } + + /// Add a file to a local archive + /// Note that this does not upload the archive to the network + pub fn add_file(&mut self, path: PathBuf, data_addr: DataAddr, meta: Metadata) { + self.map.insert(path.clone(), (data_addr, meta)); + debug!("Added a new file to the archive, path: {:?}", path); + } + + /// List all files in the archive + pub fn files(&self) -> Vec<(PathBuf, Metadata)> { + self.map + .iter() + .map(|(path, (_, meta))| (path.clone(), meta.clone())) + .collect() + } + + /// List all data addresses of the files in the archive + pub fn addresses(&self) -> Vec { + self.map.values().map(|(addr, _)| *addr).collect() + } + + /// Iterate over the archive items + /// Returns an iterator over (PathBuf, DataAddr, Metadata) + pub fn iter(&self) -> impl Iterator { + self.map + .iter() + .map(|(path, (addr, meta))| (path, addr, meta)) + } + + /// Get the underlying map + pub fn map(&self) -> &HashMap { + &self.map + } + + /// Deserialize from bytes. + pub fn from_bytes(data: Bytes) -> Result { + let root: Archive = rmp_serde::from_slice(&data[..])?; + + Ok(root) + } + + /// Serialize to bytes. + pub fn into_bytes(&self) -> Result { + let root_serialized = rmp_serde::to_vec(&self)?; + let root_serialized = Bytes::from(root_serialized); + + Ok(root_serialized) + } +} + +impl Client { + /// Fetch an archive from the network + /// + /// # Example + /// + /// ```no_run + /// # use autonomi::client::{Client, archive::ArchiveAddr}; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; + /// let client = Client::connect(&peers).await?; + /// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { + let data = self.data_get_public(addr).await?; + Ok(Archive::from_bytes(data)?) + } + + /// Upload an archive to the network + /// + /// # Example + /// + /// Create simple archive containing `file.txt` pointing to random XOR name. + /// + /// ```no_run + /// # use autonomi::client::{Client, data::DataAddr, archive::{Archive, ArchiveAddr, Metadata}}; + /// # use std::path::PathBuf; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; + /// # let client = Client::connect(&peers).await?; + /// # let wallet = todo!(); + /// let mut archive = Archive::new(); + /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); + /// let address = client.archive_put_public(archive, &wallet).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn archive_put_public( + &self, + archive: Archive, + wallet: &EvmWallet, + ) -> Result { + let bytes = archive + .into_bytes() + .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; + let result = self.data_put_public(bytes, wallet.into()).await; + debug!("Uploaded archive {archive:?} to the network and the address is {result:?}"); + result + } + + /// Get the cost to upload an archive + pub async fn archive_cost(&self, archive: Archive) -> Result { + let bytes = archive + .into_bytes() + .map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?; + let result = self.data_cost(bytes).await; + debug!("Calculated the cost to upload archive {archive:?} is {result:?}"); + result + } +} diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs index 30fd73f1a0..f5b81e7ab3 100644 --- a/autonomi/src/client/files/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -6,121 +6,68 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::data::{CostError, DataAddr, GetError, PutError}; +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::data::PrivateDataAccess; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; -use ant_networking::target_arch::{Duration, SystemTime}; use bytes::Bytes; use std::path::PathBuf; -use std::sync::LazyLock; - -use super::archive::{Archive, ArchiveAddr, Metadata}; - -/// Number of files to upload in parallel. -/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. -pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("File upload batch size: {}", batch_size); - batch_size -}); -/// Errors that can occur during the file upload operation. -#[derive(Debug, thiserror::Error)] -pub enum UploadError { - #[error("Failed to recursively traverse directory")] - WalkDir(#[from] walkdir::Error), - #[error("Input/output failure")] - IoError(#[from] std::io::Error), - #[error("Failed to upload file")] - PutError(#[from] PutError), - #[error("Failed to fetch file")] - GetError(#[from] GetError), - #[error("Failed to serialize")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), -} - -/// Errors that can occur during the download operation. -#[derive(Debug, thiserror::Error)] -pub enum DownloadError { - #[error("Failed to download file")] - GetError(#[from] GetError), - #[error("IO failure")] - IoError(#[from] std::io::Error), -} +use super::archive::{PrivateArchive, PrivateArchiveAccess}; +use super::fs_public::{DownloadError, UploadError}; -/// Errors that can occur during the file cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum FileCostError { - #[error("Cost error: {0}")] - Cost(#[from] CostError), - #[error("IO failure")] - IoError(#[from] std::io::Error), - #[error("Serialization error")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Self encryption error")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Walkdir error")] - WalkDir(#[from] walkdir::Error), -} +use super::fs_public::FILE_UPLOAD_BATCH_SIZE; impl Client { - /// Download file from network to local file system - pub async fn file_download_public( + /// Download a private file from network to local file system + pub async fn file_download( &self, - data_addr: DataAddr, + data_access: PrivateDataAccess, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let data = self.data_get_public(data_addr).await?; + let data = self.data_get(data_access).await?; if let Some(parent) = to_dest.parent() { tokio::fs::create_dir_all(parent).await?; - debug!("Created parent directories {parent:?} for {to_dest:?}"); + debug!("Created parent directories for {to_dest:?}"); } tokio::fs::write(to_dest.clone(), data).await?; - debug!("Downloaded file to {to_dest:?} from the network address {data_addr:?}"); + debug!("Downloaded file to {to_dest:?}"); Ok(()) } - /// Download directory from network to local file system - pub async fn dir_download_public( + /// Download a private directory from network to local file system + pub async fn dir_download( &self, - archive_addr: ArchiveAddr, + archive_access: PrivateArchiveAccess, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let archive = self.archive_get_public(archive_addr).await?; - debug!("Downloaded archive for the directory from the network at {archive_addr:?}"); + let archive = self.archive_get(archive_access).await?; for (path, addr, _meta) in archive.iter() { - self.file_download_public(*addr, to_dest.join(path)).await?; + self.file_download(addr.clone(), to_dest.join(path)).await?; } - debug!( - "All files in the directory downloaded to {:?} from the network address {:?}", - to_dest.parent(), - archive_addr - ); + debug!("Downloaded directory to {to_dest:?}"); Ok(()) } - /// Upload a directory to the network. The directory is recursively walked. - /// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive) - pub async fn dir_upload_public( + /// Upload a private directory to the network. The directory is recursively walked. + /// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive) + pub async fn dir_upload( &self, dir_path: PathBuf, wallet: &EvmWallet, - ) -> Result { - info!("Uploading directory: {dir_path:?}"); + ) -> Result { + info!("Uploading directory as private: {dir_path:?}"); let start = tokio::time::Instant::now(); - // start upload of files in parallel + // start upload of file in parallel let mut upload_tasks = Vec::new(); for entry in walkdir::WalkDir::new(dir_path) { let entry = entry?; @@ -128,10 +75,10 @@ impl Client { continue; } - let metadata = metadata_from_entry(&entry); + let metadata = super::fs_public::metadata_from_entry(&entry); let path = entry.path().to_path_buf(); upload_tasks.push(async move { - let file = self.file_upload_public(path.clone(), wallet).await; + let file = self.file_upload(path.clone(), wallet).await; (path, metadata, file) }); } @@ -144,7 +91,7 @@ impl Client { uploads.len(), start.elapsed() ); - let mut archive = Archive::new(); + let mut archive = PrivateArchive::new(); for (path, metadata, maybe_file) in uploads.into_iter() { match maybe_file { Ok(file) => archive.add_file(path, file, metadata), @@ -157,125 +104,32 @@ impl Client { // upload archive let archive_serialized = archive.into_bytes()?; - let arch_addr = self - .data_put_public(archive_serialized, wallet.into()) - .await?; + let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; - info!("Complete archive upload completed in {:?}", start.elapsed()); + info!( + "Complete private archive upload completed in {:?}", + start.elapsed() + ); #[cfg(feature = "loud")] println!("Upload completed in {:?}", start.elapsed()); - debug!("Directory uploaded to the network at {arch_addr:?}"); Ok(arch_addr) } - /// Upload a file to the network. - /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) - async fn file_upload_public( + /// Upload a private file to the network. + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap) + async fn file_upload( &self, path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading file: {path:?}"); #[cfg(feature = "loud")] println!("Uploading file: {path:?}"); - let data = tokio::fs::read(path.clone()).await?; + let data = tokio::fs::read(path).await?; let data = Bytes::from(data); - let addr = self.data_put_public(data, wallet.into()).await?; - debug!("File {path:?} uploaded to the network at {addr:?}"); + let addr = self.data_put(data, wallet.into()).await?; + debug!("Uploaded file successfully in the privateAchive: {addr:?}"); Ok(addr) } - - /// Get the cost to upload a file/dir to the network. - /// quick and dirty implementation, please refactor once files are cleanly implemented - pub async fn file_cost(&self, path: &PathBuf) -> Result { - let mut archive = Archive::new(); - let mut total_cost = ant_evm::Amount::ZERO; - - for entry in walkdir::WalkDir::new(path) { - let entry = entry?; - - if !entry.file_type().is_file() { - continue; - } - - let path = entry.path().to_path_buf(); - tracing::info!("Cost for file: {path:?}"); - - let data = tokio::fs::read(&path).await?; - let file_bytes = Bytes::from(data); - let file_cost = self.data_cost(file_bytes.clone()).await?; - - total_cost += file_cost.as_atto(); - - // re-do encryption to get the correct map xorname here - // this code needs refactor - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, _) = crate::self_encryption::encrypt(file_bytes)?; - tracing::debug!("Encryption took: {:.2?}", now.elapsed()); - let map_xor_name = *data_map_chunk.address().xorname(); - - let metadata = metadata_from_entry(&entry); - archive.add_file(path, map_xor_name, metadata); - } - - let root_serialized = rmp_serde::to_vec(&archive)?; - - let archive_cost = self.data_cost(Bytes::from(root_serialized)).await?; - - total_cost += archive_cost.as_atto(); - debug!("Total cost for the directory: {total_cost:?}"); - Ok(total_cost.into()) - } -} - -// Get metadata from directory entry. Defaults to `0` for creation and modification times if -// any error is encountered. Logs errors upon error. -pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { - let fs_metadata = match entry.metadata() { - Ok(metadata) => metadata, - Err(err) => { - tracing::warn!( - "Failed to get metadata for `{}`: {err}", - entry.path().display() - ); - return Metadata { - uploaded: 0, - created: 0, - modified: 0, - size: 0, - }; - } - }; - - let unix_time = |property: &'static str, time: std::io::Result| { - time.inspect_err(|err| { - tracing::warn!( - "Failed to get '{property}' metadata for `{}`: {err}", - entry.path().display() - ); - }) - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH) - .inspect_err(|err| { - tracing::warn!( - "'{property}' metadata of `{}` is before UNIX epoch: {err}", - entry.path().display() - ); - }) - .unwrap_or(Duration::from_secs(0)) - .as_secs() - }; - let created = unix_time("created", fs_metadata.created()); - let modified = unix_time("modified", fs_metadata.modified()); - - Metadata { - uploaded: SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(), - created, - modified, - size: fs_metadata.len(), - } } diff --git a/autonomi/src/client/files/fs_private.rs b/autonomi/src/client/files/fs_private.rs deleted file mode 100644 index 3fef1264b3..0000000000 --- a/autonomi/src/client/files/fs_private.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::client::data::PrivateDataAccess; -use crate::client::utils::process_tasks_with_max_concurrency; -use crate::client::Client; -use ant_evm::EvmWallet; -use bytes::Bytes; -use std::path::PathBuf; - -use super::archive_private::{PrivateArchive, PrivateArchiveAccess}; -use super::fs::{DownloadError, UploadError}; - -use super::fs::FILE_UPLOAD_BATCH_SIZE; - -impl Client { - /// Download a private file from network to local file system - pub async fn file_download( - &self, - data_access: PrivateDataAccess, - to_dest: PathBuf, - ) -> Result<(), DownloadError> { - let data = self.data_get(data_access).await?; - if let Some(parent) = to_dest.parent() { - tokio::fs::create_dir_all(parent).await?; - debug!("Created parent directories for {to_dest:?}"); - } - tokio::fs::write(to_dest.clone(), data).await?; - debug!("Downloaded file to {to_dest:?}"); - Ok(()) - } - - /// Download a private directory from network to local file system - pub async fn dir_download( - &self, - archive_access: PrivateArchiveAccess, - to_dest: PathBuf, - ) -> Result<(), DownloadError> { - let archive = self.archive_get(archive_access).await?; - for (path, addr, _meta) in archive.iter() { - self.file_download(addr.clone(), to_dest.join(path)).await?; - } - debug!("Downloaded directory to {to_dest:?}"); - Ok(()) - } - - /// Upload a private directory to the network. The directory is recursively walked. - /// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive) - pub async fn dir_upload( - &self, - dir_path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - info!("Uploading directory as private: {dir_path:?}"); - let start = tokio::time::Instant::now(); - - // start upload of file in parallel - let mut upload_tasks = Vec::new(); - for entry in walkdir::WalkDir::new(dir_path) { - let entry = entry?; - if !entry.file_type().is_file() { - continue; - } - - let metadata = super::fs::metadata_from_entry(&entry); - let path = entry.path().to_path_buf(); - upload_tasks.push(async move { - let file = self.file_upload(path.clone(), wallet).await; - (path, metadata, file) - }); - } - - // wait for all files to be uploaded - let uploads = - process_tasks_with_max_concurrency(upload_tasks, *FILE_UPLOAD_BATCH_SIZE).await; - info!( - "Upload of {} files completed in {:?}", - uploads.len(), - start.elapsed() - ); - let mut archive = PrivateArchive::new(); - for (path, metadata, maybe_file) in uploads.into_iter() { - match maybe_file { - Ok(file) => archive.add_file(path, file, metadata), - Err(err) => { - error!("Failed to upload file: {path:?}: {err:?}"); - return Err(err); - } - } - } - - // upload archive - let archive_serialized = archive.into_bytes()?; - let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; - - info!( - "Complete private archive upload completed in {:?}", - start.elapsed() - ); - #[cfg(feature = "loud")] - println!("Upload completed in {:?}", start.elapsed()); - Ok(arch_addr) - } - - /// Upload a private file to the network. - /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap) - async fn file_upload( - &self, - path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - info!("Uploading file: {path:?}"); - #[cfg(feature = "loud")] - println!("Uploading file: {path:?}"); - - let data = tokio::fs::read(path).await?; - let data = Bytes::from(data); - let addr = self.data_put(data, wallet.into()).await?; - debug!("Uploaded file successfully in the privateAchive: {addr:?}"); - Ok(addr) - } -} diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs new file mode 100644 index 0000000000..dec7ce8fcc --- /dev/null +++ b/autonomi/src/client/files/fs_public.rs @@ -0,0 +1,281 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::data::{CostError, DataAddr, GetError, PutError}; +use crate::client::utils::process_tasks_with_max_concurrency; +use crate::client::Client; +use ant_evm::EvmWallet; +use ant_networking::target_arch::{Duration, SystemTime}; +use bytes::Bytes; +use std::path::PathBuf; +use std::sync::LazyLock; + +use super::archive_public::{Archive, ArchiveAddr, Metadata}; + +/// Number of files to upload in parallel. +/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. +pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("File upload batch size: {}", batch_size); + batch_size +}); + +/// Errors that can occur during the file upload operation. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + #[error("Failed to recursively traverse directory")] + WalkDir(#[from] walkdir::Error), + #[error("Input/output failure")] + IoError(#[from] std::io::Error), + #[error("Failed to upload file")] + PutError(#[from] PutError), + #[error("Failed to fetch file")] + GetError(#[from] GetError), + #[error("Failed to serialize")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), +} + +/// Errors that can occur during the download operation. +#[derive(Debug, thiserror::Error)] +pub enum DownloadError { + #[error("Failed to download file")] + GetError(#[from] GetError), + #[error("IO failure")] + IoError(#[from] std::io::Error), +} + +/// Errors that can occur during the file cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum FileCostError { + #[error("Cost error: {0}")] + Cost(#[from] CostError), + #[error("IO failure")] + IoError(#[from] std::io::Error), + #[error("Serialization error")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Self encryption error")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Walkdir error")] + WalkDir(#[from] walkdir::Error), +} + +impl Client { + /// Download file from network to local file system + pub async fn file_download_public( + &self, + data_addr: DataAddr, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let data = self.data_get_public(data_addr).await?; + if let Some(parent) = to_dest.parent() { + tokio::fs::create_dir_all(parent).await?; + debug!("Created parent directories {parent:?} for {to_dest:?}"); + } + tokio::fs::write(to_dest.clone(), data).await?; + debug!("Downloaded file to {to_dest:?} from the network address {data_addr:?}"); + Ok(()) + } + + /// Download directory from network to local file system + pub async fn dir_download_public( + &self, + archive_addr: ArchiveAddr, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let archive = self.archive_get_public(archive_addr).await?; + debug!("Downloaded archive for the directory from the network at {archive_addr:?}"); + for (path, addr, _meta) in archive.iter() { + self.file_download_public(*addr, to_dest.join(path)).await?; + } + debug!( + "All files in the directory downloaded to {:?} from the network address {:?}", + to_dest.parent(), + archive_addr + ); + Ok(()) + } + + /// Upload a directory to the network. The directory is recursively walked. + /// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive) + pub async fn dir_upload_public( + &self, + dir_path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + info!("Uploading directory: {dir_path:?}"); + let start = tokio::time::Instant::now(); + + // start upload of files in parallel + let mut upload_tasks = Vec::new(); + for entry in walkdir::WalkDir::new(dir_path) { + let entry = entry?; + if !entry.file_type().is_file() { + continue; + } + + let metadata = metadata_from_entry(&entry); + let path = entry.path().to_path_buf(); + upload_tasks.push(async move { + let file = self.file_upload_public(path.clone(), wallet).await; + (path, metadata, file) + }); + } + + // wait for all files to be uploaded + let uploads = + process_tasks_with_max_concurrency(upload_tasks, *FILE_UPLOAD_BATCH_SIZE).await; + info!( + "Upload of {} files completed in {:?}", + uploads.len(), + start.elapsed() + ); + let mut archive = Archive::new(); + for (path, metadata, maybe_file) in uploads.into_iter() { + match maybe_file { + Ok(file) => archive.add_file(path, file, metadata), + Err(err) => { + error!("Failed to upload file: {path:?}: {err:?}"); + return Err(err); + } + } + } + + // upload archive + let archive_serialized = archive.into_bytes()?; + let arch_addr = self + .data_put_public(archive_serialized, wallet.into()) + .await?; + + info!("Complete archive upload completed in {:?}", start.elapsed()); + #[cfg(feature = "loud")] + println!("Upload completed in {:?}", start.elapsed()); + debug!("Directory uploaded to the network at {arch_addr:?}"); + Ok(arch_addr) + } + + /// Upload a file to the network. + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) + async fn file_upload_public( + &self, + path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + info!("Uploading file: {path:?}"); + #[cfg(feature = "loud")] + println!("Uploading file: {path:?}"); + + let data = tokio::fs::read(path.clone()).await?; + let data = Bytes::from(data); + let addr = self.data_put_public(data, wallet.into()).await?; + debug!("File {path:?} uploaded to the network at {addr:?}"); + Ok(addr) + } + + /// Get the cost to upload a file/dir to the network. + /// quick and dirty implementation, please refactor once files are cleanly implemented + pub async fn file_cost(&self, path: &PathBuf) -> Result { + let mut archive = Archive::new(); + let mut total_cost = ant_evm::Amount::ZERO; + + for entry in walkdir::WalkDir::new(path) { + let entry = entry?; + + if !entry.file_type().is_file() { + continue; + } + + let path = entry.path().to_path_buf(); + tracing::info!("Cost for file: {path:?}"); + + let data = tokio::fs::read(&path).await?; + let file_bytes = Bytes::from(data); + let file_cost = self.data_cost(file_bytes.clone()).await?; + + total_cost += file_cost.as_atto(); + + // re-do encryption to get the correct map xorname here + // this code needs refactor + let now = ant_networking::target_arch::Instant::now(); + let (data_map_chunk, _) = crate::self_encryption::encrypt(file_bytes)?; + tracing::debug!("Encryption took: {:.2?}", now.elapsed()); + let map_xor_name = *data_map_chunk.address().xorname(); + + let metadata = metadata_from_entry(&entry); + archive.add_file(path, map_xor_name, metadata); + } + + let root_serialized = rmp_serde::to_vec(&archive)?; + + let archive_cost = self.data_cost(Bytes::from(root_serialized)).await?; + + total_cost += archive_cost.as_atto(); + debug!("Total cost for the directory: {total_cost:?}"); + Ok(total_cost.into()) + } +} + +// Get metadata from directory entry. Defaults to `0` for creation and modification times if +// any error is encountered. Logs errors upon error. +pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { + let fs_metadata = match entry.metadata() { + Ok(metadata) => metadata, + Err(err) => { + tracing::warn!( + "Failed to get metadata for `{}`: {err}", + entry.path().display() + ); + return Metadata { + uploaded: 0, + created: 0, + modified: 0, + size: 0, + }; + } + }; + + let unix_time = |property: &'static str, time: std::io::Result| { + time.inspect_err(|err| { + tracing::warn!( + "Failed to get '{property}' metadata for `{}`: {err}", + entry.path().display() + ); + }) + .unwrap_or(SystemTime::UNIX_EPOCH) + .duration_since(SystemTime::UNIX_EPOCH) + .inspect_err(|err| { + tracing::warn!( + "'{property}' metadata of `{}` is before UNIX epoch: {err}", + entry.path().display() + ); + }) + .unwrap_or(Duration::from_secs(0)) + .as_secs() + }; + let created = unix_time("created", fs_metadata.created()); + let modified = unix_time("modified", fs_metadata.modified()); + + Metadata { + uploaded: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(), + created, + modified, + size: fs_metadata.len(), + } +} diff --git a/autonomi/src/client/files/mod.rs b/autonomi/src/client/files/mod.rs index 0f76d26d28..981c1d472c 100644 --- a/autonomi/src/client/files/mod.rs +++ b/autonomi/src/client/files/mod.rs @@ -1,8 +1,8 @@ pub mod archive; -pub mod archive_private; +pub mod archive_public; #[cfg(feature = "fs")] #[cfg_attr(docsrs, doc(cfg(feature = "fs")))] pub mod fs; #[cfg(feature = "fs")] #[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs_private; +pub mod fs_public; diff --git a/autonomi/src/client/vault/user_data.rs b/autonomi/src/client/vault/user_data.rs index c30ba5f574..a0b4069534 100644 --- a/autonomi/src/client/vault/user_data.rs +++ b/autonomi/src/client/vault/user_data.rs @@ -10,8 +10,8 @@ use std::collections::HashMap; use crate::client::data::GetError; use crate::client::data::PutError; -use crate::client::files::archive::ArchiveAddr; -use crate::client::files::archive_private::PrivateArchiveAccess; +use crate::client::files::archive::PrivateArchiveAccess; +use crate::client::files::archive_public::ArchiveAddr; use crate::client::payment::PaymentOption; use crate::client::registers::RegisterAddress; use crate::client::vault::VaultError; diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index da11ebe6c3..1447c5aa8f 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -3,7 +3,7 @@ use crate::client::{ data::PrivateDataAccess, - files::{archive::ArchiveAddr, archive_private::PrivateArchiveAccess}, + files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, payment::PaymentOption as RustPaymentOption, vault::{UserData, VaultSecretKey}, Client as RustClient, diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index 59573a6c44..d4c92bae0e 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -5,8 +5,8 @@ use alloy::providers::Provider; use ant_evm::{QuoteHash, TxHash}; use ant_logging::LogBuilder; use autonomi::client::external_signer::encrypt_data; -use autonomi::client::files::archive::Metadata; -use autonomi::client::files::archive_private::PrivateArchive; +use autonomi::client::files::archive::PrivateArchive; +use autonomi::client::files::archive_public::Metadata; use autonomi::client::payment::Receipt; use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; use autonomi::client::vault::VaultSecretKey; diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 9eb0604681..8ae37d9608 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -106,7 +106,7 @@ async fn file_into_vault() -> Result<()> { let (ap, got_version) = new_client.fetch_and_decrypt_vault(&client_sk).await?; assert_eq!(set_version, got_version); - let ap_archive_fetched = autonomi::client::files::archive::Archive::from_bytes(ap)?; + let ap_archive_fetched = autonomi::client::files::archive_public::Archive::from_bytes(ap)?; assert_eq!( archive, ap_archive_fetched, From 37814067dd561b22081dd49be98cb567de27f5f0 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 10:51:08 +0100 Subject: [PATCH 06/13] refactor(autonomi): restrict consts to crate --- autonomi/src/client/data/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autonomi/src/client/data/mod.rs b/autonomi/src/client/data/mod.rs index f333616d67..5c7077981b 100644 --- a/autonomi/src/client/data/mod.rs +++ b/autonomi/src/client/data/mod.rs @@ -25,7 +25,7 @@ pub mod public; /// Number of chunks to upload in parallel. /// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { +pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") .ok() .and_then(|s| s.parse().ok()) @@ -40,11 +40,11 @@ pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { }); /// Number of retries to upload chunks. -pub const RETRY_ATTEMPTS: usize = 3; +pub(crate) const RETRY_ATTEMPTS: usize = 3; /// Number of chunks to download in parallel. /// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { +pub(crate) static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") .ok() .and_then(|s| s.parse().ok()) From c8cfafcadcbd259dea5a7598ae677ed16df86905 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 11:15:07 +0100 Subject: [PATCH 07/13] refactor(autonomi): move structs to top-level --- ant-cli/src/access/user_data.rs | 3 +- ant-cli/src/actions/download.rs | 5 +- autonomi/src/client/data/mod.rs | 10 ++-- autonomi/src/client/files/archive.rs | 38 +++++++++++- autonomi/src/client/files/archive_public.rs | 48 +++------------- autonomi/src/client/files/fs.rs | 63 ++++++++++++++++++-- autonomi/src/client/files/fs_public.rs | 64 ++------------------- autonomi/tests/external_signer.rs | 3 +- 8 files changed, 119 insertions(+), 115 deletions(-) diff --git a/ant-cli/src/access/user_data.rs b/ant-cli/src/access/user_data.rs index 57deb85785..3fc20785cd 100644 --- a/ant-cli/src/access/user_data.rs +++ b/ant-cli/src/access/user_data.rs @@ -10,8 +10,7 @@ use std::collections::HashMap; use autonomi::client::{ address::{addr_to_str, str_to_addr}, - archive::ArchiveAddr, - archive_private::PrivateArchiveAccess, + files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, registers::{RegisterAddress, RegisterSecretKey}, vault::UserData, }; diff --git a/ant-cli/src/actions/download.rs b/ant-cli/src/actions/download.rs index f4edf8da8e..6b3bbd380c 100644 --- a/ant-cli/src/actions/download.rs +++ b/ant-cli/src/actions/download.rs @@ -8,7 +8,10 @@ use super::get_progress_bar; use autonomi::{ - client::{address::str_to_addr, archive::ArchiveAddr, archive_private::PrivateArchiveAccess}, + client::{ + address::str_to_addr, + files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, + }, Client, }; use color_eyre::{ diff --git a/autonomi/src/client/data/mod.rs b/autonomi/src/client/data/mod.rs index 5c7077981b..fb30f48ce9 100644 --- a/autonomi/src/client/data/mod.rs +++ b/autonomi/src/client/data/mod.rs @@ -24,6 +24,7 @@ use crate::{self_encryption::encrypt, Client}; pub mod public; /// Number of chunks to upload in parallel. +/// /// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") @@ -39,12 +40,10 @@ pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { batch_size }); -/// Number of retries to upload chunks. -pub(crate) const RETRY_ATTEMPTS: usize = 3; - /// Number of chunks to download in parallel. +/// /// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. -pub(crate) static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { +pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") .ok() .and_then(|s| s.parse().ok()) @@ -58,6 +57,9 @@ pub(crate) static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| batch_size }); +/// Number of retries to upload chunks. +pub(crate) const RETRY_ATTEMPTS: usize = 3; + /// Raw Data Address (points to a DataMap) pub type DataAddr = XorName; /// Raw Chunk Address (points to a [`Chunk`]) diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/files/archive.rs index c188f04043..1d71bd03f6 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -13,7 +13,6 @@ use std::{ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; -use super::archive_public::{Metadata, RenameError}; use crate::{ client::{ data::{GetError, PrivateDataAccess, PutError}, @@ -23,11 +22,48 @@ use crate::{ }; use bytes::Bytes; use serde::{Deserialize, Serialize}; +use thiserror::Error; /// The address of a private archive /// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data pub type PrivateArchiveAccess = PrivateDataAccess; +#[derive(Error, Debug, PartialEq, Eq)] +pub enum RenameError { + #[error("File not found in archive: {0}")] + FileNotFound(PathBuf), +} + +/// Metadata for a file in an archive. Time values are UNIX timestamps. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Metadata { + /// When the file was (last) uploaded to the network. + pub uploaded: u64, + /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. + pub created: u64, + /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. + pub modified: u64, + /// File size in bytes + pub size: u64, +} + +impl Metadata { + /// Create a new metadata struct with the current time as uploaded, created and modified. + pub fn new_with_size(size: u64) -> Self { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(); + + Self { + uploaded: now, + created: now, + modified: now, + size, + } + } +} + /// A private archive of files that containing file paths, their metadata and the files data maps /// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/files/archive_public.rs index 7b8323032a..2178579b4b 100644 --- a/autonomi/src/client/files/archive_public.rs +++ b/autonomi/src/client/files/archive_public.rs @@ -18,21 +18,17 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use xor_name::XorName; -/// The address of an archive on the network. Points to an [`Archive`]. -pub type ArchiveAddr = XorName; - -use thiserror::Error; - +use super::archive::Metadata; use crate::{ - client::data::{CostError, DataAddr, GetError, PutError}, + client::{ + data::{CostError, DataAddr, GetError, PutError}, + files::archive::RenameError, + }, Client, }; -#[derive(Error, Debug, PartialEq, Eq)] -pub enum RenameError { - #[error("File not found in archive: {0}")] - FileNotFound(PathBuf), -} +/// The address of an archive on the network. Points to an [`Archive`]. +pub type ArchiveAddr = XorName; /// An archive of files that containing file paths, their metadata and the files data addresses /// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. @@ -42,36 +38,6 @@ pub struct Archive { map: HashMap, } -/// Metadata for a file in an archive. Time values are UNIX timestamps. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Metadata { - /// When the file was (last) uploaded to the network. - pub uploaded: u64, - /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. - pub created: u64, - /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. - pub modified: u64, - /// File size in bytes - pub size: u64, -} - -impl Metadata { - /// Create a new metadata struct with the current time as uploaded, created and modified. - pub fn new_with_size(size: u64) -> Self { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(); - - Self { - uploaded: now, - created: now, - modified: now, - size, - } - } -} - impl Archive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs index f5b81e7ab3..b698515183 100644 --- a/autonomi/src/client/files/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -14,17 +14,72 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::data::PrivateDataAccess; +use crate::client::data::{CostError, GetError, PrivateDataAccess, PutError}; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; use bytes::Bytes; -use std::path::PathBuf; +use std::{path::PathBuf, sync::LazyLock}; use super::archive::{PrivateArchive, PrivateArchiveAccess}; -use super::fs_public::{DownloadError, UploadError}; -use super::fs_public::FILE_UPLOAD_BATCH_SIZE; +/// Number of files to upload in parallel. +/// +/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. +pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("File upload batch size: {}", batch_size); + batch_size +}); + +/// Errors that can occur during the file upload operation. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + #[error("Failed to recursively traverse directory")] + WalkDir(#[from] walkdir::Error), + #[error("Input/output failure")] + IoError(#[from] std::io::Error), + #[error("Failed to upload file")] + PutError(#[from] PutError), + #[error("Failed to fetch file")] + GetError(#[from] GetError), + #[error("Failed to serialize")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), +} + +/// Errors that can occur during the download operation. +#[derive(Debug, thiserror::Error)] +pub enum DownloadError { + #[error("Failed to download file")] + GetError(#[from] GetError), + #[error("IO failure")] + IoError(#[from] std::io::Error), +} + +/// Errors that can occur during the file cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum FileCostError { + #[error("Cost error: {0}")] + Cost(#[from] CostError), + #[error("IO failure")] + IoError(#[from] std::io::Error), + #[error("Serialization error")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Self encryption error")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Walkdir error")] + WalkDir(#[from] walkdir::Error), +} impl Client { /// Download a private file from network to local file system diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs index dec7ce8fcc..c428eabb10 100644 --- a/autonomi/src/client/files/fs_public.rs +++ b/autonomi/src/client/files/fs_public.rs @@ -6,73 +6,17 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::data::{CostError, DataAddr, GetError, PutError}; +use crate::client::data::DataAddr; +use crate::client::files::archive::Metadata; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; use ant_networking::target_arch::{Duration, SystemTime}; use bytes::Bytes; use std::path::PathBuf; -use std::sync::LazyLock; -use super::archive_public::{Archive, ArchiveAddr, Metadata}; - -/// Number of files to upload in parallel. -/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. -pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("File upload batch size: {}", batch_size); - batch_size -}); - -/// Errors that can occur during the file upload operation. -#[derive(Debug, thiserror::Error)] -pub enum UploadError { - #[error("Failed to recursively traverse directory")] - WalkDir(#[from] walkdir::Error), - #[error("Input/output failure")] - IoError(#[from] std::io::Error), - #[error("Failed to upload file")] - PutError(#[from] PutError), - #[error("Failed to fetch file")] - GetError(#[from] GetError), - #[error("Failed to serialize")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), -} - -/// Errors that can occur during the download operation. -#[derive(Debug, thiserror::Error)] -pub enum DownloadError { - #[error("Failed to download file")] - GetError(#[from] GetError), - #[error("IO failure")] - IoError(#[from] std::io::Error), -} - -/// Errors that can occur during the file cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum FileCostError { - #[error("Cost error: {0}")] - Cost(#[from] CostError), - #[error("IO failure")] - IoError(#[from] std::io::Error), - #[error("Serialization error")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Self encryption error")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Walkdir error")] - WalkDir(#[from] walkdir::Error), -} +use super::archive_public::{Archive, ArchiveAddr}; +use super::fs::*; impl Client { /// Download file from network to local file system diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index d4c92bae0e..58722c5d45 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -5,8 +5,7 @@ use alloy::providers::Provider; use ant_evm::{QuoteHash, TxHash}; use ant_logging::LogBuilder; use autonomi::client::external_signer::encrypt_data; -use autonomi::client::files::archive::PrivateArchive; -use autonomi::client::files::archive_public::Metadata; +use autonomi::client::files::archive::{Metadata, PrivateArchive}; use autonomi::client::payment::Receipt; use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; use autonomi::client::vault::VaultSecretKey; From ca218073d102c9773b5ee7ad9f8e008f654f4818 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 11:25:06 +0100 Subject: [PATCH 08/13] refactor(autonomi): rename PrivateDataAccess to DataMapChunk --- autonomi/README_PYTHON.md | 4 ++-- .../examples/autonomi_private_encryption.py | 2 +- autonomi/python/autonomi_client/__init__.py | 4 ++-- autonomi/src/client/data/mod.rs | 23 ++++++++++--------- autonomi/src/client/files/archive.rs | 19 +++++++-------- autonomi/src/client/files/fs.rs | 8 +++---- autonomi/src/client/wasm.rs | 12 +++++----- autonomi/src/python.rs | 20 ++++++++-------- 8 files changed, 47 insertions(+), 45 deletions(-) diff --git a/autonomi/README_PYTHON.md b/autonomi/README_PYTHON.md index 43e6ceaf04..6772ce14a1 100644 --- a/autonomi/README_PYTHON.md +++ b/autonomi/README_PYTHON.md @@ -43,7 +43,7 @@ print(f"Retrieved: {retrieved.decode()}") - `data_put_public(data: bytes, payment: PaymentOption)`: Upload data - `data_get_public(addr: str)`: Download data - `data_put(data: bytes, payment: PaymentOption)`: Store private data - - `data_get(access: PrivateDataAccess)`: Retrieve private data + - `data_get(access: DataMapChunk)`: Retrieve private data - `register_generate_key()`: Generate register key - `Wallet`: Ethereum wallet management @@ -56,7 +56,7 @@ print(f"Retrieved: {retrieved.decode()}") #### Private Data -- `PrivateDataAccess`: Handle private data storage +- `DataMapChunk`: Handle private data storage - `from_hex(hex: str)`: Create from hex string - `to_hex()`: Convert to hex string - `address()`: Get short reference address diff --git a/autonomi/examples/autonomi_private_encryption.py b/autonomi/examples/autonomi_private_encryption.py index 3cfdfe54a1..e95bcc5386 100644 --- a/autonomi/examples/autonomi_private_encryption.py +++ b/autonomi/examples/autonomi_private_encryption.py @@ -1,5 +1,5 @@ from autonomi_client import ( - Client, Wallet, PaymentOption, PrivateDataAccess, + Client, Wallet, PaymentOption, DataMapChunk, encrypt, hash_to_short_string ) import json diff --git a/autonomi/python/autonomi_client/__init__.py b/autonomi/python/autonomi_client/__init__.py index 11d550e79d..b1e437b894 100644 --- a/autonomi/python/autonomi_client/__init__.py +++ b/autonomi/python/autonomi_client/__init__.py @@ -1,4 +1,4 @@ -from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, PrivateDataAccess, encrypt +from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, DataMapChunk, encrypt __all__ = [ "Client", @@ -6,6 +6,6 @@ "PaymentOption", "VaultSecretKey", "UserData", - "PrivateDataAccess", + "DataMapChunk", "encrypt" ] diff --git a/autonomi/src/client/data/mod.rs b/autonomi/src/client/data/mod.rs index fb30f48ce9..b85f54a68e 100644 --- a/autonomi/src/client/data/mod.rs +++ b/autonomi/src/client/data/mod.rs @@ -127,9 +127,9 @@ pub enum CostError { /// Private data on the network can be accessed with this #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PrivateDataAccess(Chunk); +pub struct DataMapChunk(Chunk); -impl PrivateDataAccess { +impl DataMapChunk { pub fn to_hex(&self) -> String { hex::encode(self.0.value()) } @@ -139,7 +139,7 @@ impl PrivateDataAccess { Ok(Self(Chunk::new(Bytes::from(data)))) } - /// Get a private address for [`PrivateDataAccess`]. Note that this is not a network address, it is only used for refering to private data client side. + /// Get a private address for [`DataMapChunk`]. Note that this is not a network address, it is only used for refering to private data client side. pub fn address(&self) -> String { hash_to_short_string(&self.to_hex()) } @@ -153,8 +153,8 @@ fn hash_to_short_string(input: &str) -> String { } impl Client { - /// Fetch a blob of private data from the network - pub async fn data_get(&self, data_map: PrivateDataAccess) -> Result { + /// Fetch a blob of (private) data from the network + pub async fn data_get(&self, data_map: DataMapChunk) -> Result { info!( "Fetching private data from Data Map {:?}", data_map.0.address() @@ -166,13 +166,14 @@ impl Client { } /// Upload a piece of private data to the network. This data will be self-encrypted. - /// Returns the [`PrivateDataAccess`] containing the map to the encrypted chunks. - /// This data is private and only accessible with the [`PrivateDataAccess`]. + /// The [`DataMapChunk`] is not uploaded to the network, keeping the data private. + /// + /// Returns the [`DataMapChunk`] containing the map to the encrypted chunks. pub async fn data_put( &self, data: Bytes, payment_option: PaymentOption, - ) -> Result { + ) -> Result { let now = ant_networking::target_arch::Instant::now(); let (data_map_chunk, chunks) = encrypt(data)?; debug!("Encryption took: {:.2?}", now.elapsed()); @@ -220,7 +221,7 @@ impl Client { } } - Ok(PrivateDataAccess(data_map_chunk)) + Ok(DataMapChunk(data_map_chunk)) } } @@ -230,9 +231,9 @@ mod tests { #[test] fn test_hex() { - let data_map = PrivateDataAccess(Chunk::new(Bytes::from_static(b"hello"))); + let data_map = DataMapChunk(Chunk::new(Bytes::from_static(b"hello"))); let hex = data_map.to_hex(); - let data_map2 = PrivateDataAccess::from_hex(&hex).expect("Failed to decode hex"); + let data_map2 = DataMapChunk::from_hex(&hex).expect("Failed to decode hex"); assert_eq!(data_map, data_map2); } } diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/files/archive.rs index 1d71bd03f6..a33a6a2ac6 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -15,7 +15,7 @@ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; use crate::{ client::{ - data::{GetError, PrivateDataAccess, PutError}, + data::{DataMapChunk, GetError, PutError}, payment::PaymentOption, }, Client, @@ -24,9 +24,10 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use thiserror::Error; -/// The address of a private archive -/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data -pub type PrivateArchiveAccess = PrivateDataAccess; +/// The address of a private archive. +/// +/// Contains the [`DataMapChunk`] leading to the [`PrivateArchive`] data +pub type PrivateArchiveAccess = DataMapChunk; #[derive(Error, Debug, PartialEq, Eq)] pub enum RenameError { @@ -68,7 +69,7 @@ impl Metadata { /// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct PrivateArchive { - map: HashMap, + map: HashMap, } impl PrivateArchive { @@ -99,7 +100,7 @@ impl PrivateArchive { /// Add a file to a local archive /// Note that this does not upload the archive to the network - pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) { + pub fn add_file(&mut self, path: PathBuf, data_map: DataMapChunk, meta: Metadata) { self.map.insert(path.clone(), (data_map, meta)); debug!("Added a new file to the archive, path: {:?}", path); } @@ -113,7 +114,7 @@ impl PrivateArchive { } /// List all data addresses of the files in the archive - pub fn addresses(&self) -> Vec { + pub fn addresses(&self) -> Vec { self.map .values() .map(|(data_map, _)| data_map.clone()) @@ -122,14 +123,14 @@ impl PrivateArchive { /// Iterate over the archive items /// Returns an iterator over (PathBuf, SecretDataMap, Metadata) - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.map .iter() .map(|(path, (data_map, meta))| (path, data_map, meta)) } /// Get the underlying map - pub fn map(&self) -> &HashMap { + pub fn map(&self) -> &HashMap { &self.map } diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs index b698515183..6e67c0948c 100644 --- a/autonomi/src/client/files/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -14,7 +14,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::data::{CostError, GetError, PrivateDataAccess, PutError}; +use crate::client::data::{CostError, DataMapChunk, GetError, PutError}; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; @@ -85,7 +85,7 @@ impl Client { /// Download a private file from network to local file system pub async fn file_download( &self, - data_access: PrivateDataAccess, + data_access: DataMapChunk, to_dest: PathBuf, ) -> Result<(), DownloadError> { let data = self.data_get(data_access).await?; @@ -171,12 +171,12 @@ impl Client { } /// Upload a private file to the network. - /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap) + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap) async fn file_upload( &self, path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading file: {path:?}"); #[cfg(feature = "loud")] println!("Uploading file: {path:?}"); diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 8353e55ab9..d02d68e7b6 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -1,7 +1,7 @@ use super::address::{addr_to_str, str_to_addr}; #[cfg(feature = "vault")] use super::vault::UserData; -use crate::client::data_private::PrivateDataAccess; +use crate::client::data_private::DataMapChunk; use crate::client::payment::Receipt; use ant_protocol::storage::Chunk; use libp2p::Multiaddr; @@ -107,7 +107,7 @@ impl JsClient { /// Upload private data to the network. /// - /// Returns the `PrivateDataAccess` chunk of the data. + /// Returns the `DataMapChunk` chunk of the data. #[wasm_bindgen(js_name = putPrivateData)] pub async fn put_private_data( &self, @@ -124,7 +124,7 @@ impl JsClient { /// Upload private data to the network. /// Uses a `Receipt` as payment. /// - /// Returns the `PrivateDataAccess` chunk of the data. + /// Returns the `DataMapChunk` chunk of the data. #[wasm_bindgen(js_name = putPrivateDataWithReceipt)] pub async fn put_private_data_with_receipt( &self, @@ -151,7 +151,7 @@ impl JsClient { /// Fetch the data from the network. #[wasm_bindgen(js_name = getPrivateData)] pub async fn get_private_data(&self, private_data_access: JsValue) -> Result, JsError> { - let private_data_access: PrivateDataAccess = + let private_data_access: DataMapChunk = serde_wasm_bindgen::from_value(private_data_access)?; let data = self.0.data_get(private_data_access).await?; @@ -278,7 +278,7 @@ mod archive_private { use super::*; use crate::client::archive::Metadata; use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess}; - use crate::client::data_private::PrivateDataAccess; + use crate::client::data_private::DataMapChunk; use crate::client::payment::Receipt; use std::path::PathBuf; use wasm_bindgen::{JsError, JsValue}; @@ -304,7 +304,7 @@ mod archive_private { metadata: JsValue, ) -> Result<(), JsError> { let path = PathBuf::from(path); - let data_map: PrivateDataAccess = serde_wasm_bindgen::from_value(data_map)?; + let data_map: DataMapChunk = serde_wasm_bindgen::from_value(data_map)?; let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?; self.0.add_file(path, data_map, metadata); diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index 1447c5aa8f..0c28401b55 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -2,7 +2,7 @@ #![allow(non_local_definitions)] use crate::client::{ - data::PrivateDataAccess, + data::DataMapChunk, files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, payment::PaymentOption as RustPaymentOption, vault::{UserData, VaultSecretKey}, @@ -38,7 +38,7 @@ impl PyClient { Ok(Self { inner: client }) } - fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { + fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let access = rt .block_on( @@ -49,10 +49,10 @@ impl PyClient { pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}")) })?; - Ok(PyPrivateDataAccess { inner: access }) + Ok(PyDataMapChunk { inner: access }) } - fn data_get(&self, access: &PyPrivateDataAccess) -> PyResult> { + fn data_get(&self, access: &PyDataMapChunk) -> PyResult> { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let data = rt .block_on(self.inner.data_get(access.inner.clone())) @@ -294,17 +294,17 @@ impl PyUserData { } } -#[pyclass(name = "PrivateDataAccess")] +#[pyclass(name = "DataMapChunk")] #[derive(Clone)] -pub(crate) struct PyPrivateDataAccess { - inner: PrivateDataAccess, +pub(crate) struct PyDataMapChunk { + inner: DataMapChunk, } #[pymethods] -impl PyPrivateDataAccess { +impl PyDataMapChunk { #[staticmethod] fn from_hex(hex: &str) -> PyResult { - PrivateDataAccess::from_hex(hex) + DataMapChunk::from_hex(hex) .map(|access| Self { inner: access }) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex: {e}"))) } @@ -342,7 +342,7 @@ fn autonomi_client_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(encrypt, m)?)?; Ok(()) } From ce43e0b2abcd7e81d6a3ac813d158cb57da90a9d Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 11:33:24 +0100 Subject: [PATCH 09/13] docs(autonomi): fix doc link --- autonomi/src/client/files/archive_public.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/files/archive_public.rs index 2178579b4b..f52afe753c 100644 --- a/autonomi/src/client/files/archive_public.rs +++ b/autonomi/src/client/files/archive_public.rs @@ -32,7 +32,7 @@ pub type ArchiveAddr = XorName; /// An archive of files that containing file paths, their metadata and the files data addresses /// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. -/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::archive_private::PrivateArchive`]. +/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::files::archive::PrivateArchive`]. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct Archive { map: HashMap, From 458ccb24ee8309d8681df29893a447c36cc0f097 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 12:31:35 +0100 Subject: [PATCH 10/13] refactor(autonomi): renames; dir upload change Change dir_upload to return data map instead of uploading the data map --- ant-cli/src/commands/file.rs | 7 +++++- autonomi/src/client/files/archive.rs | 18 +++++++------- autonomi/src/client/files/archive_public.rs | 27 ++++++++++----------- autonomi/src/client/files/fs.rs | 16 +++--------- autonomi/src/client/files/fs_public.rs | 6 ++--- autonomi/src/lib.rs | 2 +- autonomi/tests/fs.rs | 3 ++- 7 files changed, 38 insertions(+), 41 deletions(-) diff --git a/ant-cli/src/commands/file.rs b/ant-cli/src/commands/file.rs index fde2e9e1d0..b6b2e30623 100644 --- a/ant-cli/src/commands/file.rs +++ b/ant-cli/src/commands/file.rs @@ -59,10 +59,15 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( local_addr = addr_to_str(xor_name); local_addr.clone() } else { - let private_data_access = client + let private_archive = client .dir_upload(dir_path, &wallet) .await .wrap_err("Failed to upload file")?; + let private_data_access = client + .archive_put(private_archive, (&wallet).into()) + .await + .wrap_err("Failed to upload private archive")?; + local_addr = private_data_access.address(); private_data_access.to_hex() }; diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/files/archive.rs index a33a6a2ac6..58f0788059 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -24,9 +24,7 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use thiserror::Error; -/// The address of a private archive. -/// -/// Contains the [`DataMapChunk`] leading to the [`PrivateArchive`] data +/// Private archive data map, allowing access to the [`PrivateArchive`] data. pub type PrivateArchiveAccess = DataMapChunk; #[derive(Error, Debug, PartialEq, Eq)] @@ -65,8 +63,9 @@ impl Metadata { } } -/// A private archive of files that containing file paths, their metadata and the files data maps -/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. +/// Directory structure mapping filepaths to their data maps and metadata. +/// +/// The data maps are stored within this structure instead of uploading them to the network, keeping the data private. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct PrivateArchive { map: HashMap, @@ -121,8 +120,9 @@ impl PrivateArchive { .collect() } - /// Iterate over the archive items - /// Returns an iterator over (PathBuf, SecretDataMap, Metadata) + /// Iterate over the archive items. + /// + /// Returns an iterator over ([`PathBuf`], [`DataMapChunk`], [`Metadata`]) pub fn iter(&self) -> impl Iterator { self.map .iter() @@ -151,7 +151,7 @@ impl PrivateArchive { } impl Client { - /// Fetch a private archive from the network + /// Fetch a [`PrivateArchive`] from the network pub async fn archive_get( &self, addr: PrivateArchiveAccess, @@ -160,7 +160,7 @@ impl Client { Ok(PrivateArchive::from_bytes(data)?) } - /// Upload a private archive to the network + /// Upload a [`PrivateArchive`] to the network pub async fn archive_put( &self, archive: PrivateArchive, diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/files/archive_public.rs index f52afe753c..108d220553 100644 --- a/autonomi/src/client/files/archive_public.rs +++ b/autonomi/src/client/files/archive_public.rs @@ -27,18 +27,17 @@ use crate::{ Client, }; -/// The address of an archive on the network. Points to an [`Archive`]. +/// The address of a public archive on the network. Points to an [`PublicArchive`]. pub type ArchiveAddr = XorName; -/// An archive of files that containing file paths, their metadata and the files data addresses -/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. -/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::files::archive::PrivateArchive`]. +/// Public variant of [`crate::client::files::archive::PrivateArchive`]. Differs in that data maps of files are uploaded +/// to the network, of which the addresses are stored in this archive. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] -pub struct Archive { +pub struct PublicArchive { map: HashMap, } -impl Archive { +impl PublicArchive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network pub fn new() -> Self { @@ -98,8 +97,8 @@ impl Archive { } /// Deserialize from bytes. - pub fn from_bytes(data: Bytes) -> Result { - let root: Archive = rmp_serde::from_slice(&data[..])?; + pub fn from_bytes(data: Bytes) -> Result { + let root: PublicArchive = rmp_serde::from_slice(&data[..])?; Ok(root) } @@ -128,9 +127,9 @@ impl Client { /// # Ok(()) /// # } /// ``` - pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { + pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { let data = self.data_get_public(addr).await?; - Ok(Archive::from_bytes(data)?) + Ok(PublicArchive::from_bytes(data)?) } /// Upload an archive to the network @@ -140,14 +139,14 @@ impl Client { /// Create simple archive containing `file.txt` pointing to random XOR name. /// /// ```no_run - /// # use autonomi::client::{Client, data::DataAddr, archive::{Archive, ArchiveAddr, Metadata}}; + /// # use autonomi::client::{Client, data::DataAddr, archive::{PublicArchive, ArchiveAddr, Metadata}}; /// # use std::path::PathBuf; /// # #[tokio::main] /// # async fn main() -> Result<(), Box> { /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; /// # let client = Client::connect(&peers).await?; /// # let wallet = todo!(); - /// let mut archive = Archive::new(); + /// let mut archive = PublicArchive::new(); /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); /// let address = client.archive_put_public(archive, &wallet).await?; /// # Ok(()) @@ -155,7 +154,7 @@ impl Client { /// ``` pub async fn archive_put_public( &self, - archive: Archive, + archive: PublicArchive, wallet: &EvmWallet, ) -> Result { let bytes = archive @@ -167,7 +166,7 @@ impl Client { } /// Get the cost to upload an archive - pub async fn archive_cost(&self, archive: Archive) -> Result { + pub async fn archive_cost(&self, archive: PublicArchive) -> Result { let bytes = archive .into_bytes() .map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?; diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs index 6e67c0948c..e278a1b38f 100644 --- a/autonomi/src/client/files/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -112,13 +112,13 @@ impl Client { Ok(()) } - /// Upload a private directory to the network. The directory is recursively walked. - /// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive) + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// The data maps of these (private) files are not uploaded but returned within the [`PrivateArchive`] return type. pub async fn dir_upload( &self, dir_path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading directory as private: {dir_path:?}"); let start = tokio::time::Instant::now(); @@ -157,17 +157,9 @@ impl Client { } } - // upload archive - let archive_serialized = archive.into_bytes()?; - let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; - - info!( - "Complete private archive upload completed in {:?}", - start.elapsed() - ); #[cfg(feature = "loud")] println!("Upload completed in {:?}", start.elapsed()); - Ok(arch_addr) + Ok(archive) } /// Upload a private file to the network. diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs index c428eabb10..d140f873c0 100644 --- a/autonomi/src/client/files/fs_public.rs +++ b/autonomi/src/client/files/fs_public.rs @@ -15,7 +15,7 @@ use ant_networking::target_arch::{Duration, SystemTime}; use bytes::Bytes; use std::path::PathBuf; -use super::archive_public::{Archive, ArchiveAddr}; +use super::archive_public::{ArchiveAddr, PublicArchive}; use super::fs::*; impl Client { @@ -88,7 +88,7 @@ impl Client { uploads.len(), start.elapsed() ); - let mut archive = Archive::new(); + let mut archive = PublicArchive::new(); for (path, metadata, maybe_file) in uploads.into_iter() { match maybe_file { Ok(file) => archive.add_file(path, file, metadata), @@ -133,7 +133,7 @@ impl Client { /// Get the cost to upload a file/dir to the network. /// quick and dirty implementation, please refactor once files are cleanly implemented pub async fn file_cost(&self, path: &PathBuf) -> Result { - let mut archive = Archive::new(); + let mut archive = PublicArchive::new(); let mut total_cost = ant_evm::Amount::ZERO; for entry in walkdir::WalkDir::new(path) { diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index 97fe148095..7f200df9cc 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -79,7 +79,7 @@ pub use bytes::Bytes; pub use libp2p::Multiaddr; #[doc(inline)] -pub use client::Client; +pub use client::{files::archive::PrivateArchive, Client}; #[cfg(feature = "extension-module")] mod python; diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 8ae37d9608..e9a8f77729 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -106,7 +106,8 @@ async fn file_into_vault() -> Result<()> { let (ap, got_version) = new_client.fetch_and_decrypt_vault(&client_sk).await?; assert_eq!(set_version, got_version); - let ap_archive_fetched = autonomi::client::files::archive_public::Archive::from_bytes(ap)?; + let ap_archive_fetched = + autonomi::client::files::archive_public::PublicArchive::from_bytes(ap)?; assert_eq!( archive, ap_archive_fetched, From 0950bf4a3cf8db766344ff4335e6ad176b94cd48 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 13:19:52 +0100 Subject: [PATCH 11/13] docs(autonomi): update README.md --- autonomi/README.md | 58 ++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/autonomi/README.md b/autonomi/README.md index 7c759bd315..3dbaf5f672 100644 --- a/autonomi/README.md +++ b/autonomi/README.md @@ -53,52 +53,44 @@ let wallet = Wallet::new_from_private_key(EvmNetwork::new_custom("", "< ## Running tests -### Using a local EVM testnet +To run the tests, we can run a local network: -1. If you haven't, install Foundry, to be able to run Anvil - nodes: https://book.getfoundry.sh/getting-started/installation -2. Run a local EVM node: +1. Run a local EVM node: + > Note: To run the EVM node, Foundry is required to be installed: https://book.getfoundry.sh/getting-started/installation -```sh -cargo run --bin=evm-testnet -``` - -3. Run a local network with the `local` feature and use the local evm node. - -```sh -cargo run --bin=antctl --features=local -- local run --build --clean --rewards-address= evm-local -``` + ```sh + cargo run --bin evm-testnet + ``` -4. Then run the tests with the `local` feature and pass the EVM params again: +2. Run a local network with the `local` feature and use the local EVM node. + ```sh + cargo run --bin antctl --features local -- local run --build --clean --rewards-address evm-local + ``` -```sh -EVM_NETWORK=local cargo test --package autonomi --features=local -# Or with logs -RUST_LOG=autonomi EVM_NETWORK=local cargo test --package autonomi --features local -- --nocapture -``` +3. Then run the tests with the `local` feature and pass the EVM params again: + ```sh + EVM_NETWORK=local cargo test --features local --package autonomi + ``` ### Using a live testnet or mainnet -Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and -point it to a live network. +Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and point it to a live network. 1. Run a local network with the `local` feature: ```sh -cargo run --bin=antctl --features=local -- local run --build --clean --rewards-address= evm-arbitrum-one +cargo run --bin antctl --features local -- local run --build --clean --rewards-address evm-arbitrum-one ``` -2. Then run the tests with the `local` feature. Make sure that the wallet of the private key you pass has enough gas and - payment tokens on the network (in this case Arbitrum One): +2. Then pass the private key of the wallet, and ensure it has enough gas and payment tokens on the network (in this case Arbitrum One): ```sh -EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY= cargo test --package=autonomi --features=local +EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY= cargo test --package autonomi --features local ``` ## Using funds from the Deployer Wallet -You can use the `Deployer wallet private key` printed in the EVM node output to -initialise a wallet from with almost infinite gas and payment tokens. Example: +You can use the `Deployer wallet private key` printed in the EVM node output to initialise a wallet from with almost infinite gas and payment tokens. Example: ```rust let rpc_url = "http://localhost:54370/"; @@ -107,9 +99,9 @@ let data_payments_address = "0x8464135c8F25Da09e49BC8782676a84730C318bC"; let private_key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; let network = Network::Custom(CustomNetwork::new( -rpc_url, -payment_token_address, -data_payments_address, + rpc_url, + payment_token_address, + data_payments_address, )); let deployer_wallet = Wallet::new_from_private_key(network, private_key).unwrap(); @@ -117,15 +109,15 @@ let receiving_wallet = Wallet::new_with_random_wallet(network); // Send 10 payment tokens (atto) let _ = deployer_wallet -.transfer_tokens(receiving_wallet.address(), Amount::from(10)) -.await; + .transfer_tokens(receiving_wallet.address(), Amount::from(10)) + .await; ``` Alternatively, you can provide the wallet address that should own all the gas and payment tokens to the EVM testnet startup command using the `--genesis-wallet` flag: ```sh -cargo run --bin evm-testnet -- --genesis-wallet= +cargo run --bin evm-testnet -- --genesis-wallet ``` ```shell From d6c7676849fae39e736c34d0ea27df7223bb1812 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 13:37:55 +0100 Subject: [PATCH 12/13] fix(autonomi): change WASM to account for renames --- autonomi/src/client/wasm.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index d02d68e7b6..0f9a2ea802 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -1,7 +1,7 @@ use super::address::{addr_to_str, str_to_addr}; #[cfg(feature = "vault")] use super::vault::UserData; -use crate::client::data_private::DataMapChunk; +use crate::client::data::DataMapChunk; use crate::client::payment::Receipt; use ant_protocol::storage::Chunk; use libp2p::Multiaddr; @@ -171,15 +171,14 @@ impl JsClient { mod archive { use super::*; use crate::client::{ - address::str_to_addr, - archive::{Archive, Metadata}, + address::str_to_addr, files::archive::Metadata, files::archive_public::PublicArchive, }; use std::path::PathBuf; use wasm_bindgen::JsError; /// Structure mapping paths to data addresses. #[wasm_bindgen(js_name = Archive)] - pub struct JsArchive(Archive); + pub struct JsArchive(PublicArchive); /// Create new metadata with the current time as uploaded, created and modified. /// @@ -201,7 +200,7 @@ mod archive { /// Create a new archive. #[wasm_bindgen(constructor)] pub fn new() -> Self { - Self(Archive::new()) + Self(PublicArchive::new()) } /// Add a new file to the archive. @@ -276,9 +275,8 @@ mod archive { mod archive_private { use super::*; - use crate::client::archive::Metadata; - use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess}; - use crate::client::data_private::DataMapChunk; + use crate::client::data::DataMapChunk; + use crate::client::files::archive::{Metadata, PrivateArchive, PrivateArchiveAccess}; use crate::client::payment::Receipt; use std::path::PathBuf; use wasm_bindgen::{JsError, JsValue}; @@ -388,7 +386,7 @@ mod archive_private { mod vault { use super::*; use crate::client::address::addr_to_str; - use crate::client::archive_private::PrivateArchiveAccess; + use crate::client::files::archive::PrivateArchiveAccess; use crate::client::payment::Receipt; use crate::client::vault::key::blst_to_blsttc; use crate::client::vault::key::derive_secret_key_from_seed; From 725b96792f27a3a0e9e79085978a375285d668de Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 6 Dec 2024 14:24:35 +0100 Subject: [PATCH 13/13] feat(autonomi): add convenience method/rename --- ant-cli/src/commands/file.rs | 10 +++----- autonomi/README.md | 2 +- autonomi/examples/put_and_dir_upload.rs | 2 +- autonomi/src/client/files/fs.rs | 13 ++++++++++ autonomi/src/client/files/fs_public.rs | 32 +++++++++++++++---------- autonomi/tests/fs.rs | 4 ++-- 6 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ant-cli/src/commands/file.rs b/ant-cli/src/commands/file.rs index b6b2e30623..146133e348 100644 --- a/ant-cli/src/commands/file.rs +++ b/ant-cli/src/commands/file.rs @@ -53,20 +53,16 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( let local_addr; let archive = if public { let xor_name = client - .dir_upload_public(dir_path, &wallet) + .dir_and_archive_upload_public(dir_path, &wallet) .await .wrap_err("Failed to upload file")?; local_addr = addr_to_str(xor_name); local_addr.clone() } else { - let private_archive = client - .dir_upload(dir_path, &wallet) - .await - .wrap_err("Failed to upload file")?; let private_data_access = client - .archive_put(private_archive, (&wallet).into()) + .dir_and_archive_upload(dir_path, &wallet) .await - .wrap_err("Failed to upload private archive")?; + .wrap_err("Failed to upload dir and archive")?; local_addr = private_data_access.address(); private_data_access.to_hex() diff --git a/autonomi/README.md b/autonomi/README.md index 3dbaf5f672..63235554a1 100644 --- a/autonomi/README.md +++ b/autonomi/README.md @@ -33,7 +33,7 @@ async fn main() -> Result<(), Box> { let _data_fetched = client.data_get_public(data_addr).await?; // Put and fetch directory from local file system. - let dir_addr = client.dir_upload_public("files/to/upload".into(), &wallet).await?; + let dir_addr = client.dir_and_archive_upload_public("files/to/upload".into(), &wallet).await?; client .dir_download_public(dir_addr, "files/downloaded".into()) .await?; diff --git a/autonomi/examples/put_and_dir_upload.rs b/autonomi/examples/put_and_dir_upload.rs index 45ebc96627..874ca57980 100644 --- a/autonomi/examples/put_and_dir_upload.rs +++ b/autonomi/examples/put_and_dir_upload.rs @@ -16,7 +16,7 @@ async fn main() -> Result<(), Box> { // Put and fetch directory from local file system. let dir_addr = client - .dir_upload_public("files/to/upload".into(), &wallet) + .dir_and_archive_upload_public("files/to/upload".into(), &wallet) .await?; client .dir_download_public(dir_addr, "files/downloaded".into()) diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs index e278a1b38f..37df1aa84f 100644 --- a/autonomi/src/client/files/fs.rs +++ b/autonomi/src/client/files/fs.rs @@ -162,6 +162,19 @@ impl Client { Ok(archive) } + /// Same as [`Client::dir_upload`] but also uploads the archive (privately) to the network. + /// + /// Returns the [`PrivateArchiveAccess`] allowing the private archive to be downloaded from the network. + pub async fn dir_and_archive_upload( + &self, + dir_path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + let archive = self.dir_upload(dir_path, wallet).await?; + let archive_addr = self.archive_put(archive, wallet.into()).await?; + Ok(archive_addr) + } + /// Upload a private file to the network. /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap) async fn file_upload( diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs index d140f873c0..fd9cad51ba 100644 --- a/autonomi/src/client/files/fs_public.rs +++ b/autonomi/src/client/files/fs_public.rs @@ -54,13 +54,16 @@ impl Client { Ok(()) } - /// Upload a directory to the network. The directory is recursively walked. - /// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive) + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// + /// The data maps of these files are uploaded on the network, making the individual files publicly available. + /// + /// This returns, but does not upload (!),the [`PublicArchive`] containing the data maps of the uploaded files. pub async fn dir_upload_public( &self, dir_path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading directory: {dir_path:?}"); let start = tokio::time::Instant::now(); @@ -99,17 +102,22 @@ impl Client { } } - // upload archive - let archive_serialized = archive.into_bytes()?; - let arch_addr = self - .data_put_public(archive_serialized, wallet.into()) - .await?; - - info!("Complete archive upload completed in {:?}", start.elapsed()); #[cfg(feature = "loud")] println!("Upload completed in {:?}", start.elapsed()); - debug!("Directory uploaded to the network at {arch_addr:?}"); - Ok(arch_addr) + Ok(archive) + } + + /// Same as [`Client::dir_upload_public`] but also uploads the archive to the network. + /// + /// Returns the [`ArchiveAddr`] of the uploaded archive. + pub async fn dir_and_archive_upload_public( + &self, + dir_path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + let archive = self.dir_upload_public(dir_path, wallet).await?; + let archive_addr = self.archive_put_public(archive, wallet).await?; + Ok(archive_addr) } /// Upload a file to the network. diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index e9a8f77729..1b8b59f801 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -30,7 +30,7 @@ async fn dir_upload_download() -> Result<()> { let wallet = get_funded_wallet(); let addr = client - .dir_upload_public("tests/file/test_dir".into(), &wallet) + .dir_and_archive_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(10)).await; @@ -86,7 +86,7 @@ async fn file_into_vault() -> Result<()> { let client_sk = bls::SecretKey::random(); let addr = client - .dir_upload_public("tests/file/test_dir".into(), &wallet) + .dir_and_archive_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(2)).await;