From d46174670bf7caf09137fbdbb2a1432aabc8577e Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Thu, 7 Nov 2024 18:02:08 +0100 Subject: [PATCH 1/6] feat(autonomi): keep filesize in metadata --- autonomi/examples/metamask/index.js | 2 +- autonomi/src/client/archive.rs | 20 +++++----------- autonomi/src/client/archive_private.rs | 6 ----- autonomi/src/client/fs.rs | 5 +++- autonomi/src/client/wasm.rs | 33 ++++++++++++++++++++------ autonomi/tests-js/index.js | 4 ++-- autonomi/tests/external_signer.rs | 6 ++++- 7 files changed, 44 insertions(+), 32 deletions(-) diff --git a/autonomi/examples/metamask/index.js b/autonomi/examples/metamask/index.js index b8ec63a5bd..66bf524037 100644 --- a/autonomi/examples/metamask/index.js +++ b/autonomi/examples/metamask/index.js @@ -40,7 +40,7 @@ export async function externalSignerPrivateDataPutToVault(peerAddr) { const privateArchive = new autonomi.PrivateArchive(); // Add our data's data map chunk to the private archive - privateArchive.addNewFile("test", privateDataAccess); + privateArchive.addFile("test", privateDataAccess, autonomi.createMetadata(data.length)); // Get the private archive's bytes const privateArchiveBytes = privateArchive.bytes(); diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/archive.rs index 9d5f1de78a..24a8fae99e 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/archive.rs @@ -50,29 +50,27 @@ pub struct Metadata { pub created: u64, /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. pub modified: u64, + /// File size in bytes + pub size: u64, } impl Metadata { - /// Create a new metadata struct - pub fn new() -> Self { + /// Create a new metadata struct with the current time as uploaded, created and modified. + pub fn new_with_size(size: u64) -> Self { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or(Duration::from_secs(0)) .as_secs(); + Self { uploaded: now, created: now, modified: now, + size, } } } -impl Default for Metadata { - fn default() -> Self { - Self::new() - } -} - impl Archive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network @@ -104,12 +102,6 @@ impl Archive { self.map.insert(path, (data_addr, meta)); } - /// Add a file to a local archive, with default metadata - /// Note that this does not upload the archive to the network - pub fn add_new_file(&mut self, path: PathBuf, data_addr: DataAddr) { - self.map.insert(path, (data_addr, Metadata::new())); - } - /// List all files in the archive pub fn files(&self) -> Vec<(PathBuf, Metadata)> { self.map diff --git a/autonomi/src/client/archive_private.rs b/autonomi/src/client/archive_private.rs index 7354634140..4bcf4c5ca9 100644 --- a/autonomi/src/client/archive_private.rs +++ b/autonomi/src/client/archive_private.rs @@ -65,12 +65,6 @@ impl PrivateArchive { self.map.insert(path, (data_map, meta)); } - /// Add a file to a local archive, with default metadata - /// Note that this does not upload the archive to the network - pub fn add_new_file(&mut self, path: PathBuf, data_map: PrivateDataAccess) { - self.map.insert(path, (data_map, Metadata::new())); - } - /// List all files in the archive pub fn files(&self) -> Vec<(PathBuf, Metadata)> { self.map diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/fs.rs index 40a43b9fba..b91efbb865 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/fs.rs @@ -208,7 +208,8 @@ impl Client { tracing::debug!("Encryption took: {:.2?}", now.elapsed()); let map_xor_name = *data_map_chunk.address().xorname(); - archive.add_file(path, map_xor_name, Metadata::new()); + let metadata = metadata_from_entry(&entry); + archive.add_file(path, map_xor_name, metadata); } let root_serialized = rmp_serde::to_vec(&archive)?; @@ -234,6 +235,7 @@ pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { uploaded: 0, created: 0, modified: 0, + size: 0, }; } }; @@ -266,5 +268,6 @@ pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { .as_secs(), created, modified, + size: fs_metadata.len(), } } diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 18d7ffa49d..425463d91c 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -18,7 +18,7 @@ use wasm_bindgen::prelude::*; /// const dataAddr = await client.putData(new Uint8Array([0, 1, 2, 3]), wallet); /// /// const archive = new Archive(); -/// archive.addNewFile("foo", dataAddr); +/// archive.addFile("foo", dataAddr, createMetadata(4)); /// /// const archiveAddr = await client.putArchive(archive, wallet); /// const archiveFetched = await client.getArchive(archiveAddr); @@ -178,6 +178,13 @@ mod archive { #[wasm_bindgen(js_name = Archive)] pub struct JsArchive(Archive); + /// Create new metadata with the current time as uploaded, created and modified. + #[wasm_bindgen(js_name = createMetadata)] + pub fn create_metadata(size: u64) -> Result { + let metadata = Metadata::new_with_size(size); + Ok(serde_wasm_bindgen::to_value(&metadata)?) + } + #[wasm_bindgen(js_class = Archive)] impl JsArchive { /// Create a new archive. @@ -187,11 +194,17 @@ mod archive { } /// Add a new file to the archive. - #[wasm_bindgen(js_name = addNewFile)] - pub fn add_new_file(&mut self, path: String, data_addr: String) -> Result<(), JsError> { + #[wasm_bindgen(js_name = addFile)] + pub fn add_file( + &mut self, + path: String, + data_addr: String, + metadata: JsValue, + ) -> Result<(), JsError> { let path = PathBuf::from(path); let data_addr = str_to_addr(&data_addr)?; - self.0.add_new_file(path, data_addr); + let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?; + self.0.add_file(path, data_addr, metadata); Ok(()) } @@ -268,11 +281,17 @@ mod archive_private { } /// Add a new file to the private archive. - #[wasm_bindgen(js_name = addNewFile)] - pub fn add_new_file(&mut self, path: String, data_map: JsValue) -> Result<(), JsError> { + #[wasm_bindgen(js_name = addFile)] + pub fn add_file( + &mut self, + path: String, + data_map: JsValue, + metadata: JsValue, + ) -> Result<(), JsError> { let path = PathBuf::from(path); let data_map: PrivateDataAccess = serde_wasm_bindgen::from_value(data_map)?; - self.0.add_new_file(path, data_map); + let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?; + self.0.add_file(path, data_map, metadata); Ok(()) } diff --git a/autonomi/tests-js/index.js b/autonomi/tests-js/index.js index a2c38d3836..31ea4e1dc5 100644 --- a/autonomi/tests-js/index.js +++ b/autonomi/tests-js/index.js @@ -45,7 +45,7 @@ describe('autonomi', function () { const data = randomData(32); const addr = await client.putData(data, wallet); const archive = new atnm.Archive(); - archive.addNewFile("foo", addr); + archive.addFile("foo", addr, atnm.createMetadata(data.length)); const archiveAddr = await client.putArchive(archive, wallet); const archiveFetched = await client.getArchive(archiveAddr); @@ -59,7 +59,7 @@ describe('autonomi', function () { const secretKey = atnm.genSecretKey(); const archive = new atnm.Archive(); - archive.addNewFile('foo', addr); + archive.addFile('foo', addr, atnm.createMetadata(data.length)); const archiveAddr = await client.putArchive(archive, wallet); const userData = new atnm.UserData(); diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index 161e881cad..89c9cd4d48 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -116,7 +116,11 @@ async fn external_signer_put() -> eyre::Result<()> { .await?; let mut private_archive = PrivateArchive::new(); - private_archive.add_file("test-file".into(), private_data_access, Metadata::default()); + private_archive.add_file( + "test-file".into(), + private_data_access, + Metadata::new_with_size(data.len() as u64), + ); let archive_serialized = private_archive.into_bytes()?; From 91146eee26ba983b59ad2d694c4c1f3b36f81820 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 8 Nov 2024 10:50:34 +0100 Subject: [PATCH 2/6] fix(autonomi): missing import in wasm binding --- autonomi/src/client/wasm.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 425463d91c..7032bfea69 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -170,7 +170,10 @@ impl JsClient { mod archive { use super::*; - use crate::client::{address::str_to_addr, archive::Archive}; + use crate::client::{ + address::str_to_addr, + archive::{Archive, Metadata}, + }; use std::path::PathBuf; use wasm_bindgen::JsError; From 78b846c561e95a1921b027d558926a10fecc016a Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 8 Nov 2024 11:09:32 +0100 Subject: [PATCH 3/6] fix(autonomi): add import for ARchive --- autonomi/src/client/wasm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index 7032bfea69..edf3358689 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -265,6 +265,7 @@ mod archive { mod archive_private { use super::*; + use crate::client::archive::Metadata; use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess}; use crate::client::data_private::PrivateDataAccess; use crate::client::payment::Receipt; From 12235a5fd37a8bf8a48a0068ccb45ef3be696f47 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 8 Nov 2024 11:58:08 +0100 Subject: [PATCH 4/6] fix(autonomi): use bigint for u64 --- autonomi/src/client/wasm.rs | 8 ++++++++ autonomi/tests-js/index.js | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index edf3358689..f79708aa53 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -182,6 +182,14 @@ mod archive { pub struct JsArchive(Archive); /// Create new metadata with the current time as uploaded, created and modified. + /// + /// # Example + /// + /// ```js + /// const metadata = createMetadata(BigInt(3)); + /// const archive = new atnm.Archive(); + /// archive.addFile("foo", addr, metadata); + /// ``` #[wasm_bindgen(js_name = createMetadata)] pub fn create_metadata(size: u64) -> Result { let metadata = Metadata::new_with_size(size); diff --git a/autonomi/tests-js/index.js b/autonomi/tests-js/index.js index 31ea4e1dc5..2a63039f15 100644 --- a/autonomi/tests-js/index.js +++ b/autonomi/tests-js/index.js @@ -45,12 +45,12 @@ describe('autonomi', function () { const data = randomData(32); const addr = await client.putData(data, wallet); const archive = new atnm.Archive(); - archive.addFile("foo", addr, atnm.createMetadata(data.length)); + archive.addFile("foo", addr, atnm.createMetadata(BigInt(data.length))); const archiveAddr = await client.putArchive(archive, wallet); const archiveFetched = await client.getArchive(archiveAddr); - assert.deepEqual(archive, archiveFetched); + assert.deepEqual(archive.map(), archiveFetched.map()); }); it('writes archive to vault and fetches it', async () => { @@ -59,7 +59,7 @@ describe('autonomi', function () { const secretKey = atnm.genSecretKey(); const archive = new atnm.Archive(); - archive.addFile('foo', addr, atnm.createMetadata(data.length)); + archive.addFile('foo', addr, atnm.createMetadata(BigInt(data.length))); const archiveAddr = await client.putArchive(archive, wallet); const userData = new atnm.UserData(); From 7285adfda938f211ca7586c63cc3b12108d4f74e Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 8 Nov 2024 13:47:21 +0100 Subject: [PATCH 5/6] fix(sn_networking): conditional import removal --- sn_networking/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index b831658632..fdb11e1e0e 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -74,7 +74,6 @@ use tokio::sync::{ oneshot, }; use tokio::time::Duration; -#[cfg(not(target_arch = "wasm32"))] use { sn_protocol::storage::{ try_deserialize_record, try_serialize_record, RecordHeader, RecordKind, From 84c6d5f18f7e4556d868c29c4aa8b2501d540eed Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Fri, 8 Nov 2024 14:22:38 +0100 Subject: [PATCH 6/6] ci: fix cargo doc check --- .github/workflows/merge.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 14c2e55821..afbf008f8c 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -79,9 +79,12 @@ jobs: run: cargo clippy --all-targets --all-features -- -Dwarnings - name: Check documentation - # Deny certain `rustdoc` lints that are unwanted. - # See https://doc.rust-lang.org/rustdoc/lints.html for lints that are 'warning' by default. - run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps + # Deny certain `rustdoc` lints that are unwanted with `RUSTDOCFLAGS`. See + # https://doc.rust-lang.org/rustdoc/lints.html for lints that are 'warning' by default. + # + # We exclude autonomi-cli because it is not published and conflicts with the `autonomi` crate name, + # resulting in an error when building docs. + run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps --workspace --exclude=autonomi-cli - name: Check local is not a default feature shell: bash