diff --git a/wnfs/Cargo.toml b/wnfs/Cargo.toml index 73a6ef1e..f42b3f0b 100644 --- a/wnfs/Cargo.toml +++ b/wnfs/Cargo.toml @@ -25,12 +25,13 @@ async-std = { version = "1.11", features = ["attributes"] } async-stream = "0.3" async-trait = "0.1" bitvec = { version = "1.0", features = ["serde"] } -chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } +chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] } either = "1.8" futures = "0.3" futures-util = "0.3" libipld = { version = "0.15", features = ["dag-cbor", "derive", "serde-codec"] } log = "0.4" +multihash = "0.16" once_cell = "1.16" proptest = { version = "1.0", optional = true } rand = { version = "0.8", optional = true } diff --git a/wnfs/examples/private.rs b/wnfs/examples/private.rs index 899f5f2a..9a2064f6 100644 --- a/wnfs/examples/private.rs +++ b/wnfs/examples/private.rs @@ -41,11 +41,10 @@ async fn main() { println!("{:#?}", dir); } -async fn get_forest_cid_and_private_ref(store: &mut B, rng: &mut R) -> (Cid, PrivateRef) -where - B: BlockStore, - R: RngCore, -{ +async fn get_forest_cid_and_private_ref( + store: &mut impl BlockStore, + rng: &mut impl RngCore, +) -> (Cid, PrivateRef) { // Create the private forest (a HAMT), a map-like structure where file and directory ciphertexts are stored. let forest = Rc::new(PrivateForest::new()); diff --git a/wnfs/src/common/blockstore.rs b/wnfs/src/common/blockstore.rs index 9f71348e..15fc9acd 100644 --- a/wnfs/src/common/blockstore.rs +++ b/wnfs/src/common/blockstore.rs @@ -41,16 +41,12 @@ pub trait BlockStore { self.put_block(bytes, IpldCodec::DagCbor).await } - async fn put_private_serializable( + async fn put_private_serializable( &mut self, value: &V, key: &Key, - rng: &mut R, - ) -> Result - where - V: Serialize, - R: RngCore, - { + rng: &mut impl RngCore, + ) -> Result { let ipld = ipld_serde::to_ipld(value)?; let mut bytes = Vec::new(); ipld.encode(DagCborCodec, &mut bytes)?; diff --git a/wnfs/src/common/encoding.rs b/wnfs/src/common/encoding.rs index c31003d0..2ad5cbd4 100644 --- a/wnfs/src/common/encoding.rs +++ b/wnfs/src/common/encoding.rs @@ -22,9 +22,9 @@ pub mod dagcbor { } /// Encodes an async serializable value into DagCbor bytes. - pub async fn async_encode( + pub async fn async_encode( value: &V, - store: &mut B, + store: &mut impl BlockStore, ) -> Result> { let ipld = value.async_serialize_ipld(store).await?; let mut bytes = Vec::new(); diff --git a/wnfs/src/common/link.rs b/wnfs/src/common/link.rs index 7f0a03b1..ba08fcad 100644 --- a/wnfs/src/common/link.rs +++ b/wnfs/src/common/link.rs @@ -55,7 +55,7 @@ impl Link { } /// Gets the value stored in link. It attempts to get it from the store if it is not present in link. - pub async fn resolve_value(&self, store: &B) -> Result<&T> + pub async fn resolve_value(&self, store: &B) -> Result<&T> where T: DeserializeOwned, { @@ -123,7 +123,7 @@ impl Link { } /// Compares two links for equality. Attempts to get them from store if they are not already cached. - pub async fn deep_eq(&self, other: &Link, store: &mut B) -> Result + pub async fn deep_eq(&self, other: &Link, store: &mut impl BlockStore) -> Result where T: PartialEq + AsyncSerialize, { @@ -137,7 +137,7 @@ impl Link { #[async_trait(?Send)] impl IpldEq for Link { - async fn eq(&self, other: &Link, store: &mut B) -> Result { + async fn eq(&self, other: &Link, store: &mut B) -> Result { if self == other { return Ok(true); } diff --git a/wnfs/src/private/directory.rs b/wnfs/src/private/directory.rs index 64b908dd..67488921 100644 --- a/wnfs/src/private/directory.rs +++ b/wnfs/src/private/directory.rs @@ -1,20 +1,22 @@ -use std::{collections::BTreeMap, rc::Rc}; - -use anyhow::{bail, ensure, Result}; -use chrono::{DateTime, Utc}; -use rand_core::RngCore; -use semver::Version; -use serde::{de::Error as DeError, ser::Error as SerError, Deserialize, Deserializer, Serialize}; - use super::{ - namefilter::Namefilter, Key, PrivateFile, PrivateForest, PrivateNode, PrivateNodeHeader, - PrivateRef, PrivateRefSerializable, RevisionKey, + encrypted::Encrypted, namefilter::Namefilter, Key, PrivateFile, PrivateForest, PrivateNode, + PrivateNodeHeader, PrivateRef, PrivateRefSerializable, RevisionKey, }; - use crate::{ dagcbor, error, utils, BlockStore, FsError, HashOutput, Id, Metadata, NodeType, PathNodes, PathNodesResult, }; +use anyhow::{bail, ensure, Result}; +use async_once_cell::OnceCell; +use chrono::{DateTime, Utc}; +use libipld::{cbor::DagCborCodec, prelude::Encode, Cid}; +use rand_core::RngCore; +use semver::Version; +use serde::{de::Error as DeError, ser::Error as SerError, Deserialize, Deserializer, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + rc::Rc, +}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -41,10 +43,12 @@ pub type PrivatePathNodesResult = PathNodesResult; /// /// println!("dir = {:?}", dir); /// ``` -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug)] pub struct PrivateDirectory { + persisted_as: OnceCell, pub version: Version, pub header: PrivateNodeHeader, + pub previous: Option>>, pub metadata: Metadata, pub entries: BTreeMap, } @@ -54,6 +58,7 @@ struct PrivateDirectorySerializable { pub r#type: NodeType, pub version: Version, pub header: Vec, + pub previous: Option>>, pub metadata: Metadata, pub entries: BTreeMap, } @@ -92,10 +97,12 @@ impl PrivateDirectory { /// /// println!("dir = {:?}", dir); /// ``` - pub fn new(parent_bare_name: Namefilter, time: DateTime, rng: &mut R) -> Self { + pub fn new(parent_bare_name: Namefilter, time: DateTime, rng: &mut impl RngCore) -> Self { Self { + persisted_as: OnceCell::new(), version: Version::new(0, 2, 0), header: PrivateNodeHeader::new(parent_bare_name, rng), + previous: None, metadata: Metadata::new(time), entries: BTreeMap::new(), } @@ -127,9 +134,11 @@ impl PrivateDirectory { inumber: HashOutput, ) -> Self { Self { + persisted_as: OnceCell::new(), version: Version::new(0, 2, 0), header: PrivateNodeHeader::with_seed(parent_bare_name, ratchet_seed, inumber), metadata: Metadata::new(time), + previous: None, entries: BTreeMap::new(), } } @@ -159,17 +168,12 @@ impl PrivateDirectory { &self.metadata } - /// Advances the ratchet. - pub(crate) fn advance_ratchet(&mut self) { - self.header.advance_ratchet(); - } - /// Creates a new `PathNodes` that is not based on an existing file tree. - pub(crate) fn create_path_nodes( + pub(crate) fn create_path_nodes( path_segments: &[String], time: DateTime, parent_bare_name: Namefilter, - rng: &mut R, + rng: &mut impl RngCore, ) -> PrivatePathNodes { let mut working_parent_bare_name = parent_bare_name; let path: Vec<(Rc, String)> = path_segments @@ -201,12 +205,12 @@ impl PrivateDirectory { /// Uses specified path segments and their existence in the file tree to generate `PathNodes`. /// /// Supports cases where the entire path does not exist. - pub(crate) async fn get_path_nodes( + pub(crate) async fn get_path_nodes( self: Rc, path_segments: &[String], search_latest: bool, forest: &PrivateForest, - store: &B, + store: &impl BlockStore, ) -> Result { use PathNodesResult::*; let mut working_node = self; @@ -247,14 +251,14 @@ impl PrivateDirectory { } /// Uses specified path segments to generate `PathNodes`. Creates missing directories as needed. - pub(crate) async fn get_or_create_path_nodes( + pub(crate) async fn get_or_create_path_nodes( self: Rc, path_segments: &[String], search_latest: bool, time: DateTime, forest: &PrivateForest, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result { use PathNodesResult::*; match self @@ -287,23 +291,64 @@ impl PrivateDirectory { } } + /// This should be called to prepare a node for modifications, + /// if it's meant to be a successor revision of the current revision. + /// + /// It will store the current revision in the given `BlockStore` to + /// retrieve its CID and put that into the `previous` links, + /// as well as advancing the ratchet and resetting the `persisted_as` pointer. + pub(crate) async fn prepare_next_revision( + self: Rc, + store: &mut impl BlockStore, + rng: &mut impl RngCore, + ) -> Result { + let cid = self.store(store, rng).await?; + + let mut cloned = Rc::try_unwrap(self).unwrap_or_else(|rc| (*rc).clone()); + cloned.persisted_as = OnceCell::new(); // Also done in `.clone()`, but need this to work in case try_unwrap optimizes. + let key = cloned.header.get_private_ref().revision_key.0; + let previous = Encrypted::from_value(BTreeSet::from([cid]), &key, rng)?; + + cloned.previous = Some(previous); + cloned.header.advance_ratchet(); + + Ok(cloned) + } + + /// This prepares this file for key rotation, usually for moving or + /// copying the file to some other place. + /// + /// Will reset the ratchet, so a different key is necessary for read access, + /// will reset the inumber to reset write access, + /// will update the bare namefilter to match the new parent's namefilter, + /// so it inherits the write access rules from the new parent and + /// resets the `persisted_as` pointer. + pub(crate) fn prepare_key_rotation( + &mut self, + parent_bare_name: Namefilter, + rng: &mut impl RngCore, + ) { + self.header.inumber = utils::get_random_bytes(rng); + self.header.update_bare_name(parent_bare_name); + self.header.reset_ratchet(rng); + self.persisted_as = OnceCell::new(); + } + /// Fix up `PathNodes` so that parents refer to the newly updated children. - async fn fix_up_path_nodes( + async fn fix_up_path_nodes( path_nodes: PrivatePathNodes, - forest: Rc, - store: &mut B, - rng: &mut R, + mut forest: Rc, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result<(Rc, Rc)> { - let mut working_forest = Rc::clone(&forest); - let mut working_child_dir = { - let mut tmp = (*path_nodes.tail).clone(); - tmp.advance_ratchet(); - Rc::new(tmp) - }; + let mut working_child_dir = + Rc::new(path_nodes.tail.prepare_next_revision(store, rng).await?); for (parent_dir, segment) in path_nodes.path.iter().rev() { - let mut parent_dir = (**parent_dir).clone(); - parent_dir.advance_ratchet(); + let mut parent_dir = Rc::clone(parent_dir) + .prepare_next_revision(store, rng) + .await?; + let child_private_ref = working_child_dir.header.get_private_ref(); parent_dir @@ -312,7 +357,7 @@ impl PrivateDirectory { let parent_dir = Rc::new(parent_dir); - working_forest = working_forest + forest = forest .put( working_child_dir.header.get_saturated_name(), &child_private_ref, @@ -325,7 +370,7 @@ impl PrivateDirectory { working_child_dir = parent_dir; } - working_forest = working_forest + forest = forest .put( working_child_dir.header.get_saturated_name(), &working_child_dir.header.get_private_ref(), @@ -335,7 +380,7 @@ impl PrivateDirectory { ) .await?; - Ok((working_child_dir, working_forest)) + Ok((working_child_dir, forest)) } /// Follows a path and fetches the node at the end of the path. @@ -378,12 +423,12 @@ impl PrivateDirectory { /// assert!(result.is_some()); /// } /// ``` - pub async fn get_node( + pub async fn get_node( self: Rc, path_segments: &[String], search_latest: bool, forest: Rc, - store: &B, + store: &impl BlockStore, ) -> Result>> { use PathNodesResult::*; let root_dir = Rc::clone(&self); @@ -468,12 +513,12 @@ impl PrivateDirectory { /// assert_eq!(&result, content); /// } /// ``` - pub async fn read( + pub async fn read( self: Rc, path_segments: &[String], search_latest: bool, forest: Rc, - store: &B, + store: &impl BlockStore, ) -> Result>> { let root_dir = Rc::clone(&self); let (path, filename) = utils::split_last(path_segments)?; @@ -555,15 +600,15 @@ impl PrivateDirectory { /// } /// ``` #[allow(clippy::too_many_arguments)] - pub async fn write( + pub async fn write( self: Rc, path_segments: &[String], search_latest: bool, time: DateTime, content: Vec, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { let (directory_path, filename) = utils::split_last(path_segments)?; @@ -580,8 +625,7 @@ impl PrivateDirectory { .await? { Some(PrivateNode::File(file_before)) => { - let mut file = (*file_before).clone(); - + let mut file = file_before.prepare_next_revision(store, rng).await?; let (content, forest) = PrivateFile::prepare_content( &file.header.bare_name, content, @@ -592,11 +636,10 @@ impl PrivateDirectory { .await?; file.content = content; file.metadata.upsert_mtime(time); - file.header.advance_ratchet(); (file, forest) } - Some(PrivateNode::Dir(_)) => bail!(FsError::DirectoryAlreadyExists), + Some(PrivateNode::Dir(_)) => bail!(FsError::NotAFile), None => { PrivateFile::with_content( directory.header.bare_name.clone(), @@ -678,12 +721,12 @@ impl PrivateDirectory { /// assert!(node.is_some()); /// } /// ``` - pub async fn lookup_node<'a, B: BlockStore>( + pub async fn lookup_node( &self, path_segment: &str, search_latest: bool, forest: &PrivateForest, - store: &B, + store: &impl BlockStore, ) -> Result> { Ok(match self.entries.get(path_segment) { Some(private_ref) => { @@ -738,14 +781,14 @@ impl PrivateDirectory { /// assert!(node.is_some()); /// } /// ``` - pub async fn mkdir( + pub async fn mkdir( self: Rc, path_segments: &[String], search_latest: bool, time: DateTime, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { let path_nodes = self .get_or_create_path_nodes(path_segments, search_latest, time, &forest, store, rng) @@ -817,12 +860,12 @@ impl PrivateDirectory { /// ); /// } /// ``` - pub async fn ls( + pub async fn ls( self: Rc, path_segments: &[String], search_latest: bool, forest: Rc, - store: &B, + store: &impl BlockStore, ) -> Result>> { let root_dir = Rc::clone(&self); match self @@ -915,13 +958,13 @@ impl PrivateDirectory { /// assert_eq!(result.len(), 0); /// } /// ``` - pub async fn rm( + pub async fn rm( self: Rc, path_segments: &[String], search_latest: bool, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { let (directory_path, node_name) = utils::split_last(path_segments)?; @@ -960,15 +1003,15 @@ impl PrivateDirectory { /// /// Fixes up the subtree bare names to refer to the new parent. #[allow(clippy::too_many_arguments)] - async fn attach( + async fn attach( self: Rc, node: PrivateNode, path_segments: &[String], search_latest: bool, time: DateTime, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { let (directory_path, filename) = utils::split_last(path_segments)?; @@ -1070,15 +1113,15 @@ impl PrivateDirectory { /// } /// ``` #[allow(clippy::too_many_arguments)] - pub async fn basic_mv( + pub async fn basic_mv( self: Rc, path_segments_from: &[String], path_segments_to: &[String], search_latest: bool, time: DateTime, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { let PrivateOpResult { root_dir, @@ -1163,15 +1206,15 @@ impl PrivateDirectory { /// } /// ``` #[allow(clippy::too_many_arguments)] - pub async fn cp( + pub async fn cp( self: Rc, path_segments_from: &[String], path_segments_to: &[String], search_latest: bool, time: DateTime, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { let PrivateOpResult { root_dir, @@ -1195,10 +1238,10 @@ impl PrivateDirectory { } /// Serializes the directory with provided Serde serialilzer. - pub(crate) fn serialize( + pub(crate) fn serialize( &self, serializer: S, - rng: &mut R, + rng: &mut impl RngCore, ) -> Result where S: serde::Serializer, @@ -1214,15 +1257,18 @@ impl PrivateDirectory { entries.insert(name.clone(), private_ref_serializable); } + let header = { + let cbor_bytes = dagcbor::encode(&self.header).map_err(SerError::custom)?; + key.0 + .encrypt(&Key::generate_nonce(rng), &cbor_bytes) + .map_err(SerError::custom)? + }; + (PrivateDirectorySerializable { r#type: NodeType::PrivateDirectory, version: self.version.clone(), - header: { - let cbor_bytes = dagcbor::encode(&self.header).map_err(SerError::custom)?; - key.0 - .encrypt(&Key::generate_nonce(rng), &cbor_bytes) - .map_err(SerError::custom)? - }, + header, + previous: self.previous.clone(), metadata: self.metadata.clone(), entries, }) @@ -1230,7 +1276,11 @@ impl PrivateDirectory { } /// Deserializes the directory with provided Serde deserializer and key. - pub(crate) fn deserialize<'de, D>(deserializer: D, key: &RevisionKey) -> Result + pub(crate) fn deserialize<'de, D>( + deserializer: D, + key: &RevisionKey, + from_cid: Cid, + ) -> Result where D: Deserializer<'de>, { @@ -1238,6 +1288,7 @@ impl PrivateDirectory { version, metadata, header, + previous, entries: entries_encrypted, .. } = PrivateDirectorySerializable::deserialize(deserializer)?; @@ -1251,15 +1302,66 @@ impl PrivateDirectory { } Ok(Self { + persisted_as: OnceCell::new_with(Some(from_cid)), version, metadata, header: { let cbor_bytes = key.0.decrypt(&header).map_err(DeError::custom)?; dagcbor::decode(&cbor_bytes).map_err(DeError::custom)? }, + previous, entries, }) } + + pub async fn store(&self, store: &mut impl BlockStore, rng: &mut impl RngCore) -> Result { + let cid = self + .persisted_as + .get_or_try_init::(async { + // TODO(matheus23) deduplicate when reworking serialization + let private_ref = &self.header.get_private_ref(); + + // Serialize node to cbor. + let ipld = self.serialize(libipld::serde::Serializer, rng)?; + let mut bytes = Vec::new(); + ipld.encode(DagCborCodec, &mut bytes)?; + + // Encrypt bytes with content key. + let enc_bytes = private_ref + .content_key + .0 + .encrypt(&Key::generate_nonce(rng), &bytes)?; + + // Store content section in blockstore and get Cid. + store.put_block(enc_bytes, libipld::IpldCodec::Raw).await + }) + .await?; + + Ok(*cid) + } +} + +impl PartialEq for PrivateDirectory { + fn eq(&self, other: &Self) -> bool { + self.version == other.version + && self.header == other.header + && self.previous == other.previous + && self.metadata == other.metadata + && self.entries == other.entries + } +} + +impl Clone for PrivateDirectory { + fn clone(&self) -> Self { + Self { + persisted_as: OnceCell::new_with(self.persisted_as.get().cloned()), + version: self.version.clone(), + header: self.header.clone(), + previous: self.previous.clone(), + metadata: self.metadata.clone(), + entries: self.entries.clone(), + } + } } impl Id for PrivateDirectory { @@ -2063,4 +2165,43 @@ mod tests { assert!(result.is_err()); } + + #[async_std::test] + async fn write_generates_previous_link() { + let rng = &mut TestRng::deterministic_rng(RngAlgorithm::ChaCha); + let store = &mut MemoryBlockStore::new(); + let hamt = Rc::new(PrivateForest::new()); + let old_dir = Rc::new(PrivateDirectory::new( + Namefilter::default(), + Utc::now(), + rng, + )); + + let PrivateOpResult { + root_dir: new_dir, .. + } = Rc::clone(&old_dir) + .write( + &["file.txt".into()], + false, + Utc::now(), + b"Hello".to_vec(), + hamt, + store, + rng, + ) + .await + .unwrap(); + + assert!(old_dir.previous.is_none()); + + let old_key = RevisionKey::from(&old_dir.header.ratchet); + let previous_encrypted = new_dir.previous.clone(); + let previous_links = previous_encrypted + .unwrap() + .resolve_value(&old_key.0) + .cloned() + .unwrap(); + + assert_eq!(previous_links.len(), 1); + } } diff --git a/wnfs/src/private/encrypted.rs b/wnfs/src/private/encrypted.rs new file mode 100644 index 00000000..64d69665 --- /dev/null +++ b/wnfs/src/private/encrypted.rs @@ -0,0 +1,118 @@ +use std::io::Cursor; + +use anyhow::Result; +use libipld::{cbor::DagCborCodec, codec::Decode, prelude::Encode, Ipld}; +use once_cell::sync::OnceCell; +use rand_core::RngCore; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +use crate::FsError; + +use super::Key; + +/// A wrapper for encrypted data. +/// +/// When serialized or deserialized this will only +/// ever emit or consume ciphertexts. +/// +/// It can be resolved to the plaintext value `T`, when +/// you call `resolve_value`. Any subsequent calls to +/// `resolve_value` will re-use a cached, decrypted value. +#[derive(Debug, Clone, Eq)] +pub struct Encrypted { + ciphertext: Vec, + value_cache: OnceCell, +} + +impl Encrypted { + /// Constructs an `Encrypted` wrapper from a given plaintext value. + /// + /// This will compute a ciphertext by serializing the value and encrypting the + /// serialized value given the key and randomness. + /// + /// To ensure confidentiality, the randomness should be cryptographically secure + /// randomness. + pub fn from_value(value: T, key: &Key, rng: &mut impl RngCore) -> Result + where + T: Serialize, + { + let ipld = value.serialize(libipld::serde::Serializer)?; + let mut bytes = Vec::new(); + ipld.encode(DagCborCodec, &mut bytes)?; + let ciphertext = key.encrypt(&Key::generate_nonce(rng), &bytes)?; + + Ok(Self { + value_cache: OnceCell::from(value), + ciphertext, + }) + } + + /// Constructs an `Encrypted` wrapper from some serialized ciphertext. + /// + /// This won't compute the decrypted value inside. That has to be lazily + /// computed via `resolve_value`. + pub fn from_ciphertext(ciphertext: Vec) -> Self { + Self { + ciphertext, + value_cache: OnceCell::new(), + } + } + + /// Decrypts and deserializes the value inside the `Encrypted` wrapper using + /// given key. + /// + /// This operation may fail if given key doesn't decrypt the ciphertext or + /// deserializing the value from the encrypted plaintext doesn't work. + pub fn resolve_value(&self, key: &Key) -> Result<&T> + where + T: DeserializeOwned, + { + self.value_cache.get_or_try_init(|| { + let bytes = key.decrypt(&self.ciphertext)?; + let ipld = Ipld::decode(DagCborCodec, &mut Cursor::new(bytes))?; + libipld::serde::from_ipld::(ipld) + .map_err(|e| FsError::InvalidDeserialization(e.to_string()).into()) + }) + } + + /// Gets the ciphertext + pub fn get_ciphertext(&self) -> &Vec { + &self.ciphertext + } + + /// Consumes the ciphertext + pub fn take_ciphertext(self) -> Vec { + self.ciphertext + } + + /// Looks up the cached value. If `resolve_value` has never + /// been called, then the cache will be unpopulated and this will + /// return `None`. + pub fn get_value(&self) -> Option<&T> { + self.value_cache.get() + } +} + +impl<'de, T> Deserialize<'de> for Encrypted { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Ok(Self::from_ciphertext(Vec::::deserialize(deserializer)?)) + } +} + +impl Serialize for Encrypted { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.ciphertext.serialize(serializer) + } +} + +impl PartialEq for Encrypted { + fn eq(&self, other: &Self) -> bool { + self.get_ciphertext() == other.get_ciphertext() + } +} diff --git a/wnfs/src/private/file.rs b/wnfs/src/private/file.rs index 311a1bb7..fa6bbdb8 100644 --- a/wnfs/src/private/file.rs +++ b/wnfs/src/private/file.rs @@ -1,19 +1,22 @@ use super::{ - namefilter::Namefilter, Key, PrivateForest, PrivateNodeHeader, RevisionKey, NONCE_SIZE, + encrypted::Encrypted, namefilter::Namefilter, Key, PrivateForest, PrivateNodeHeader, + RevisionKey, NONCE_SIZE, }; use crate::{ - dagcbor, utils::get_random_bytes, BlockStore, FsError, Hasher, Id, Metadata, NodeType, + dagcbor, utils, utils::get_random_bytes, BlockStore, FsError, Hasher, Id, Metadata, NodeType, MAX_BLOCK_SIZE, }; use anyhow::Result; +use async_once_cell::OnceCell; use async_stream::try_stream; use chrono::{DateTime, Utc}; use futures::{Stream, StreamExt}; +use libipld::{cbor::DagCborCodec, prelude::Encode, Cid}; use rand_core::RngCore; use semver::Version; use serde::{de::Error as DeError, ser::Error as SerError, Deserialize, Deserializer, Serialize}; use sha3::Sha3_256; -use std::{iter, rc::Rc}; +use std::{collections::BTreeSet, iter, rc::Rc}; //-------------------------------------------------------------------------------------------------- // Constants @@ -65,10 +68,12 @@ pub const MAX_BLOCK_CONTENT_SIZE: usize = MAX_BLOCK_SIZE - NONCE_SIZE; /// println!("file = {:?}", file); /// } /// ``` -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug)] pub struct PrivateFile { + persisted_as: OnceCell, pub version: Version, pub header: PrivateNodeHeader, + pub previous: Option>>, pub metadata: Metadata, pub(crate) content: FileContent, } @@ -92,6 +97,7 @@ struct PrivateFileSerializable { pub r#type: NodeType, pub version: Version, pub header: Vec, + pub previous: Option>>, pub metadata: Metadata, pub content: FileContent, } @@ -119,11 +125,13 @@ impl PrivateFile { /// /// println!("file = {:?}", file); /// ``` - pub fn new(parent_bare_name: Namefilter, time: DateTime, rng: &mut R) -> Self { + pub fn new(parent_bare_name: Namefilter, time: DateTime, rng: &mut impl RngCore) -> Self { Self { + persisted_as: OnceCell::new(), version: Version::new(0, 2, 0), metadata: Metadata::new(time), header: PrivateNodeHeader::new(parent_bare_name, rng), + previous: None, content: FileContent::Inline { data: vec![] }, } } @@ -162,13 +170,13 @@ impl PrivateFile { /// println!("file = {:?}", file); /// } /// ``` - pub async fn with_content( + pub async fn with_content( parent_bare_name: Namefilter, time: DateTime, content: Vec, forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result<(Self, Rc)> { let header = PrivateNodeHeader::new(parent_bare_name, rng); let (content, forest) = @@ -176,9 +184,11 @@ impl PrivateFile { Ok(( Self { + persisted_as: OnceCell::new(), version: Version::new(0, 2, 0), metadata: Metadata::new(time), header, + previous: None, content, }, forest, @@ -227,11 +237,11 @@ impl PrivateFile { /// assert_eq!(content, stream_content); /// } /// ``` - pub fn stream_content<'a, B: BlockStore>( + pub fn stream_content<'a>( &'a self, index: usize, forest: &'a PrivateForest, - store: &'a B, + store: &'a impl BlockStore, ) -> impl Stream>> + 'a { Box::pin(try_stream! { match &self.content { @@ -294,10 +304,10 @@ impl PrivateFile { /// assert_eq!(content, all_content); /// } /// ``` - pub async fn get_content( + pub async fn get_content( &self, forest: &PrivateForest, - store: &B, + store: &impl BlockStore, ) -> Result> { let mut content = Vec::with_capacity(self.get_content_size_upper_bound()); let mut stream = self.stream_content(0, forest, store); @@ -308,12 +318,12 @@ impl PrivateFile { } /// Determines where to put the content of a file. This can either be inline or stored up in chunks in a private forest. - pub(super) async fn prepare_content( + pub(super) async fn prepare_content( bare_name: &Namefilter, content: Vec, mut forest: Rc, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result<(FileContent, Rc)> { // TODO(appcypher): Use a better heuristic to determine when to use external storage. let key = Key(get_random_bytes(rng)); @@ -355,11 +365,11 @@ impl PrivateFile { } /// Decrypts a block of a file's content. - async fn decrypt_block( + async fn decrypt_block( key: &Key, label: &Namefilter, forest: &PrivateForest, - store: &B, + store: &impl BlockStore, ) -> Result> { let label_hash = &Sha3_256::hash(&label.as_bytes()); @@ -405,15 +415,70 @@ impl PrivateFile { let mut label = bare_name.clone(); label.add(&key_bytes); label.add(&key_hash); + label.saturate(); label } + /// This should be called to prepare a node for modifications, + /// if it's meant to be a successor revision of the current revision. + /// + /// It will store the current revision in the given `BlockStore` to + /// retrieve its CID and put that into the `previous` links, + /// as well as advancing the ratchet and resetting the `persisted_as` pointer. + pub(crate) async fn prepare_next_revision( + self: Rc, + store: &mut impl BlockStore, + rng: &mut impl RngCore, + ) -> Result { + let cid = self.store(store, rng).await?; + + let mut cloned = Rc::try_unwrap(self).unwrap_or_else(|rc| (*rc).clone()); + cloned.persisted_as = OnceCell::new(); // Also done in `.clone()`, but need this to work in case try_unwrap optimizes. + let key = cloned.header.get_private_ref().revision_key.0; + let previous = Encrypted::from_value(BTreeSet::from([cid]), &key, rng)?; + + cloned.previous = Some(previous); + cloned.header.advance_ratchet(); + + Ok(cloned) + } + + /// This prepares this file for key rotation, usually for moving or + /// copying the file to some other place. + /// + /// Will reset the ratchet, so a different key is necessary for read access, + /// will reset the inumber to reset write access, + /// will update the bare namefilter to match the new parent's namefilter, + /// so it inherits the write access rules from the new parent and + /// resets the `persisted_as` pointer. + /// Will copy and re-encrypt all external content. + pub(crate) async fn prepare_key_rotation( + &mut self, + parent_bare_name: Namefilter, + forest: Rc, + store: &mut impl BlockStore, + rng: &mut impl RngCore, + ) -> Result> { + let content = self.get_content(&forest, store).await?; + + self.header.inumber = utils::get_random_bytes(rng); + self.header.update_bare_name(parent_bare_name); + self.header.reset_ratchet(rng); + self.persisted_as = OnceCell::new(); + + let (content, forest) = + Self::prepare_content(&self.header.bare_name, content, forest, store, rng).await?; + self.content = content; + + Ok(forest) + } + /// Serializes the file with provided Serde serialilzer. - pub(crate) fn serialize( + pub(crate) fn serialize( &self, serializer: S, - rng: &mut R, + rng: &mut impl RngCore, ) -> Result where S: serde::Serializer, @@ -429,6 +494,7 @@ impl PrivateFile { .encrypt(&Key::generate_nonce(rng), &cbor_bytes) .map_err(SerError::custom)? }, + previous: self.previous.clone(), metadata: self.metadata.clone(), content: self.content.clone(), }) @@ -436,7 +502,11 @@ impl PrivateFile { } /// Deserializes the file with provided Serde deserializer and key. - pub(crate) fn deserialize<'de, D>(deserializer: D, key: &RevisionKey) -> Result + pub(crate) fn deserialize<'de, D>( + deserializer: D, + key: &RevisionKey, + from_cid: Cid, + ) -> Result where D: Deserializer<'de>, { @@ -444,12 +514,15 @@ impl PrivateFile { version, metadata, header, + previous, content, .. } = PrivateFileSerializable::deserialize(deserializer)?; Ok(Self { + persisted_as: OnceCell::new_with(Some(from_cid)), version, + previous, metadata, header: { let cbor_bytes = key.0.decrypt(&header).map_err(DeError::custom)?; @@ -458,6 +531,59 @@ impl PrivateFile { content, }) } + + pub(crate) async fn store( + &self, + store: &mut impl BlockStore, + rng: &mut impl RngCore, + ) -> Result { + let cid = self + .persisted_as + .get_or_try_init::(async { + // TODO(matheus23) deduplicate when reworking serialization + let private_ref = &self.header.get_private_ref(); + + // Serialize node to cbor. + let ipld = self.serialize(libipld::serde::Serializer, rng)?; + let mut bytes = Vec::new(); + ipld.encode(DagCborCodec, &mut bytes)?; + + // Encrypt bytes with content key. + let enc_bytes = private_ref + .content_key + .0 + .encrypt(&Key::generate_nonce(rng), &bytes)?; + + // Store content section in blockstore and get Cid. + store.put_block(enc_bytes, libipld::IpldCodec::Raw).await + }) + .await?; + + Ok(*cid) + } +} + +impl PartialEq for PrivateFile { + fn eq(&self, other: &Self) -> bool { + self.header == other.header + && self.version == other.version + && self.previous == other.previous + && self.metadata == other.metadata + && self.content == other.content + } +} + +impl Clone for PrivateFile { + fn clone(&self) -> Self { + Self { + persisted_as: OnceCell::new(), + version: self.version.clone(), + header: self.header.clone(), + previous: self.previous.clone(), + metadata: self.metadata.clone(), + content: self.content.clone(), + } + } } impl Id for PrivateFile { diff --git a/wnfs/src/private/forest.rs b/wnfs/src/private/forest.rs index d37be0b5..79d459c2 100644 --- a/wnfs/src/private/forest.rs +++ b/wnfs/src/private/forest.rs @@ -1,7 +1,7 @@ use super::{ hamt::{self, Hamt}, namefilter::Namefilter, - Key, PrivateNode, PrivateRef, + PrivateNode, PrivateRef, }; use crate::{utils::UnwrapOrClone, BlockStore, HashOutput, Hasher, Link}; use anyhow::Result; @@ -38,11 +38,6 @@ pub type PrivateForest = Hamt>; //-------------------------------------------------------------------------------------------------- impl PrivateForest { - /// Encrypts supplied bytes with a random nonce and AES key. - pub(crate) fn encrypt(key: &Key, data: &[u8], rng: &mut R) -> Result> { - key.encrypt(&Key::generate_nonce(rng), data) - } - /// Puts a new value at the given key. /// /// # Examples @@ -75,24 +70,17 @@ impl PrivateForest { /// assert_eq!(forest.get(private_ref, PrivateForest::resolve_lowest, store).await.unwrap(), Some(node)); /// } /// ``` - pub async fn put( + pub async fn put( self: Rc, saturated_name: Namefilter, private_ref: &PrivateRef, value: &PrivateNode, - store: &mut B, - rng: &mut R, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { debug!("Private Forest Set: PrivateRef: {:?}", private_ref); - // Serialize node to cbor. - let cbor_bytes = value.serialize_to_cbor(rng)?; - - // Encrypt bytes with content key. - let enc_bytes = Self::encrypt(&private_ref.content_key.0, &cbor_bytes, rng)?; - - // Store content section in blockstore and get Cid. - let content_cid = store.put_block(enc_bytes, libipld::IpldCodec::Raw).await?; + let content_cid = value.store(store, rng).await?; // Store header and Cid in root node. self.put_encrypted(saturated_name, content_cid, store).await @@ -141,11 +129,11 @@ impl PrivateForest { /// assert_eq!(forest.get(private_ref, PrivateForest::resolve_lowest, store).await.unwrap(), Some(node)); /// } /// ``` - pub async fn get( + pub async fn get( &self, private_ref: &PrivateRef, resolve_bias: impl FnOnce(&BTreeSet) -> Option<&Cid>, - store: &B, + store: &impl BlockStore, ) -> Result> { debug!("Private Forest Get: PrivateRef: {:?}", private_ref); @@ -163,17 +151,7 @@ impl PrivateForest { None => return Ok(None), }; - // Fetch encrypted bytes from blockstore. - let enc_bytes = store.get_block(cid).await?; - - // Decrypt bytes - let cbor_bytes = private_ref.content_key.0.decrypt(&enc_bytes)?; - - // Deserialize bytes. - Ok(Some(PrivateNode::deserialize_from_cbor( - &cbor_bytes, - &private_ref.revision_key, - )?)) + Ok(Some(PrivateNode::load(*cid, private_ref, store).await?)) } /// Checks that a value with the given saturated name hash key exists. @@ -211,10 +189,10 @@ impl PrivateForest { /// assert!(forest.has(name_hash, store).await.unwrap()); /// } /// ``` - pub async fn has( + pub async fn has( &self, saturated_name_hash: &HashOutput, - store: &B, + store: &impl BlockStore, ) -> Result { Ok(self .root @@ -224,11 +202,11 @@ impl PrivateForest { } /// Sets a new encrypted value at the given key. - pub async fn put_encrypted( + pub async fn put_encrypted( self: Rc, name: Namefilter, value: Cid, - store: &mut B, + store: &mut impl BlockStore, ) -> Result> { // TODO(matheus23): This iterates the path in the HAMT twice. // We could consider implementing something like upsert instead. @@ -238,6 +216,7 @@ impl PrivateForest { .await? .cloned() .unwrap_or_default(); + values.insert(value); let mut forest = self.unwrap_or_clone()?; @@ -247,19 +226,19 @@ impl PrivateForest { /// Gets the encrypted value at the given key. #[inline] - pub async fn get_encrypted<'b, B: BlockStore>( + pub async fn get_encrypted<'b>( &'b self, name_hash: &HashOutput, - store: &B, + store: &impl BlockStore, ) -> Result>> { self.root.get_by_hash(name_hash, store).await } /// Removes the encrypted value at the given key. - pub async fn remove_encrypted( + pub async fn remove_encrypted( self: Rc, name_hash: &HashOutput, - store: &mut B, + store: &mut impl BlockStore, ) -> Result<(Rc, Option>)> { let mut cloned = (*self).clone(); let (root, pair) = cloned.root.remove_by_hash(name_hash, store).await?; diff --git a/wnfs/src/private/hamt/node.rs b/wnfs/src/private/hamt/node.rs index 50f42db5..f7fcecba 100644 --- a/wnfs/src/private/hamt/node.rs +++ b/wnfs/src/private/hamt/node.rs @@ -84,7 +84,7 @@ where /// assert_eq!(node.get(&String::from("key"), store).await.unwrap(), Some(&42)); /// } /// ``` - pub async fn set(self: Rc, key: K, value: V, store: &B) -> Result> + pub async fn set(self: Rc, key: K, value: V, store: &impl BlockStore) -> Result> where K: DeserializeOwned + Clone + AsRef<[u8]>, V: DeserializeOwned + Clone, @@ -112,7 +112,7 @@ where /// assert_eq!(node.get(&String::from("key"), store).await.unwrap(), Some(&42)); /// } /// ``` - pub async fn get<'a, B: BlockStore>(&'a self, key: &K, store: &B) -> Result> + pub async fn get<'a>(&'a self, key: &K, store: &impl BlockStore) -> Result> where K: DeserializeOwned + AsRef<[u8]>, V: DeserializeOwned, @@ -146,10 +146,10 @@ where /// assert_eq!(node.get(&String::from("key"), store).await.unwrap(), None); /// } /// ``` - pub async fn remove( + pub async fn remove( self: Rc, key: &K, - store: &B, + store: &impl BlockStore, ) -> Result<(Rc, Option>)> where K: DeserializeOwned + Clone + AsRef<[u8]>, @@ -180,10 +180,10 @@ where /// assert_eq!(node.get_by_hash(key_hash, store).await.unwrap(), Some(&42)); /// } /// ``` - pub async fn get_by_hash<'a, B: BlockStore>( + pub async fn get_by_hash<'a>( &'a self, hash: &HashOutput, - store: &B, + store: &impl BlockStore, ) -> Result> where K: DeserializeOwned + AsRef<[u8]>, @@ -220,10 +220,10 @@ where /// assert_eq!(node.get(&String::from("key"), store).await.unwrap(), None); /// } /// ``` - pub async fn remove_by_hash( + pub async fn remove_by_hash( self: Rc, hash: &HashOutput, - store: &B, + store: &impl BlockStore, ) -> Result<(Rc, Option>)> where K: DeserializeOwned + Clone + AsRef<[u8]>, @@ -269,12 +269,12 @@ where (mask & self.bitmask).count_ones() } - pub(crate) fn set_value<'a, B: BlockStore>( + pub(crate) fn set_value<'a>( self: Rc, hashnibbles: &'a mut HashNibbles, key: K, value: V, - store: &'a B, + store: &'a impl BlockStore, ) -> LocalBoxFuture<'a, Result>> where K: DeserializeOwned + Clone + AsRef<[u8]> + 'a, @@ -350,10 +350,10 @@ where } #[async_recursion(?Send)] - pub(crate) async fn get_value<'a, B: BlockStore>( + pub(crate) async fn get_value<'a>( &'a self, hashnibbles: &mut HashNibbles, - store: &B, + store: &impl BlockStore, ) -> Result>> where K: DeserializeOwned + AsRef<[u8]>, @@ -382,10 +382,10 @@ where // It's internal and is only more complex because async_recursion doesn't work here #[allow(clippy::type_complexity)] - pub(crate) fn remove_value<'k, 'v, 'a, B: BlockStore>( + pub(crate) fn remove_value<'k, 'v, 'a>( self: Rc, hashnibbles: &'a mut HashNibbles, - store: &'a B, + store: &'a impl BlockStore, ) -> LocalBoxFuture<'a, Result<(Rc>, Option>)>> where K: DeserializeOwned + Clone + AsRef<[u8]> + 'k, diff --git a/wnfs/src/private/hamt/pointer.rs b/wnfs/src/private/hamt/pointer.rs index 089a47ef..11c73a27 100644 --- a/wnfs/src/private/hamt/pointer.rs +++ b/wnfs/src/private/hamt/pointer.rs @@ -71,7 +71,7 @@ impl Pair { impl Pointer { /// Converts a Link pointer to a canonical form to ensure consistent tree representation after deletes. - pub async fn canonicalize(self, store: &B) -> Result> + pub async fn canonicalize(self, store: &impl BlockStore) -> Result> where K: DeserializeOwned + Clone + AsRef<[u8]>, V: DeserializeOwned + Clone, diff --git a/wnfs/src/private/hamt/strategies/operations.rs b/wnfs/src/private/hamt/strategies/operations.rs index 7936fa87..af560ca6 100644 --- a/wnfs/src/private/hamt/strategies/operations.rs +++ b/wnfs/src/private/hamt/strategies/operations.rs @@ -176,9 +176,9 @@ where /// println!("{:?}", node); /// } /// ``` -pub async fn node_from_operations( +pub async fn node_from_operations( operations: &Operations, - store: &mut B, + store: &mut impl BlockStore, ) -> Result>> where K: DeserializeOwned + Serialize + Clone + Debug + AsRef<[u8]>, diff --git a/wnfs/src/private/key.rs b/wnfs/src/private/key.rs index 55413bc1..2509a07a 100644 --- a/wnfs/src/private/key.rs +++ b/wnfs/src/private/key.rs @@ -124,10 +124,7 @@ impl Key { /// println!("Nonce: {:?}", nonce); /// ``` #[inline] - pub fn generate_nonce(rng: &mut R) -> [u8; NONCE_SIZE] - where - R: RngCore, - { + pub fn generate_nonce(rng: &mut impl RngCore) -> [u8; NONCE_SIZE] { utils::get_random_bytes::(rng) } diff --git a/wnfs/src/private/mod.rs b/wnfs/src/private/mod.rs index 3fafc565..f0769c07 100644 --- a/wnfs/src/private/mod.rs +++ b/wnfs/src/private/mod.rs @@ -1,4 +1,5 @@ mod directory; +mod encrypted; mod file; mod forest; pub mod hamt; diff --git a/wnfs/src/private/node.rs b/wnfs/src/private/node.rs index 588971f7..45958b21 100644 --- a/wnfs/src/private/node.rs +++ b/wnfs/src/private/node.rs @@ -1,21 +1,17 @@ use super::{ - hamt::Hasher, namefilter::Namefilter, Key, PrivateDirectory, PrivateFile, PrivateForest, - PrivateRef, + encrypted::Encrypted, hamt::Hasher, namefilter::Namefilter, Key, PrivateDirectory, PrivateFile, + PrivateForest, PrivateRef, }; use crate::{utils, BlockStore, FsError, HashOutput, Id, NodeType, HASH_BYTE_SIZE}; use anyhow::{bail, Result}; use async_recursion::async_recursion; use chrono::{DateTime, Utc}; -use libipld::{ - cbor::DagCborCodec, - prelude::{Decode, Encode}, - serde as ipld_serde, Ipld, -}; +use libipld::{cbor::DagCborCodec, prelude::Decode, Cid, Ipld}; use rand_core::RngCore; use serde::{Deserialize, Serialize}; use sha3::Sha3_256; use skip_ratchet::{seek::JumpSize, Ratchet, RatchetSeeker}; -use std::{cmp::Ordering, fmt::Debug, io::Cursor, rc::Rc}; +use std::{cmp::Ordering, collections::BTreeSet, fmt::Debug, io::Cursor, rc::Rc}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -138,7 +134,7 @@ impl PrivateNode { } /// Generates two random set of bytes. - pub(crate) fn generate_double_random(rng: &mut R) -> (HashOutput, HashOutput) { + pub(crate) fn generate_double_random(rng: &mut impl RngCore) -> (HashOutput, HashOutput) { const _DOUBLE_SIZE: usize = HASH_BYTE_SIZE * 2; let [first, second] = unsafe { std::mem::transmute::<[u8; _DOUBLE_SIZE], [[u8; HASH_BYTE_SIZE]; 2]>( @@ -150,48 +146,43 @@ impl PrivateNode { /// Updates bare name ancestry of private sub tree. #[async_recursion(?Send)] - pub(crate) async fn update_ancestry( + pub(crate) async fn update_ancestry( &mut self, parent_bare_name: Namefilter, - forest: Rc, - store: &mut B, - rng: &mut R, + mut forest: Rc, + store: &mut impl BlockStore, + rng: &mut impl RngCore, ) -> Result> { - let forest = match self { + match self { Self::File(file) => { let mut file = (**file).clone(); - file.header.update_bare_name(parent_bare_name); - file.header.reset_ratchet(rng); + forest = file + .prepare_key_rotation(parent_bare_name, forest, store, rng) + .await?; *self = Self::File(Rc::new(file)); - - forest } Self::Dir(old_dir) => { let mut dir = (**old_dir).clone(); - let mut working_forest = Rc::clone(&forest); for (name, private_ref) in &old_dir.entries { let mut node = forest .get(private_ref, PrivateForest::resolve_lowest, store) .await? .ok_or(FsError::NotFound)?; - working_forest = node - .update_ancestry(dir.header.bare_name.clone(), working_forest, store, rng) + forest = node + .update_ancestry(dir.header.bare_name.clone(), forest, store, rng) .await?; dir.entries .insert(name.clone(), node.get_header().get_private_ref()); } - dir.header.update_bare_name(parent_bare_name); - dir.header.reset_ratchet(rng); + dir.prepare_key_rotation(parent_bare_name, rng); *self = Self::Dir(Rc::new(dir)); - - working_forest } }; @@ -235,6 +226,30 @@ impl PrivateNode { } } + /// Gets the previous links of the node. + /// + /// The previous links are encrypted with the previous revision's + /// revision key, so you need to know an 'older' revision of the + /// skip ratchet to decrypt these. + /// + /// The previous links is exactly one Cid in most cases and refers + /// to the ciphertext Cid from the previous revision that this + /// node is an update of. + /// + /// If this node is a merge-node, it has two or more previous Cids. + /// A single previous Cid must be from the previous revision, but all + /// other Cids may appear in even older revisions. + /// + /// The previous links is `None`, it doesn't have previous Cids. + /// The node is malformed if the previous links are `Some`, but + /// the `BTreeSet` inside is empty. + pub fn get_previous(&self) -> &Option>> { + match self { + Self::File(file) => &file.previous, + Self::Dir(dir) => &dir.previous, + } + } + /// Casts a node to a directory. /// /// # Examples @@ -338,10 +353,10 @@ impl PrivateNode { } /// Gets the latest version of the node using exponential search. - pub(crate) async fn search_latest( + pub(crate) async fn search_latest( &self, forest: &PrivateForest, - store: &B, + store: &impl BlockStore, ) -> Result { let header = self.get_header(); @@ -389,42 +404,49 @@ impl PrivateNode { } } - /// Serializes the node with provided Serde serializer. - pub(crate) fn serialize( + pub(crate) async fn load( + cid: Cid, + private_ref: &PrivateRef, + store: &impl BlockStore, + ) -> Result { + // Fetch encrypted bytes from blockstore. + let enc_bytes = store.get_block(&cid).await?; + + // Decrypt bytes + let cbor_bytes = private_ref.content_key.0.decrypt(&enc_bytes)?; + + // Deserialize + PrivateNode::deserialize_from_cbor(&cbor_bytes, &private_ref.revision_key, cid) + } + + pub(crate) async fn store( &self, - serializer: S, - rng: &mut R, - ) -> Result - where - S: serde::Serializer, - { + store: &mut impl BlockStore, + rng: &mut impl RngCore, + ) -> Result { match self { - PrivateNode::File(file) => file.serialize(serializer, rng), - PrivateNode::Dir(dir) => dir.serialize(serializer, rng), + PrivateNode::File(file) => file.store(store, rng).await, + PrivateNode::Dir(dir) => dir.store(store, rng).await, } } - /// Serializes the node to dag-cbor bytes. - pub(crate) fn serialize_to_cbor(&self, rng: &mut R) -> Result> { - let ipld = self.serialize(ipld_serde::Serializer, rng)?; - let mut bytes = Vec::new(); - ipld.encode(DagCborCodec, &mut bytes)?; - Ok(bytes) - } - /// Deserializes the node from dag-cbor bytes. - pub(crate) fn deserialize_from_cbor(bytes: &[u8], key: &RevisionKey) -> Result { + pub(crate) fn deserialize_from_cbor( + bytes: &[u8], + key: &RevisionKey, + from_cid: Cid, + ) -> Result { let ipld = Ipld::decode(DagCborCodec, &mut Cursor::new(bytes))?; - (ipld, key).try_into() + (ipld, key, from_cid).try_into() } } -impl TryFrom<(Ipld, &RevisionKey)> for PrivateNode { +impl TryFrom<(Ipld, &RevisionKey, Cid)> for PrivateNode { type Error = anyhow::Error; - fn try_from(pair: (Ipld, &RevisionKey)) -> Result { - match pair { - (Ipld::Map(map), key) => { + fn try_from(triple: (Ipld, &RevisionKey, Cid)) -> Result { + match triple { + (Ipld::Map(map), key, from_cid) => { let r#type: NodeType = map .get("type") .ok_or(FsError::MissingNodeType)? @@ -432,11 +454,13 @@ impl TryFrom<(Ipld, &RevisionKey)> for PrivateNode { Ok(match r#type { NodeType::PrivateFile => { - PrivateNode::from(PrivateFile::deserialize(Ipld::Map(map), key)?) - } - NodeType::PrivateDirectory => { - PrivateNode::from(PrivateDirectory::deserialize(Ipld::Map(map), key)?) + PrivateNode::from(PrivateFile::deserialize(Ipld::Map(map), key, from_cid)?) } + NodeType::PrivateDirectory => PrivateNode::from(PrivateDirectory::deserialize( + Ipld::Map(map), + key, + from_cid, + )?), other => bail!(FsError::UnexpectedNodeType(other)), }) } @@ -468,7 +492,7 @@ impl From for PrivateNode { impl PrivateNodeHeader { /// Creates a new PrivateNodeHeader. - pub(crate) fn new(parent_bare_name: Namefilter, rng: &mut R) -> Self { + pub(crate) fn new(parent_bare_name: Namefilter, rng: &mut impl RngCore) -> Self { let (inumber, ratchet_seed) = PrivateNode::generate_double_random(rng); Self { bare_name: { @@ -513,7 +537,7 @@ impl PrivateNodeHeader { } /// Resets the ratchet. - pub(crate) fn reset_ratchet(&mut self, rng: &mut R) { + pub(crate) fn reset_ratchet(&mut self, rng: &mut impl RngCore) { self.ratchet = Ratchet::zero(utils::get_random_bytes(rng)) } @@ -578,8 +602,8 @@ impl PrivateNodeHeader { /// ``` #[inline] pub fn get_saturated_name(&self) -> Namefilter { - let revision_key = Key::new(self.ratchet.derive_key()); - self.get_saturated_name_with_key(&revision_key) + let revision_key = RevisionKey::from(&self.ratchet); + self.get_saturated_name_with_key(&revision_key.0) } } @@ -589,6 +613,12 @@ impl From for RevisionKey { } } +impl From<&Ratchet> for RevisionKey { + fn from(ratchet: &Ratchet) -> Self { + Self::from(Key::new(ratchet.derive_key())) + } +} + impl From for Key { fn from(key: RevisionKey) -> Self { key.0 @@ -646,10 +676,9 @@ mod tests { let file = PrivateNode::File(Rc::new(file)); let private_ref = file.get_header().get_private_ref(); - let bytes = file.serialize_to_cbor(rng).unwrap(); + let cid = file.store(store, rng).await.unwrap(); - let deserialized_node = - PrivateNode::deserialize_from_cbor(&bytes, &private_ref.revision_key).unwrap(); + let deserialized_node = PrivateNode::load(cid, &private_ref, store).await.unwrap(); assert_eq!(file, deserialized_node); } diff --git a/wnfs/src/private/previous.rs b/wnfs/src/private/previous.rs index 886eb8c2..b2d0c148 100644 --- a/wnfs/src/private/previous.rs +++ b/wnfs/src/private/previous.rs @@ -1,9 +1,13 @@ -use std::rc::Rc; +use std::{collections::BTreeSet, rc::Rc}; use anyhow::{bail, Result}; +use libipld::Cid; use skip_ratchet::{ratchet::PreviousIterator, Ratchet}; -use super::{PrivateDirectory, PrivateFile, PrivateForest, PrivateNode, PrivateNodeHeader}; +use super::{ + encrypted::Encrypted, PrivateDirectory, PrivateFile, PrivateForest, PrivateNode, + PrivateNodeHeader, RevisionKey, +}; use crate::{BlockStore, FsError, PathNodes, PathNodesResult}; @@ -47,6 +51,8 @@ pub struct PrivateNodeHistory { /// This will always be the header of the *next* version after what's retrieved from /// the `ratchets` iterator. header: PrivateNodeHeader, + /// The private node tracks which previous revision's value it was a modification of. + previous: Option>>, /// The iterator for previous revision ratchets. ratchets: PreviousIterator, } @@ -66,6 +72,7 @@ impl PrivateNodeHistory { ) -> Result { Self::from_header( node.get_header().clone(), + node.get_previous().clone(), past_ratchet, discrepancy_budget, forest, @@ -77,6 +84,7 @@ impl PrivateNodeHistory { /// See also `PrivateNodeHistory::of`. pub fn from_header( header: PrivateNodeHeader, + previous: Option>>, past_ratchet: &Ratchet, discrepancy_budget: usize, forest: Rc, @@ -87,6 +95,7 @@ impl PrivateNodeHistory { Ok(PrivateNodeHistory { forest, header, + previous, ratchets, }) } @@ -95,23 +104,43 @@ impl PrivateNodeHistory { /// previous point in history. /// /// Returns `None` if there is no such node in the `PrivateForest` at that point in time. - pub async fn get_previous_node( + pub async fn get_previous_node( &mut self, - store: &B, + store: &impl BlockStore, ) -> Result> { - match self.ratchets.next() { - None => Ok(None), - Some(previous_ratchet) => { - // TODO(matheus23) Make the `resolve_bias` be biased towards eventual `previous` backpointers. - self.header.ratchet = previous_ratchet; - self.forest - .get( - &self.header.get_private_ref(), - PrivateForest::resolve_lowest, - store, - ) - .await - } + let Some(previous_ratchet) = self.ratchets.next() + else { + return Ok(None); + }; + + let previous_cids = self.resolve_previous_cids(&previous_ratchet)?; + + self.header.ratchet = previous_ratchet; + + let previous_node = self + .forest + .get( + &self.header.get_private_ref(), + PrivateForest::resolve_one_of::) -> Option<&Cid>>(&previous_cids), + store, + ) + .await?; + + if let Some(previous_node) = previous_node { + self.previous = previous_node.get_previous().clone(); + Ok(Some(previous_node)) + } else { + Ok(None) + } + } + + fn resolve_previous_cids(&self, previous_ratchet: &Ratchet) -> Result> { + if let Some(encrypted) = &self.previous { + let revision_key = RevisionKey::from(previous_ratchet); + // Cloning here because otherwise lifetimes are hard + Ok(encrypted.resolve_value(&revision_key.0)?.clone()) + } else { + Ok(BTreeSet::new()) } } @@ -120,9 +149,9 @@ impl PrivateNodeHistory { /// Returns `None` if there is no previous node with that revision in the `PrivateForest`, /// throws `FsError::NotADirectory` if the previous node happens to not be a directory. /// That should only happen for all nodes or for none. - pub async fn get_previous_dir( + pub async fn get_previous_dir( &mut self, - store: &B, + store: &impl BlockStore, ) -> Result>> { match self.get_previous_node(store).await? { Some(PrivateNode::Dir(dir)) => Ok(Some(dir)), @@ -136,9 +165,9 @@ impl PrivateNodeHistory { /// Returns `None` if there is no previous node with that revision in the `PrivateForest`, /// throws `FsError::NotAFile` if the previous node happens to not be a file. /// That should only happen for all nodes or for none. - pub async fn get_previous_file( + pub async fn get_previous_file( &mut self, - store: &B, + store: &impl BlockStore, ) -> Result>> { match self.get_previous_node(store).await? { Some(PrivateNode::File(file)) => Ok(Some(file)), @@ -159,14 +188,14 @@ impl PrivateNodeOnPathHistory { /// When `search_latest` is true, it follow the path in the current revision /// down to the child, and then look for the latest revision of the target node, /// including all in-between versions in the history. - pub async fn of( + pub async fn of( directory: Rc, past_ratchet: &Ratchet, discrepancy_budget: usize, path_segments: &[String], search_latest: bool, forest: Rc, - store: &B, + store: &impl BlockStore, ) -> Result { // To get the history on a node on a path from a given directory that we // know its newest and oldest ratchet of, we need to generate @@ -232,14 +261,14 @@ impl PrivateNodeOnPathHistory { /// until the current revision. /// /// If `search_latest` is false, the target history is empty. - async fn path_nodes_and_target_history( + async fn path_nodes_and_target_history( dir: Rc, discrepancy_budget: usize, path_segments: &[String], target_path_segment: &String, search_latest: bool, forest: Rc, - store: &B, + store: &impl BlockStore, ) -> Result<(Vec<(Rc, String)>, PrivateNodeHistory)> { // We only search for the latest revision in the private node. // It may have been deleted in future versions of its ancestor directories. @@ -252,13 +281,9 @@ impl PrivateNodeOnPathHistory { PathNodesResult::NotADirectory(_, _) => bail!(FsError::NotADirectory), }; - // TODO(matheus23) refactor using let-else once rust stable 1.65 released (Nov 3rd) - let target = match (*path_nodes.tail) - .lookup_node(target_path_segment, false, &forest, store) - .await? - { - Some(target) => target, - None => bail!(FsError::NotFound), + let Some(target) = (*path_nodes.tail).lookup_node(target_path_segment, false, &forest, store).await? + else { + bail!(FsError::NotFound); }; let target_latest = if search_latest { @@ -307,7 +332,7 @@ impl PrivateNodeOnPathHistory { /// Step the history one revision back and retrieve the node at the configured path. /// /// Returns `None` if there is no more previous revisions. - pub async fn get_previous(&mut self, store: &B) -> Result> { + pub async fn get_previous(&mut self, store: &impl BlockStore) -> Result> { // Finding the previous revision of a node works by trying to get // the previous revision of the path elements starting on the deepest // path node working upwards, in case the history of lower nodes @@ -322,10 +347,9 @@ impl PrivateNodeOnPathHistory { return Ok(Some(node)); } - // TODO(matheus23) refactor using let-else once rust stable 1.65 released (Nov 3rd) - let working_stack = match self.find_and_step_segment_history(store).await? { - Some(stack) => stack, - None => return Ok(None), + let Some(working_stack) = self.find_and_step_segment_history(store).await? + else { + return Ok(None); }; if !self @@ -339,18 +363,17 @@ impl PrivateNodeOnPathHistory { "Should not happen: path stack was empty after call to repopulate_segment_histories", ); - // TODO(matheus23) refactor using let-else once rust stable 1.65 released (Nov 3rd) - let older_node = match ancestor + let Some(older_node) = ancestor .dir .lookup_node(&ancestor.path_segment, false, &self.forest, store) .await? - { - Some(older_node) => older_node, - None => return Ok(None), + else { + return Ok(None); }; self.target = match PrivateNodeHistory::from_header( self.target.header.clone(), + self.target.previous.clone(), &older_node.get_header().ratchet, self.discrepancy_budget, Rc::clone(&self.forest), @@ -376,9 +399,9 @@ impl PrivateNodeOnPathHistory { /// /// Returns None if the no path segment history in the stack has any /// more history entries. - async fn find_and_step_segment_history( + async fn find_and_step_segment_history( &mut self, - store: &B, + store: &impl BlockStore, ) -> Result, String)>>> { let mut working_stack = Vec::with_capacity(self.path.len()); @@ -412,10 +435,10 @@ impl PrivateNodeOnPathHistory { /// a steppable history entry on top. /// /// Returns false if there's no corresponding path in the previous revision. - async fn repopulate_segment_histories( + async fn repopulate_segment_histories( &mut self, working_stack: Vec<(Rc, String)>, - store: &B, + store: &impl BlockStore, ) -> Result { // Work downwards from the previous history entry of a path segment we found for (directory, path_segment) in working_stack { @@ -425,14 +448,12 @@ impl PrivateNodeOnPathHistory { .expect("Should not happen: repopulate_segment_histories called when the path stack was empty."); // Go down from the older ancestor directory parallel to the new revision's path - // TODO(matheus23) refactor using let-else once rust stable 1.65 released (Nov 3rd) - let older_directory = match ancestor + let Some(PrivateNode::Dir(older_directory)) = ancestor .dir .lookup_node(&ancestor.path_segment, false, &self.forest, store) .await? - { - Some(PrivateNode::Dir(older_directory)) => older_directory, - _ => return Ok(false), + else { + return Ok(false); }; let mut directory_history = match PrivateNodeHistory::of( @@ -452,10 +473,9 @@ impl PrivateNodeOnPathHistory { }; // We need to find the in-between history entry! See the test case `previous_with_multiple_child_changes`. - // TODO(matheus23) refactor using let-else once rust stable 1.65 released (Nov 3rd) - let directory_prev = match directory_history.get_previous_dir(store).await? { - Some(dir) => dir, - _ => return Ok(false), + let Some(directory_prev) = directory_history.get_previous_dir(store).await? + else { + return Ok(false); }; self.path.push(PathSegmentHistory { @@ -1011,11 +1031,7 @@ mod tests { let past_ratchet = root_dir.header.ratchet.clone(); - let root_dir = { - let mut tmp = (*root_dir).clone(); - tmp.advance_ratchet(); - Rc::new(tmp) - }; + let root_dir = Rc::new(root_dir.prepare_next_revision(store, rng).await.unwrap()); let forest = forest .put( diff --git a/wnfs/src/public/directory.rs b/wnfs/src/public/directory.rs index 1aae1eae..a7232e01 100644 --- a/wnfs/src/public/directory.rs +++ b/wnfs/src/public/directory.rs @@ -146,10 +146,10 @@ impl PublicDirectory { /// Uses specified path segments and their existence in the file tree to generate `PathNodes`. /// /// Supports cases where the entire path does not exist. - pub(crate) async fn get_path_nodes( + pub(crate) async fn get_path_nodes( self: Rc, path_segments: &[String], - store: &B, + store: &impl BlockStore, ) -> Result { use PathNodesResult::*; let mut working_node = self; @@ -187,11 +187,11 @@ impl PublicDirectory { } /// Uses specified path segments to generate `PathNodes`. Creates missing directories as needed. - pub(crate) async fn get_or_create_path_nodes( + pub(crate) async fn get_or_create_path_nodes( self: Rc, path_segments: &[String], time: DateTime, - store: &B, + store: &impl BlockStore, ) -> Result { use PathNodesResult::*; match self.get_path_nodes(path_segments, store).await? { @@ -258,10 +258,10 @@ impl PublicDirectory { /// assert!(result.is_some()); /// } /// ``` - pub async fn get_node( + pub async fn get_node( self: Rc, path_segments: &[String], - store: &B, + store: &impl BlockStore, ) -> Result>> { use PathNodesResult::*; let root_dir = Rc::clone(&self); @@ -311,10 +311,10 @@ impl PublicDirectory { /// assert!(node.is_some()); /// } /// ``` - pub async fn lookup_node( + pub async fn lookup_node( &self, path_segment: &str, - store: &B, + store: &impl BlockStore, ) -> Result> { Ok(match self.userland.get(path_segment) { Some(link) => Some(link.resolve_value(store).await?.clone()), @@ -347,7 +347,7 @@ impl PublicDirectory { /// } /// ``` #[inline(always)] - pub async fn store(&self, store: &mut B) -> Result { + pub async fn store(&self, store: &mut impl BlockStore) -> Result { store.put_async_serializable(self).await } @@ -385,10 +385,10 @@ impl PublicDirectory { /// assert_eq!(result, cid); /// } /// ``` - pub async fn read( + pub async fn read( self: Rc, path_segments: &[String], - store: &mut B, + store: &mut impl BlockStore, ) -> Result> { let root_dir = Rc::clone(&self); let (path, filename) = utils::split_last(path_segments)?; @@ -434,12 +434,12 @@ impl PublicDirectory { /// .unwrap(); /// } /// ``` - pub async fn write( + pub async fn write( self: Rc, path_segments: &[String], content_cid: Cid, time: DateTime, - store: &B, + store: &impl BlockStore, ) -> Result> { let (directory_path, filename) = utils::split_last(path_segments)?; @@ -505,11 +505,11 @@ impl PublicDirectory { /// ``` /// /// This method acts like `mkdir -p` in Unix because it creates intermediate directories if they do not exist. - pub async fn mkdir( + pub async fn mkdir( self: Rc, path_segments: &[String], time: DateTime, - store: &B, + store: &impl BlockStore, ) -> Result> { let path_nodes = self .get_or_create_path_nodes(path_segments, time, store) @@ -555,10 +555,10 @@ impl PublicDirectory { /// assert_eq!(result[0].0, "tabby.png"); /// } /// ``` - pub async fn ls( + pub async fn ls( self: Rc, path_segments: &[String], - store: &B, + store: &impl BlockStore, ) -> Result>> { let root_dir = Rc::clone(&self); match self.get_path_nodes(path_segments, store).await? { @@ -625,10 +625,10 @@ impl PublicDirectory { /// assert_eq!(result.len(), 0); /// } /// ``` - pub async fn rm( + pub async fn rm( self: Rc, path_segments: &[String], - store: &B, + store: &impl BlockStore, ) -> Result> { let (directory_path, node_name) = utils::split_last(path_segments)?; @@ -698,12 +698,12 @@ impl PublicDirectory { /// assert_eq!(result.len(), 2); /// } /// ``` - pub async fn basic_mv( + pub async fn basic_mv( self: Rc, path_segments_from: &[String], path_segments_to: &[String], time: DateTime, - store: &B, + store: &impl BlockStore, ) -> Result> { let root_dir = Rc::clone(&self); let (directory_path, filename) = utils::split_last(path_segments_to)?; @@ -781,10 +781,10 @@ impl PublicDirectory { /// .unwrap(); /// } /// ``` - pub async fn base_history_on( + pub async fn base_history_on( self: Rc, base: Rc, - store: &mut B, + store: &mut impl BlockStore, ) -> Result> { if Rc::ptr_eq(&self, &base) { return Ok(PublicOpResult { @@ -814,10 +814,10 @@ impl PublicDirectory { /// Constructs a tree from directory with `base` as the historical ancestor. #[async_recursion(?Send)] - pub(crate) async fn base_history_on_helper( + pub(crate) async fn base_history_on_helper( link: &PublicLink, base_link: &PublicLink, - store: &mut B, + store: &mut impl BlockStore, ) -> Result> { if link.deep_eq(base_link, store).await? { return Ok(None); @@ -877,10 +877,7 @@ impl AsyncSerialize for PublicDirectory { for (name, link) in self.userland.iter() { map.insert( name.clone(), - *link - .resolve_cid(store) - .await - .map_err(|e| SerError::custom(format!("{e}")))?, + *link.resolve_cid(store).await.map_err(SerError::custom)?, ); } map diff --git a/wnfs/src/public/file.rs b/wnfs/src/public/file.rs index 784c8d49..e61626c4 100644 --- a/wnfs/src/public/file.rs +++ b/wnfs/src/public/file.rs @@ -112,7 +112,7 @@ impl PublicFile { /// } /// ``` #[inline(always)] - pub async fn store(&self, store: &mut B) -> Result { + pub async fn store(&self, store: &mut impl BlockStore) -> Result { store.put_serializable(self).await } } diff --git a/wnfs/src/public/node.rs b/wnfs/src/public/node.rs index 6493cd4a..a8176b50 100644 --- a/wnfs/src/public/node.rs +++ b/wnfs/src/public/node.rs @@ -187,7 +187,7 @@ impl PublicNode { } /// Serializes a node to the block store. - pub async fn store(&self, store: &mut B) -> Result { + pub async fn store(&self, store: &mut impl BlockStore) -> Result { Ok(match self { Self::File(file) => file.store(store).await?, Self::Dir(dir) => dir.store(store).await?,