From c679662b06b6af9d2b5ba2b47933552b7b21b21b Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 8 Nov 2024 01:10:59 +0300 Subject: [PATCH 01/19] deduplicate collection logic to a base type --- crates/storage/src/collections.rs | 243 +++++++++++++++++- .../storage/src/collections/unordered_map.rs | 2 + .../storage/src/collections/unordered_set.rs | 2 + crates/storage/src/collections/vector.rs | 2 + 4 files changed, 245 insertions(+), 4 deletions(-) diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 1122bb681..ae6acf01f 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -1,14 +1,249 @@ //! High-level data structures for storage. -/// This module provides functionality for hashmap data structures. +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; + +use borsh::{BorshDeserialize, BorshSerialize}; + pub mod unordered_map; pub use unordered_map::UnorderedMap; -/// This module provides functionality for hashset data structures. pub mod unordered_set; pub use unordered_set::UnorderedSet; -/// This module provides functionality for vector data structures. pub mod vector; pub use vector::Vector; -/// This module provides functionality for handling errors. pub mod error; pub use error::StoreError; + +use crate as calimero_storage; +use crate::address::{Id, Path}; +use crate::entities::{Data, Element}; +use crate::interface::{Interface, StorageError}; +use crate::{AtomicUnit, Collection}; + +#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +#[type_id(255)] +#[root] +struct Collection { + /// The entries in the collection. + #[collection] + entries: Entries, + /// The storage element for the map. + #[storage] + storage: Element, +} + +/// A collection of entries in a map. +#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] +#[children(Entry)] +struct Entries { + /// Helper to associate the generic types with the collection. + _priv: PhantomData, +} + +/// An entry in a map. +#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +#[type_id(254)] +struct Entry { + /// The item in the entry. + item: T, + /// The storage element for the entry. + #[storage] + storage: Element, +} + +#[derive(Debug)] +struct CollectionMut<'a, T: BorshSerialize + BorshDeserialize> { + collection: &'a mut Collection, +} + +#[derive(Debug)] +struct EntryMut<'a, T: BorshSerialize + BorshDeserialize> { + collection: CollectionMut<'a, T>, + entry: Entry, +} + +type StoreResult = std::result::Result; + +impl Collection { + /// Creates a new collection. + fn new() -> Self { + let mut this = Self { + entries: Entries { _priv: PhantomData }, + storage: Element::new(&Path::new("::unused").expect("valid path"), None), + }; + + let _ = Interface::save(&mut this).expect("save collection"); + + this + } + + /// Inserts an item into the collection. + fn insert(&mut self, id: Option, item: T) -> StoreResult<()> { + let path = self.path(); + + let mut collection = CollectionMut::new(self); + + collection.insert(Entry { + item, + storage: Element::new(&path, id), + })?; + + Ok(()) + } + + fn get(&self, id: Id) -> StoreResult> { + let entry = Interface::find_by_id::>(id)?; + + Ok(entry.map(|entry| entry.item)) + } + + fn get_mut(&mut self, id: Id) -> StoreResult>> { + let entry = Interface::find_by_id::>(id)?; + + Ok(entry.map(|entry| EntryMut { + collection: CollectionMut::new(self), + entry, + })) + } + + fn entries(&self) -> StoreResult>> { + let children = Interface::child_info_for(self.id(), &self.entries)?; + + let iter = children.into_iter().map(|child| { + let id = child.id(); + + let entry = Interface::find_by_id::>(id)? + .ok_or(StoreError::StorageError(StorageError::NotFound(id)))?; + + Ok(entry.item) + }); + + Ok(iter) + } + + fn clear(&mut self) -> StoreResult<()> { + let mut collection = CollectionMut::new(self); + + collection.clear()?; + + Ok(()) + } +} + +impl EntryMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + fn remove(mut self) -> StoreResult { + let old = self + .collection + .get(self.entry.id())? + .ok_or(StoreError::StorageError(StorageError::NotFound( + self.entry.id(), + )))?; + + let _ = Interface::remove_child_from( + self.collection.id(), + &mut self.collection.entries, + self.entry.id(), + )?; + + Ok(old) + } +} + +impl Deref for EntryMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.entry.item + } +} + +impl DerefMut for EntryMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.entry.item + } +} + +impl Drop for EntryMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + fn drop(&mut self) { + self.entry.element_mut().update(); + let _ignored = Interface::save(&mut self.entry); + } +} + +impl<'a, T> CollectionMut<'a, T> +where + T: BorshSerialize + BorshDeserialize, +{ + fn new(collection: &'a mut Collection) -> Self { + Self { collection } + } + + fn insert(&mut self, item: Entry) -> StoreResult<()> { + let mut item = item; + + let _ = Interface::add_child_to( + self.collection.id(), + &mut self.collection.entries, + &mut item, + )?; + + Ok(()) + } + + fn clear(&mut self) -> StoreResult<()> { + let children = + Interface::child_info_for(self.collection.id(), &mut self.collection.entries)?; + + for child in children { + let _ = Interface::remove_child_from( + self.collection.id(), + &mut self.collection.entries, + child.id(), + )?; + } + + Ok(()) + } +} + +impl Deref for CollectionMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + type Target = Collection; + + fn deref(&self) -> &Self::Target { + self.collection + } +} + +impl DerefMut for CollectionMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + self.collection + } +} + +impl Drop for CollectionMut<'_, T> +where + T: BorshSerialize + BorshDeserialize, +{ + fn drop(&mut self) { + self.collection.element_mut().update(); + let _ignored = Interface::save(self.collection); + } +} diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index 7eaea3726..eaab359e0 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -1,3 +1,5 @@ +//! This module provides functionality for the unordered map data structure. + use core::borrow::Borrow; use core::marker::PhantomData; diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 378ef5bb7..eceb08a04 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -1,3 +1,5 @@ +//! This module provides functionality for the unordered set data structure. + use core::borrow::Borrow; use core::marker::PhantomData; diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index 4454d337b..b82c74a67 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -1,3 +1,5 @@ +//! This module provides functionality for the vector data structure. + use core::borrow::Borrow; use core::marker::PhantomData; From 1393a7c4aee5cfe0f491158519b93855addc1add Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 8 Nov 2024 01:11:42 +0300 Subject: [PATCH 02/19] update unordered map to use collection --- .../storage/src/collections/unordered_map.rs | 230 ++++++++---------- 1 file changed, 95 insertions(+), 135 deletions(-) diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index eaab359e0..79d217a3f 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -1,52 +1,25 @@ //! This module provides functionality for the unordered map data structure. use core::borrow::Borrow; -use core::marker::PhantomData; +use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; -// fixme! macro expects `calimero_storage` to be in deps -use crate as calimero_storage; -use crate::address::{Id, Path}; +use super::Collection; +use crate::address::Id; use crate::collections::error::StoreError; -use crate::entities::{Data, Element}; -use crate::interface::Interface; -use crate::{AtomicUnit, Collection}; +use crate::entities::Data; /// A map collection that stores key-value pairs. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(255)] -#[root] +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] pub struct UnorderedMap { - /// The id used for the map's entries. - id: Id, - /// The entries in the map. - entries: Entries, - /// The storage element for the map. - #[storage] - storage: Element, -} - -/// A collection of entries in a map. -#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[children(Entry)] -struct Entries { - /// Helper to associate the generic types with the collection. - _priv: PhantomData<(K, V)>, -} - -/// An entry in a map. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(254)] -pub struct Entry { - /// The key for the entry. - key: K, - /// The value for the entry. - value: V, - /// The storage element for the entry. - #[storage] - storage: Element, + #[borsh(bound( + serialize = "K: BorshSerialize, V: BorshSerialize", + deserialize = "K: BorshDeserialize, V: BorshDeserialize" + ))] + // rename to inner? + inner: Collection<(K, V)>, } impl @@ -60,45 +33,20 @@ impl /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn new() -> Result { - let id = Id::random(); - let mut this = Self { - id: id, - entries: Entries::default(), - storage: Element::new(&Path::new(format!("::unused::map::{id}::path"))?, Some(id)), - }; - - let _ = Interface::save(&mut this)?; - - Ok(this) + pub fn new() -> Self { + Self { + entries: Collection::new(), + } } /// Compute the ID for a key. fn compute_id(&self, key: &[u8]) -> Id { let mut hasher = Sha256::new(); - hasher.update(self.id.as_bytes()); + hasher.update(self.entries.id().as_bytes()); hasher.update(key); Id::new(hasher.finalize().into()) } - /// Get the raw entry for a key in the map. - /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, or a child - /// [`Element`](crate::entities::Element) cannot be found, an error will be - /// returned. - /// - fn get_raw(&self, key: &Q) -> Result>, StoreError> - where - K: Borrow, - Q: PartialEq + AsRef<[u8]> + ?Sized, - { - Ok(Interface::find_by_id::>( - self.compute_id(key.as_ref()), - )?) - } - /// Insert a key-value pair into the map. /// /// # Errors @@ -111,26 +59,16 @@ impl where K: AsRef<[u8]> + PartialEq, { - if let Some(mut entry) = self.get_raw(&key)? { - let ret_value = entry.value; - entry.value = value; - // has to be called to update the entry - entry.element_mut().update(); - let _ = Interface::save(&mut entry)?; - return Ok(Some(ret_value)); + let id = self.compute_id(key.as_ref()); + + if let Some(mut entry) = self.entries.get_mut(id)? { + dbg!(&entry); + let (_, v) = &mut *entry; + + return Ok(Some(mem::replace(v, value))); } - let path = self.path(); - let storage = Element::new(&path, Some(self.compute_id(key.as_ref()))); - let _ = Interface::add_child_to( - self.storage.id(), - &mut self.entries, - &mut Entry { - key, - value, - storage, - }, - )?; + self.inner.insert(Some(id), (key, value))?; Ok(None) } @@ -144,9 +82,11 @@ impl /// returned. /// pub fn entries(&self) -> Result, StoreError> { - let entries = Interface::children_of(self.id(), &self.entries)?; + let iter = self.inner.entries()?; - Ok(entries.into_iter().map(|entry| (entry.key, entry.value))) + let iter = iter.flat_map(|entry| entry.ok()); + + Ok(iter.fuse()) } /// Get the number of entries in the map. @@ -158,7 +98,7 @@ impl /// returned. /// pub fn len(&self) -> Result { - Ok(Interface::child_info_for(self.id(), &self.entries)?.len()) + Ok(self.inner.entries()?.len()) } /// Get the value for a key in the map. @@ -174,10 +114,9 @@ impl K: Borrow, Q: PartialEq + AsRef<[u8]> + ?Sized, { - let entry = Interface::find_by_id::>(self.compute_id(key.as_ref()))?; - let value = entry.map(|e| e.value); + let id = self.compute_id(key.as_ref()); - Ok(value) + Ok(self.inner.get(id)?.map(|(_, v)| v)) } /// Check if the map contains a key. @@ -193,7 +132,7 @@ impl K: Borrow + PartialEq, Q: PartialEq + AsRef<[u8]> + ?Sized, { - Ok(self.get_raw(key)?.is_some()) + self.get(key).map(|v| v.is_some()) } /// Remove a key from the map, returning the value at the key if it previously existed. @@ -204,18 +143,18 @@ impl /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn remove(&mut self, key: &Q) -> Result + pub fn remove(&mut self, key: &Q) -> Result, StoreError> where K: Borrow, Q: PartialEq + AsRef<[u8]> + ?Sized, { - let entry = Element::new(&self.path(), Some(self.compute_id(key.as_ref()))); + let id = self.compute_id(key.as_ref()); + + let Some(entry) = self.inner.get_mut(id)? else { + return Ok(None); + }; - Ok(Interface::remove_child_from( - self.id(), - &mut self.entries, - entry.id(), - )?) + entry.remove().map(|(_, v)| Some(v)) } /// Clear the map, removing all entries. @@ -227,23 +166,32 @@ impl /// returned. /// pub fn clear(&mut self) -> Result<(), StoreError> { - let entries = Interface::children_of(self.id(), &self.entries)?; - - for entry in entries { - let _ = Interface::remove_child_from(self.id(), &mut self.entries, entry.id())?; - } - - Ok(()) + self.inner.clear() } } #[cfg(test)] mod tests { use crate::collections::unordered_map::UnorderedMap; + use crate::entities::Data; + + #[test] + fn doo() { + let mut map = UnorderedMap::::new(); + + dbg!(map.entries.element().merkle_hash()); + + assert!(map + .insert("key".to_string(), "value".to_string()) + .expect("insert failed") + .is_none()); + + dbg!(map.entries.element().merkle_hash()); + } #[test] fn test_unordered_map_basic_operations() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key".to_string(), "value".to_string()) @@ -251,18 +199,19 @@ mod tests { .is_none()); assert_eq!( - map.get("key").expect("get failed"), - Some("value".to_string()) + map.get("key").expect("get failed").as_deref(), + Some("value") ); assert_ne!( - map.get("key").expect("get failed"), - Some("value2".to_string()) + map.get("key").expect("get failed").as_deref(), + Some("value2") ); assert_eq!( map.insert("key".to_string(), "value2".to_string()) - .expect("insert failed"), - Some("value".to_string()) + .expect("insert failed") + .as_deref(), + Some("value") ); assert!(map .insert("key2".to_string(), "value".to_string()) @@ -270,23 +219,28 @@ mod tests { .is_none()); assert_eq!( - map.get("key").expect("get failed"), - Some("value2".to_string()) + map.get("key").expect("get failed").as_deref(), + Some("value2") ); assert_eq!( - map.get("key2").expect("get failed"), - Some("value".to_string()) + map.get("key2").expect("get failed").as_deref(), + Some("value") ); - assert_eq!(map.remove("key").expect("error while removing key"), true); - assert_eq!(map.remove("key").expect("error while removing key"), false); + assert_eq!( + map.remove("key") + .expect("error while removing key") + .as_deref(), + Some("value2") + ); + assert_eq!(map.remove("key").expect("error while removing key"), None); assert_eq!(map.get("key").expect("get failed"), None); } #[test] fn test_unordered_map_insert_and_get() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key1".to_string(), "value1".to_string()) @@ -298,18 +252,18 @@ mod tests { .is_none()); assert_eq!( - map.get("key1").expect("get failed"), - Some("value1".to_string()) + map.get("key1").expect("get failed").as_deref(), + Some("value1") ); assert_eq!( - map.get("key2").expect("get failed"), - Some("value2".to_string()) + map.get("key2").expect("get failed").as_deref(), + Some("value2") ); } #[test] fn test_unordered_map_update_value() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key".to_string(), "value".to_string()) @@ -321,27 +275,30 @@ mod tests { .is_none()); assert_eq!( - map.get("key").expect("get failed"), - Some("new_value".to_string()) + map.get("key").expect("get failed").as_deref(), + Some("new_value") ); } #[test] fn test_remove() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key".to_string(), "value".to_string()) .expect("insert failed") .is_none()); - assert_eq!(map.remove("key").expect("remove failed"), true); + assert_eq!( + map.remove("key").expect("remove failed").as_deref(), + Some("value") + ); assert_eq!(map.get("key").expect("get failed"), None); } #[test] fn test_clear() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key1".to_string(), "value1".to_string()) @@ -360,7 +317,7 @@ mod tests { #[test] fn test_unordered_map_len() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert_eq!(map.len().expect("len failed"), 0); @@ -379,14 +336,17 @@ mod tests { assert_eq!(map.len().expect("len failed"), 2); - assert_eq!(map.remove("key1").expect("remove failed"), true); + assert_eq!( + map.remove("key1").expect("remove failed").as_deref(), + Some("value1") + ); assert_eq!(map.len().expect("len failed"), 1); } #[test] fn test_unordered_map_contains() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key".to_string(), "value".to_string()) @@ -399,7 +359,7 @@ mod tests { #[test] fn test_unordered_map_entries() { - let mut map = UnorderedMap::::new().expect("failed to create map"); + let mut map = UnorderedMap::::new(); assert!(map .insert("key1".to_string(), "value1".to_string()) From 35ab8c4d5696273ee59c614d7854490ea6eaef63 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 8 Nov 2024 01:26:32 +0300 Subject: [PATCH 03/19] update unordered set to use collection --- .../storage/src/collections/unordered_set.rs | 113 ++++++------------ 1 file changed, 35 insertions(+), 78 deletions(-) diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index eceb08a04..32b598279 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -1,50 +1,20 @@ //! This module provides functionality for the unordered set data structure. use core::borrow::Borrow; -use core::marker::PhantomData; use borsh::{BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; +use super::Collection; // fixme! macro expects `calimero_storage` to be in deps -use crate as calimero_storage; -use crate::address::{Id, Path}; +use crate::address::Id; use crate::collections::error::StoreError; -use crate::entities::{Data, Element}; -use crate::interface::Interface; -use crate::{AtomicUnit, Collection}; +use crate::entities::Data; /// A set collection that stores unqiue values once. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(253)] -#[root] +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd)] pub struct UnorderedSet { - /// The prefix used for the set's entries. - id: Id, - /// The entries in the set. - entries: Entries, - /// The storage element for the set. - #[storage] - storage: Element, -} - -/// A collection of entries in a set. -#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[children(Entry)] -struct Entries { - /// Helper to associate the generic types with the collection. - _priv: PhantomData, -} - -/// An entry in a set. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(252)] -pub struct Entry { - /// The value for the entry. - value: V, - /// The storage element for the entry. - #[storage] - storage: Element, + inner: Collection, } impl UnorderedSet { @@ -56,23 +26,16 @@ impl UnorderedSet { /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn new() -> Result { - let id = Id::random(); - let mut this = Self { - id: id, - entries: Entries::default(), - storage: Element::new(&Path::new(format!("::unused::set::{id}::path"))?, Some(id)), - }; - - let _ = Interface::save(&mut this)?; - - Ok(this) + pub fn new() -> Self { + Self { + inner: Collection::new(), + } } /// Compute the ID for a value in the set. fn compute_id(&self, value: &[u8]) -> Id { let mut hasher = Sha256::new(); - hasher.update(self.id.as_bytes()); + hasher.update(self.inner.id().as_bytes()); hasher.update(value); Id::new(hasher.finalize().into()) } @@ -89,18 +52,13 @@ impl UnorderedSet { where V: AsRef<[u8]> + PartialEq, { - let path = self.path(); + let id = self.compute_id(value.as_ref()); - if self.contains(&value)? { + if self.inner.get_mut(id)?.is_some() { return Ok(false); - } + }; - let storage = Element::new(&path, Some(self.compute_id(value.as_ref()))); - let _ = Interface::add_child_to( - self.storage.id(), - &mut self.entries, - &mut Entry { value, storage }, - )?; + self.inner.insert(Some(id), value)?; Ok(true) } @@ -114,9 +72,11 @@ impl UnorderedSet { /// returned. /// pub fn entries(&self) -> Result, StoreError> { - let entries = Interface::children_of(self.id(), &self.entries)?; + let iter = self.inner.entries()?; - Ok(entries.into_iter().map(|entry| entry.value)) + let iter = iter.flat_map(|entry| entry.ok()); + + Ok(iter) } /// Get the number of entries in the set. @@ -128,7 +88,7 @@ impl UnorderedSet { /// returned. /// pub fn len(&self) -> Result { - Ok(Interface::child_info_for(self.id(), &self.entries)?.len()) + Ok(self.inner.entries()?.len()) } /// Get the value for a key in the set. @@ -144,8 +104,9 @@ impl UnorderedSet { V: Borrow, Q: PartialEq + ?Sized + AsRef<[u8]>, { - let entry = Interface::find_by_id::>(self.compute_id(value.as_ref()))?; - Ok(entry.is_some()) + let id = self.compute_id(value.as_ref()); + + Ok(self.inner.get(id)?.is_some()) } /// Remove a key from the set, returning the value at the key if it previously existed. @@ -161,13 +122,15 @@ impl UnorderedSet { V: Borrow, Q: PartialEq + AsRef<[u8]> + ?Sized, { - let entry = Element::new(&self.path(), Some(self.compute_id(value.as_ref()))); + let id = self.compute_id(value.as_ref()); + + let Some(entry) = self.inner.get_mut(id)? else { + return Ok(false); + }; - Ok(Interface::remove_child_from( - self.id(), - &mut self.entries, - entry.id(), - )?) + let _ignored = entry.remove()?; + + Ok(true) } /// Clear the set, removing all entries. @@ -179,13 +142,7 @@ impl UnorderedSet { /// returned. /// pub fn clear(&mut self) -> Result<(), StoreError> { - let entries = Interface::children_of(self.id(), &self.entries)?; - - for entry in entries { - let _ = Interface::remove_child_from(self.id(), &mut self.entries, entry.id())?; - } - - Ok(()) + self.inner.clear() } } @@ -195,7 +152,7 @@ mod tests { #[test] fn test_unordered_set_operations() { - let mut set = UnorderedSet::::new().expect("failed to create set"); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); @@ -223,7 +180,7 @@ mod tests { #[test] fn test_unordered_set_len() { - let mut set = UnorderedSet::::new().expect("failed to create set"); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); assert!(set.insert("value2".to_string()).expect("insert failed")); @@ -238,7 +195,7 @@ mod tests { #[test] fn test_unordered_set_clear() { - let mut set = UnorderedSet::::new().expect("failed to create set"); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); assert!(set.insert("value2".to_string()).expect("insert failed")); @@ -254,7 +211,7 @@ mod tests { #[test] fn test_unordered_set_entries() { - let mut set = UnorderedSet::::new().expect("failed to create set"); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); assert!(set.insert("value2".to_string()).expect("insert failed")); From e6d70bb5dc71748f45bd363ec0203a3adce667cd Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 8 Nov 2024 14:58:07 +0300 Subject: [PATCH 04/19] cache child infos --- crates/storage/src/collections.rs | 75 +++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index ae6acf01f..deaac9eef 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -1,9 +1,12 @@ //! High-level data structures for storage. +use std::cell::RefCell; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; +use std::ptr; use borsh::{BorshDeserialize, BorshSerialize}; +use indexmap::IndexSet; pub mod unordered_map; pub use unordered_map::UnorderedMap; @@ -14,26 +17,32 @@ pub use vector::Vector; pub mod error; pub use error::StoreError; +// fixme! macro expects `calimero_storage` to be in deps use crate as calimero_storage; use crate::address::{Id, Path}; use crate::entities::{Data, Element}; use crate::interface::{Interface, StorageError}; use crate::{AtomicUnit, Collection}; -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] #[type_id(255)] #[root] struct Collection { /// The entries in the collection. #[collection] entries: Entries, + /// The storage element for the map. #[storage] storage: Element, + + #[skip] + #[borsh(skip)] + children_ids: RefCell>>, } /// A collection of entries in a map. -#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] +#[derive(Collection, Copy, Clone, Debug, Eq, PartialEq)] #[children(Entry)] struct Entries { /// Helper to associate the generic types with the collection. @@ -41,7 +50,7 @@ struct Entries { } /// An entry in a map. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] #[type_id(254)] struct Entry { /// The item in the entry. @@ -68,7 +77,8 @@ impl Collection { /// Creates a new collection. fn new() -> Self { let mut this = Self { - entries: Entries { _priv: PhantomData }, + entries: Entries::default(), + children_ids: RefCell::new(None), storage: Element::new(&Path::new("::unused").expect("valid path"), None), }; @@ -106,14 +116,10 @@ impl Collection { })) } - fn entries(&self) -> StoreResult>> { - let children = Interface::child_info_for(self.id(), &self.entries)?; - - let iter = children.into_iter().map(|child| { - let id = child.id(); - - let entry = Interface::find_by_id::>(id)? - .ok_or(StoreError::StorageError(StorageError::NotFound(id)))?; + fn entries(&self) -> StoreResult> + '_> { + let iter = self.children_cache()?.iter().copied().map(|child| { + let entry = Interface::find_by_id::>(child)? + .ok_or(StoreError::StorageError(StorageError::NotFound(child)))?; Ok(entry.item) }); @@ -128,13 +134,32 @@ impl Collection { Ok(()) } + + fn children_cache(&self) -> StoreResult<&mut IndexSet> { + let mut cache = self.children_ids.borrow_mut(); + + if cache.is_none() { + let children = Interface::child_info_for(self.id(), &self.entries)?; + + let children = children.into_iter().map(|c| c.id()).collect(); + + *cache = Some(children); + } + + let children = cache.as_mut().expect("children"); + + #[allow(unsafe_code)] + let children = unsafe { &mut *ptr::from_mut(children) }; + + Ok(children) + } } impl EntryMut<'_, T> where T: BorshSerialize + BorshDeserialize, { - fn remove(mut self) -> StoreResult { + fn remove(self) -> StoreResult { let old = self .collection .get(self.entry.id())? @@ -144,10 +169,15 @@ where let _ = Interface::remove_child_from( self.collection.id(), - &mut self.collection.entries, + &self.collection.entries, self.entry.id(), )?; + let _ = self + .collection + .children_cache()? + .shift_remove(&self.entry.id()); + Ok(old) } } @@ -193,24 +223,21 @@ where fn insert(&mut self, item: Entry) -> StoreResult<()> { let mut item = item; - let _ = Interface::add_child_to( - self.collection.id(), - &mut self.collection.entries, - &mut item, - )?; + let _ = Interface::add_child_to(self.collection.id(), &self.entries, &mut item)?; + + let _ignored = self.collection.children_cache()?.insert(item.id()); Ok(()) } fn clear(&mut self) -> StoreResult<()> { - let children = - Interface::child_info_for(self.collection.id(), &mut self.collection.entries)?; + let children = self.collection.children_cache()?; - for child in children { + for child in children.drain(..) { let _ = Interface::remove_child_from( self.collection.id(), - &mut self.collection.entries, - child.id(), + &self.collection.entries, + child, )?; } From 97ccc7cd2787a1ffed051cfe3f56d0b2c65c741d Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 06:58:42 +0300 Subject: [PATCH 05/19] tick forward --- crates/storage-macros/src/lib.rs | 125 ++---------------- crates/storage/src/collections.rs | 38 +++++- crates/storage/src/collections/error.rs | 3 +- .../storage/src/collections/unordered_map.rs | 74 +++++++---- .../storage/src/collections/unordered_set.rs | 41 +++++- crates/storage/src/collections/vector.rs | 1 - crates/storage/src/interface.rs | 4 +- crates/storage/src/lib.rs | 2 +- 8 files changed, 136 insertions(+), 152 deletions(-) diff --git a/crates/storage-macros/src/lib.rs b/crates/storage-macros/src/lib.rs index 4cd1edcfa..0e9b0cb57 100644 --- a/crates/storage-macros/src/lib.rs +++ b/crates/storage-macros/src/lib.rs @@ -1,6 +1,6 @@ use borsh as _; use proc_macro::TokenStream; -use quote::{format_ident, quote}; +use quote::quote; use syn::{parse_macro_input, Data, DeriveInput, Fields, LitInt, Type}; #[cfg(test)] @@ -189,60 +189,6 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { .expect("You must designate one field with #[storage] for the Element"); let storage_ident = storage_field.ident.as_ref().unwrap(); - let storage_ty = &storage_field.ty; - - let field_implementations = named_fields - .iter() - .filter_map(|f| { - let ident = f.ident.as_ref().unwrap(); - let ty = &f.ty; - - let private = f.attrs.iter().any(|attr| attr.path().is_ident("private")); - let skip = f.attrs.iter().any(|attr| attr.path().is_ident("skip")); - - if skip || ident == storage_ident { - None - } else { - let setter = format_ident!("set_{}", ident); - - let setter_action = if private { - quote! { - self.#ident = value; - } - } else { - quote! { - self.#ident = value; - self.#storage_ident.update(); - } - }; - - Some(quote! { - #[doc = "Setter for the "] - #[doc = stringify!(#ident)] - #[doc = " field."] - pub fn #setter(&mut self, value: #ty) -> bool { - if self.#ident == value { - false - } else { - #setter_action - true - } - } - }) - } - }) - .collect::>(); - - let serializable_fields: Vec<_> = named_fields - .iter() - .filter(|f| { - !f.attrs - .iter() - .any(|attr| attr.path().is_ident("skip") || attr.path().is_ident("private")) - && f.ident.as_ref().unwrap() != storage_ident - }) - .map(|f| f.ident.as_ref().unwrap()) - .collect(); let regular_fields: Vec<_> = named_fields .iter() @@ -277,49 +223,16 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { .map(|f| f.ty.clone()) .collect(); - let skipped_fields: Vec<_> = named_fields - .iter() - .filter(|f| { - f.attrs - .iter() - .any(|attr| attr.path().is_ident("skip") || attr.path().is_ident("private")) - }) - .map(|f| f.ident.as_ref().unwrap()) - .collect(); - - let mut data_where_clause = where_clause.clone(); + let mut serde_where_clause = where_clause.clone(); for ty in input.generics.type_params() { let ident = &ty.ident; - data_where_clause.predicates.push(syn::parse_quote!( + serde_where_clause.predicates.push(syn::parse_quote!( #ident: calimero_sdk::borsh::BorshSerialize + calimero_sdk::borsh::BorshDeserialize )); } - let deserialize_impl = quote! { - impl #impl_generics calimero_sdk::borsh::BorshDeserialize for #name #ty_generics #data_where_clause { - fn deserialize_reader(reader: &mut R) -> std::io::Result { - let #storage_ident = #storage_ty::deserialize_reader(reader)?; - Ok(Self { - #storage_ident, - #(#serializable_fields: calimero_sdk::borsh::BorshDeserialize::deserialize_reader(reader)?,)* - #(#skipped_fields: Default::default(),)* - }) - } - } - }; - - let serialize_impl = quote! { - impl #impl_generics calimero_sdk::borsh::BorshSerialize for #name #ty_generics #data_where_clause { - fn serialize(&self, writer: &mut W) -> std::io::Result<()> { - calimero_sdk::borsh::BorshSerialize::serialize(&self.#storage_ident, writer)?; - #(calimero_sdk::borsh::BorshSerialize::serialize(&self.#serializable_fields, writer)?;)* - Ok(()) - } - } - }; - let root_impl = if is_root { quote! { fn is_root() -> bool { @@ -334,21 +247,8 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { } }; - let mut local_where_clause = where_clause; - - for ty in input.generics.type_params() { - let ident = &ty.ident; - local_where_clause.predicates.push(syn::parse_quote!( - #ident: PartialEq - )); - } - let expanded = quote! { - impl #impl_generics #name #ty_generics #local_where_clause { - #(#field_implementations)* - } - - impl #impl_generics calimero_storage::entities::Data for #name #ty_generics #data_where_clause { + impl #impl_generics calimero_storage::entities::Data for #name #ty_generics #serde_where_clause { fn calculate_merkle_hash(&self) -> Result<[u8; 32], calimero_storage::interface::StorageError> { use calimero_storage::exports::Digest; let mut hasher = calimero_storage::exports::Sha256::new(); @@ -375,7 +275,7 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { match collection { #( stringify!(#collection_fields) => { - let child = <#collection_field_types #ty_generics as calimero_storage::entities::Collection>::Child::try_from_slice(slice) + let child = <#collection_field_types as calimero_storage::entities::Collection>::Child::try_from_slice(slice) .map_err(|e| calimero_storage::interface::StorageError::DeserializationError(e))?; child.calculate_merkle_hash() }, @@ -411,14 +311,15 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { } } - impl #impl_generics calimero_storage::entities::AtomicUnit for #name #ty_generics #data_where_clause {} - - #deserialize_impl - - #serialize_impl + impl #impl_generics calimero_storage::entities::AtomicUnit for #name #ty_generics #serde_where_clause {} }; - TokenStream::from(expanded) + TokenStream::from(quote! { + #[allow(unused_mut)] + const _: () = { + #expanded + }; + }) } /// Derives the [`Collection`](calimero_storage::entities::Collection) trait for @@ -540,7 +441,7 @@ pub fn collection_derive(input: TokenStream) -> TokenStream { let data = (!data.is_empty()).then(|| quote! { { #(#data)* } }); let default_impl = quote! { - impl #impl_generics ::core::default::Default for #name #ty_generics #where_clause { + impl #impl_generics ::core::default::Default for #name #ty_generics { fn default() -> Self { Self #data } diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index deaac9eef..7abf63ad8 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -12,8 +12,8 @@ pub mod unordered_map; pub use unordered_map::UnorderedMap; pub mod unordered_set; pub use unordered_set::UnorderedSet; -pub mod vector; -pub use vector::Vector; +// pub mod vector; +// pub use vector::Vector; pub mod error; pub use error::StoreError; @@ -24,7 +24,7 @@ use crate::entities::{Data, Element}; use crate::interface::{Interface, StorageError}; use crate::{AtomicUnit, Collection}; -#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug)] #[type_id(255)] #[root] struct Collection { @@ -41,6 +41,35 @@ struct Collection { children_ids: RefCell>>, } +impl Eq for Collection {} + +impl PartialEq for Collection { + fn eq(&self, other: &Self) -> bool { + self.entries() + .unwrap() + .flatten() + .eq(other.entries().unwrap().flatten()) + } +} + +impl Ord for Collection { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.entries() + .unwrap() + .flatten() + .cmp(other.entries().unwrap().flatten()) + } +} + +impl PartialOrd for Collection { + fn partial_cmp(&self, other: &Self) -> Option { + self.entries() + .unwrap() + .flatten() + .partial_cmp(other.entries().unwrap().flatten()) + } +} + /// A collection of entries in a map. #[derive(Collection, Copy, Clone, Debug, Eq, PartialEq)] #[children(Entry)] @@ -50,7 +79,7 @@ struct Entries { } /// An entry in a map. -#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug)] #[type_id(254)] struct Entry { /// The item in the entry. @@ -71,6 +100,7 @@ struct EntryMut<'a, T: BorshSerialize + BorshDeserialize> { entry: Entry, } +#[expect(unused_qualifications, reason = "AtomicUnit macro is unsanitized")] type StoreResult = std::result::Result; impl Collection { diff --git a/crates/storage/src/collections/error.rs b/crates/storage/src/collections/error.rs index bd91c67ff..1181db385 100644 --- a/crates/storage/src/collections/error.rs +++ b/crates/storage/src/collections/error.rs @@ -1,6 +1,7 @@ +//! Error types for storage operations. + use thiserror::Error; -// fixme! macro expects `calimero_storage` to be in deps use crate::address::PathError; use crate::interface::StorageError; diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index 79d217a3f..a8c88ebb8 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -12,18 +12,15 @@ use crate::collections::error::StoreError; use crate::entities::Data; /// A map collection that stores key-value pairs. -#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] pub struct UnorderedMap { - #[borsh(bound( - serialize = "K: BorshSerialize, V: BorshSerialize", - deserialize = "K: BorshDeserialize, V: BorshDeserialize" - ))] - // rename to inner? inner: Collection<(K, V)>, } -impl - UnorderedMap +impl UnorderedMap +where + K: BorshSerialize + BorshDeserialize, + V: BorshSerialize + BorshDeserialize, { /// Create a new map collection. /// @@ -35,14 +32,14 @@ impl /// pub fn new() -> Self { Self { - entries: Collection::new(), + inner: Collection::new(), } } /// Compute the ID for a key. fn compute_id(&self, key: &[u8]) -> Id { let mut hasher = Sha256::new(); - hasher.update(self.entries.id().as_bytes()); + hasher.update(self.inner.id().as_bytes()); hasher.update(key); Id::new(hasher.finalize().into()) } @@ -61,8 +58,7 @@ impl { let id = self.compute_id(key.as_ref()); - if let Some(mut entry) = self.entries.get_mut(id)? { - dbg!(&entry); + if let Some(mut entry) = self.inner.get_mut(id)? { let (_, v) = &mut *entry; return Ok(Some(mem::replace(v, value))); @@ -81,7 +77,7 @@ impl /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn entries(&self) -> Result, StoreError> { + pub fn entries(&self) -> Result + '_, StoreError> { let iter = self.inner.entries()?; let iter = iter.flat_map(|entry| entry.ok()); @@ -170,24 +166,48 @@ impl } } -#[cfg(test)] -mod tests { - use crate::collections::unordered_map::UnorderedMap; - use crate::entities::Data; - - #[test] - fn doo() { - let mut map = UnorderedMap::::new(); +impl Eq for UnorderedMap +where + K: Eq + BorshSerialize + BorshDeserialize, + V: Eq + BorshSerialize + BorshDeserialize, +{ +} - dbg!(map.entries.element().merkle_hash()); +impl PartialEq for UnorderedMap +where + K: PartialEq + BorshSerialize + BorshDeserialize, + V: PartialEq + BorshSerialize + BorshDeserialize, +{ + fn eq(&self, other: &Self) -> bool { + self.entries().unwrap().eq(other.entries().unwrap()) + } +} - assert!(map - .insert("key".to_string(), "value".to_string()) - .expect("insert failed") - .is_none()); +impl Ord for UnorderedMap +where + K: Ord + BorshSerialize + BorshDeserialize, + V: Ord + BorshSerialize + BorshDeserialize, +{ + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.entries().unwrap().cmp(other.entries().unwrap()) + } +} - dbg!(map.entries.element().merkle_hash()); +impl PartialOrd for UnorderedMap +where + K: PartialOrd + BorshSerialize + BorshDeserialize, + V: PartialOrd + BorshSerialize + BorshDeserialize, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.entries() + .unwrap() + .partial_cmp(other.entries().unwrap()) } +} + +#[cfg(test)] +mod tests { + use crate::collections::unordered_map::UnorderedMap; #[test] fn test_unordered_map_basic_operations() { diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 32b598279..9655395f1 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -6,18 +6,20 @@ use borsh::{BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; use super::Collection; -// fixme! macro expects `calimero_storage` to be in deps use crate::address::Id; use crate::collections::error::StoreError; use crate::entities::Data; /// A set collection that stores unqiue values once. -#[derive(Clone, Debug, Eq, PartialEq, PartialOrd)] +#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] pub struct UnorderedSet { inner: Collection, } -impl UnorderedSet { +impl UnorderedSet +where + V: BorshSerialize + BorshDeserialize, +{ /// Create a new set collection. /// /// # Errors @@ -71,7 +73,7 @@ impl UnorderedSet { /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn entries(&self) -> Result, StoreError> { + pub fn entries(&self) -> Result + '_, StoreError> { let iter = self.inner.entries()?; let iter = iter.flat_map(|entry| entry.ok()); @@ -146,6 +148,37 @@ impl UnorderedSet { } } +impl Eq for UnorderedSet where V: Eq + BorshSerialize + BorshDeserialize {} + +impl PartialEq for UnorderedSet +where + V: PartialEq + BorshSerialize + BorshDeserialize, +{ + fn eq(&self, other: &Self) -> bool { + self.entries().unwrap().eq(other.entries().unwrap()) + } +} + +impl Ord for UnorderedSet +where + V: Ord + BorshSerialize + BorshDeserialize, +{ + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.entries().unwrap().cmp(other.entries().unwrap()) + } +} + +impl PartialOrd for UnorderedSet +where + V: PartialOrd + BorshSerialize + BorshDeserialize, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.entries() + .unwrap() + .partial_cmp(other.entries().unwrap()) + } +} + #[cfg(test)] mod tests { use crate::collections::UnorderedSet; diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index b82c74a67..e6f46fdaa 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -5,7 +5,6 @@ use core::marker::PhantomData; use borsh::{BorshDeserialize, BorshSerialize}; -// fixme! macro expects `calimero_storage` to be in deps use crate::address::{Id, Path}; use crate::collections::error::StoreError; use crate::entities::{Data, Element}; diff --git a/crates/storage/src/interface.rs b/crates/storage/src/interface.rs index cf3f34d13..902d01bc5 100644 --- a/crates/storage/src/interface.rs +++ b/crates/storage/src/interface.rs @@ -385,7 +385,7 @@ impl MainInterface { /// pub fn add_child_to( parent_id: Id, - collection: &mut C, + collection: &C, child: &mut D, ) -> Result { let own_hash = child.calculate_merkle_hash()?; @@ -950,7 +950,7 @@ impl MainInterface { /// pub fn remove_child_from( parent_id: Id, - collection: &mut C, + collection: &C, child_id: Id, ) -> Result { let child_exists = >::get_children_of(parent_id, collection.name())? diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index bcbed3a05..363685d6c 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -6,11 +6,11 @@ #![forbid( unreachable_pub, - unsafe_code, unsafe_op_in_unsafe_fn, clippy::missing_docs_in_private_items )] #![deny( + unsafe_code, clippy::expect_used, clippy::missing_errors_doc, clippy::missing_panics_doc, From 6d90d77d6279f4ca3cd01b796cca8dca456d03f0 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 07:23:18 +0300 Subject: [PATCH 06/19] update vector to use collection --- crates/storage/src/collections.rs | 20 +- .../storage/src/collections/unordered_map.rs | 14 +- .../storage/src/collections/unordered_set.rs | 14 +- crates/storage/src/collections/vector.rs | 174 +++++------------- 4 files changed, 67 insertions(+), 155 deletions(-) diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 7abf63ad8..67a95b59c 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -12,8 +12,8 @@ pub mod unordered_map; pub use unordered_map::UnorderedMap; pub mod unordered_set; pub use unordered_set::UnorderedSet; -// pub mod vector; -// pub use vector::Vector; +pub mod vector; +pub use vector::Vector; pub mod error; pub use error::StoreError; @@ -146,7 +146,13 @@ impl Collection { })) } - fn entries(&self) -> StoreResult> + '_> { + fn len(&self) -> StoreResult { + Ok(self.children_cache()?.len()) + } + + fn entries( + &self, + ) -> StoreResult> + DoubleEndedIterator + '_> { let iter = self.children_cache()?.iter().copied().map(|child| { let entry = Interface::find_by_id::>(child)? .ok_or(StoreError::StorageError(StorageError::NotFound(child)))?; @@ -157,6 +163,14 @@ impl Collection { Ok(iter) } + fn nth(&self, index: usize) -> StoreResult> { + Ok(self.children_cache()?.get_index(index).copied()) + } + + fn last(&self) -> StoreResult> { + Ok(self.children_cache()?.last().copied()) + } + fn clear(&mut self) -> StoreResult<()> { let mut collection = CollectionMut::new(self); diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index a8c88ebb8..1006de623 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -24,12 +24,6 @@ where { /// Create a new map collection. /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, or a child - /// [`Element`](crate::entities::Element) cannot be found, an error will be - /// returned. - /// pub fn new() -> Self { Self { inner: Collection::new(), @@ -78,11 +72,7 @@ where /// returned. /// pub fn entries(&self) -> Result + '_, StoreError> { - let iter = self.inner.entries()?; - - let iter = iter.flat_map(|entry| entry.ok()); - - Ok(iter.fuse()) + Ok(self.inner.entries()?.flatten().fuse()) } /// Get the number of entries in the map. @@ -94,7 +84,7 @@ where /// returned. /// pub fn len(&self) -> Result { - Ok(self.inner.entries()?.len()) + self.inner.len() } /// Get the value for a key in the map. diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 9655395f1..4f0a8be79 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -22,12 +22,6 @@ where { /// Create a new set collection. /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, or a child - /// [`Element`](crate::entities::Element) cannot be found, an error will be - /// returned. - /// pub fn new() -> Self { Self { inner: Collection::new(), @@ -74,11 +68,7 @@ where /// returned. /// pub fn entries(&self) -> Result + '_, StoreError> { - let iter = self.inner.entries()?; - - let iter = iter.flat_map(|entry| entry.ok()); - - Ok(iter) + Ok(self.inner.entries()?.flatten().fuse()) } /// Get the number of entries in the set. @@ -90,7 +80,7 @@ where /// returned. /// pub fn len(&self) -> Result { - Ok(self.inner.entries()?.len()) + self.inner.len() } /// Get the value for a key in the set. diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index e6f46fdaa..fb5510ab2 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -1,69 +1,30 @@ //! This module provides functionality for the vector data structure. use core::borrow::Borrow; -use core::marker::PhantomData; +use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; -use crate::address::{Id, Path}; use crate::collections::error::StoreError; -use crate::entities::{Data, Element}; -use crate::interface::{Interface, StorageError}; -use crate::{self as calimero_storage, AtomicUnit, Collection}; + +use super::Collection; /// A vector collection that stores key-value pairs. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(251)] -#[root] +#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] pub struct Vector { - /// The entries in the vector. - entries: Entries, - /// The storage element for the vector. - #[storage] - storage: Element, -} - -/// A collection of entries in a vector. -#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[children(Entry)] -struct Entries { - /// Helper to associate the generic types with the collection. - _priv: PhantomData, -} - -/// An entry in a vector. -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(250)] -pub struct Entry { - /// The value for the entry. - value: V, - /// The storage element for the entry. - #[storage] - storage: Element, + inner: Collection, } -impl Vector { +impl Vector +where + V: BorshSerialize + BorshDeserialize, +{ /// Create a new vector collection. /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, or a child - /// [`Element`](crate::entities::Element) cannot be found, an error will be - /// returned. - /// - pub fn new() -> Result { - let id = Id::random(); - let mut this = Self { - entries: Entries::default(), - storage: Element::new( - &Path::new(format!("::unused::vector::{id}::path"))?, - Some(id), - ), - }; - - let _ = Interface::save(&mut this)?; - - Ok(this) + pub fn new() -> Self { + Self { + inner: Collection::new(), + } } /// Add a value to the end of the vector. @@ -75,18 +36,7 @@ impl Vector { /// returned. /// pub fn push(&mut self, value: V) -> Result<(), StoreError> { - let mut entry = Entry { - value, - storage: Element::new(&self.path(), None), - }; - - if Interface::add_child_to(self.id(), &mut self.entries, &mut entry)? { - return Ok(()); - } - - Err(StoreError::StorageError(StorageError::ActionNotAllowed( - "Failed to add child".to_owned(), - ))) + self.inner.insert(None, value) } /// Remove and return the last value from the vector. @@ -98,35 +48,17 @@ impl Vector { /// returned. /// pub fn pop(&mut self) -> Result, StoreError> { - let id = match Interface::child_info_for(self.id(), &self.entries)?.pop() { - Some(info) => info.id(), - None => return Ok(None), + let Some(last) = self.inner.last()? else { + return Ok(None); }; - let entry = Interface::find_by_id::>(id)?; - let _ = Interface::remove_child_from(self.id(), &mut self.entries, id)?; - Ok(entry.map(|e| e.value)) - } - /// Get the raw storage entry at a specific index in the vector. - /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, or a child - /// [`Element`](crate::entities::Element) cannot be found, an error will be - /// returned. - /// - fn get_raw(&self, index: usize) -> Result>, StoreError> { - let id = match Interface::child_info_for(self.id(), &self.entries)?.get(index) { - Some(info) => info.id(), - None => return Ok(None), + let Some(entry) = self.inner.get_mut(last)? else { + return Ok(None); }; - let entry = match Interface::find_by_id::>(id) { - Ok(entry) => entry, - Err(_) => return Ok(None), - }; + let last = entry.remove()?; - Ok(entry) + Ok(Some(last)) } /// Get the value at a specific index in the vector. @@ -138,7 +70,7 @@ impl Vector { /// returned. /// pub fn get(&self, index: usize) -> Result, StoreError> { - Ok(self.get_raw(index)?.map(|entry| entry.value)) + self.inner.entries()?.nth(index).transpose() } /// Update the value at a specific index in the vector. @@ -149,18 +81,18 @@ impl Vector { /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn update(&mut self, index: usize, value: V) -> Result<(), StoreError> { - let mut entry = self.get_raw(index)?.ok_or(StoreError::StorageError( - StorageError::ActionNotAllowed("error".to_owned()), - ))?; + pub fn update(&mut self, index: usize, value: V) -> Result, StoreError> { + let Some(id) = self.inner.nth(index)? else { + return Ok(None); + }; - // has to be called to update the entry - entry.value = value; - entry.element_mut().update(); + let Some(mut entry) = self.inner.get_mut(id)? else { + return Ok(None); + }; - let _ = Interface::save::>(&mut entry)?; + let old = mem::replace(&mut *entry, value); - Ok(()) + Ok(Some(old)) } /// Get an iterator over the entries in the vector. @@ -171,10 +103,8 @@ impl Vector { /// [`Element`](crate::entities::Element) cannot be found, an error will be /// returned. /// - pub fn entries(&self) -> Result, StoreError> { - let entries = Interface::children_of(self.id(), &self.entries)?; - - Ok(entries.into_iter().map(|entry| (entry.value))) + pub fn entries(&self) -> Result + '_, StoreError> { + Ok(self.inner.entries()?.flatten().fuse()) } /// Get the number of entries in the vector. @@ -187,7 +117,7 @@ impl Vector { /// #[expect(clippy::len_without_is_empty, reason = "TODO: will be implemented")] pub fn len(&self) -> Result { - Ok(Interface::child_info_for(self.id(), &self.entries)?.len()) + self.inner.len() } /// Get the value for a key in the vector. @@ -201,10 +131,10 @@ impl Vector { pub fn contains(&self, value: &Q) -> Result where V: Borrow, - Q: PartialEq + ?Sized, + Q: PartialEq, { for entry in self.entries()? { - if value.borrow() == &entry { + if value == entry.borrow() { return Ok(true); } } @@ -221,30 +151,17 @@ impl Vector { /// returned. /// pub fn clear(&mut self) -> Result<(), StoreError> { - let entries = Interface::children_of(self.id(), &self.entries)?; - - for entry in entries { - let _ = Interface::remove_child_from(self.id(), &mut self.entries, entry.id())?; - } - - Ok(()) + self.inner.clear() } } #[cfg(test)] mod tests { - use crate::collections::error::StoreError; use crate::collections::vector::Vector; - #[test] - fn test_vector_new() { - let vector: Result, StoreError> = Vector::new(); - assert!(vector.is_ok()); - } - #[test] fn test_vector_push() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let result = vector.push(value.clone()); assert!(result.is_ok()); @@ -253,7 +170,7 @@ mod tests { #[test] fn test_vector_get() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); let retrieved_value = vector.get(0).unwrap(); @@ -262,18 +179,19 @@ mod tests { #[test] fn test_vector_update() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value1 = "test_data1".to_string(); let value2 = "test_data2".to_string(); let _ = vector.push(value1.clone()).unwrap(); - let _ = vector.update(0, value2.clone()).unwrap(); + let old = vector.update(0, value2.clone()).unwrap(); let retrieved_value = vector.get(0).unwrap(); assert_eq!(retrieved_value, Some(value2)); + assert_eq!(old, Some(value1)); } #[test] fn test_vector_get_non_existent() { - let vector: Vector = Vector::new().unwrap(); + let vector: Vector = Vector::new(); match vector.get(0) { Ok(retrieved_value) => assert_eq!(retrieved_value, None), Err(e) => panic!("Error occurred: {:?}", e), @@ -282,7 +200,7 @@ mod tests { #[test] fn test_vector_pop() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); let popped_value = vector.pop().unwrap(); @@ -292,7 +210,7 @@ mod tests { #[test] fn test_vector_entries() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value1 = "test_data1".to_string(); let value2 = "test_data2".to_string(); let _ = vector.push(value1.clone()).unwrap(); @@ -303,7 +221,7 @@ mod tests { #[test] fn test_vector_contains() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); assert!(vector.contains(&value).unwrap()); @@ -313,7 +231,7 @@ mod tests { #[test] fn test_vector_clear() { - let mut vector: Vector = Vector::new().unwrap(); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); vector.clear().unwrap(); From aeb8d6e3a804053d9ebfa0e2b2ef05d85dc6eb5c Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 07:41:49 +0300 Subject: [PATCH 07/19] add root structure --- crates/storage/src/collections.rs | 20 ++++--- crates/storage/src/collections/root.rs | 55 +++++++++++++++++++ .../storage/src/collections/unordered_map.rs | 2 +- .../storage/src/collections/unordered_set.rs | 2 +- crates/storage/src/collections/vector.rs | 7 ++- 5 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 crates/storage/src/collections/root.rs diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 67a95b59c..952b20ae5 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -14,6 +14,8 @@ pub mod unordered_set; pub use unordered_set::UnorderedSet; pub mod vector; pub use vector::Vector; +pub mod root; +pub use root::Root; pub mod error; pub use error::StoreError; @@ -118,17 +120,19 @@ impl Collection { } /// Inserts an item into the collection. - fn insert(&mut self, id: Option, item: T) -> StoreResult<()> { + fn insert(&mut self, id: Option, item: T) -> StoreResult { let path = self.path(); let mut collection = CollectionMut::new(self); - collection.insert(Entry { + let mut entry = Entry { item, storage: Element::new(&path, id), - })?; + }; - Ok(()) + collection.insert(&mut entry)?; + + Ok(entry.item) } fn get(&self, id: Id) -> StoreResult> { @@ -192,7 +196,7 @@ impl Collection { let children = cache.as_mut().expect("children"); - #[allow(unsafe_code)] + #[expect(unsafe_code, reason = "necessary for caching")] let children = unsafe { &mut *ptr::from_mut(children) }; Ok(children) @@ -264,10 +268,8 @@ where Self { collection } } - fn insert(&mut self, item: Entry) -> StoreResult<()> { - let mut item = item; - - let _ = Interface::add_child_to(self.collection.id(), &self.entries, &mut item)?; + fn insert(&mut self, item: &mut Entry) -> StoreResult<()> { + let _ = Interface::add_child_to(self.collection.id(), &self.entries, item)?; let _ignored = self.collection.children_cache()?.insert(item.id()); diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs new file mode 100644 index 000000000..5a2280e61 --- /dev/null +++ b/crates/storage/src/collections/root.rs @@ -0,0 +1,55 @@ +//! A root collection that stores a single value. + +use std::cell::RefCell; +use std::ptr; + +use borsh::{BorshDeserialize, BorshSerialize}; +use calimero_sdk::env; + +use super::Collection; +use crate::address::Id; + +/// A set collection that stores unqiue values once. +#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] +pub struct Root { + id: Id, + inner: Collection, + value: RefCell>, +} + +impl Root +where + T: BorshSerialize + BorshDeserialize, +{ + /// Creates a new root collection with the given value. + pub fn new(value: T) -> Self { + let id = Id::new(env::context_id()); + + let mut inner = Collection::new(); + + let value = inner.insert(Some(id), value).unwrap(); + + Self { + id, + inner, + value: RefCell::new(Some(value)), + } + } + + /// Gets the value of the root collection. + pub fn get(&self) -> &T { + self.get_mut() + } + + /// Gets the value of the root collection mutably. + pub fn get_mut(&self) -> &mut T { + let mut value = self.value.borrow_mut(); + + let value = value.get_or_insert_with(|| self.inner.get(self.id).unwrap().unwrap()); + + #[expect(unsafe_code, reason = "necessary for caching")] + let value = unsafe { &mut *ptr::from_mut(value) }; + + value + } +} diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index 1006de623..017261514 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -58,7 +58,7 @@ where return Ok(Some(mem::replace(v, value))); } - self.inner.insert(Some(id), (key, value))?; + let _ignored = self.inner.insert(Some(id), (key, value))?; Ok(None) } diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 4f0a8be79..4781ba5f0 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -54,7 +54,7 @@ where return Ok(false); }; - self.inner.insert(Some(id), value)?; + let _ignored = self.inner.insert(Some(id), value)?; Ok(true) } diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index fb5510ab2..b797e69fd 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -5,9 +5,8 @@ use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; -use crate::collections::error::StoreError; - use super::Collection; +use crate::collections::error::StoreError; /// A vector collection that stores key-value pairs. #[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] @@ -36,7 +35,9 @@ where /// returned. /// pub fn push(&mut self, value: V) -> Result<(), StoreError> { - self.inner.insert(None, value) + let _ignored = self.inner.insert(None, value)?; + + Ok(()) } /// Remove and return the last value from the vector. From b57f1759e7c90f2c8f43f2b988dd8c0ad1a088e6 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 11:36:39 +0300 Subject: [PATCH 08/19] do away with type IDs --- crates/storage-macros/src/lib.rs | 89 +------ crates/storage/src/address.rs | 5 + crates/storage/src/entities.rs | 183 ++++++-------- crates/storage/src/index.rs | 51 ++-- crates/storage/src/integration.rs | 15 +- crates/storage/src/interface.rs | 344 ++++++++++++++------------ crates/storage/src/tests/common.rs | 108 +------- crates/storage/src/tests/entities.rs | 122 ++------- crates/storage/src/tests/index.rs | 234 +++++++++++------- crates/storage/src/tests/interface.rs | 146 +++++------ 10 files changed, 513 insertions(+), 784 deletions(-) diff --git a/crates/storage-macros/src/lib.rs b/crates/storage-macros/src/lib.rs index 0e9b0cb57..c6363b858 100644 --- a/crates/storage-macros/src/lib.rs +++ b/crates/storage-macros/src/lib.rs @@ -1,7 +1,7 @@ use borsh as _; use proc_macro::TokenStream; use quote::quote; -use syn::{parse_macro_input, Data, DeriveInput, Fields, LitInt, Type}; +use syn::{parse_macro_input, Data, DeriveInput, Fields, Type}; #[cfg(test)] mod integration_tests_package_usage { @@ -160,15 +160,6 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { let name = &input.ident; let where_clause = input.generics.make_where_clause().clone(); let (impl_generics, ty_generics, _) = input.generics.split_for_impl(); - let is_root = input.attrs.iter().any(|attr| attr.path().is_ident("root")); - let type_id = input - .attrs - .iter() - .find(|attr| attr.path().is_ident("type_id")) - .and_then(|attr| attr.parse_args::().ok()) - .expect("AtomicUnit derive requires a #[type_id(n)] attribute, where n is a u8") - .base10_parse::() - .expect("type_id must be a valid u8"); let fields = match &input.data { Data::Struct(data) => &data.fields, @@ -190,19 +181,6 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { let storage_ident = storage_field.ident.as_ref().unwrap(); - let regular_fields: Vec<_> = named_fields - .iter() - .filter(|f| { - !f.attrs.iter().any(|attr| { - attr.path().is_ident("skip") - || attr.path().is_ident("private") - || attr.path().is_ident("collection") - || attr.path().is_ident("storage") - }) - }) - .map(|f| f.ident.as_ref().unwrap()) - .collect(); - let collection_fields: Vec<_> = named_fields .iter() .filter(|f| { @@ -213,16 +191,6 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { .map(|f| f.ident.as_ref().unwrap()) .collect(); - let collection_field_types: Vec<_> = named_fields - .iter() - .filter(|f| { - f.attrs - .iter() - .any(|attr| attr.path().is_ident("collection")) - }) - .map(|f| f.ty.clone()) - .collect(); - let mut serde_where_clause = where_clause.clone(); for ty in input.generics.type_params() { @@ -233,57 +201,8 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { )); } - let root_impl = if is_root { - quote! { - fn is_root() -> bool { - true - } - } - } else { - quote! { - fn is_root() -> bool { - false - } - } - }; - let expanded = quote! { impl #impl_generics calimero_storage::entities::Data for #name #ty_generics #serde_where_clause { - fn calculate_merkle_hash(&self) -> Result<[u8; 32], calimero_storage::interface::StorageError> { - use calimero_storage::exports::Digest; - let mut hasher = calimero_storage::exports::Sha256::new(); - hasher.update(self.element().id().as_bytes()); - #( - hasher.update( - &calimero_sdk::borsh::to_vec(&self.#regular_fields) - .map_err(calimero_storage::interface::StorageError::SerializationError)? - ); - )* - hasher.update( - &calimero_sdk::borsh::to_vec(&self.element().metadata()) - .map_err(calimero_storage::interface::StorageError::SerializationError)? - ); - Ok(hasher.finalize().into()) - } - - fn calculate_merkle_hash_for_child( - &self, - collection: &str, - slice: &[u8], - ) -> Result<[u8; 32], calimero_storage::interface::StorageError> { - use calimero_sdk::borsh::BorshDeserialize; - match collection { - #( - stringify!(#collection_fields) => { - let child = <#collection_field_types as calimero_storage::entities::Collection>::Child::try_from_slice(slice) - .map_err(|e| calimero_storage::interface::StorageError::DeserializationError(e))?; - child.calculate_merkle_hash() - }, - )* - _ => Err(calimero_storage::interface::StorageError::UnknownCollectionType(collection.to_owned())), - } - } - fn collections(&self) -> std::collections::BTreeMap> { use calimero_storage::entities::Collection; let mut collections = std::collections::BTreeMap::new(); @@ -303,12 +222,6 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { fn element_mut(&mut self) -> &mut calimero_storage::entities::Element { &mut self.#storage_ident } - - #root_impl - - fn type_id() -> u8 { - #type_id - } } impl #impl_generics calimero_storage::entities::AtomicUnit for #name #ty_generics #serde_where_clause {} diff --git a/crates/storage/src/address.rs b/crates/storage/src/address.rs index b8eb8e7e9..c1516076e 100644 --- a/crates/storage/src/address.rs +++ b/crates/storage/src/address.rs @@ -106,6 +106,11 @@ impl Id { pub const fn as_bytes(&self) -> &[u8; 32] { &self.bytes } + + /// Checks if the ID is the root. + pub fn is_root(&self) -> bool { + self.bytes == context_id() + } } impl Display for Id { diff --git a/crates/storage/src/entities.rs b/crates/storage/src/entities.rs index 5de099efa..01f5b3af5 100644 --- a/crates/storage/src/entities.rs +++ b/crates/storage/src/entities.rs @@ -214,13 +214,12 @@ mod tests; use core::fmt::{self, Debug, Display, Formatter}; use std::collections::BTreeMap; +use std::ops::{Deref, DerefMut}; use borsh::{BorshDeserialize, BorshSerialize}; -use serde::{Deserialize, Serialize}; use crate::address::{Id, Path}; use crate::env::time_now; -use crate::interface::StorageError; /// Represents an atomic unit in the storage system. /// @@ -233,10 +232,11 @@ use crate::interface::StorageError; /// # Examples /// /// ``` +/// use borsh::{BorshSerialize, BorshDeserialize}; /// use calimero_storage::entities::Element; /// use calimero_storage_macros::AtomicUnit; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// #[type_id(43)] /// struct Page { /// title: String, @@ -258,10 +258,11 @@ pub trait AtomicUnit: Data {} /// # Examples /// /// ``` +/// use borsh::{BorshSerialize, BorshDeserialize}; /// use calimero_storage_macros::{AtomicUnit, Collection}; /// use calimero_storage::entities::{ChildInfo, Data, Element}; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// #[type_id(42)] /// struct Book { /// title: String, @@ -274,7 +275,7 @@ pub trait AtomicUnit: Data {} /// #[children(Page)] /// struct Pages; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// #[type_id(43)] /// struct Page { /// content: String, @@ -314,56 +315,6 @@ pub trait Collection { /// contentious methods are included, to keep the interface simple and focused. /// pub trait Data: BorshDeserialize + BorshSerialize { - /// Calculates the Merkle hash of the [`Element`]. - /// - /// This method calculates the Merkle hash of the [`Data`] for the - /// [`Element`], which should be based on any regular fields, but ignore - /// skipped fields, private fields, collections, and the storage field (but - /// include the metadata). - /// - /// **IMPORTANT NOTE**: Collection fields do need to be included in the hash - /// calculation, but that is a job for the caller to combine, and this - /// method therefore only calculates the hash of available data (as hashing - /// the children would involve recursive lookups). - /// - /// # Errors - /// - /// This method will return an error if there is a problem calculating the - /// hash. - /// - /// # See also - /// - /// * [`calculate_merkle_hash_for_child()`](Data::calculate_merkle_hash_for_child()) - /// - fn calculate_merkle_hash(&self) -> Result<[u8; 32], StorageError>; - - /// Calculates the Merkle hash of a child of the [`Element`]. - /// - /// This method calculates the Merkle hash of the specified child of the - /// [`Element`]. - /// - /// # Parameters - /// - /// * `collection` - The name of the collection to calculate the hash for. - /// * `slice` - The slice of data to calculate the hash for. This will - /// get deserialised into the appropriate type, and the - /// hash will be calculated based on the data. - /// - /// # Errors - /// - /// This method will return an error if there is a problem calculating the - /// hash, or looking up children. - /// - /// # See also - /// - /// * [`calculate_merkle_hash()`](Data::calculate_merkle_hash()) - /// - fn calculate_merkle_hash_for_child( - &self, - collection: &str, - slice: &[u8], - ) -> Result<[u8; 32], StorageError>; - /// Information about the [`Collection`]s present in the [`Data`]. /// /// This method allows details about the subtree structure and children to @@ -409,14 +360,6 @@ pub trait Data: BorshDeserialize + BorshSerialize { self.element().id() } - /// Whether the [`Element`] is a root. - /// - /// This should return `true` for any types that should sit at the top of - /// the hierarchy; and `false` for all other types, i.e. ones that can have - /// parents. - /// - fn is_root() -> bool; - /// The path to the [`Element`] in the hierarchy. /// /// This is a convenience function that passes through to @@ -430,19 +373,6 @@ pub trait Data: BorshDeserialize + BorshSerialize { fn path(&self) -> Path { self.element().path() } - - /// The type identifier of the entity. - /// - /// This is noted so that the entity can be deserialised correctly in the - /// absence of other semantic information. It is intended that the [`Path`] - /// will be used to help with this at some point, but at present paths are - /// not fully utilised. - /// - /// The value returned is arbitrary, and is up to the implementer to decide - /// what it should be. It is recommended that it be unique for each type of - /// entity. - /// - fn type_id() -> u8; } /// Summary information for the child of an [`Element`] in the storage. @@ -454,18 +384,7 @@ pub trait Data: BorshDeserialize + BorshSerialize { /// and prevent the need for repeated lookups. /// #[derive( - BorshDeserialize, - BorshSerialize, - Clone, - Copy, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, + BorshDeserialize, BorshSerialize, Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, )] #[non_exhaustive] pub struct ChildInfo { @@ -476,13 +395,19 @@ pub struct ChildInfo { /// of the significant data in the "scope" of the child [`Element`], and is /// used to determine whether the data has changed and is valid. pub(crate) merkle_hash: [u8; 32], + + pub(crate) metadata: Metadata, } impl ChildInfo { /// Creates a new [`ChildInfo`]. #[must_use] - pub const fn new(id: Id, merkle_hash: [u8; 32]) -> Self { - Self { id, merkle_hash } + pub const fn new(id: Id, merkle_hash: [u8; 32], metadata: Metadata) -> Self { + Self { + id, + merkle_hash, + metadata, + } } /// The unique identifier for the child [`Element`]. @@ -502,6 +427,18 @@ impl ChildInfo { pub const fn merkle_hash(&self) -> [u8; 32] { self.merkle_hash } + + /// The timestamp when the child was created. + #[must_use] + pub const fn created_at(&self) -> u64 { + self.metadata.created_at + } + + /// The timestamp when the child was last updated. + #[must_use] + pub fn updated_at(&self) -> u64 { + *self.metadata.updated_at + } } impl Display for ChildInfo { @@ -602,6 +539,7 @@ pub struct Element { /// Whether the [`Element`] is dirty, i.e. has been modified since it was /// last saved. + #[borsh(skip)] pub(crate) is_dirty: bool, /// The Merkle hash of the [`Element`]. This is a cryptographic hash of the @@ -610,8 +548,10 @@ pub struct Element { /// hashing the substantive data in the [`Element`], along with the hashes /// of the children of the [`Element`], thereby representing the state of /// the entire hierarchy below the [`Element`]. + #[borsh(skip)] pub(crate) merkle_hash: [u8; 32], + #[borsh(skip)] /// The metadata for the [`Element`]. This represents a range of /// system-managed properties that are used to process the [`Element`], but /// are not part of the primary data. @@ -660,7 +600,7 @@ impl Element { is_dirty: true, metadata: Metadata { created_at: timestamp, - updated_at: timestamp, + updated_at: timestamp.into(), }, merkle_hash: [0; 32], path: path.clone(), @@ -677,7 +617,7 @@ impl Element { is_dirty: true, metadata: Metadata { created_at: timestamp, - updated_at: timestamp, + updated_at: timestamp.into(), }, merkle_hash: [0; 32], #[expect(clippy::unwrap_used, reason = "This is expected to be valid")] @@ -780,13 +720,13 @@ impl Element { /// pub fn update(&mut self) { self.is_dirty = true; - self.metadata.updated_at = time_now(); + *self.metadata.updated_at = time_now(); } /// The timestamp when the [`Element`] was last updated. #[must_use] - pub const fn updated_at(&self) -> u64 { - self.metadata.updated_at + pub fn updated_at(&self) -> u64 { + *self.metadata.updated_at } } @@ -821,26 +761,61 @@ impl Display for Element { /// Using a [`u64`] timestamp allows for 585 years from the Unix epoch, at /// nanosecond precision. This is more than sufficient for our current needs. /// -#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive( + BorshDeserialize, + BorshSerialize, + Copy, + Clone, + Debug, + Default, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, +)] #[non_exhaustive] pub struct Metadata { /// When the [`Element`] was first created. Note that this is a global /// creation time, and does not reflect the time that the [`Element`] was /// added to the local storage. - created_at: u64, + pub(crate) created_at: u64, /// When the [`Element`] was last updated. This is the time that the /// [`Element`] was last modified in any way, and is used to determine the /// freshness of the data. It is critical for the "last write wins" strategy /// that is used to resolve conflicts. - pub(crate) updated_at: u64, + pub(crate) updated_at: UpdatedAt, } -#[cfg(test)] -impl Metadata { - /// Sets the created timestamp of the [`Element`]. This is **ONLY** for use - /// in tests. - pub fn set_created_at(&mut self, created_at: u64) { - self.created_at = created_at; +#[derive( + BorshDeserialize, BorshSerialize, Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialOrd, +)] +pub struct UpdatedAt(u64); + +impl PartialEq for UpdatedAt { + fn eq(&self, _other: &Self) -> bool { + // we don't care + true + } +} + +impl Deref for UpdatedAt { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for UpdatedAt { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From for UpdatedAt { + fn from(value: u64) -> Self { + Self(value) } } diff --git a/crates/storage/src/index.rs b/crates/storage/src/index.rs index ed8e431db..548d33698 100644 --- a/crates/storage/src/index.rs +++ b/crates/storage/src/index.rs @@ -11,7 +11,8 @@ use borsh::{to_vec, BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; use crate::address::Id; -use crate::entities::ChildInfo; +use crate::entities::{ChildInfo, Metadata}; +use crate::env; use crate::interface::StorageError; use crate::store::{Key, StorageAdaptor}; @@ -35,11 +36,8 @@ struct EntityIndex { /// the hashes of its children to form the full hash. own_hash: [u8; 32], - /// Type identifier of the entity. This is noted so that the entity can be - /// deserialised correctly in the absence of other semantic information. It - /// is intended that the [`Path`](crate::address::Path) will be used to help - /// with this at some point, but at present paths are not fully utilised. - type_id: u8, + /// Metadata about the entity. + metadata: Metadata, } /// Manages the indexing system for efficient tree navigation. @@ -73,7 +71,6 @@ impl Index { parent_id: Id, collection: &str, child: ChildInfo, - type_id: u8, ) -> Result<(), StorageError> { let mut parent_index = Self::get_index(parent_id)?.ok_or(StorageError::IndexNotFound(parent_id))?; @@ -84,7 +81,7 @@ impl Index { children: BTreeMap::new(), full_hash: [0; 32], own_hash: [0; 32], - type_id, + metadata: child.metadata, }); child_index.parent_id = Some(parent_id); child_index.own_hash = child.merkle_hash(); @@ -96,7 +93,11 @@ impl Index { .children .entry(collection.to_owned()) .or_insert_with(Vec::new) - .push(ChildInfo::new(child.id(), child_index.full_hash)); + .push(ChildInfo::new( + child.id(), + child_index.full_hash, + child.metadata, + )); Self::save_index(&parent_index)?; parent_index.full_hash = Self::calculate_full_merkle_hash_for(parent_id, false)?; Self::save_index(&parent_index)?; @@ -123,14 +124,14 @@ impl Index { /// /// * [`add_child_to()`](Index::add_child_to()) /// - pub(crate) fn add_root(root: ChildInfo, type_id: u8) -> Result<(), StorageError> { + pub(crate) fn add_root(root: ChildInfo) -> Result<(), StorageError> { let mut index = Self::get_index(root.id())?.unwrap_or_else(|| EntityIndex { id: root.id(), parent_id: None, children: BTreeMap::new(), full_hash: [0; 32], own_hash: [0; 32], - type_id, + metadata: root.metadata, }); index.own_hash = root.merkle_hash(); Self::save_index(&index)?; @@ -231,13 +232,19 @@ impl Index { while let Some(parent_id) = Self::get_parent_id(current_id)? { let (parent_full_hash, _) = Self::get_hashes_for(parent_id)?.ok_or(StorageError::IndexNotFound(parent_id))?; - ancestors.push(ChildInfo::new(parent_id, parent_full_hash)); + let metadata = + Self::get_metadata(parent_id)?.ok_or(StorageError::IndexNotFound(parent_id))?; + ancestors.push(ChildInfo::new(parent_id, parent_full_hash, metadata)); current_id = parent_id; } Ok(ancestors) } + pub(crate) fn get_metadata(id: Id) -> Result, StorageError> { + Ok(Self::get_index(id)?.map(|index| index.metadata)) + } + /// Retrieves the children of a given entity. /// /// # Parameters @@ -341,23 +348,6 @@ impl Index { Ok(Self::get_index(child_id)?.and_then(|index| index.parent_id)) } - /// Retrieves the type of the given entity. - /// - /// # Parameters - /// - /// * `id` - The [`Id`] of the entity whose type is to be retrieved. - /// - /// # Errors - /// - /// If there's an issue retrieving or deserialising the index information, - /// an error will be returned. - /// - pub(crate) fn get_type_id(id: Id) -> Result { - Ok(Self::get_index(id)? - .ok_or(StorageError::IndexNotFound(id))? - .type_id) - } - /// Whether the collection has children. /// /// # Parameters @@ -404,7 +394,7 @@ impl Index { if let Some(child) = children.iter_mut().find(|c| c.id() == current_id) { let new_child_hash = Self::calculate_full_merkle_hash_for(current_id, false)?; if child.merkle_hash() != new_child_hash { - *child = ChildInfo::new(current_id, new_child_hash); + *child = ChildInfo::new(current_id, new_child_hash, child.metadata); } break; } @@ -513,6 +503,7 @@ impl Index { index.own_hash = merkle_hash; Self::save_index(&index)?; index.full_hash = Self::calculate_full_merkle_hash_for(id, false)?; + index.metadata.updated_at = env::time_now().into(); Self::save_index(&index)?; Ok(index.full_hash) } diff --git a/crates/storage/src/integration.rs b/crates/storage/src/integration.rs index 0baea1231..b9281b1c2 100644 --- a/crates/storage/src/integration.rs +++ b/crates/storage/src/integration.rs @@ -1,24 +1,11 @@ //! Types used for integration with the runtime. use borsh::{BorshDeserialize, BorshSerialize}; -use calimero_sdk::serde::{Deserialize, Serialize}; use crate::interface::ComparisonData; /// Comparison data for synchronisation. -#[derive( - BorshDeserialize, - BorshSerialize, - Clone, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[expect(clippy::exhaustive_structs, reason = "Exhaustive")] pub struct Comparison { /// The type of the entity. diff --git a/crates/storage/src/interface.rs b/crates/storage/src/interface.rs index 902d01bc5..f75ad708d 100644 --- a/crates/storage/src/interface.rs +++ b/crates/storage/src/interface.rs @@ -218,14 +218,15 @@ use core::marker::PhantomData; use std::collections::BTreeMap; use std::io::Error as IoError; -use borsh::{to_vec, BorshDeserialize, BorshSerialize}; +use borsh::{from_slice, to_vec, BorshDeserialize, BorshSerialize}; use eyre::Report; use indexmap::IndexMap; -use serde::{Deserialize, Serialize}; +use serde::Serialize; +use sha2::{Digest, Sha256}; use thiserror::Error as ThisError; use crate::address::{Id, Path}; -use crate::entities::{ChildInfo, Collection, Data}; +use crate::entities::{ChildInfo, Collection, Data, Metadata}; use crate::index::Index; use crate::store::{Key, MainStorage, StorageAdaptor}; use crate::sync; @@ -262,19 +263,7 @@ pub type Interface = MainInterface; /// Note: This enum contains the entity type, for passing to the guest for /// processing along with the ID and data. /// -#[derive( - BorshDeserialize, - BorshSerialize, - Clone, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[expect(clippy::exhaustive_enums, reason = "Exhaustive")] pub enum Action { /// Add an entity with the given ID, type, and data. @@ -282,14 +271,14 @@ pub enum Action { /// Unique identifier of the entity. id: Id, - /// Type identifier of the entity. - type_id: u8, - /// Serialised data of the entity. data: Vec, /// Details of the ancestors of the entity. ancestors: Vec, + + /// The metadata of the entity. + metadata: Metadata, }, /// Compare the entity with the given ID and type. Note that this results in @@ -316,31 +305,19 @@ pub enum Action { /// Unique identifier of the entity. id: Id, - /// Type identifier of the entity. - type_id: u8, - /// Serialised data of the entity. data: Vec, /// Details of the ancestors of the entity. ancestors: Vec, + + /// The metadata of the entity. + metadata: Metadata, }, } /// Data that is used for comparison between two nodes. -#[derive( - BorshDeserialize, - BorshSerialize, - Clone, - Debug, - Deserialize, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, -)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct ComparisonData { /// The unique identifier of the entity being compared. id: Id, @@ -359,6 +336,9 @@ pub struct ComparisonData { /// The list of children of the entity, with their IDs and hashes, /// organised by collection name. children: BTreeMap>, + + /// The metadata of the entity. + metadata: Metadata, } /// The primary interface for the storage system. @@ -388,15 +368,28 @@ impl MainInterface { collection: &C, child: &mut D, ) -> Result { - let own_hash = child.calculate_merkle_hash()?; + if !child.element().is_dirty() { + return Ok(false); + } + + let data = to_vec(child).map_err(|e| StorageError::SerializationError(e.into()))?; + + let own_hash = Sha256::digest(&data).into(); + >::add_child_to( parent_id, collection.name(), - ChildInfo::new(child.id(), own_hash), - D::type_id(), + ChildInfo::new(child.id(), own_hash, child.element().metadata), )?; - child.element_mut().merkle_hash = >::update_hash_for(child.id(), own_hash)?; - Self::save(child) + + let Some(hash) = Self::save_raw(child.id(), data, child.element().metadata)? else { + return Ok(false); + }; + + child.element_mut().is_dirty = false; + child.element_mut().merkle_hash = hash; + + Ok(true) } /// Applies an [`Action`] to the storage system. @@ -427,17 +420,21 @@ impl MainInterface { /// If there is an error when deserialising into the specified type, or when /// applying the [`Action`], an error will be returned. /// - pub fn apply_action(action: Action) -> Result, StorageError> { + pub fn apply_action(action: Action) -> Result, StorageError> { let ancestors = match action { Action::Add { - data, ancestors, .. + id, + data, + ancestors, + metadata, } | Action::Update { - data, ancestors, .. + id, + data, + ancestors, + metadata, } => { - let mut entity = - D::try_from_slice(&data).map_err(StorageError::DeserializationError)?; - _ = Self::save(&mut entity)?; + _ = Self::save_raw(id, data, metadata)?; ancestors } Action::Compare { .. } => { @@ -574,34 +571,34 @@ impl MainInterface { /// This function will return an error if there are issues accessing local /// data or if there are problems during the comparison process. /// - pub fn compare_trees( + pub fn compare_trees( foreign_entity_data: Option>, foreign_index_data: &ComparisonData, ) -> Result<(Vec, Vec), StorageError> { let mut actions = (vec![], vec![]); - let foreign_entity = foreign_entity_data - .as_ref() - .map(|d| D::try_from_slice(d)) - .transpose() - .map_err(StorageError::DeserializationError)?; + let id = foreign_index_data.id; + + let local_metadata = >::get_metadata(id)?; - let Some(local_entity) = Self::find_by_id::(foreign_index_data.id)? else { + let Some(local_entity) = Self::find_by_id_raw(id)? else { if let Some(foreign_entity) = foreign_entity_data { // Local entity doesn't exist, so we need to add it actions.0.push(Action::Add { - id: foreign_index_data.id, - type_id: D::type_id(), + id, data: foreign_entity, ancestors: foreign_index_data.ancestors.clone(), + metadata: foreign_index_data.metadata, }); } return Ok(actions); }; - let (local_full_hash, local_own_hash) = >::get_hashes_for(local_entity.id())? - .ok_or(StorageError::IndexNotFound(local_entity.id()))?; + let local_metadata = local_metadata.ok_or(StorageError::IndexNotFound(id))?; + + let (local_full_hash, local_own_hash) = + >::get_hashes_for(id)?.ok_or(StorageError::IndexNotFound(id))?; // Compare full Merkle hashes if local_full_hash == foreign_index_data.full_hash { @@ -610,24 +607,23 @@ impl MainInterface { // Compare own hashes and timestamps if local_own_hash != foreign_index_data.own_hash { - match (foreign_entity, foreign_entity_data) { - (Some(foreign_entity), Some(foreign_entity_data)) - if local_entity.element().updated_at() - <= foreign_entity.element().updated_at() => + match foreign_entity_data { + Some(foreign_entity_data) + if local_metadata.updated_at <= foreign_index_data.metadata.updated_at => { actions.0.push(Action::Update { - id: local_entity.id(), - type_id: D::type_id(), + id, data: foreign_entity_data, ancestors: foreign_index_data.ancestors.clone(), + metadata: foreign_index_data.metadata, }); } _ => { actions.1.push(Action::Update { - id: foreign_index_data.id, - type_id: D::type_id(), - data: to_vec(&local_entity).map_err(StorageError::SerializationError)?, - ancestors: >::get_ancestors_of(local_entity.id())?, + id, + data: local_entity, + ancestors: >::get_ancestors_of(id)?, + metadata: local_metadata, }); } } @@ -635,7 +631,16 @@ impl MainInterface { // The list of collections from the type will be the same on both sides, as // the type is the same. - let local_collections = local_entity.collections(); + + let local_collection_names = >::get_collection_names_for(id)?; + + let local_collections = local_collection_names + .iter() + .map(|name| { + let children = >::get_children_of(id, name)?; + Ok((name.clone(), children)) + }) + .collect::, StorageError>>()?; // Compare children for (local_coll_name, local_children) in &local_collections { @@ -649,19 +654,22 @@ impl MainInterface { .map(|child| (child.id(), child.merkle_hash())) .collect(); - for (id, local_hash) in &local_child_map { - match foreign_child_map.get(id) { + for (child_id, local_hash) in &local_child_map { + match foreign_child_map.get(child_id) { Some(foreign_hash) if local_hash != foreign_hash => { - actions.0.push(Action::Compare { id: *id }); - actions.1.push(Action::Compare { id: *id }); + actions.0.push(Action::Compare { id: *child_id }); + actions.1.push(Action::Compare { id: *child_id }); } None => { - if let Some(local_child) = Self::find_by_id_raw(*id)? { + if let Some(local_child) = Self::find_by_id_raw(*child_id)? { + let metadata = >::get_metadata(*child_id)? + .ok_or(StorageError::IndexNotFound(*child_id))?; + actions.1.push(Action::Add { - id: *id, - type_id: >::get_type_id(*id)?, + id: *child_id, data: local_child, - ancestors: >::get_ancestors_of(local_entity.id())?, + ancestors: >::get_ancestors_of(id)?, + metadata, }); } } @@ -682,11 +690,14 @@ impl MainInterface { // The entire collection is missing from the foreign entity for child in local_children { if let Some(local_child) = Self::find_by_id_raw(child.id())? { + let metadata = >::get_metadata(child.id())? + .ok_or(StorageError::IndexNotFound(child.id()))?; + actions.1.push(Action::Add { id: child.id(), - type_id: >::get_type_id(child.id())?, data: local_child, - ancestors: >::get_ancestors_of(local_entity.id())?, + ancestors: >::get_ancestors_of(child.id())?, + metadata, }); } } @@ -714,14 +725,14 @@ impl MainInterface { /// This function will return an error if there are issues accessing local /// data or if there are problems during the comparison process. /// - pub fn compare_affective( + pub fn compare_affective( data: Option>, comparison_data: ComparisonData, ) -> Result<(), StorageError> { - let (local, remote) = Interface::compare_trees::(data, &comparison_data)?; + let (local, remote) = Interface::compare_trees(data, &comparison_data)?; for action in local { - let _ignored = Interface::apply_action::(action)?; + let _ignored = Interface::apply_action(action)?; } for action in remote { @@ -750,18 +761,21 @@ impl MainInterface { pub fn find_by_id(id: Id) -> Result, StorageError> { let value = S::storage_read(Key::Entry(id)); - match value { - Some(slice) => { - let mut entity = - D::try_from_slice(&slice).map_err(StorageError::DeserializationError)?; - // TODO: This is needed for now, as the field gets stored. Later we will - // TODO: implement a custom serialiser that will skip this field along with - // TODO: any others that should not be stored. - entity.element_mut().is_dirty = false; - Ok(Some(entity)) - } - None => Ok(None), - } + let Some(slice) = value else { + return Ok(None); + }; + + let mut item = from_slice::(&slice).map_err(StorageError::DeserializationError)?; + + let (full_hash, _) = + >::get_hashes_for(id)?.ok_or(StorageError::IndexNotFound(id))?; + + item.element_mut().merkle_hash = full_hash; + + item.element_mut().metadata = + >::get_metadata(id)?.ok_or(StorageError::IndexNotFound(id))?; + + Ok(Some(item)) } /// Finds an [`Element`](crate::entities::Element) by its unique identifier @@ -859,38 +873,42 @@ impl MainInterface { /// If an error occurs when interacting with the storage system, an error /// will be returned. /// - pub fn generate_comparison_data( - entity: Option<&D>, - ) -> Result { - let Some(entity) = entity else { + pub fn generate_comparison_data(id: Option) -> Result { + let Some(id) = id else { return Ok(ComparisonData { id: Id::root(), own_hash: [0; 32], full_hash: [0; 32], ancestors: Vec::new(), children: BTreeMap::new(), + metadata: Metadata::default(), }); }; - let (full_hash, own_hash) = >::get_hashes_for(entity.id())? - .ok_or(StorageError::IndexNotFound(entity.id()))?; + let (full_hash, own_hash) = + >::get_hashes_for(id)?.ok_or(StorageError::IndexNotFound(id))?; - let ancestors = >::get_ancestors_of(entity.id())?; - let children = entity - .collections() - .into_keys() + let metadata = >::get_metadata(id)?.ok_or(StorageError::IndexNotFound(id))?; + + let ancestors = >::get_ancestors_of(id)?; + + let collection_names = >::get_collection_names_for(id)?; + + let children = collection_names + .into_iter() .map(|collection_name| { - >::get_children_of(entity.id(), &collection_name) + >::get_children_of(id, &collection_name) .map(|children| (collection_name.clone(), children)) }) .collect::, _>>()?; Ok(ComparisonData { - id: entity.id(), + id, own_hash, full_hash, ancestors, children, + metadata, }) } @@ -965,7 +983,9 @@ impl MainInterface { let (parent_full_hash, _) = >::get_hashes_for(parent_id)?.ok_or(StorageError::IndexNotFound(parent_id))?; let mut ancestors = >::get_ancestors_of(parent_id)?; - ancestors.insert(0, ChildInfo::new(parent_id, parent_full_hash)); + let metadata = + >::get_metadata(parent_id)?.ok_or(StorageError::IndexNotFound(parent_id))?; + ancestors.insert(0, ChildInfo::new(parent_id, parent_full_hash, metadata)); _ = S::storage_remove(Key::Entry(child_id)); @@ -1002,19 +1022,21 @@ impl MainInterface { /// This function will return an error if there are issues accessing local /// data or if there are problems during the comparison process. /// - pub fn commit_root(mut root: D) -> Result<(), StorageError> { + pub fn commit_root(root: D) -> Result<(), StorageError> { if root.id() != Id::root() { return Err(StorageError::UnexpectedId(root.id())); } - // fixme! mutations (action application) doesn't propagate to the root - // fixme! so, a best-attempt approach is to force a save on the root - // fixme! deeply nested entries have undefined behaviour - root.element_mut().is_dirty = true; + if !root.element().is_dirty() { + return Ok(()); + } + + let data = to_vec(&root).map_err(|e| StorageError::SerializationError(e.into()))?; - let _ = Self::save(&mut root)?; + let hash = Self::save_raw(root.id(), data, root.element().metadata)? + .expect("root should always be successfully committed"); - sync::commit_root(&root.element().merkle_hash())?; + sync::commit_root(&hash)?; Ok(()) } @@ -1083,78 +1105,76 @@ impl MainInterface { /// pub fn save(entity: &mut D) -> Result { if !entity.element().is_dirty() { - return Ok(true); + return Ok(false); } - let id = entity.id(); - if !D::is_root() && >::get_parent_id(id)?.is_none() { + let data = to_vec(entity).map_err(|e| StorageError::SerializationError(e.into()))?; + + let Some(hash) = Self::save_raw(entity.id(), data, entity.element().metadata)? else { + return Ok(false); + }; + + entity.element_mut().is_dirty = false; + entity.element_mut().merkle_hash = hash; + + Ok(true) + } + + /// Saves raw data to the storage system. + /// + /// # Errors + /// + /// If an error occurs when serialising data or interacting with the storage + /// system, an error will be returned. + /// + pub fn save_raw( + id: Id, + data: Vec, + metadata: Metadata, + ) -> Result, StorageError> { + if !id.is_root() && >::get_parent_id(id)?.is_none() { return Err(StorageError::CannotCreateOrphan(id)); } - let is_new = Self::find_by_id::(id)?.is_none(); - if !is_new { - if let Some(existing) = Self::find_by_id::(id)? { - if existing.element().metadata.updated_at >= entity.element().metadata.updated_at { - return Ok(false); - } + let last_metadata = >::get_metadata(id)?; + + let is_new = last_metadata.is_none(); + + if let Some(last_metadata) = &last_metadata { + if last_metadata.updated_at > metadata.updated_at { + return Ok(None); } - } else if D::is_root() { - >::add_root(ChildInfo::new(id, [0_u8; 32]), D::type_id())?; + } else if id.is_root() { + >::add_root(ChildInfo::new(id, [0_u8; 32], metadata))?; } - let own_hash = entity.calculate_merkle_hash()?; - entity.element_mut().merkle_hash = >::update_hash_for(id, own_hash)?; + let own_hash = Sha256::digest(&data).into(); - _ = S::storage_write( - Key::Entry(id), - &to_vec(entity).map_err(StorageError::SerializationError)?, - ); + let full_hash = >::update_hash_for(id, own_hash)?; - entity.element_mut().is_dirty = false; + _ = S::storage_write(Key::Entry(id), &data); + + let ancestors = >::get_ancestors_of(id)?; let action = if is_new { Action::Add { id, - type_id: D::type_id(), - data: to_vec(entity).map_err(StorageError::SerializationError)?, - ancestors: >::get_ancestors_of(id)?, + data, + ancestors, + metadata, } } else { Action::Update { id, - type_id: D::type_id(), - data: to_vec(entity).map_err(StorageError::SerializationError)?, - ancestors: >::get_ancestors_of(id)?, + data, + ancestors, + metadata, } }; sync::push_action(action); - Ok(true) - } - - /// Type identifier of the entity. - /// - /// This is noted so that the entity can be deserialised correctly in the - /// absence of other semantic information. It is intended that the [`Path`] - /// will be used to help with this at some point, but at present paths are - /// not fully utilised. - /// - /// The value returned is arbitrary, and is up to the implementer to decide - /// what it should be. It is recommended that it be unique for each type of - /// entity. - /// - /// # Parameters - /// - /// * `id` - The [`Id`] of the entity whose type is to be retrieved. - /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, an error - /// will be returned. - /// - pub fn type_of(id: Id) -> Result { - >::get_type_id(id) + Ok(Some(full_hash)) } /// Validates the stored state. diff --git a/crates/storage/src/tests/common.rs b/crates/storage/src/tests/common.rs index 41bd46cb2..7f6feba2b 100644 --- a/crates/storage/src/tests/common.rs +++ b/crates/storage/src/tests/common.rs @@ -1,11 +1,10 @@ use std::collections::BTreeMap; -use borsh::{to_vec, BorshDeserialize, BorshSerialize}; -use sha2::{Digest, Sha256}; +use borsh::{BorshDeserialize, BorshSerialize}; use velcro::btree_map; use crate::entities::{AtomicUnit, ChildInfo, Collection, Data, Element}; -use crate::interface::{Interface, StorageError}; +use crate::interface::Interface; /// For tests against empty data structs. #[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, PartialEq, PartialOrd)] @@ -14,21 +13,6 @@ pub struct EmptyData { } impl Data for EmptyData { - fn calculate_merkle_hash(&self) -> Result<[u8; 32], StorageError> { - let mut hasher = Sha256::new(); - hasher.update(self.element().id().as_bytes()); - hasher.update(&to_vec(&self.element().metadata).map_err(StorageError::SerializationError)?); - Ok(hasher.finalize().into()) - } - - fn calculate_merkle_hash_for_child( - &self, - collection: &str, - _slice: &[u8], - ) -> Result<[u8; 32], StorageError> { - Err(StorageError::UnknownCollectionType(collection.to_owned())) - } - fn collections(&self) -> BTreeMap> { BTreeMap::new() } @@ -40,14 +24,6 @@ impl Data for EmptyData { fn element_mut(&mut self) -> &mut Element { &mut self.storage } - - fn is_root() -> bool { - true - } - - fn type_id() -> u8 { - 101 - } } /// A simple page with a title, and paragraphs as children. @@ -72,29 +48,6 @@ impl Page { impl AtomicUnit for Page {} impl Data for Page { - fn calculate_merkle_hash(&self) -> Result<[u8; 32], StorageError> { - let mut hasher = Sha256::new(); - hasher.update(self.element().id().as_bytes()); - hasher.update(&to_vec(&self.title).map_err(StorageError::SerializationError)?); - hasher.update(&to_vec(&self.element().metadata).map_err(StorageError::SerializationError)?); - Ok(hasher.finalize().into()) - } - - fn calculate_merkle_hash_for_child( - &self, - collection: &str, - slice: &[u8], - ) -> Result<[u8; 32], StorageError> { - match collection { - "Paragraphs" => { - let child = ::Child::try_from_slice(slice) - .map_err(|e| StorageError::DeserializationError(e))?; - child.calculate_merkle_hash() - } - _ => Err(StorageError::UnknownCollectionType(collection.to_owned())), - } - } - fn collections(&self) -> BTreeMap> { btree_map! { "Paragraphs".to_owned(): Interface::child_info_for(self.id(), &self.paragraphs).unwrap_or_default(), @@ -108,14 +61,6 @@ impl Data for Page { fn element_mut(&mut self) -> &mut Element { &mut self.storage } - - fn is_root() -> bool { - true - } - - fn type_id() -> u8 { - 102 - } } /// A simple paragraph with text. No children. Belongs to a page. @@ -138,22 +83,6 @@ impl Paragraph { impl AtomicUnit for Paragraph {} impl Data for Paragraph { - fn calculate_merkle_hash(&self) -> Result<[u8; 32], StorageError> { - let mut hasher = Sha256::new(); - hasher.update(self.element().id().as_bytes()); - hasher.update(&to_vec(&self.text).map_err(StorageError::SerializationError)?); - hasher.update(&to_vec(&self.element().metadata).map_err(StorageError::SerializationError)?); - Ok(hasher.finalize().into()) - } - - fn calculate_merkle_hash_for_child( - &self, - collection: &str, - _slice: &[u8], - ) -> Result<[u8; 32], StorageError> { - Err(StorageError::UnknownCollectionType(collection.to_owned())) - } - fn collections(&self) -> BTreeMap> { BTreeMap::new() } @@ -165,14 +94,6 @@ impl Data for Paragraph { fn element_mut(&mut self) -> &mut Element { &mut self.storage } - - fn is_root() -> bool { - false - } - - fn type_id() -> u8 { - 103 - } } /// A collection of paragraphs for a page. @@ -203,23 +124,6 @@ pub struct Person { } impl Data for Person { - fn calculate_merkle_hash(&self) -> Result<[u8; 32], StorageError> { - let mut hasher = Sha256::new(); - hasher.update(self.element().id().as_bytes()); - hasher.update(&to_vec(&self.name).map_err(StorageError::SerializationError)?); - hasher.update(&to_vec(&self.age).map_err(StorageError::SerializationError)?); - hasher.update(&to_vec(&self.element().metadata).map_err(StorageError::SerializationError)?); - Ok(hasher.finalize().into()) - } - - fn calculate_merkle_hash_for_child( - &self, - collection: &str, - _slice: &[u8], - ) -> Result<[u8; 32], StorageError> { - Err(StorageError::UnknownCollectionType(collection.to_owned())) - } - fn collections(&self) -> BTreeMap> { BTreeMap::new() } @@ -231,12 +135,4 @@ impl Data for Person { fn element_mut(&mut self) -> &mut Element { &mut self.storage } - - fn is_root() -> bool { - true - } - - fn type_id() -> u8 { - 104 - } } diff --git a/crates/storage/src/tests/entities.rs b/crates/storage/src/tests/entities.rs index 6db091247..1dd23cc9b 100644 --- a/crates/storage/src/tests/entities.rs +++ b/crates/storage/src/tests/entities.rs @@ -1,14 +1,11 @@ use std::time::{SystemTime, UNIX_EPOCH}; -use borsh::to_vec; use claims::{assert_ge, assert_le}; use sha2::{Digest, Sha256}; use velcro::btree_map; use super::*; -use crate::index::Index; use crate::interface::Interface; -use crate::store::MainStorage; use crate::tests::common::{Page, Paragraph, Paragraphs, Person}; #[cfg(test)] @@ -26,81 +23,6 @@ mod collection__public_methods { mod data__public_methods { use super::*; - #[test] - fn calculate_merkle_hash() { - let element = Element::new(&Path::new("::root::node::leaf").unwrap(), None); - let person = Person { - name: "Alice".to_owned(), - age: 30, - storage: element.clone(), - }; - - let mut hasher = Sha256::new(); - hasher.update(person.id().as_bytes()); - hasher.update(&to_vec(&person.name).unwrap()); - hasher.update(&to_vec(&person.age).unwrap()); - hasher.update(&to_vec(&person.element().metadata).unwrap()); - let expected_hash: [u8; 32] = hasher.finalize().into(); - - assert_eq!(person.calculate_merkle_hash().unwrap(), expected_hash); - } - - #[test] - fn calculate_merkle_hash_for_child__valid() { - let parent = Element::new(&Path::new("::root::node").unwrap(), None); - let mut page = Page::new_from_element("Node", parent); - let child1 = Element::new(&Path::new("::root::node::leaf").unwrap(), None); - let mut para1 = Paragraph::new_from_element("Leaf1", child1); - assert!(Interface::save(&mut page).unwrap()); - - assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para1).unwrap()); - let para1_slice = to_vec(¶1).unwrap(); - let para1_hash = page - .calculate_merkle_hash_for_child("Paragraphs", ¶1_slice) - .unwrap(); - let expected_hash1 = para1.calculate_merkle_hash().unwrap(); - assert_eq!(para1_hash, expected_hash1); - - let child2 = Element::new(&Path::new("::root::node::leaf").unwrap(), None); - let para2 = Paragraph::new_from_element("Leaf2", child2); - let para2_slice = to_vec(¶2).unwrap(); - let para2_hash = page - .calculate_merkle_hash_for_child("Paragraphs", ¶2_slice) - .unwrap(); - assert_ne!(para2_hash, para1_hash); - } - - #[test] - fn calculate_merkle_hash_for_child__invalid() { - let parent = Element::new(&Path::new("::root::node").unwrap(), None); - let mut page = Page::new_from_element("Node", parent); - let child1 = Element::new(&Path::new("::root::node::leaf").unwrap(), None); - let mut para1 = Paragraph::new_from_element("Leaf1", child1); - assert!(Interface::save(&mut page).unwrap()); - - assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para1).unwrap()); - let invalid_slice = &[0, 1, 2, 3]; - let result = page.calculate_merkle_hash_for_child("Paragraphs", invalid_slice); - assert!(matches!(result, Err(StorageError::DeserializationError(_)))); - } - - #[test] - fn calculate_merkle_hash_for_child__unknown_collection() { - let parent = Element::new(&Path::new("::root::node").unwrap(), None); - let mut page = Page::new_from_element("Node", parent); - let child = Element::new(&Path::new("::root::node::leaf").unwrap(), None); - let mut para = Paragraph::new_from_element("Leaf", child); - assert!(Interface::save(&mut page).unwrap()); - - assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para).unwrap()); - let para_slice = to_vec(¶).unwrap(); - let result = page.calculate_merkle_hash_for_child("unknown_collection", ¶_slice); - assert!(matches!( - result, - Err(StorageError::UnknownCollectionType(_)) - )); - } - #[test] fn collections() { let parent = Element::new(&Path::new("::root::node").unwrap(), None); @@ -179,7 +101,7 @@ mod child_info__constructor { fn new() { let id = Id::random(); let hash = Sha256::digest(b"1").into(); - let info = ChildInfo::new(id, hash); + let info = ChildInfo::new(id, hash, Metadata::default()); assert_eq!(info.id, id); assert_eq!(info.merkle_hash, hash); } @@ -191,13 +113,21 @@ mod child_info__public_methods { #[test] fn id() { - let info = ChildInfo::new(Id::random(), Sha256::digest(b"1").into()); + let info = ChildInfo::new( + Id::random(), + Sha256::digest(b"1").into(), + Metadata::default(), + ); assert_eq!(info.id(), info.id); } #[test] fn merkle_hash() { - let info = ChildInfo::new(Id::random(), Sha256::digest(b"1").into()); + let info = ChildInfo::new( + Id::random(), + Sha256::digest(b"1").into(), + Metadata::default(), + ); assert_eq!(info.merkle_hash(), info.merkle_hash); } } @@ -208,7 +138,11 @@ mod child_info__traits { #[test] fn display() { - let info = ChildInfo::new(Id::random(), Sha256::digest(b"1").into()); + let info = ChildInfo::new( + Id::random(), + Sha256::digest(b"1").into(), + Metadata::default(), + ); assert_eq!( format!("{info}"), format!( @@ -247,8 +181,8 @@ mod element__constructor { assert_eq!(element.path, path); assert_ge!(element.metadata.created_at, timestamp1); assert_le!(element.metadata.created_at, timestamp2); - assert_ge!(element.metadata.updated_at, timestamp1); - assert_le!(element.metadata.updated_at, timestamp2); + assert_ge!(*element.metadata.updated_at, timestamp1); + assert_le!(*element.metadata.updated_at, timestamp2); assert!(element.is_dirty); } } @@ -281,7 +215,7 @@ mod element__public_methods { #[test] fn is_dirty() { - let element = Element::new(&Path::new("::root::node::leaf").unwrap(), None); + let element = Element::root(); assert!(element.is_dirty()); let mut person = Person { @@ -296,22 +230,6 @@ mod element__public_methods { assert!(person.element().is_dirty()); } - #[test] - fn merkle_hash() { - let element = Element::new(&Path::new("::root::node::leaf").unwrap(), None); - let mut person = Person { - name: "Steve".to_owned(), - age: 50, - storage: element.clone(), - }; - assert_eq!(person.element().merkle_hash(), [0_u8; 32]); - - assert!(Interface::save(&mut person).unwrap()); - let expected_hash = - >::calculate_full_merkle_hash_for(person.id(), false).unwrap(); - assert_eq!(person.element().merkle_hash(), expected_hash); - } - #[test] #[ignore] fn metadata() { @@ -327,7 +245,7 @@ mod element__public_methods { #[test] fn update() { - let element = Element::new(&Path::new("::root::node::leaf").unwrap(), None); + let element = Element::root(); let updated_at = element.metadata.updated_at; let mut person = Person { name: "Bob".to_owned(), diff --git a/crates/storage/src/tests/index.rs b/crates/storage/src/tests/index.rs index 4c4d25995..c3dd39ccf 100644 --- a/crates/storage/src/tests/index.rs +++ b/crates/storage/src/tests/index.rs @@ -9,7 +9,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.id, root_id); @@ -29,8 +34,7 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection_name, - ChildInfo::new(child_id, child_own_hash), - 2, + ChildInfo::new(child_id, child_own_hash, Metadata::default()), ) .is_ok()); @@ -41,7 +45,7 @@ mod index__public_methods { assert_eq!(updated_root_index.children.len(), 1); assert_eq!( updated_root_index.children[collection_name][0], - ChildInfo::new(child_id, child_full_hash) + ChildInfo::new(child_id, child_full_hash, Metadata::default()) ); let child_index = >::get_index(child_id).unwrap().unwrap(); @@ -56,7 +60,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.id, root_id); @@ -73,35 +82,41 @@ mod index__public_methods { let grandchild_collection_name = "Pages"; let greatgrandchild_collection_name = "Paragraphs"; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let child_id = Id::random(); let child_hash = [2_u8; 32]; - let child_info = ChildInfo::new(child_id, child_hash); + let child_info = ChildInfo::new(child_id, child_hash, Metadata::default()); assert!( - >::add_child_to(root_id, child_collection_name, child_info, 2) - .is_ok() + >::add_child_to(root_id, child_collection_name, child_info).is_ok() ); let grandchild_id = Id::random(); let grandchild_hash = [3_u8; 32]; - let grandchild_info = ChildInfo::new(grandchild_id, grandchild_hash); + let grandchild_info = ChildInfo::new(grandchild_id, grandchild_hash, Metadata::default()); assert!(>::add_child_to( child_id, grandchild_collection_name, grandchild_info, - 3, ) .is_ok()); let greatgrandchild_id = Id::random(); let greatgrandchild_hash = [4_u8; 32]; - let greatgrandchild_info = ChildInfo::new(greatgrandchild_id, greatgrandchild_hash); + let greatgrandchild_info = ChildInfo::new( + greatgrandchild_id, + greatgrandchild_hash, + Metadata::default(), + ); assert!(>::add_child_to( grandchild_id, greatgrandchild_collection_name, greatgrandchild_info, - 4, ) .is_ok()); @@ -114,7 +129,8 @@ mod index__public_methods { >::get_hashes_for(grandchild_id) .unwrap() .unwrap() - .0 + .0, + Metadata::default() ) ); assert_eq!( @@ -124,7 +140,8 @@ mod index__public_methods { >::get_hashes_for(child_id) .unwrap() .unwrap() - .0 + .0, + Metadata::default() ) ); assert_eq!( @@ -134,7 +151,8 @@ mod index__public_methods { >::get_hashes_for(root_id) .unwrap() .unwrap() - .0 + .0, + Metadata::default() ) ); } @@ -144,7 +162,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let collection_name = "Books"; let child1_id = Id::random(); @@ -166,22 +189,26 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection_name, - ChildInfo::new(child1_id, child1_own_hash), - 2, + ChildInfo::new(child1_id, child1_own_hash, Metadata::default()), ) .is_ok()); assert!(>::add_child_to( root_id, collection_name, - ChildInfo::new(child2_id, child2_own_hash), - 2, + ChildInfo::new(child2_id, child2_own_hash, Metadata::default()), ) .is_ok()); let children = >::get_children_of(root_id, collection_name).unwrap(); assert_eq!(children.len(), 2); - assert_eq!(children[0], ChildInfo::new(child1_id, child1_full_hash)); - assert_eq!(children[1], ChildInfo::new(child2_id, child2_full_hash)); + assert_eq!( + children[0], + ChildInfo::new(child1_id, child1_full_hash, Metadata::default()) + ); + assert_eq!( + children[1], + ChildInfo::new(child2_id, child2_full_hash, Metadata::default()) + ); } #[test] @@ -189,7 +216,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let collection1_name = "Pages"; let child1_id = Id::random(); @@ -219,32 +251,38 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection1_name, - ChildInfo::new(child1_id, child1_own_hash), - 2, + ChildInfo::new(child1_id, child1_own_hash, Metadata::default()), ) .is_ok()); assert!(>::add_child_to( root_id, collection1_name, - ChildInfo::new(child2_id, child2_own_hash), - 2, + ChildInfo::new(child2_id, child2_own_hash, Metadata::default()), ) .is_ok()); assert!(>::add_child_to( root_id, collection2_name, - ChildInfo::new(child3_id, child3_own_hash), - 2, + ChildInfo::new(child3_id, child3_own_hash, Metadata::default()), ) .is_ok()); let children1 = >::get_children_of(root_id, collection1_name).unwrap(); assert_eq!(children1.len(), 2); - assert_eq!(children1[0], ChildInfo::new(child1_id, child1_full_hash)); - assert_eq!(children1[1], ChildInfo::new(child2_id, child2_full_hash)); + assert_eq!( + children1[0], + ChildInfo::new(child1_id, child1_full_hash, Metadata::default()) + ); + assert_eq!( + children1[1], + ChildInfo::new(child2_id, child2_full_hash, Metadata::default()) + ); let children2 = >::get_children_of(root_id, collection2_name).unwrap(); assert_eq!(children2.len(), 1); - assert_eq!(children2[0], ChildInfo::new(child3_id, child3_full_hash)); + assert_eq!( + children2[0], + ChildInfo::new(child3_id, child3_full_hash, Metadata::default()) + ); } #[test] @@ -252,7 +290,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let collection1_name = "Pages"; let collection2_name = "Chapters"; @@ -266,15 +309,13 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection1_name, - ChildInfo::new(child1_id, child1_own_hash), - 2, + ChildInfo::new(child1_id, child1_own_hash, Metadata::default()), ) .is_ok()); assert!(>::add_child_to( root_id, collection2_name, - ChildInfo::new(child2_id, child2_own_hash), - 2, + ChildInfo::new(child2_id, child2_own_hash, Metadata::default()), ) .is_ok()); @@ -290,7 +331,12 @@ mod index__public_methods { let root_own_hash = [1_u8; 32]; let root_full_hash = [0_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_own_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_own_hash, + Metadata::default() + ),) + .is_ok()); assert_eq!( >::get_hashes_for(root_id) @@ -305,7 +351,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.id, root_id); @@ -320,8 +371,7 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection_name, - ChildInfo::new(child_id, child_own_hash), - 2, + ChildInfo::new(child_id, child_own_hash, Metadata::default()), ) .is_ok()); @@ -332,28 +382,18 @@ mod index__public_methods { assert_eq!(>::get_parent_id(root_id).unwrap(), None); } - #[test] - fn get_type_id() { - let root_id = Id::random(); - let root_hash = [1_u8; 32]; - - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 99).is_ok()); - - let root_index = >::get_index(root_id).unwrap().unwrap(); - assert_eq!(root_index.id, root_id); - assert_eq!(root_index.own_hash, root_hash); - assert_eq!(root_index.type_id, 99); - - assert_eq!(>::get_type_id(root_id).unwrap(), 99,); - } - #[test] fn has_children() { let root_id = Id::random(); let root_hash = [1_u8; 32]; let collection_name = "Books"; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); assert!(!>::has_children(root_id, collection_name).unwrap()); let child_id = Id::random(); @@ -362,8 +402,7 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection_name, - ChildInfo::new(child_id, child_own_hash), - 2, + ChildInfo::new(child_id, child_own_hash, Metadata::default()), ) .is_ok()); assert!(>::has_children(root_id, collection_name).unwrap()); @@ -374,7 +413,12 @@ mod index__public_methods { let root_id = Id::random(); let root_hash = [1_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.id, root_id); @@ -389,8 +433,7 @@ mod index__public_methods { assert!(>::add_child_to( root_id, collection_name, - ChildInfo::new(child_id, child_own_hash), - 2, + ChildInfo::new(child_id, child_own_hash, Metadata::default()), ) .is_ok()); assert!( @@ -419,7 +462,7 @@ mod index__private_methods { children: BTreeMap::new(), full_hash: hash1, own_hash: hash2, - type_id: 1, + metadata: Metadata::default(), }; >::save_index(&index).unwrap(); @@ -439,7 +482,7 @@ mod index__private_methods { children: BTreeMap::new(), full_hash: hash1, own_hash: hash2, - type_id: 1, + metadata: Metadata::default(), }; >::save_index(&index).unwrap(); assert_eq!(>::get_index(id).unwrap().unwrap(), index); @@ -456,27 +499,26 @@ mod hashing { #[test] fn calculate_full_merkle_hash_for__with_children() { let root_id = Id::random(); - assert!(>::add_root(ChildInfo::new(root_id, [0_u8; 32]), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + [0_u8; 32], + Metadata::default() + ),) + .is_ok()); let collection_name = "Children"; let child1_id = Id::random(); let child1_hash = [1_u8; 32]; - let child1_info = ChildInfo::new(child1_id, child1_hash); - assert!( - >::add_child_to(root_id, collection_name, child1_info, 2).is_ok() - ); + let child1_info = ChildInfo::new(child1_id, child1_hash, Metadata::default()); + assert!(>::add_child_to(root_id, collection_name, child1_info).is_ok()); let child2_id = Id::random(); let child2_hash = [2_u8; 32]; - let child2_info = ChildInfo::new(child2_id, child2_hash); - assert!( - >::add_child_to(root_id, collection_name, child2_info, 2).is_ok() - ); + let child2_info = ChildInfo::new(child2_id, child2_hash, Metadata::default()); + assert!(>::add_child_to(root_id, collection_name, child2_info).is_ok()); let child3_id = Id::random(); let child3_hash = [3_u8; 32]; - let child3_info = ChildInfo::new(child3_id, child3_hash); - assert!( - >::add_child_to(root_id, collection_name, child3_info, 2).is_ok() - ); + let child3_info = ChildInfo::new(child3_id, child3_hash, Metadata::default()); + assert!(>::add_child_to(root_id, collection_name, child3_info).is_ok()); assert_eq!( hex::encode( @@ -512,17 +554,21 @@ mod hashing { let grandchild_collection_name = "Pages"; let greatgrandchild_collection_name = "Paragraphs"; - assert!(>::add_root(ChildInfo::new(root_id, root_hash), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.full_hash, [0_u8; 32]); let child_id = Id::random(); let child_hash = [2_u8; 32]; - let child_info = ChildInfo::new(child_id, child_hash); + let child_info = ChildInfo::new(child_id, child_hash, Metadata::default()); assert!( - >::add_child_to(root_id, child_collection_name, child_info, 2) - .is_ok() + >::add_child_to(root_id, child_collection_name, child_info).is_ok() ); let root_index_with_child = >::get_index(root_id).unwrap().unwrap(); @@ -538,12 +584,11 @@ mod hashing { let grandchild_id = Id::random(); let grandchild_hash = [3_u8; 32]; - let grandchild_info = ChildInfo::new(grandchild_id, grandchild_hash); + let grandchild_info = ChildInfo::new(grandchild_id, grandchild_hash, Metadata::default()); assert!(>::add_child_to( child_id, grandchild_collection_name, grandchild_info, - 3, ) .is_ok()); @@ -568,12 +613,15 @@ mod hashing { let greatgrandchild_id = Id::random(); let greatgrandchild_hash = [4_u8; 32]; - let greatgrandchild_info = ChildInfo::new(greatgrandchild_id, greatgrandchild_hash); + let greatgrandchild_info = ChildInfo::new( + greatgrandchild_id, + greatgrandchild_hash, + Metadata::default(), + ); assert!(>::add_child_to( grandchild_id, greatgrandchild_collection_name, greatgrandchild_info, - 4, ) .is_ok()); @@ -691,7 +739,12 @@ mod hashing { .try_into() .unwrap(); - assert!(>::add_root(ChildInfo::new(root_id, root_hash1), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash1, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.id, root_id); @@ -709,7 +762,12 @@ mod hashing { let root_hash1 = [1_u8; 32]; let root_hash2 = [2_u8; 32]; - assert!(>::add_root(ChildInfo::new(root_id, root_hash1), 1).is_ok()); + assert!(>::add_root(ChildInfo::new( + root_id, + root_hash1, + Metadata::default() + ),) + .is_ok()); let root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(root_index.id, root_id); diff --git a/crates/storage/src/tests/interface.rs b/crates/storage/src/tests/interface.rs index 3a44d1484..860ad9804 100644 --- a/crates/storage/src/tests/interface.rs +++ b/crates/storage/src/tests/interface.rs @@ -14,7 +14,7 @@ mod interface__public_methods { #[test] fn children_of() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut page = Page::new_from_element("Node", element); assert!(Interface::save(&mut page).unwrap()); assert_eq!( @@ -28,7 +28,9 @@ mod interface__public_methods { let mut para1 = Paragraph::new_from_element("Leaf1", child1); let mut para2 = Paragraph::new_from_element("Leaf2", child2); let mut para3 = Paragraph::new_from_element("Leaf3", child3); - assert!(Interface::save(&mut page).unwrap()); + + assert!(!Interface::save(&mut page).unwrap()); + assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para1).unwrap()); assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para2).unwrap()); assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para3).unwrap()); @@ -40,7 +42,7 @@ mod interface__public_methods { #[test] fn find_by_id__existent() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut page = Page::new_from_element("Leaf", element); let id = page.id(); assert!(Interface::save(&mut page).unwrap()); @@ -73,28 +75,15 @@ mod interface__public_methods { #[test] fn save__basic() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut page = Page::new_from_element("Node", element); assert_ok!(Interface::save(&mut page)); } - #[test] - fn save__multiple() { - let element1 = Element::new(&Path::new("::root::node1").unwrap(), None); - let element2 = Element::new(&Path::new("::root::node2").unwrap(), None); - let mut page1 = Page::new_from_element("Node1", element1); - let mut page2 = Page::new_from_element("Node2", element2); - - assert!(Interface::save(&mut page1).unwrap()); - assert!(Interface::save(&mut page2).unwrap()); - assert_eq!(Interface::find_by_id(page1.id()).unwrap(), Some(page1)); - assert_eq!(Interface::find_by_id(page2.id()).unwrap(), Some(page2)); - } - #[test] fn save__not_dirty() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut page = Page::new_from_element("Node", element); assert!(Interface::save(&mut page).unwrap()); @@ -104,13 +93,13 @@ mod interface__public_methods { #[test] fn save__too_old() { - let element1 = Element::new(&Path::new("::root::node").unwrap(), None); + let element1 = Element::root(); let mut page1 = Page::new_from_element("Node", element1); let mut page2 = page1.clone(); assert!(Interface::save(&mut page1).unwrap()); page2.element_mut().update(); - sleep(Duration::from_millis(1)); + sleep(Duration::from_millis(2)); page1.element_mut().update(); assert!(Interface::save(&mut page1).unwrap()); assert!(!Interface::save(&mut page2).unwrap()); @@ -118,11 +107,13 @@ mod interface__public_methods { #[test] fn save__update_existing() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut page = Page::new_from_element("Node", element); let id = page.id(); assert!(Interface::save(&mut page).unwrap()); + page.storage.update(); + // TODO: Modify the element's data and check it changed assert!(Interface::save(&mut page).unwrap()); @@ -163,19 +154,16 @@ mod interface__apply_actions { #[test] fn apply_action__add() { - let page = Page::new_from_element( - "Test Page", - Element::new(&Path::new("::test").unwrap(), None), - ); + let page = Page::new_from_element("Test Page", Element::root()); let serialized = to_vec(&page).unwrap(); let action = Action::Add { id: page.id(), - type_id: 102, data: serialized, ancestors: vec![], + metadata: page.element().metadata, }; - assert!(Interface::apply_action::(action).is_ok()); + assert!(Interface::apply_action(action).is_ok()); // Verify the page was added let retrieved_page = Interface::find_by_id::(page.id()).unwrap(); @@ -185,10 +173,7 @@ mod interface__apply_actions { #[test] fn apply_action__update() { - let mut page = Page::new_from_element( - "Old Title", - Element::new(&Path::new("::test").unwrap(), None), - ); + let mut page = Page::new_from_element("Old Title", Element::root()); assert!(Interface::save(&mut page).unwrap()); page.title = "New Title".to_owned(); @@ -196,12 +181,12 @@ mod interface__apply_actions { let serialized = to_vec(&page).unwrap(); let action = Action::Update { id: page.id(), - type_id: 102, data: serialized, ancestors: vec![], + metadata: page.element().metadata, }; - assert!(Interface::apply_action::(action).is_ok()); + assert!(Interface::apply_action(action).is_ok()); // Verify the page was updated let retrieved_page = Interface::find_by_id::(page.id()).unwrap().unwrap(); @@ -210,10 +195,7 @@ mod interface__apply_actions { #[test] fn apply_action__delete() { - let mut page = Page::new_from_element( - "Test Page", - Element::new(&Path::new("::test").unwrap(), None), - ); + let mut page = Page::new_from_element("Test Page", Element::root()); assert!(Interface::save(&mut page).unwrap()); let action = Action::Delete { @@ -221,7 +203,7 @@ mod interface__apply_actions { ancestors: vec![], }; - assert!(Interface::apply_action::(action).is_ok()); + assert!(Interface::apply_action(action).is_ok()); // Verify the page was deleted let retrieved_page = Interface::find_by_id::(page.id()).unwrap(); @@ -230,50 +212,26 @@ mod interface__apply_actions { #[test] fn apply_action__compare() { - let page = Page::new_from_element( - "Test Page", - Element::new(&Path::new("::test").unwrap(), None), - ); + let page = Page::new_from_element("Test Page", Element::root()); let action = Action::Compare { id: page.id() }; // Compare should fail - assert!(Interface::apply_action::(action).is_err()); - } - - #[test] - fn apply_action__wrong_type() { - let page = Page::new_from_element( - "Test Page", - Element::new(&Path::new("::test").unwrap(), None), - ); - let serialized = to_vec(&page).unwrap(); - let action = Action::Add { - id: page.id(), - type_id: 102, - data: serialized, - ancestors: vec![], - }; - - // Trying to apply a Page action as if it were a Paragraph should fail - assert!(Interface::apply_action::(action).is_err()); + assert!(Interface::apply_action(action).is_err()); } #[test] fn apply_action__non_existent_update() { - let page = Page::new_from_element( - "Test Page", - Element::new(&Path::new("::test").unwrap(), None), - ); + let page = Page::new_from_element("Test Page", Element::root()); let serialized = to_vec(&page).unwrap(); let action = Action::Update { id: page.id(), - type_id: 102, data: serialized, ancestors: vec![], + metadata: page.element().metadata, }; // Updating a non-existent page should still succeed (it will be added) - assert!(Interface::apply_action::(action).is_ok()); + assert!(Interface::apply_action(action).is_ok()); // Verify the page was added let retrieved_page = Interface::find_by_id::(page.id()).unwrap(); @@ -292,7 +250,7 @@ mod interface__comparison { foreign: Option<&D>, comparison_data: &ComparisonData, ) -> Result<(Vec, Vec), StorageError> { - Interface::compare_trees::( + Interface::compare_trees( foreign .map(to_vec) .transpose() @@ -303,7 +261,7 @@ mod interface__comparison { #[test] fn compare_trees__identical() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut local = Page::new_from_element("Test Page", element); let mut foreign = local.clone(); @@ -316,7 +274,7 @@ mod interface__comparison { let result = compare_trees( Some(&foreign), - &ForeignInterface::generate_comparison_data(Some(&foreign)).unwrap(), + &ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), ) .unwrap(); assert_eq!(result, (vec![], vec![])); @@ -324,7 +282,7 @@ mod interface__comparison { #[test] fn compare_trees__local_newer() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut local = Page::new_from_element("Test Page", element.clone()); let mut foreign = Page::new_from_element("Old Test Page", element); @@ -337,7 +295,7 @@ mod interface__comparison { let result = compare_trees( Some(&foreign), - &ForeignInterface::generate_comparison_data(Some(&foreign)).unwrap(), + &ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), ) .unwrap(); assert_eq!( @@ -346,9 +304,9 @@ mod interface__comparison { vec![], vec![Action::Update { id: local.id(), - type_id: 102, data: to_vec(&local).unwrap(), - ancestors: vec![] + ancestors: vec![], + metadata: local.element().metadata, }] ) ); @@ -356,7 +314,7 @@ mod interface__comparison { #[test] fn compare_trees__foreign_newer() { - let element = Element::new(&Path::new("::root::node").unwrap(), None); + let element = Element::root(); let mut local = Page::new_from_element("Old Test Page", element.clone()); let mut foreign = Page::new_from_element("Test Page", element); @@ -369,7 +327,7 @@ mod interface__comparison { let result = compare_trees( Some(&foreign), - &ForeignInterface::generate_comparison_data(Some(&foreign)).unwrap(), + &ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), ) .unwrap(); assert_eq!( @@ -377,9 +335,9 @@ mod interface__comparison { ( vec![Action::Update { id: foreign.id(), - type_id: 102, data: to_vec(&foreign).unwrap(), - ancestors: vec![] + ancestors: vec![], + metadata: foreign.element().metadata, }], vec![] ) @@ -388,7 +346,7 @@ mod interface__comparison { #[test] fn compare_trees__with_collections() { - let page_element = Element::new(&Path::new("::root::node").unwrap(), None); + let page_element = Element::root(); let para1_element = Element::new(&Path::new("::root::node::leaf1").unwrap(), None); let para2_element = Element::new(&Path::new("::root::node::leaf2").unwrap(), None); let para3_element = Element::new(&Path::new("::root::node::leaf3").unwrap(), None); @@ -432,7 +390,7 @@ mod interface__comparison { let (local_actions, foreign_actions) = compare_trees( Some(&foreign_page), - &ForeignInterface::generate_comparison_data(Some(&foreign_page)).unwrap(), + &ForeignInterface::generate_comparison_data(Some(foreign_page.id())).unwrap(), ) .unwrap(); @@ -442,9 +400,9 @@ mod interface__comparison { // Page needs update due to different child structure Action::Update { id: foreign_page.id(), - type_id: 102, data: to_vec(&foreign_page).unwrap(), - ancestors: vec![] + ancestors: vec![], + metadata: foreign_page.element().metadata, }, // Para1 needs comparison due to different hash Action::Compare { @@ -463,9 +421,9 @@ mod interface__comparison { // Para2 needs to be added to foreign Action::Add { id: local_para2.id(), - type_id: 103, data: to_vec(&local_para2).unwrap(), - ancestors: vec![] + ancestors: vec![], + metadata: local_para2.element().metadata, }, // Para3 needs to be added locally, but we don't have the data, so we compare Action::Compare { @@ -477,7 +435,7 @@ mod interface__comparison { // Compare the updated para1 let (local_para1_actions, foreign_para1_actions) = compare_trees( Some(&foreign_para1), - &ForeignInterface::generate_comparison_data(Some(&foreign_para1)).unwrap(), + &ForeignInterface::generate_comparison_data(Some(foreign_para1.id())).unwrap(), ) .unwrap(); @@ -498,9 +456,13 @@ mod interface__comparison { local_para1_actions, vec![Action::Update { id: foreign_para1.id(), - type_id: 103, data: to_vec(&foreign_para1).unwrap(), - ancestors: vec![ChildInfo::new(foreign_page.id(), local_para1_ancestor_hash,)], + ancestors: vec![ChildInfo::new( + foreign_page.id(), + local_para1_ancestor_hash, + local_page.element().metadata + )], + metadata: foreign_para1.element().metadata, }] ); assert_eq!(foreign_para1_actions, vec![]); @@ -508,7 +470,7 @@ mod interface__comparison { // Compare para3 which doesn't exist locally let (local_para3_actions, foreign_para3_actions) = compare_trees( Some(&foreign_para3), - &ForeignInterface::generate_comparison_data(Some(&foreign_para3)).unwrap(), + &ForeignInterface::generate_comparison_data(Some(foreign_para3.id())).unwrap(), ) .unwrap(); @@ -529,9 +491,13 @@ mod interface__comparison { local_para3_actions, vec![Action::Add { id: foreign_para3.id(), - type_id: 103, data: to_vec(&foreign_para3).unwrap(), - ancestors: vec![ChildInfo::new(foreign_page.id(), local_para3_ancestor_hash,)], + ancestors: vec![ChildInfo::new( + foreign_page.id(), + local_para3_ancestor_hash, + foreign_page.element().metadata + )], + metadata: foreign_para3.element().metadata, }] ); assert_eq!(foreign_para3_actions, vec![]); From a27f5de9520719200de0ce8511887d93ddab249d Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 12:36:22 +0300 Subject: [PATCH 09/19] update collections --- crates/storage/src/collections.rs | 22 ++++++---- crates/storage/src/collections/root.rs | 44 +++++++++++++++++-- .../storage/src/collections/unordered_map.rs | 19 +++++++- .../storage/src/collections/unordered_set.rs | 12 ++++- crates/storage/src/collections/vector.rs | 19 +++++++- crates/storage/src/entities.rs | 1 + 6 files changed, 102 insertions(+), 15 deletions(-) diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 952b20ae5..1686eaf26 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -20,15 +20,12 @@ pub mod error; pub use error::StoreError; // fixme! macro expects `calimero_storage` to be in deps -use crate as calimero_storage; use crate::address::{Id, Path}; use crate::entities::{Data, Element}; use crate::interface::{Interface, StorageError}; -use crate::{AtomicUnit, Collection}; +use crate::{self as calimero_storage, AtomicUnit, Collection}; #[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug)] -#[type_id(255)] -#[root] struct Collection { /// The entries in the collection. #[collection] @@ -82,7 +79,6 @@ struct Entries { /// An entry in a map. #[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug)] -#[type_id(254)] struct Entry { /// The item in the entry. item: T, @@ -107,14 +103,24 @@ type StoreResult = std::result::Result; impl Collection { /// Creates a new collection. - fn new() -> Self { + fn new(id: Option) -> Self { + let id = id.unwrap_or_else(|| Id::random()); + let mut this = Self { entries: Entries::default(), children_ids: RefCell::new(None), - storage: Element::new(&Path::new("::unused").expect("valid path"), None), + storage: Element::new(&Path::new("::unused").expect("valid path"), Some(id)), }; - let _ = Interface::save(&mut this).expect("save collection"); + if id.is_root() { + let _ignored = Interface::save(&mut this).expect("save"); + } else { + let root = root::ROOT + .with(|root| root.borrow().clone()) + .expect("no root??"); + + let _ = Interface::add_child_to(root.id, &root, &mut this).expect("add child"); + } this } diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs index 5a2280e61..2d5d8823c 100644 --- a/crates/storage/src/collections/root.rs +++ b/crates/storage/src/collections/root.rs @@ -1,13 +1,45 @@ //! A root collection that stores a single value. use std::cell::RefCell; +use std::marker::PhantomData; use std::ptr; use borsh::{BorshDeserialize, BorshSerialize}; -use calimero_sdk::env; -use super::Collection; +use super::{Collection, Entry}; use crate::address::Id; +use crate::entities::Data; +use crate::env; + +/// Thing. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct RootHandle { + /// The ID of the root collection. + pub id: Id, + _priv: PhantomData, +} + +impl RootHandle { + fn new(id: Id) -> Self { + Self { + id, + _priv: PhantomData, + } + } +} + +impl crate::entities::Collection for RootHandle { + type Child = T; + + fn name(&self) -> &str { + "RootHandle" + } +} + +thread_local! { + /// The root collection handle. + pub static ROOT: RefCell>>> = RefCell::new(None); +} /// A set collection that stores unqiue values once. #[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] @@ -25,7 +57,13 @@ where pub fn new(value: T) -> Self { let id = Id::new(env::context_id()); - let mut inner = Collection::new(); + let old = ROOT.with(|root| root.borrow_mut().replace(RootHandle::new(id))); + + if old.is_some() { + panic!("root collection already defined"); + } + + let mut inner = Collection::new(Some(id)); let value = inner.insert(Some(id), value).unwrap(); diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index 017261514..d122baf3c 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -26,7 +26,7 @@ where /// pub fn new() -> Self { Self { - inner: Collection::new(), + inner: Collection::new(None), } } @@ -198,9 +198,12 @@ where #[cfg(test)] mod tests { use crate::collections::unordered_map::UnorderedMap; + use crate::collections::Root; #[test] fn test_unordered_map_basic_operations() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map @@ -250,6 +253,8 @@ mod tests { #[test] fn test_unordered_map_insert_and_get() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map @@ -273,6 +278,8 @@ mod tests { #[test] fn test_unordered_map_update_value() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map @@ -292,6 +299,8 @@ mod tests { #[test] fn test_remove() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map @@ -308,6 +317,8 @@ mod tests { #[test] fn test_clear() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map @@ -327,6 +338,8 @@ mod tests { #[test] fn test_unordered_map_len() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert_eq!(map.len().expect("len failed"), 0); @@ -356,6 +369,8 @@ mod tests { #[test] fn test_unordered_map_contains() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map @@ -369,6 +384,8 @@ mod tests { #[test] fn test_unordered_map_entries() { + let _root = Root::new(()); + let mut map = UnorderedMap::::new(); assert!(map diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 4781ba5f0..cb36a2599 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -24,7 +24,7 @@ where /// pub fn new() -> Self { Self { - inner: Collection::new(), + inner: Collection::new(None), } } @@ -171,10 +171,12 @@ where #[cfg(test)] mod tests { - use crate::collections::UnorderedSet; + use crate::collections::{Root, UnorderedSet}; #[test] fn test_unordered_set_operations() { + let _root = Root::new(()); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); @@ -203,6 +205,8 @@ mod tests { #[test] fn test_unordered_set_len() { + let _root = Root::new(()); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); @@ -218,6 +222,8 @@ mod tests { #[test] fn test_unordered_set_clear() { + let _root = Root::new(()); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); @@ -234,6 +240,8 @@ mod tests { #[test] fn test_unordered_set_entries() { + let _root = Root::new(()); + let mut set = UnorderedSet::::new(); assert!(set.insert("value1".to_string()).expect("insert failed")); diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index b797e69fd..c3310e095 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -22,7 +22,7 @@ where /// pub fn new() -> Self { Self { - inner: Collection::new(), + inner: Collection::new(None), } } @@ -159,9 +159,12 @@ where #[cfg(test)] mod tests { use crate::collections::vector::Vector; + use crate::collections::Root; #[test] fn test_vector_push() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let result = vector.push(value.clone()); @@ -171,6 +174,8 @@ mod tests { #[test] fn test_vector_get() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); @@ -180,6 +185,8 @@ mod tests { #[test] fn test_vector_update() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value1 = "test_data1".to_string(); let value2 = "test_data2".to_string(); @@ -192,6 +199,8 @@ mod tests { #[test] fn test_vector_get_non_existent() { + let _root = Root::new(()); + let vector: Vector = Vector::new(); match vector.get(0) { Ok(retrieved_value) => assert_eq!(retrieved_value, None), @@ -201,6 +210,8 @@ mod tests { #[test] fn test_vector_pop() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); @@ -211,6 +222,8 @@ mod tests { #[test] fn test_vector_entries() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value1 = "test_data1".to_string(); let value2 = "test_data2".to_string(); @@ -222,6 +235,8 @@ mod tests { #[test] fn test_vector_contains() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); @@ -232,6 +247,8 @@ mod tests { #[test] fn test_vector_clear() { + let _root = Root::new(()); + let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); diff --git a/crates/storage/src/entities.rs b/crates/storage/src/entities.rs index 01f5b3af5..211ba676e 100644 --- a/crates/storage/src/entities.rs +++ b/crates/storage/src/entities.rs @@ -788,6 +788,7 @@ pub struct Metadata { pub(crate) updated_at: UpdatedAt, } +/// The timestamp when the [`Element`] was last updated. #[derive( BorshDeserialize, BorshSerialize, Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialOrd, )] From 8f5aab5c49d60f2ba4be4170e37a19e67b48583e Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 18:00:32 +0300 Subject: [PATCH 10/19] update kv-store --- apps/kv-store/src/__private.rs | 55 +++-------- apps/kv-store/src/lib.rs | 18 ++-- crates/sdk/macros/src/logic/method.rs | 56 ++++++----- crates/storage/src/collections.rs | 4 +- crates/storage/src/collections/root.rs | 98 +++++++++++++++---- .../storage/src/collections/unordered_map.rs | 32 ++---- .../storage/src/collections/unordered_set.rs | 16 +-- crates/storage/src/collections/vector.rs | 25 ++--- crates/storage/src/env.rs | 2 +- crates/storage/src/integration.rs | 3 - 10 files changed, 155 insertions(+), 154 deletions(-) diff --git a/apps/kv-store/src/__private.rs b/apps/kv-store/src/__private.rs index 471b84c67..0ddeb6178 100644 --- a/apps/kv-store/src/__private.rs +++ b/apps/kv-store/src/__private.rs @@ -1,7 +1,6 @@ -use calimero_sdk::borsh::{from_slice, to_vec}; +use calimero_sdk::borsh::from_slice; use calimero_sdk::{app, env}; -use calimero_storage::collections::unordered_map::{Entry, UnorderedMap}; -use calimero_storage::entities::Data; +use calimero_storage::address::Id; use calimero_storage::integration::Comparison; use calimero_storage::interface::{Action, Interface, StorageError}; use calimero_storage::sync::{self, SyncArtifact}; @@ -16,66 +15,36 @@ impl KvStore { let artifact = from_slice::(&args).map_err(StorageError::DeserializationError)?; - let this = Interface::root::()?; - match artifact { SyncArtifact::Actions(actions) => { for action in actions { let _ignored = match action { - Action::Add { type_id, .. } | Action::Update { type_id, .. } => { - match type_id { - 1 => Interface::apply_action::(action)?, - 254 => Interface::apply_action::>(action)?, - 255 => { - Interface::apply_action::>(action)? - } - _ => return Err(StorageError::UnknownType(type_id)), - } - } - Action::Delete { .. } => { - todo!("how are we supposed to identify the entity to delete???????") + Action::Compare { id } => { + sync::push_comparison(Comparison { + data: Interface::find_by_id_raw(id)?, + comparison_data: Interface::generate_comparison_data(Some(id))?, + }); } - Action::Compare { .. } => { - todo!("how are we supposed to compare when `Comparison` needs `type_id`???????") + Action::Add { .. } | Action::Update { .. } | Action::Delete { .. } => { + Interface::apply_action(action)?; } }; } - - if let Some(this) = this { - return Interface::commit_root(this); - } } SyncArtifact::Comparisons(comparisons) => { if comparisons.is_empty() { sync::push_comparison(Comparison { - type_id: ::type_id(), - data: this - .as_ref() - .map(to_vec) - .transpose() - .map_err(StorageError::SerializationError)?, - comparison_data: Interface::generate_comparison_data(this.as_ref())?, + data: Interface::find_by_id_raw(Id::root())?, + comparison_data: Interface::generate_comparison_data(None)?, }); } for Comparison { - type_id, data, comparison_data, } in comparisons { - match type_id { - 1 => Interface::compare_affective::(data, comparison_data)?, - 254 => Interface::compare_affective::>( - data, - comparison_data, - )?, - 255 => Interface::compare_affective::>( - data, - comparison_data, - )?, - _ => return Err(StorageError::UnknownType(type_id)), - }; + Interface::compare_affective(data, comparison_data)?; } } } diff --git a/apps/kv-store/src/lib.rs b/apps/kv-store/src/lib.rs index ebe163683..476727983 100644 --- a/apps/kv-store/src/lib.rs +++ b/apps/kv-store/src/lib.rs @@ -2,22 +2,18 @@ use std::collections::BTreeMap; +use calimero_sdk::borsh::{BorshDeserialize, BorshSerialize}; use calimero_sdk::types::Error; use calimero_sdk::{app, env}; use calimero_storage::collections::UnorderedMap; -use calimero_storage::entities::Element; -use calimero_storage::AtomicUnit; mod __private; #[app::state(emits = for<'a> Event<'a>)] -#[derive(AtomicUnit, Clone, Debug, PartialEq, PartialOrd)] -#[root] -#[type_id(1)] +#[derive(Clone, Debug, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] +#[borsh(crate = "calimero_sdk::borsh")] pub struct KvStore { items: UnorderedMap, - #[storage] - storage: Element, } #[app::event] @@ -33,8 +29,7 @@ impl KvStore { #[app::init] pub fn init() -> KvStore { KvStore { - items: UnorderedMap::new().unwrap(), - storage: Element::root(), + items: UnorderedMap::new(), } } @@ -93,7 +88,10 @@ impl KvStore { app::emit!(Event::Removed { key }); - self.items.remove(key).map_err(Into::into) + self.items + .remove(key) + .map(|v| v.is_some()) + .map_err(Into::into) } pub fn clear(&mut self) -> Result<(), Error> { diff --git a/crates/sdk/macros/src/logic/method.rs b/crates/sdk/macros/src/logic/method.rs index 4f3b83e2d..b41bd4ad8 100644 --- a/crates/sdk/macros/src/logic/method.rs +++ b/crates/sdk/macros/src/logic/method.rs @@ -1,5 +1,6 @@ use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; +use quote::{quote, quote_spanned, ToTokens}; +use syn::spanned::Spanned; use syn::{Error as SynError, GenericParam, Ident, ImplItemFn, Path, ReturnType, Visibility}; use crate::errors::{Errors, ParseError}; @@ -61,12 +62,14 @@ impl ToTokens for PublicLogicMethod<'_> { let input_lifetime = if self.has_refs { let lifetime = lifetimes::input(); - quote! { <#lifetime> } + quote_spanned! { name.span()=> + <#lifetime> + } } else { quote! {} }; - quote! { + quote_spanned! {name.span()=> #[derive(::calimero_sdk::serde::Deserialize)] #[serde(crate = "::calimero_sdk::serde")] struct #input_ident #input_lifetime { @@ -94,38 +97,41 @@ impl ToTokens for PublicLogicMethod<'_> { let (def, mut call) = match &self.self_type { Some(type_) => ( { - let mutability = match type_ { - SelfType::Mutable(_) => Some(quote! {mut}), - SelfType::Owned(_) | SelfType::Immutable(_) => None, + let (mutability, ty) = match type_ { + SelfType::Mutable(ty) => (Some(quote! {mut}), ty), + SelfType::Owned(ty) | SelfType::Immutable(ty) => (None, ty), }; - quote! { - let Some(#mutability app) = ::calimero_storage::interface::Interface::root::<#self_>().ok().flatten() + quote_spanned! {ty.span()=> + let Some(#mutability app) = ::calimero_storage::collections::Root::<#self_>::fetch() else { ::calimero_sdk::env::panic_str("Failed to find or read app state") }; } }, - quote! { app.#name(#(#arg_idents),*); }, + quote_spanned! {name.span()=> + app.#name(#(#arg_idents),*) + }, ), None => ( if init_method { - quote! { - if let Some(mut app) = ::calimero_storage::interface::Interface::root::<#self_>().ok().flatten() { + quote_spanned! {name.span()=> + if let Some(mut app) = ::calimero_storage::collections::Root::<#self_>::fetch() { ::calimero_sdk::env::panic_str("Cannot initialize over already existing state.") }; - let mut app: #self_ = + let mut app = } } else { quote! {} }, - quote! { <#self_>::#name(#(#arg_idents),*); }, + quote_spanned! {name.span()=> + <#self_>::#name(#(#arg_idents),*) + }, ), }; - if let (Some(_), false) = (&self.ret, init_method) { - //only when it's not init - call = quote! { + if let (Some(ret), false) = (&self.ret, init_method) { + call = quote_spanned! {ret.ty.span()=> let output = #call; let output = { #[expect(unused_imports)] @@ -140,22 +146,24 @@ impl ToTokens for PublicLogicMethod<'_> { ), } }; - ::calimero_sdk::env::value_return(&output); - }; + ::calimero_sdk::env::value_return(&output) + } } let state_finalizer = match (&self.self_type, init_method) { (Some(SelfType::Mutable(_)), _) | (_, true) => quote! { - if let Err(_) = ::calimero_storage::interface::Interface::commit_root(app) { - ::calimero_sdk::env::panic_str("Failed to commit app state") - } + app.commit(); }, _ => quote! {}, }; // todo! when generics are present, strip them let init_impl = if init_method { - quote! { + call = quote_spanned! {name.span()=> + ::calimero_storage::collections::Root::new(|| #call) + }; + + quote_spanned! {name.span()=> impl ::calimero_sdk::state::AppStateInit for #self_ { type Return = #ret; } @@ -164,7 +172,7 @@ impl ToTokens for PublicLogicMethod<'_> { quote! {} }; - quote! { + quote_spanned! {name.span()=> #[cfg(target_arch = "wasm32")] #[no_mangle] pub extern "C" fn #name() { @@ -176,7 +184,7 @@ impl ToTokens for PublicLogicMethod<'_> { #def - #call + #call; #state_finalizer } diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 1686eaf26..bc90ed095 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -14,7 +14,8 @@ pub mod unordered_set; pub use unordered_set::UnorderedSet; pub mod vector; pub use vector::Vector; -pub mod root; +mod root; +#[doc(hidden)] pub use root::Root; pub mod error; pub use error::StoreError; @@ -323,6 +324,5 @@ where { fn drop(&mut self) { self.collection.element_mut().update(); - let _ignored = Interface::save(self.collection); } } diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs index 2d5d8823c..0a7ea7020 100644 --- a/crates/storage/src/collections/root.rs +++ b/crates/storage/src/collections/root.rs @@ -2,7 +2,9 @@ use std::cell::RefCell; use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; use std::ptr; +use std::sync::LazyLock; use borsh::{BorshDeserialize, BorshSerialize}; @@ -10,10 +12,11 @@ use super::{Collection, Entry}; use crate::address::Id; use crate::entities::Data; use crate::env; +use crate::interface::Interface; /// Thing. #[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub struct RootHandle { +pub(super) struct RootHandle { /// The ID of the root collection. pub id: Id, _priv: PhantomData, @@ -41,12 +44,23 @@ thread_local! { pub static ROOT: RefCell>>> = RefCell::new(None); } +static ID: LazyLock = LazyLock::new(|| Id::new(env::context_id())); + +/// Prepares the root collection. +fn employ_root_guard() { + let old = ROOT.with(|root| root.borrow_mut().replace(RootHandle::new(*ID))); + + if old.is_some() { + panic!("root collection already defined"); + } +} + /// A set collection that stores unqiue values once. -#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] +#[derive(Debug)] pub struct Root { - id: Id, inner: Collection, value: RefCell>, + dirty: bool, } impl Root @@ -54,40 +68,86 @@ where T: BorshSerialize + BorshDeserialize, { /// Creates a new root collection with the given value. - pub fn new(value: T) -> Self { - let id = Id::new(env::context_id()); + pub fn new T>(f: F) -> Self { + employ_root_guard(); - let old = ROOT.with(|root| root.borrow_mut().replace(RootHandle::new(id))); + let mut inner = Collection::new(Some(*ID)); - if old.is_some() { - panic!("root collection already defined"); - } - - let mut inner = Collection::new(Some(id)); + let id = Self::entry_id(); - let value = inner.insert(Some(id), value).unwrap(); + let value = inner.insert(Some(id), f()).unwrap(); Self { - id, inner, + dirty: false, value: RefCell::new(Some(value)), } } - /// Gets the value of the root collection. - pub fn get(&self) -> &T { - self.get_mut() + fn entry_id() -> Id { + Id::new([118; 32]) } - /// Gets the value of the root collection mutably. - pub fn get_mut(&self) -> &mut T { + fn get(&self) -> &mut T { let mut value = self.value.borrow_mut(); - let value = value.get_or_insert_with(|| self.inner.get(self.id).unwrap().unwrap()); + let id = Self::entry_id(); + + let value = value.get_or_insert_with(|| self.inner.get(id).unwrap().unwrap()); #[expect(unsafe_code, reason = "necessary for caching")] let value = unsafe { &mut *ptr::from_mut(value) }; value } + + /// Fetches the root collection. + pub fn fetch() -> Option { + employ_root_guard(); + + let inner = Interface::root().unwrap()?; + + Some(Self { + inner, + dirty: false, + value: RefCell::new(None), + }) + } + + /// Commits the root collection. + pub fn commit(mut self) { + let _ignored = ROOT.with(|root| root.borrow_mut().take()); + + if self.dirty { + if let Some(value) = self.value.into_inner() { + if let Some(mut entry) = self.inner.get_mut(Self::entry_id()).unwrap() { + *entry = value; + } + } + } + + Interface::commit_root(self.inner).unwrap(); + } +} + +impl Deref for Root +where + T: BorshSerialize + BorshDeserialize, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + self.get() + } +} + +impl DerefMut for Root +where + T: BorshSerialize + BorshDeserialize, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + self.dirty = true; + + self.get() + } } diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index d122baf3c..005a5a057 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -202,9 +202,7 @@ mod tests { #[test] fn test_unordered_map_basic_operations() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key".to_string(), "value".to_string()) @@ -253,9 +251,7 @@ mod tests { #[test] fn test_unordered_map_insert_and_get() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key1".to_string(), "value1".to_string()) @@ -278,9 +274,7 @@ mod tests { #[test] fn test_unordered_map_update_value() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key".to_string(), "value".to_string()) @@ -299,9 +293,7 @@ mod tests { #[test] fn test_remove() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key".to_string(), "value".to_string()) @@ -317,9 +309,7 @@ mod tests { #[test] fn test_clear() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key1".to_string(), "value1".to_string()) @@ -338,9 +328,7 @@ mod tests { #[test] fn test_unordered_map_len() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert_eq!(map.len().expect("len failed"), 0); @@ -369,9 +357,7 @@ mod tests { #[test] fn test_unordered_map_contains() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key".to_string(), "value".to_string()) @@ -384,9 +370,7 @@ mod tests { #[test] fn test_unordered_map_entries() { - let _root = Root::new(()); - - let mut map = UnorderedMap::::new(); + let mut map = Root::new(|| UnorderedMap::new()); assert!(map .insert("key1".to_string(), "value1".to_string()) diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index cb36a2599..195dcdb1c 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -175,9 +175,7 @@ mod tests { #[test] fn test_unordered_set_operations() { - let _root = Root::new(()); - - let mut set = UnorderedSet::::new(); + let mut set = Root::new(|| UnorderedSet::new()); assert!(set.insert("value1".to_string()).expect("insert failed")); @@ -205,9 +203,7 @@ mod tests { #[test] fn test_unordered_set_len() { - let _root = Root::new(()); - - let mut set = UnorderedSet::::new(); + let mut set = Root::new(|| UnorderedSet::new()); assert!(set.insert("value1".to_string()).expect("insert failed")); assert!(set.insert("value2".to_string()).expect("insert failed")); @@ -222,9 +218,7 @@ mod tests { #[test] fn test_unordered_set_clear() { - let _root = Root::new(()); - - let mut set = UnorderedSet::::new(); + let mut set = Root::new(|| UnorderedSet::new()); assert!(set.insert("value1".to_string()).expect("insert failed")); assert!(set.insert("value2".to_string()).expect("insert failed")); @@ -240,9 +234,7 @@ mod tests { #[test] fn test_unordered_set_entries() { - let _root = Root::new(()); - - let mut set = UnorderedSet::::new(); + let mut set = Root::new(|| UnorderedSet::new()); assert!(set.insert("value1".to_string()).expect("insert failed")); assert!(set.insert("value2".to_string()).expect("insert failed")); diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index c3310e095..701193be9 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -11,6 +11,7 @@ use crate::collections::error::StoreError; /// A vector collection that stores key-value pairs. #[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] pub struct Vector { + // Borrow/ToOwned inner: Collection, } @@ -163,9 +164,8 @@ mod tests { #[test] fn test_vector_push() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let result = vector.push(value.clone()); assert!(result.is_ok()); @@ -174,9 +174,8 @@ mod tests { #[test] fn test_vector_get() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); let retrieved_value = vector.get(0).unwrap(); @@ -185,9 +184,8 @@ mod tests { #[test] fn test_vector_update() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value1 = "test_data1".to_string(); let value2 = "test_data2".to_string(); let _ = vector.push(value1.clone()).unwrap(); @@ -199,9 +197,8 @@ mod tests { #[test] fn test_vector_get_non_existent() { - let _root = Root::new(()); + let vector = Root::new(|| Vector::::new()); - let vector: Vector = Vector::new(); match vector.get(0) { Ok(retrieved_value) => assert_eq!(retrieved_value, None), Err(e) => panic!("Error occurred: {:?}", e), @@ -210,9 +207,8 @@ mod tests { #[test] fn test_vector_pop() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); let popped_value = vector.pop().unwrap(); @@ -222,9 +218,8 @@ mod tests { #[test] fn test_vector_entries() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value1 = "test_data1".to_string(); let value2 = "test_data2".to_string(); let _ = vector.push(value1.clone()).unwrap(); @@ -235,9 +230,8 @@ mod tests { #[test] fn test_vector_contains() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); assert!(vector.contains(&value).unwrap()); @@ -247,9 +241,8 @@ mod tests { #[test] fn test_vector_clear() { - let _root = Root::new(()); + let mut vector = Root::new(|| Vector::new()); - let mut vector: Vector = Vector::new(); let value = "test_data".to_string(); let _ = vector.push(value.clone()).unwrap(); vector.clear().unwrap(); diff --git a/crates/storage/src/env.rs b/crates/storage/src/env.rs index 44e74874f..1423e57a8 100644 --- a/crates/storage/src/env.rs +++ b/crates/storage/src/env.rs @@ -163,7 +163,7 @@ mod mocked { /// Return the context id. pub(super) const fn context_id() -> [u8; 32] { - [0; 32] + [236; 32] } /// Gets the current time. diff --git a/crates/storage/src/integration.rs b/crates/storage/src/integration.rs index b9281b1c2..aa3a0452e 100644 --- a/crates/storage/src/integration.rs +++ b/crates/storage/src/integration.rs @@ -8,9 +8,6 @@ use crate::interface::ComparisonData; #[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[expect(clippy::exhaustive_structs, reason = "Exhaustive")] pub struct Comparison { - /// The type of the entity. - pub type_id: u8, - /// The serialised data of the entity. pub data: Option>, From f4c4ff569af6e0861bd8a47efdc5434327dc8f10 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 18:34:09 +0300 Subject: [PATCH 11/19] add Debug + Eq + Ord for collection types --- .../storage/src/collections/unordered_map.rs | 19 ++++++- .../storage/src/collections/unordered_set.rs | 18 ++++++- crates/storage/src/collections/vector.rs | 49 ++++++++++++++++++- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index 005a5a057..d80cc76ea 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -1,6 +1,7 @@ //! This module provides functionality for the unordered map data structure. use core::borrow::Borrow; +use core::fmt; use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; @@ -12,7 +13,7 @@ use crate::collections::error::StoreError; use crate::entities::Data; /// A map collection that stores key-value pairs. -#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] +#[derive(Clone, BorshSerialize, BorshDeserialize)] pub struct UnorderedMap { inner: Collection<(K, V)>, } @@ -195,6 +196,22 @@ where } } +impl fmt::Debug for UnorderedMap +where + K: fmt::Debug + BorshSerialize + BorshDeserialize, + V: fmt::Debug + BorshSerialize + BorshDeserialize, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if f.alternate() { + f.debug_struct("UnorderedMap") + .field("entries", &self.inner) + .finish() + } else { + f.debug_map().entries(self.entries().unwrap()).finish() + } + } +} + #[cfg(test)] mod tests { use crate::collections::unordered_map::UnorderedMap; diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 195dcdb1c..d64e4b189 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -1,6 +1,7 @@ //! This module provides functionality for the unordered set data structure. use core::borrow::Borrow; +use core::fmt; use borsh::{BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; @@ -11,7 +12,7 @@ use crate::collections::error::StoreError; use crate::entities::Data; /// A set collection that stores unqiue values once. -#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] +#[derive(Clone, BorshSerialize, BorshDeserialize)] pub struct UnorderedSet { inner: Collection, } @@ -169,6 +170,21 @@ where } } +impl fmt::Debug for UnorderedSet +where + V: fmt::Debug + BorshSerialize + BorshDeserialize, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if f.alternate() { + f.debug_struct("UnorderedSet") + .field("entries", &self.inner) + .finish() + } else { + f.debug_set().entries(self.entries().unwrap()).finish() + } + } +} + #[cfg(test)] mod tests { use crate::collections::{Root, UnorderedSet}; diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index 701193be9..4507f674e 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -1,6 +1,7 @@ //! This module provides functionality for the vector data structure. use core::borrow::Borrow; +use core::fmt; use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; @@ -9,7 +10,7 @@ use super::Collection; use crate::collections::error::StoreError; /// A vector collection that stores key-value pairs. -#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)] +#[derive(Clone, BorshSerialize, BorshDeserialize)] pub struct Vector { // Borrow/ToOwned inner: Collection, @@ -157,6 +158,52 @@ where } } +impl Eq for Vector where V: Eq + BorshSerialize + BorshDeserialize {} + +impl PartialEq for Vector +where + V: PartialEq + BorshSerialize + BorshDeserialize, +{ + fn eq(&self, other: &Self) -> bool { + self.entries().unwrap().eq(other.entries().unwrap()) + } +} + +impl Ord for Vector +where + V: Ord + BorshSerialize + BorshDeserialize, +{ + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.entries().unwrap().cmp(other.entries().unwrap()) + } +} + +impl PartialOrd for Vector +where + V: PartialOrd + BorshSerialize + BorshDeserialize, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.entries() + .unwrap() + .partial_cmp(other.entries().unwrap()) + } +} + +impl fmt::Debug for Vector +where + V: fmt::Debug + BorshSerialize + BorshDeserialize, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if f.alternate() { + f.debug_struct("Vector") + .field("entries", &self.inner) + .finish() + } else { + f.debug_list().entries(self.entries().unwrap()).finish() + } + } +} + #[cfg(test)] mod tests { use crate::collections::vector::Vector; From 26958e512368b9c2dd1d77f8e99521b735568a46 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 18:35:32 +0300 Subject: [PATCH 12/19] fix guard --- crates/storage/src/collections/root.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs index 0a7ea7020..036b0b110 100644 --- a/crates/storage/src/collections/root.rs +++ b/crates/storage/src/collections/root.rs @@ -103,10 +103,10 @@ where /// Fetches the root collection. pub fn fetch() -> Option { - employ_root_guard(); - let inner = Interface::root().unwrap()?; + employ_root_guard(); + Some(Self { inner, dirty: false, From 11af29696e11cba74a1a7088247a71008510a3b2 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 15 Nov 2024 18:39:59 +0300 Subject: [PATCH 13/19] clean macro --- crates/sdk/macros/src/logic/method.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/sdk/macros/src/logic/method.rs b/crates/sdk/macros/src/logic/method.rs index b41bd4ad8..7d6fc7eee 100644 --- a/crates/sdk/macros/src/logic/method.rs +++ b/crates/sdk/macros/src/logic/method.rs @@ -115,11 +115,11 @@ impl ToTokens for PublicLogicMethod<'_> { None => ( if init_method { quote_spanned! {name.span()=> - if let Some(mut app) = ::calimero_storage::collections::Root::<#self_>::fetch() { + if ::calimero_storage::collections::Root::<#self_>::fetch().is_some() { ::calimero_sdk::env::panic_str("Cannot initialize over already existing state.") }; - let mut app = + let app = } } else { quote! {} From 7e04ba954a282a6d24cf306662641c597dfa6971 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Thu, 21 Nov 2024 23:05:15 +0300 Subject: [PATCH 14/19] patches, patches --- apps/kv-store/src/__private.rs | 24 +- apps/kv-store/src/lib.rs | 2 +- crates/node/src/lib.rs | 23 +- crates/node/src/sync/state.rs | 18 ++ crates/storage-macros/tests/atomic_unit.rs | 8 +- crates/storage/src/address.rs | 5 +- crates/storage/src/collections.rs | 222 ++++++++++++------ crates/storage/src/collections/root.rs | 100 ++++---- .../storage/src/collections/unordered_map.rs | 43 +++- .../storage/src/collections/unordered_set.rs | 22 +- crates/storage/src/collections/vector.rs | 25 +- crates/storage/src/entities.rs | 22 +- crates/storage/src/index.rs | 41 ++-- crates/storage/src/interface.rs | 171 ++++++++------ crates/storage/src/store.rs | 2 +- crates/storage/src/tests/common.rs | 4 +- crates/storage/src/tests/entities.rs | 8 +- crates/storage/src/tests/index.rs | 24 +- crates/storage/src/tests/interface.rs | 92 ++++---- 19 files changed, 506 insertions(+), 350 deletions(-) diff --git a/apps/kv-store/src/__private.rs b/apps/kv-store/src/__private.rs index 0ddeb6178..0dc76417b 100644 --- a/apps/kv-store/src/__private.rs +++ b/apps/kv-store/src/__private.rs @@ -1,15 +1,20 @@ use calimero_sdk::borsh::from_slice; use calimero_sdk::{app, env}; use calimero_storage::address::Id; +use calimero_storage::collections::Root; use calimero_storage::integration::Comparison; -use calimero_storage::interface::{Action, Interface, StorageError}; +use calimero_storage::interface::{Action, MainInterface, StorageError}; use calimero_storage::sync::{self, SyncArtifact}; use crate::KvStore; #[app::logic] impl KvStore { - pub fn __calimero_sync_next() -> Result<(), StorageError> { + pub fn __calimero_sync_next() { + Self::___calimero_sync_next().expect("fatal: sync failed"); + } + + fn ___calimero_sync_next() -> Result<(), StorageError> { let args = env::input().expect("fatal: missing input"); let artifact = @@ -21,12 +26,13 @@ impl KvStore { let _ignored = match action { Action::Compare { id } => { sync::push_comparison(Comparison { - data: Interface::find_by_id_raw(id)?, - comparison_data: Interface::generate_comparison_data(Some(id))?, + data: MainInterface::find_by_id_raw(id), + comparison_data: MainInterface::generate_comparison_data(Some(id)) + .unwrap(), }); } Action::Add { .. } | Action::Update { .. } | Action::Delete { .. } => { - Interface::apply_action(action)?; + MainInterface::apply_action(action).unwrap(); } }; } @@ -34,8 +40,8 @@ impl KvStore { SyncArtifact::Comparisons(comparisons) => { if comparisons.is_empty() { sync::push_comparison(Comparison { - data: Interface::find_by_id_raw(Id::root())?, - comparison_data: Interface::generate_comparison_data(None)?, + data: MainInterface::find_by_id_raw(Id::root()), + comparison_data: MainInterface::generate_comparison_data(None)?, }); } @@ -44,11 +50,13 @@ impl KvStore { comparison_data, } in comparisons { - Interface::compare_affective(data, comparison_data)?; + MainInterface::compare_affective(data, comparison_data)?; } } } + Root::::commit_headless(); + Ok(()) } } diff --git a/apps/kv-store/src/lib.rs b/apps/kv-store/src/lib.rs index 476727983..3cc210b56 100644 --- a/apps/kv-store/src/lib.rs +++ b/apps/kv-store/src/lib.rs @@ -10,7 +10,7 @@ use calimero_storage::collections::UnorderedMap; mod __private; #[app::state(emits = for<'a> Event<'a>)] -#[derive(Clone, Debug, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] +#[derive(Debug, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] #[borsh(crate = "calimero_sdk::borsh")] pub struct KvStore { items: UnorderedMap, diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 010ad204d..dfae19713 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -343,17 +343,12 @@ impl Node { let shared_key = SharedKey::from_sk(&sender_key); - let artifact = &shared_key + let artifact = shared_key .decrypt(artifact, [0; 12]) .ok_or_eyre("failed to decrypt message")?; let Some(outcome) = self - .execute( - &mut context, - "apply_state_delta", - to_vec(&artifact)?, - author_id, - ) + .execute(&mut context, "__calimero_sync_next", artifact, author_id) .await? else { bail!("application not installed"); @@ -500,11 +495,13 @@ impl Node { })?; } - if let Err(err) = self - .send_state_delta(&context, &outcome, executor_public_key) - .await - { - error!(%err, "Failed to send state delta."); + if !outcome.artifact.is_empty() { + if let Err(err) = self + .send_state_delta(&context, &outcome, executor_public_key) + .await + { + error!(%err, "Failed to send state delta."); + } } Ok(outcome) @@ -539,7 +536,7 @@ impl Node { if outcome.returns.is_ok() { if let Some(root_hash) = outcome.root_hash { - if outcome.artifact.is_empty() { + if outcome.artifact.is_empty() && method != "__calimero_sync_next" { eyre::bail!("context state changed, but no actions were generated, discarding execution outcome to mitigate potential state inconsistency"); } diff --git a/crates/node/src/sync/state.rs b/crates/node/src/sync/state.rs index 1761ca813..967927b3a 100644 --- a/crates/node/src/sync/state.rs +++ b/crates/node/src/sync/state.rs @@ -102,6 +102,13 @@ impl Node { }; if root_hash == context.root_hash { + debug!( + context_id=%context.id, + our_identity=%our_identity, + their_identity=%their_identity, + "Root hashes match, up to date", + ); + return Ok(()); } @@ -206,6 +213,13 @@ impl Node { .await?; if their_root_hash == context.root_hash { + debug!( + context_id=%context.id, + our_identity=%our_identity, + their_identity=%their_identity, + "Root hashes match, up to date", + ); + return Ok(()); } @@ -294,6 +308,10 @@ impl Node { Some(shared_key), ) .await?; + + if outcome.artifact.is_empty() { + break; + } } debug!( diff --git a/crates/storage-macros/tests/atomic_unit.rs b/crates/storage-macros/tests/atomic_unit.rs index 08b0508e8..b15ca4eb2 100644 --- a/crates/storage-macros/tests/atomic_unit.rs +++ b/crates/storage-macros/tests/atomic_unit.rs @@ -29,7 +29,7 @@ use borsh::{to_vec, BorshDeserialize}; use calimero_storage::address::Path; use calimero_storage::entities::{Data, Element}; use calimero_storage::exports::{Digest, Sha256}; -use calimero_storage::interface::Interface; +use calimero_storage::interface::MainInterface; use calimero_storage_macros::AtomicUnit; #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] @@ -161,7 +161,7 @@ mod basics { fn setters__confirm_set_dirty() { let path = Path::new("::root::node").unwrap(); let mut unit = Simple::new(&path); - assert!(Interface::save(&mut unit).unwrap()); + assert!(MainInterface::save(&mut unit).unwrap()); assert!(!unit.element().is_dirty()); assert!(unit.set_name("Test Name".to_owned())); @@ -172,12 +172,12 @@ mod basics { fn setters__confirm_not_set_not_dirty() { let path = Path::new("::root::node").unwrap(); let mut unit = Simple::new(&path); - assert!(Interface::save(&mut unit).unwrap()); + assert!(MainInterface::save(&mut unit).unwrap()); assert!(!unit.element().is_dirty()); assert!(unit.set_name("Test Name".to_owned())); assert!(unit.element().is_dirty()); - assert!(Interface::save(&mut unit).unwrap()); + assert!(MainInterface::save(&mut unit).unwrap()); assert!(!unit.set_name("Test Name".to_owned())); assert!(!unit.element().is_dirty()); } diff --git a/crates/storage/src/address.rs b/crates/storage/src/address.rs index c1516076e..4ad9cc3f2 100644 --- a/crates/storage/src/address.rs +++ b/crates/storage/src/address.rs @@ -13,7 +13,6 @@ use core::fmt::{self, Debug, Display, Formatter}; use std::io::{Error as IoError, ErrorKind as IoErrorKind, Read, Write}; use borsh::{BorshDeserialize, BorshSerialize}; -use calimero_sdk::serde::{Deserialize, Serialize}; use fixedstr::Flexstr; use thiserror::Error as ThisError; @@ -40,15 +39,13 @@ use crate::env::{context_id, random_bytes}; Copy, Clone, Debug, - Deserialize, Eq, PartialEq, Hash, Ord, PartialOrd, - Serialize, + Default, )] -#[serde(crate = "calimero_sdk::serde")] pub struct Id { /// The byte array representation of the ID. bytes: [u8; 32], diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index bc90ed095..9ad41ce7c 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -1,9 +1,12 @@ //! High-level data structures for storage. +use core::fmt; use std::cell::RefCell; +use std::collections::BTreeMap; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::ptr; +use std::sync::LazyLock; use borsh::{BorshDeserialize, BorshSerialize}; use indexmap::IndexSet; @@ -21,52 +24,74 @@ pub mod error; pub use error::StoreError; // fixme! macro expects `calimero_storage` to be in deps +use crate as calimero_storage; use crate::address::{Id, Path}; -use crate::entities::{Data, Element}; +use crate::entities::{ChildInfo, Data, Element}; use crate::interface::{Interface, StorageError}; -use crate::{self as calimero_storage, AtomicUnit, Collection}; +use crate::store::{MainStorage, StorageAdaptor}; +use crate::{AtomicUnit, Collection}; -#[derive(AtomicUnit, BorshSerialize, BorshDeserialize, Clone, Debug)] -struct Collection { - /// The entries in the collection. - #[collection] - entries: Entries, +mod compat { + use std::collections::BTreeMap; - /// The storage element for the map. - #[storage] + use borsh::{BorshDeserialize, BorshSerialize}; + + use crate::entities::{ChildInfo, Collection, Data, Element}; + + /// Thing. + #[derive(Copy, Clone, Debug)] + pub(super) struct RootHandle; + + #[derive(BorshSerialize, BorshDeserialize)] + pub(super) struct RootChildDud; + + impl Data for RootChildDud { + fn collections(&self) -> BTreeMap> { + unimplemented!() + } + + fn element(&self) -> &Element { + unimplemented!() + } + + fn element_mut(&mut self) -> &mut Element { + unimplemented!() + } + } + + impl Collection for RootHandle { + type Child = RootChildDud; + + fn name(&self) -> &str { + "no collection, remove this nonsense" + } + } +} + +use compat::RootHandle; + +#[derive(BorshSerialize, BorshDeserialize)] +struct Collection { storage: Element, - #[skip] #[borsh(skip)] children_ids: RefCell>>, -} -impl Eq for Collection {} + #[borsh(skip)] + _priv: PhantomData<(T, S)>, +} -impl PartialEq for Collection { - fn eq(&self, other: &Self) -> bool { - self.entries() - .unwrap() - .flatten() - .eq(other.entries().unwrap().flatten()) +impl Data for Collection { + fn collections(&self) -> BTreeMap> { + BTreeMap::new() } -} -impl Ord for Collection { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.entries() - .unwrap() - .flatten() - .cmp(other.entries().unwrap().flatten()) + fn element(&self) -> &Element { + &self.storage } -} -impl PartialOrd for Collection { - fn partial_cmp(&self, other: &Self) -> Option { - self.entries() - .unwrap() - .flatten() - .partial_cmp(other.entries().unwrap().flatten()) + fn element_mut(&mut self) -> &mut Element { + &mut self.storage } } @@ -88,39 +113,27 @@ struct Entry { storage: Element, } -#[derive(Debug)] -struct CollectionMut<'a, T: BorshSerialize + BorshDeserialize> { - collection: &'a mut Collection, -} - -#[derive(Debug)] -struct EntryMut<'a, T: BorshSerialize + BorshDeserialize> { - collection: CollectionMut<'a, T>, - entry: Entry, -} - #[expect(unused_qualifications, reason = "AtomicUnit macro is unsanitized")] type StoreResult = std::result::Result; -impl Collection { +static ROOT_ID: LazyLock = LazyLock::new(|| Id::root()); + +impl Collection { /// Creates a new collection. fn new(id: Option) -> Self { let id = id.unwrap_or_else(|| Id::random()); let mut this = Self { - entries: Entries::default(), children_ids: RefCell::new(None), storage: Element::new(&Path::new("::unused").expect("valid path"), Some(id)), + _priv: PhantomData, }; if id.is_root() { - let _ignored = Interface::save(&mut this).expect("save"); + let _ignored = >::save(&mut this).expect("save"); } else { - let root = root::ROOT - .with(|root| root.borrow().clone()) - .expect("no root??"); - - let _ = Interface::add_child_to(root.id, &root, &mut this).expect("add child"); + let _ = + >::add_child_to(*ROOT_ID, &RootHandle, &mut this).expect("add child"); } this @@ -142,14 +155,15 @@ impl Collection { Ok(entry.item) } + #[inline(never)] fn get(&self, id: Id) -> StoreResult> { - let entry = Interface::find_by_id::>(id)?; + let entry = >::find_by_id::>(id)?; Ok(entry.map(|entry| entry.item)) } - fn get_mut(&mut self, id: Id) -> StoreResult>> { - let entry = Interface::find_by_id::>(id)?; + fn get_mut(&mut self, id: Id) -> StoreResult>> { + let entry = >::find_by_id::>(id)?; Ok(entry.map(|entry| EntryMut { collection: CollectionMut::new(self), @@ -165,7 +179,7 @@ impl Collection { &self, ) -> StoreResult> + DoubleEndedIterator + '_> { let iter = self.children_cache()?.iter().copied().map(|child| { - let entry = Interface::find_by_id::>(child)? + let entry = >::find_by_id::>(child)? .ok_or(StoreError::StorageError(StorageError::NotFound(child)))?; Ok(entry.item) @@ -194,7 +208,7 @@ impl Collection { let mut cache = self.children_ids.borrow_mut(); if cache.is_none() { - let children = Interface::child_info_for(self.id(), &self.entries)?; + let children = >::child_info_for(self.id(), &RootHandle)?; let children = children.into_iter().map(|c| c.id()).collect(); @@ -210,9 +224,16 @@ impl Collection { } } -impl EntryMut<'_, T> +#[derive(Debug)] +struct EntryMut<'a, T: BorshSerialize + BorshDeserialize, S: StorageAdaptor> { + collection: CollectionMut<'a, T, S>, + entry: Entry, +} + +impl EntryMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn remove(self) -> StoreResult { let old = self @@ -222,11 +243,8 @@ where self.entry.id(), )))?; - let _ = Interface::remove_child_from( - self.collection.id(), - &self.collection.entries, - self.entry.id(), - )?; + let _ = + >::remove_child_from(self.collection.id(), &RootHandle, self.entry.id())?; let _ = self .collection @@ -237,9 +255,10 @@ where } } -impl Deref for EntryMut<'_, T> +impl Deref for EntryMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { type Target = T; @@ -248,7 +267,7 @@ where } } -impl DerefMut for EntryMut<'_, T> +impl DerefMut for EntryMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, { @@ -257,26 +276,33 @@ where } } -impl Drop for EntryMut<'_, T> +impl Drop for EntryMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn drop(&mut self) { self.entry.element_mut().update(); - let _ignored = Interface::save(&mut self.entry); + let _ignored = >::save(&mut self.entry); } } -impl<'a, T> CollectionMut<'a, T> +#[derive(Debug)] +struct CollectionMut<'a, T: BorshSerialize + BorshDeserialize, S: StorageAdaptor> { + collection: &'a mut Collection, +} + +impl<'a, T, S> CollectionMut<'a, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { - fn new(collection: &'a mut Collection) -> Self { + fn new(collection: &'a mut Collection) -> Self { Self { collection } } fn insert(&mut self, item: &mut Entry) -> StoreResult<()> { - let _ = Interface::add_child_to(self.collection.id(), &self.entries, item)?; + let _ = >::add_child_to(self.collection.id(), &RootHandle, item)?; let _ignored = self.collection.children_cache()?.insert(item.id()); @@ -287,42 +313,82 @@ where let children = self.collection.children_cache()?; for child in children.drain(..) { - let _ = Interface::remove_child_from( - self.collection.id(), - &self.collection.entries, - child, - )?; + let _ = >::remove_child_from(self.collection.id(), &RootHandle, child)?; } Ok(()) } } -impl Deref for CollectionMut<'_, T> +impl Deref for CollectionMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { - type Target = Collection; + type Target = Collection; fn deref(&self) -> &Self::Target { self.collection } } -impl DerefMut for CollectionMut<'_, T> +impl DerefMut for CollectionMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn deref_mut(&mut self) -> &mut Self::Target { self.collection } } -impl Drop for CollectionMut<'_, T> +impl Drop for CollectionMut<'_, T, S> where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn drop(&mut self) { self.collection.element_mut().update(); } } + +impl fmt::Debug for Collection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Collection") + .field("element", &self.storage) + .finish() + } +} + +impl Eq for Collection {} + +impl PartialEq + for Collection +{ + fn eq(&self, other: &Self) -> bool { + let l = self.entries().unwrap().flatten(); + let r = other.entries().unwrap().flatten(); + + l.eq(r) + } +} + +impl Ord for Collection { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let l = self.entries().unwrap().flatten(); + let r = other.entries().unwrap().flatten(); + + l.cmp(r) + } +} + +impl PartialOrd + for Collection +{ + fn partial_cmp(&self, other: &Self) -> Option { + let l = self.entries().unwrap().flatten(); + let r = other.entries().unwrap().flatten(); + + l.partial_cmp(r) + } +} diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs index 036b0b110..f698cb0db 100644 --- a/crates/storage/src/collections/root.rs +++ b/crates/storage/src/collections/root.rs @@ -1,77 +1,56 @@ //! A root collection that stores a single value. +use core::fmt; use std::cell::RefCell; -use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::ptr; -use std::sync::LazyLock; use borsh::{BorshDeserialize, BorshSerialize}; -use super::{Collection, Entry}; +use super::{Collection, ROOT_ID}; use crate::address::Id; -use crate::entities::Data; -use crate::env; use crate::interface::Interface; +use crate::store::{MainStorage, StorageAdaptor}; -/// Thing. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub(super) struct RootHandle { - /// The ID of the root collection. - pub id: Id, - _priv: PhantomData, -} - -impl RootHandle { - fn new(id: Id) -> Self { - Self { - id, - _priv: PhantomData, - } - } +/// A set collection that stores unqiue values once. +pub struct Root { + inner: Collection, + value: RefCell>, + dirty: bool, } -impl crate::entities::Collection for RootHandle { - type Child = T; - - fn name(&self) -> &str { - "RootHandle" +impl fmt::Debug for Root +where + T: BorshSerialize + BorshDeserialize + fmt::Debug, + S: StorageAdaptor, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Root") + .field("inner", &self.inner) + .field("value", &self.value) + .field("dirty", &self.dirty) + .finish() } } -thread_local! { - /// The root collection handle. - pub static ROOT: RefCell>>> = RefCell::new(None); -} - -static ID: LazyLock = LazyLock::new(|| Id::new(env::context_id())); - -/// Prepares the root collection. -fn employ_root_guard() { - let old = ROOT.with(|root| root.borrow_mut().replace(RootHandle::new(*ID))); - - if old.is_some() { - panic!("root collection already defined"); +impl Root +where + T: BorshSerialize + BorshDeserialize, +{ + /// Creates a new root collection with the given value. + pub fn new T>(f: F) -> Self { + Self::new_internal(f) } } -/// A set collection that stores unqiue values once. -#[derive(Debug)] -pub struct Root { - inner: Collection, - value: RefCell>, - dirty: bool, -} - -impl Root +impl Root where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { /// Creates a new root collection with the given value. - pub fn new T>(f: F) -> Self { - employ_root_guard(); - - let mut inner = Collection::new(Some(*ID)); + pub fn new_internal T>(f: F) -> Self { + let mut inner = Collection::new(Some(*ROOT_ID)); let id = Self::entry_id(); @@ -103,9 +82,7 @@ where /// Fetches the root collection. pub fn fetch() -> Option { - let inner = Interface::root().unwrap()?; - - employ_root_guard(); + let inner = >::root().unwrap()?; Some(Self { inner, @@ -116,8 +93,6 @@ where /// Commits the root collection. pub fn commit(mut self) { - let _ignored = ROOT.with(|root| root.borrow_mut().take()); - if self.dirty { if let Some(value) = self.value.into_inner() { if let Some(mut entry) = self.inner.get_mut(Self::entry_id()).unwrap() { @@ -126,13 +101,19 @@ where } } - Interface::commit_root(self.inner).unwrap(); + >::commit_root(Some(self.inner)).unwrap(); + } + + /// Commits the root collection without an instance of the root state. + pub fn commit_headless() { + >::commit_root::>(None).unwrap(); } } -impl Deref for Root +impl Deref for Root where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { type Target = T; @@ -141,9 +122,10 @@ where } } -impl DerefMut for Root +impl DerefMut for Root where T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn deref_mut(&mut self) -> &mut Self::Target { self.dirty = true; diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index d80cc76ea..c0ce7a78c 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -7,25 +7,38 @@ use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; -use super::Collection; +use super::{Collection, StorageAdaptor}; use crate::address::Id; use crate::collections::error::StoreError; use crate::entities::Data; +use crate::store::MainStorage; /// A map collection that stores key-value pairs. -#[derive(Clone, BorshSerialize, BorshDeserialize)] -pub struct UnorderedMap { - inner: Collection<(K, V)>, +#[derive(BorshSerialize, BorshDeserialize)] +pub struct UnorderedMap { + #[borsh(bound(serialize = "", deserialize = ""))] + inner: Collection<(K, V), S>, } -impl UnorderedMap +impl UnorderedMap where K: BorshSerialize + BorshDeserialize, V: BorshSerialize + BorshDeserialize, { /// Create a new map collection. - /// pub fn new() -> Self { + Self::new_internal() + } +} + +impl UnorderedMap +where + K: BorshSerialize + BorshDeserialize, + V: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, +{ + /// Create a new map collection. + fn new_internal() -> Self { Self { inner: Collection::new(None), } @@ -157,37 +170,41 @@ where } } -impl Eq for UnorderedMap +impl Eq for UnorderedMap where K: Eq + BorshSerialize + BorshDeserialize, V: Eq + BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { } -impl PartialEq for UnorderedMap +impl PartialEq for UnorderedMap where K: PartialEq + BorshSerialize + BorshDeserialize, V: PartialEq + BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn eq(&self, other: &Self) -> bool { self.entries().unwrap().eq(other.entries().unwrap()) } } -impl Ord for UnorderedMap +impl Ord for UnorderedMap where K: Ord + BorshSerialize + BorshDeserialize, V: Ord + BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.entries().unwrap().cmp(other.entries().unwrap()) } } -impl PartialOrd for UnorderedMap +impl PartialOrd for UnorderedMap where K: PartialOrd + BorshSerialize + BorshDeserialize, V: PartialOrd + BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn partial_cmp(&self, other: &Self) -> Option { self.entries() @@ -196,10 +213,11 @@ where } } -impl fmt::Debug for UnorderedMap +impl fmt::Debug for UnorderedMap where K: fmt::Debug + BorshSerialize + BorshDeserialize, V: fmt::Debug + BorshSerialize + BorshDeserialize, + S: StorageAdaptor, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if f.alternate() { @@ -214,8 +232,7 @@ where #[cfg(test)] mod tests { - use crate::collections::unordered_map::UnorderedMap; - use crate::collections::Root; + use crate::collections::{Root, UnorderedMap}; #[test] fn test_unordered_map_basic_operations() { diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index d64e4b189..5bd135cdd 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -10,20 +10,32 @@ use super::Collection; use crate::address::Id; use crate::collections::error::StoreError; use crate::entities::Data; +use crate::store::{MainStorage, StorageAdaptor}; /// A set collection that stores unqiue values once. -#[derive(Clone, BorshSerialize, BorshDeserialize)] -pub struct UnorderedSet { - inner: Collection, +#[derive(BorshSerialize, BorshDeserialize)] +pub struct UnorderedSet { + #[borsh(bound(serialize = "", deserialize = ""))] + inner: Collection, } -impl UnorderedSet +impl UnorderedSet where V: BorshSerialize + BorshDeserialize, { /// Create a new set collection. - /// pub fn new() -> Self { + Self::new_internal() + } +} + +impl UnorderedSet +where + V: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, +{ + /// Create a new set collection. + fn new_internal() -> Self { Self { inner: Collection::new(None), } diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index 4507f674e..9582e115a 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -8,21 +8,33 @@ use borsh::{BorshDeserialize, BorshSerialize}; use super::Collection; use crate::collections::error::StoreError; +use crate::store::{MainStorage, StorageAdaptor}; /// A vector collection that stores key-value pairs. -#[derive(Clone, BorshSerialize, BorshDeserialize)] -pub struct Vector { +#[derive(BorshSerialize, BorshDeserialize)] +pub struct Vector { // Borrow/ToOwned - inner: Collection, + #[borsh(bound(serialize = "", deserialize = ""))] + inner: Collection, } -impl Vector +impl Vector where V: BorshSerialize + BorshDeserialize, { /// Create a new vector collection. - /// pub fn new() -> Self { + Self::new_internal() + } +} + +impl Vector +where + V: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, +{ + /// Create a new vector collection. + fn new_internal() -> Self { Self { inner: Collection::new(None), } @@ -206,8 +218,7 @@ where #[cfg(test)] mod tests { - use crate::collections::vector::Vector; - use crate::collections::Root; + use crate::collections::{Root, Vector}; #[test] fn test_vector_push() { diff --git a/crates/storage/src/entities.rs b/crates/storage/src/entities.rs index 211ba676e..dec94aab7 100644 --- a/crates/storage/src/entities.rs +++ b/crates/storage/src/entities.rs @@ -383,9 +383,7 @@ pub trait Data: BorshDeserialize + BorshSerialize { /// purpose is to make information such as the Merkle hash trivially available /// and prevent the need for repeated lookups. /// -#[derive( - BorshDeserialize, BorshSerialize, Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, -)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Copy, Debug, Eq, Hash, PartialEq)] #[non_exhaustive] pub struct ChildInfo { /// The unique identifier for the child [`Element`]. @@ -399,6 +397,20 @@ pub struct ChildInfo { pub(crate) metadata: Metadata, } +impl Ord for ChildInfo { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.created_at() + .cmp(&other.created_at()) + .then_with(|| self.id.cmp(&other.id)) + } +} + +impl PartialOrd for ChildInfo { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + impl ChildInfo { /// Creates a new [`ChildInfo`]. #[must_use] @@ -535,7 +547,7 @@ impl Display for ChildInfo { #[non_exhaustive] pub struct Element { /// The unique identifier for the [`Element`]. - id: Id, + pub(crate) id: Id, /// Whether the [`Element`] is dirty, i.e. has been modified since it was /// last saved. @@ -551,10 +563,10 @@ pub struct Element { #[borsh(skip)] pub(crate) merkle_hash: [u8; 32], - #[borsh(skip)] /// The metadata for the [`Element`]. This represents a range of /// system-managed properties that are used to process the [`Element`], but /// are not part of the primary data. + #[borsh(skip)] pub(crate) metadata: Metadata, /// The path to the [`Element`] in the hierarchy of the storage. diff --git a/crates/storage/src/index.rs b/crates/storage/src/index.rs index 548d33698..9fb446e15 100644 --- a/crates/storage/src/index.rs +++ b/crates/storage/src/index.rs @@ -5,14 +5,13 @@ mod tests; use core::marker::PhantomData; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use borsh::{to_vec, BorshDeserialize, BorshSerialize}; use sha2::{Digest, Sha256}; use crate::address::Id; -use crate::entities::{ChildInfo, Metadata}; -use crate::env; +use crate::entities::{ChildInfo, Metadata, UpdatedAt}; use crate::interface::StorageError; use crate::store::{Key, StorageAdaptor}; @@ -89,15 +88,21 @@ impl Index { child_index.full_hash = Self::calculate_full_merkle_hash_for(child.id(), false)?; Self::save_index(&child_index)?; - parent_index + let children = parent_index .children .entry(collection.to_owned()) - .or_insert_with(Vec::new) - .push(ChildInfo::new( - child.id(), - child_index.full_hash, - child.metadata, - )); + .or_insert_with(Vec::new); + + let mut ordered = children.drain(..).collect::>(); + + let _ignored = ordered.replace(ChildInfo::new( + child.id(), + child_index.full_hash, + child.metadata, + )); + + children.extend(ordered.into_iter()); + Self::save_index(&parent_index)?; parent_index.full_hash = Self::calculate_full_merkle_hash_for(parent_id, false)?; Self::save_index(&parent_index)?; @@ -284,9 +289,8 @@ impl Index { /// pub(crate) fn get_collection_names_for(parent_id: Id) -> Result, StorageError> { Ok(Self::get_index(parent_id)? - .ok_or(StorageError::IndexNotFound(parent_id))? - .children - .keys() + .iter() + .flat_map(|e| e.children.keys()) .cloned() .collect()) } @@ -498,13 +502,20 @@ impl Index { /// If there's an issue updating or saving the index, an error will be /// returned. /// - pub(crate) fn update_hash_for(id: Id, merkle_hash: [u8; 32]) -> Result<[u8; 32], StorageError> { + pub(crate) fn update_hash_for( + id: Id, + merkle_hash: [u8; 32], + updated_at: Option, + ) -> Result<[u8; 32], StorageError> { let mut index = Self::get_index(id)?.ok_or(StorageError::IndexNotFound(id))?; index.own_hash = merkle_hash; Self::save_index(&index)?; index.full_hash = Self::calculate_full_merkle_hash_for(id, false)?; - index.metadata.updated_at = env::time_now().into(); + if let Some(updated_at) = updated_at { + index.metadata.updated_at = updated_at; + } Self::save_index(&index)?; + >::recalculate_ancestor_hashes_for(id)?; Ok(index.full_hash) } } diff --git a/crates/storage/src/interface.rs b/crates/storage/src/interface.rs index f75ad708d..bce5202d4 100644 --- a/crates/storage/src/interface.rs +++ b/crates/storage/src/interface.rs @@ -232,7 +232,7 @@ use crate::store::{Key, MainStorage, StorageAdaptor}; use crate::sync; /// Convenient type alias for the main storage system. -pub type Interface = MainInterface; +pub type MainInterface = Interface; /// Actions to be taken during synchronisation. /// @@ -343,12 +343,10 @@ pub struct ComparisonData { /// The primary interface for the storage system. #[derive(Debug, Default, Clone)] -#[expect(private_bounds, reason = "The StorageAdaptor is for internal use only")] #[non_exhaustive] -pub struct MainInterface(PhantomData); +pub struct Interface(PhantomData); -#[expect(private_bounds, reason = "The StorageAdaptor is for internal use only")] -impl MainInterface { +impl Interface { /// Adds a child to a collection. /// /// # Parameters @@ -420,11 +418,12 @@ impl MainInterface { /// If there is an error when deserialising into the specified type, or when /// applying the [`Action`], an error will be returned. /// - pub fn apply_action(action: Action) -> Result, StorageError> { - let ancestors = match action { + pub fn apply_action(action: Action) -> Result<(), StorageError> { + match action { Action::Add { id, data, + // todo! we only need parent_id ancestors, metadata, } @@ -434,26 +433,35 @@ impl MainInterface { ancestors, metadata, } => { - _ = Self::save_raw(id, data, metadata)?; - ancestors + if let Some(parent) = ancestors.first() { + let own_hash = Sha256::digest(&data).into(); + + >::add_child_to( + parent.id(), + "no collection, remove this nonsense", + ChildInfo::new(id, own_hash, metadata), + )?; + } + + if Self::save_internal(id, &data, metadata)?.is_none() { + // we didn't save anything, so we skip updating the ancestors + return Ok(()); + } + + sync::push_action(Action::Compare { id }); } Action::Compare { .. } => { return Err(StorageError::ActionNotAllowed("Compare".to_owned())) } - Action::Delete { id, ancestors, .. } => { - _ = S::storage_remove(Key::Entry(id)); - ancestors + Action::Delete { + id, ancestors: _, .. + } => { + // todo! remove_child_from here + let _ignored = S::storage_remove(Key::Entry(id)); } }; - for ancestor in &ancestors { - let (current_hash, _) = >::get_hashes_for(ancestor.id())? - .ok_or(StorageError::IndexNotFound(ancestor.id()))?; - if current_hash != ancestor.merkle_hash() { - return Ok(Some(ancestor.id())); - } - } - Ok(None) + Ok(()) } /// The children of the [`Collection`]. @@ -573,7 +581,7 @@ impl MainInterface { /// pub fn compare_trees( foreign_entity_data: Option>, - foreign_index_data: &ComparisonData, + foreign_index_data: ComparisonData, ) -> Result<(Vec, Vec), StorageError> { let mut actions = (vec![], vec![]); @@ -581,13 +589,13 @@ impl MainInterface { let local_metadata = >::get_metadata(id)?; - let Some(local_entity) = Self::find_by_id_raw(id)? else { + let Some(local_entity) = Self::find_by_id_raw(id) else { if let Some(foreign_entity) = foreign_entity_data { // Local entity doesn't exist, so we need to add it actions.0.push(Action::Add { id, data: foreign_entity, - ancestors: foreign_index_data.ancestors.clone(), + ancestors: foreign_index_data.ancestors, metadata: foreign_index_data.metadata, }); } @@ -614,7 +622,7 @@ impl MainInterface { actions.0.push(Action::Update { id, data: foreign_entity_data, - ancestors: foreign_index_data.ancestors.clone(), + ancestors: foreign_index_data.ancestors, metadata: foreign_index_data.metadata, }); } @@ -635,10 +643,10 @@ impl MainInterface { let local_collection_names = >::get_collection_names_for(id)?; let local_collections = local_collection_names - .iter() + .into_iter() .map(|name| { - let children = >::get_children_of(id, name)?; - Ok((name.clone(), children)) + let children = >::get_children_of(id, &name)?; + Ok((name, children)) }) .collect::, StorageError>>()?; @@ -661,7 +669,7 @@ impl MainInterface { actions.1.push(Action::Compare { id: *child_id }); } None => { - if let Some(local_child) = Self::find_by_id_raw(*child_id)? { + if let Some(local_child) = Self::find_by_id_raw(*child_id) { let metadata = >::get_metadata(*child_id)? .ok_or(StorageError::IndexNotFound(*child_id))?; @@ -689,7 +697,7 @@ impl MainInterface { } else { // The entire collection is missing from the foreign entity for child in local_children { - if let Some(local_child) = Self::find_by_id_raw(child.id())? { + if let Some(local_child) = Self::find_by_id_raw(child.id()) { let metadata = >::get_metadata(child.id())? .ok_or(StorageError::IndexNotFound(child.id()))?; @@ -729,10 +737,14 @@ impl MainInterface { data: Option>, comparison_data: ComparisonData, ) -> Result<(), StorageError> { - let (local, remote) = Interface::compare_trees(data, &comparison_data)?; + let (local, remote) = >::compare_trees(data, comparison_data)?; for action in local { - let _ignored = Interface::apply_action(action)?; + if let Action::Compare { .. } = &action { + continue; + } + + >::apply_action(action)?; } for action in remote { @@ -793,13 +805,8 @@ impl MainInterface { /// * `id` - The unique identifier of the [`Element`](crate::entities::Element) /// to find. /// - /// # Errors - /// - /// If an error occurs when interacting with the storage system, an error - /// will be returned. - /// - pub fn find_by_id_raw(id: Id) -> Result>, StorageError> { - Ok(S::storage_read(Key::Entry(id))) + pub fn find_by_id_raw(id: Id) -> Option> { + S::storage_read(Key::Entry(id)) } /// Finds one or more [`Element`](crate::entities::Element)s by path in the @@ -874,21 +881,11 @@ impl MainInterface { /// will be returned. /// pub fn generate_comparison_data(id: Option) -> Result { - let Some(id) = id else { - return Ok(ComparisonData { - id: Id::root(), - own_hash: [0; 32], - full_hash: [0; 32], - ancestors: Vec::new(), - children: BTreeMap::new(), - metadata: Metadata::default(), - }); - }; + let id = id.unwrap_or_else(Id::root); - let (full_hash, own_hash) = - >::get_hashes_for(id)?.ok_or(StorageError::IndexNotFound(id))?; + let (full_hash, own_hash) = >::get_hashes_for(id)?.unwrap_or_default(); - let metadata = >::get_metadata(id)?.ok_or(StorageError::IndexNotFound(id))?; + let metadata = >::get_metadata(id)?.unwrap_or_default(); let ancestors = >::get_ancestors_of(id)?; @@ -1022,21 +1019,28 @@ impl MainInterface { /// This function will return an error if there are issues accessing local /// data or if there are problems during the comparison process. /// - pub fn commit_root(root: D) -> Result<(), StorageError> { - if root.id() != Id::root() { - return Err(StorageError::UnexpectedId(root.id())); - } + pub fn commit_root(root: Option) -> Result<(), StorageError> { + let id: Id = Id::root(); - if !root.element().is_dirty() { - return Ok(()); - } + let hash = if let Some(root) = root { + if root.id() != id { + return Err(StorageError::UnexpectedId(root.id())); + } - let data = to_vec(&root).map_err(|e| StorageError::SerializationError(e.into()))?; + if !root.element().is_dirty() { + return Ok(()); + } - let hash = Self::save_raw(root.id(), data, root.element().metadata)? - .expect("root should always be successfully committed"); + let data = to_vec(&root).map_err(|e| StorageError::SerializationError(e.into()))?; - sync::commit_root(&hash)?; + Self::save_raw(id, data, root.element().metadata)? + } else { + >::get_hashes_for(id)?.map(|(full_hash, _)| full_hash) + }; + + if let Some(hash) = hash { + sync::commit_root(&hash)?; + } Ok(()) } @@ -1127,19 +1131,13 @@ impl MainInterface { /// If an error occurs when serialising data or interacting with the storage /// system, an error will be returned. /// - pub fn save_raw( + fn save_internal( id: Id, - data: Vec, + data: &[u8], metadata: Metadata, - ) -> Result, StorageError> { - if !id.is_root() && >::get_parent_id(id)?.is_none() { - return Err(StorageError::CannotCreateOrphan(id)); - } - + ) -> Result, StorageError> { let last_metadata = >::get_metadata(id)?; - let is_new = last_metadata.is_none(); - if let Some(last_metadata) = &last_metadata { if last_metadata.updated_at > metadata.updated_at { return Ok(None); @@ -1148,11 +1146,36 @@ impl MainInterface { >::add_root(ChildInfo::new(id, [0_u8; 32], metadata))?; } - let own_hash = Sha256::digest(&data).into(); + let own_hash = Sha256::digest(data).into(); + + let full_hash = >::update_hash_for(id, own_hash, Some(metadata.updated_at))?; - let full_hash = >::update_hash_for(id, own_hash)?; + _ = S::storage_write(Key::Entry(id), data); - _ = S::storage_write(Key::Entry(id), &data); + let is_new = metadata.created_at == *metadata.updated_at; + + Ok(Some((is_new, full_hash))) + } + + /// Saves raw data to the storage system. + /// + /// # Errors + /// + /// If an error occurs when serialising data or interacting with the storage + /// system, an error will be returned. + /// + pub fn save_raw( + id: Id, + data: Vec, + metadata: Metadata, + ) -> Result, StorageError> { + if !id.is_root() && >::get_parent_id(id)?.is_none() { + return Err(StorageError::CannotCreateOrphan(id)); + } + + let Some((is_new, full_hash)) = Self::save_internal(id, &data, metadata)? else { + return Ok(None); + }; let ancestors = >::get_ancestors_of(id)?; diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index bcc839472..e54ed5149 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -41,7 +41,7 @@ impl Key { /// used for key operations during testing, such as modelling a foreign node's /// data store. /// -pub(crate) trait StorageAdaptor { +pub trait StorageAdaptor { /// Reads data from persistent storage. /// /// # Parameters diff --git a/crates/storage/src/tests/common.rs b/crates/storage/src/tests/common.rs index 7f6feba2b..3d5adc46d 100644 --- a/crates/storage/src/tests/common.rs +++ b/crates/storage/src/tests/common.rs @@ -4,7 +4,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use velcro::btree_map; use crate::entities::{AtomicUnit, ChildInfo, Collection, Data, Element}; -use crate::interface::Interface; +use crate::interface::MainInterface; /// For tests against empty data structs. #[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, PartialEq, PartialOrd)] @@ -50,7 +50,7 @@ impl AtomicUnit for Page {} impl Data for Page { fn collections(&self) -> BTreeMap> { btree_map! { - "Paragraphs".to_owned(): Interface::child_info_for(self.id(), &self.paragraphs).unwrap_or_default(), + "Paragraphs".to_owned(): MainInterface::child_info_for(self.id(), &self.paragraphs).unwrap_or_default(), } } diff --git a/crates/storage/src/tests/entities.rs b/crates/storage/src/tests/entities.rs index 1dd23cc9b..63d094364 100644 --- a/crates/storage/src/tests/entities.rs +++ b/crates/storage/src/tests/entities.rs @@ -5,7 +5,7 @@ use sha2::{Digest, Sha256}; use velcro::btree_map; use super::*; -use crate::interface::Interface; +use crate::interface::MainInterface; use crate::tests::common::{Page, Paragraph, Paragraphs, Person}; #[cfg(test)] @@ -30,7 +30,7 @@ mod data__public_methods { assert_eq!( page.collections(), btree_map! { - "Paragraphs".to_owned(): Interface::child_info_for(page.id(), &page.paragraphs).unwrap_or_default(), + "Paragraphs".to_owned(): MainInterface::child_info_for(page.id(), &page.paragraphs).unwrap_or_default(), } ); @@ -223,7 +223,7 @@ mod element__public_methods { age: 30, storage: element, }; - assert!(Interface::save(&mut person).unwrap()); + assert!(MainInterface::save(&mut person).unwrap()); assert!(!person.element().is_dirty()); person.element_mut().update(); @@ -252,7 +252,7 @@ mod element__public_methods { age: 40, storage: element, }; - assert!(Interface::save(&mut person).unwrap()); + assert!(MainInterface::save(&mut person).unwrap()); assert!(!person.element().is_dirty); person.element_mut().update(); diff --git a/crates/storage/src/tests/index.rs b/crates/storage/src/tests/index.rs index c3dd39ccf..92b9292b8 100644 --- a/crates/storage/src/tests/index.rs +++ b/crates/storage/src/tests/index.rs @@ -170,7 +170,7 @@ mod index__public_methods { .is_ok()); let collection_name = "Books"; - let child1_id = Id::random(); + let child1_id = Id::from([2; 32]); let child1_own_hash = [2_u8; 32]; let child1_full_hash: [u8; 32] = hex::decode("75877bb41d393b5fb8455ce60ecd8dda001d06316496b14dfa7f895656eeca4a") @@ -178,7 +178,7 @@ mod index__public_methods { .try_into() .unwrap(); - let child2_id = Id::random(); + let child2_id = Id::from([3; 32]); let child2_own_hash = [3_u8; 32]; let child2_full_hash: [u8; 32] = hex::decode("648aa5c579fb30f38af744d97d6ec840c7a91277a499a0d780f3e7314eca090b") @@ -213,7 +213,7 @@ mod index__public_methods { #[test] fn get_children_of__two_collections() { - let root_id = Id::random(); + let root_id = Id::from([1; 32]); let root_hash = [1_u8; 32]; assert!(>::add_root(ChildInfo::new( @@ -224,14 +224,14 @@ mod index__public_methods { .is_ok()); let collection1_name = "Pages"; - let child1_id = Id::random(); + let child1_id = Id::from([2; 32]); let child1_own_hash = [2_u8; 32]; let child1_full_hash: [u8; 32] = hex::decode("75877bb41d393b5fb8455ce60ecd8dda001d06316496b14dfa7f895656eeca4a") .unwrap() .try_into() .unwrap(); - let child2_id = Id::random(); + let child2_id = Id::from([3; 32]); let child2_own_hash = [3_u8; 32]; let child2_full_hash: [u8; 32] = hex::decode("648aa5c579fb30f38af744d97d6ec840c7a91277a499a0d780f3e7314eca090b") @@ -240,7 +240,7 @@ mod index__public_methods { .unwrap(); let collection2_name = "Reviews"; - let child3_id = Id::random(); + let child3_id = Id::from([4; 32]); let child3_own_hash = [4_u8; 32]; let child3_full_hash: [u8; 32] = hex::decode("9f4fb68f3e1dac82202f9aa581ce0bbf1f765df0e9ac3c8c57e20f685abab8ed") @@ -498,7 +498,7 @@ mod hashing { #[test] fn calculate_full_merkle_hash_for__with_children() { - let root_id = Id::random(); + let root_id = Id::from([0; 32]); assert!(>::add_root(ChildInfo::new( root_id, [0_u8; 32], @@ -507,15 +507,15 @@ mod hashing { .is_ok()); let collection_name = "Children"; - let child1_id = Id::random(); + let child1_id = Id::from([1; 32]); let child1_hash = [1_u8; 32]; let child1_info = ChildInfo::new(child1_id, child1_hash, Metadata::default()); assert!(>::add_child_to(root_id, collection_name, child1_info).is_ok()); - let child2_id = Id::random(); + let child2_id = Id::from([2; 32]); let child2_hash = [2_u8; 32]; let child2_info = ChildInfo::new(child2_id, child2_hash, Metadata::default()); assert!(>::add_child_to(root_id, collection_name, child2_info).is_ok()); - let child3_id = Id::random(); + let child3_id = Id::from([3; 32]); let child3_hash = [3_u8; 32]; let child3_info = ChildInfo::new(child3_id, child3_hash, Metadata::default()); assert!(>::add_child_to(root_id, collection_name, child3_info).is_ok()); @@ -750,7 +750,7 @@ mod hashing { assert_eq!(root_index.id, root_id); assert_eq!(root_index.full_hash, root_hash0); - assert!(>::update_hash_for(root_id, root_hash2).is_ok()); + assert!(>::update_hash_for(root_id, root_hash2, None).is_ok()); let updated_root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(updated_root_index.id, root_id); assert_eq!(updated_root_index.full_hash, root_full_hash); @@ -773,7 +773,7 @@ mod hashing { assert_eq!(root_index.id, root_id); assert_eq!(root_index.own_hash, root_hash1); - assert!(>::update_hash_for(root_id, root_hash2).is_ok()); + assert!(>::update_hash_for(root_id, root_hash2, None).is_ok()); let updated_root_index = >::get_index(root_id).unwrap().unwrap(); assert_eq!(updated_root_index.id, root_id); assert_eq!(updated_root_index.own_hash, root_hash2); diff --git a/crates/storage/src/tests/interface.rs b/crates/storage/src/tests/interface.rs index 860ad9804..219cb8ade 100644 --- a/crates/storage/src/tests/interface.rs +++ b/crates/storage/src/tests/interface.rs @@ -16,9 +16,9 @@ mod interface__public_methods { fn children_of() { let element = Element::root(); let mut page = Page::new_from_element("Node", element); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); assert_eq!( - Interface::children_of(page.id(), &page.paragraphs).unwrap(), + MainInterface::children_of(page.id(), &page.paragraphs).unwrap(), vec![] ); @@ -29,13 +29,13 @@ mod interface__public_methods { let mut para2 = Paragraph::new_from_element("Leaf2", child2); let mut para3 = Paragraph::new_from_element("Leaf3", child3); - assert!(!Interface::save(&mut page).unwrap()); + assert!(!MainInterface::save(&mut page).unwrap()); - assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para1).unwrap()); - assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para2).unwrap()); - assert!(Interface::add_child_to(page.id(), &mut page.paragraphs, &mut para3).unwrap()); + assert!(MainInterface::add_child_to(page.id(), &mut page.paragraphs, &mut para1).unwrap()); + assert!(MainInterface::add_child_to(page.id(), &mut page.paragraphs, &mut para2).unwrap()); + assert!(MainInterface::add_child_to(page.id(), &mut page.paragraphs, &mut para3).unwrap()); assert_eq!( - Interface::children_of(page.id(), &page.paragraphs).unwrap(), + MainInterface::children_of(page.id(), &page.paragraphs).unwrap(), vec![para1, para2, para3] ); } @@ -45,14 +45,14 @@ mod interface__public_methods { let element = Element::root(); let mut page = Page::new_from_element("Leaf", element); let id = page.id(); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); - assert_eq!(Interface::find_by_id(id).unwrap(), Some(page)); + assert_eq!(MainInterface::find_by_id(id).unwrap(), Some(page)); } #[test] fn find_by_id__non_existent() { - assert_none!(Interface::find_by_id::(Id::random()).unwrap()); + assert_none!(MainInterface::find_by_id::(Id::random()).unwrap()); } #[test] @@ -78,7 +78,7 @@ mod interface__public_methods { let element = Element::root(); let mut page = Page::new_from_element("Node", element); - assert_ok!(Interface::save(&mut page)); + assert_ok!(MainInterface::save(&mut page)); } #[test] @@ -86,9 +86,9 @@ mod interface__public_methods { let element = Element::root(); let mut page = Page::new_from_element("Node", element); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); page.element_mut().update(); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); } #[test] @@ -97,12 +97,12 @@ mod interface__public_methods { let mut page1 = Page::new_from_element("Node", element1); let mut page2 = page1.clone(); - assert!(Interface::save(&mut page1).unwrap()); + assert!(MainInterface::save(&mut page1).unwrap()); page2.element_mut().update(); sleep(Duration::from_millis(2)); page1.element_mut().update(); - assert!(Interface::save(&mut page1).unwrap()); - assert!(!Interface::save(&mut page2).unwrap()); + assert!(MainInterface::save(&mut page1).unwrap()); + assert!(!MainInterface::save(&mut page2).unwrap()); } #[test] @@ -110,14 +110,14 @@ mod interface__public_methods { let element = Element::root(); let mut page = Page::new_from_element("Node", element); let id = page.id(); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); page.storage.update(); // TODO: Modify the element's data and check it changed - assert!(Interface::save(&mut page).unwrap()); - assert_eq!(Interface::find_by_id(id).unwrap(), Some(page)); + assert!(MainInterface::save(&mut page).unwrap()); + assert_eq!(MainInterface::find_by_id(id).unwrap(), Some(page)); } #[test] @@ -163,10 +163,10 @@ mod interface__apply_actions { metadata: page.element().metadata, }; - assert!(Interface::apply_action(action).is_ok()); + assert!(MainInterface::apply_action(action).is_ok()); // Verify the page was added - let retrieved_page = Interface::find_by_id::(page.id()).unwrap(); + let retrieved_page = MainInterface::find_by_id::(page.id()).unwrap(); assert!(retrieved_page.is_some()); assert_eq!(retrieved_page.unwrap().title, "Test Page"); } @@ -174,7 +174,7 @@ mod interface__apply_actions { #[test] fn apply_action__update() { let mut page = Page::new_from_element("Old Title", Element::root()); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); page.title = "New Title".to_owned(); page.element_mut().update(); @@ -186,27 +186,29 @@ mod interface__apply_actions { metadata: page.element().metadata, }; - assert!(Interface::apply_action(action).is_ok()); + assert!(MainInterface::apply_action(action).is_ok()); // Verify the page was updated - let retrieved_page = Interface::find_by_id::(page.id()).unwrap().unwrap(); + let retrieved_page = MainInterface::find_by_id::(page.id()) + .unwrap() + .unwrap(); assert_eq!(retrieved_page.title, "New Title"); } #[test] fn apply_action__delete() { let mut page = Page::new_from_element("Test Page", Element::root()); - assert!(Interface::save(&mut page).unwrap()); + assert!(MainInterface::save(&mut page).unwrap()); let action = Action::Delete { id: page.id(), ancestors: vec![], }; - assert!(Interface::apply_action(action).is_ok()); + assert!(MainInterface::apply_action(action).is_ok()); // Verify the page was deleted - let retrieved_page = Interface::find_by_id::(page.id()).unwrap(); + let retrieved_page = MainInterface::find_by_id::(page.id()).unwrap(); assert!(retrieved_page.is_none()); } @@ -216,7 +218,7 @@ mod interface__apply_actions { let action = Action::Compare { id: page.id() }; // Compare should fail - assert!(Interface::apply_action(action).is_err()); + assert!(MainInterface::apply_action(action).is_err()); } #[test] @@ -231,10 +233,10 @@ mod interface__apply_actions { }; // Updating a non-existent page should still succeed (it will be added) - assert!(Interface::apply_action(action).is_ok()); + assert!(MainInterface::apply_action(action).is_ok()); // Verify the page was added - let retrieved_page = Interface::find_by_id::(page.id()).unwrap(); + let retrieved_page = MainInterface::find_by_id::(page.id()).unwrap(); assert!(retrieved_page.is_some()); assert_eq!(retrieved_page.unwrap().title, "Test Page"); } @@ -244,13 +246,13 @@ mod interface__apply_actions { mod interface__comparison { use super::*; - type ForeignInterface = MainInterface>; + type ForeignInterface = Interface>; fn compare_trees( foreign: Option<&D>, - comparison_data: &ComparisonData, + comparison_data: ComparisonData, ) -> Result<(Vec, Vec), StorageError> { - Interface::compare_trees( + MainInterface::compare_trees( foreign .map(to_vec) .transpose() @@ -265,7 +267,7 @@ mod interface__comparison { let mut local = Page::new_from_element("Test Page", element); let mut foreign = local.clone(); - assert!(Interface::save(&mut local).unwrap()); + assert!(MainInterface::save(&mut local).unwrap()); assert!(ForeignInterface::save(&mut foreign).unwrap()); assert_eq!( local.element().merkle_hash(), @@ -274,7 +276,7 @@ mod interface__comparison { let result = compare_trees( Some(&foreign), - &ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), + ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), ) .unwrap(); assert_eq!(result, (vec![], vec![])); @@ -291,11 +293,11 @@ mod interface__comparison { // Make local newer sleep(Duration::from_millis(10)); local.element_mut().update(); - assert!(Interface::save(&mut local).unwrap()); + assert!(MainInterface::save(&mut local).unwrap()); let result = compare_trees( Some(&foreign), - &ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), + ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), ) .unwrap(); assert_eq!( @@ -318,7 +320,7 @@ mod interface__comparison { let mut local = Page::new_from_element("Old Test Page", element.clone()); let mut foreign = Page::new_from_element("Test Page", element); - assert!(Interface::save(&mut local).unwrap()); + assert!(MainInterface::save(&mut local).unwrap()); // Make foreign newer sleep(Duration::from_millis(10)); @@ -327,7 +329,7 @@ mod interface__comparison { let result = compare_trees( Some(&foreign), - &ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), + ForeignInterface::generate_comparison_data(Some(foreign.id())).unwrap(), ) .unwrap(); assert_eq!( @@ -360,14 +362,14 @@ mod interface__comparison { let mut foreign_para1 = Paragraph::new_from_element("Updated Paragraph 1", para1_element); let mut foreign_para3 = Paragraph::new_from_element("Foreign Paragraph 3", para3_element); - assert!(Interface::save(&mut local_page).unwrap()); - assert!(Interface::add_child_to( + assert!(MainInterface::save(&mut local_page).unwrap()); + assert!(MainInterface::add_child_to( local_page.id(), &mut local_page.paragraphs, &mut local_para1 ) .unwrap()); - assert!(Interface::add_child_to( + assert!(MainInterface::add_child_to( local_page.id(), &mut local_page.paragraphs, &mut local_para2 @@ -390,7 +392,7 @@ mod interface__comparison { let (local_actions, foreign_actions) = compare_trees( Some(&foreign_page), - &ForeignInterface::generate_comparison_data(Some(foreign_page.id())).unwrap(), + ForeignInterface::generate_comparison_data(Some(foreign_page.id())).unwrap(), ) .unwrap(); @@ -435,7 +437,7 @@ mod interface__comparison { // Compare the updated para1 let (local_para1_actions, foreign_para1_actions) = compare_trees( Some(&foreign_para1), - &ForeignInterface::generate_comparison_data(Some(foreign_para1.id())).unwrap(), + ForeignInterface::generate_comparison_data(Some(foreign_para1.id())).unwrap(), ) .unwrap(); @@ -470,7 +472,7 @@ mod interface__comparison { // Compare para3 which doesn't exist locally let (local_para3_actions, foreign_para3_actions) = compare_trees( Some(&foreign_para3), - &ForeignInterface::generate_comparison_data(Some(foreign_para3.id())).unwrap(), + ForeignInterface::generate_comparison_data(Some(foreign_para3.id())).unwrap(), ) .unwrap(); From c44d16863cede99614a4f22de05520b718d20545 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 22 Nov 2024 00:15:53 +0300 Subject: [PATCH 15/19] clippy --- apps/visited/src/lib.rs | 18 +++++----------- crates/storage/src/collections.rs | 12 +++++++++-- crates/storage/src/collections/root.rs | 10 +++++++++ .../storage/src/collections/unordered_map.rs | 20 +++++++++++++----- .../storage/src/collections/unordered_set.rs | 20 +++++++++++++----- crates/storage/src/collections/vector.rs | 20 +++++++++++++----- crates/storage/src/entities.rs | 21 +++++-------------- crates/storage/src/index.rs | 3 ++- crates/storage/src/integration.rs | 2 +- crates/storage/src/interface.rs | 4 ++-- crates/storage/src/lib.rs | 7 +------ 11 files changed, 81 insertions(+), 56 deletions(-) diff --git a/apps/visited/src/lib.rs b/apps/visited/src/lib.rs index 563425900..22638befd 100644 --- a/apps/visited/src/lib.rs +++ b/apps/visited/src/lib.rs @@ -3,19 +3,15 @@ use std::result::Result; use calimero_sdk::app; +use calimero_sdk::borsh::{BorshDeserialize, BorshSerialize}; use calimero_sdk::types::Error; use calimero_storage::collections::{UnorderedMap, UnorderedSet}; -use calimero_storage::entities::Element; -use calimero_storage::AtomicUnit; #[app::state] -#[derive(AtomicUnit, Clone, Debug, PartialEq, PartialOrd)] -#[root] -#[type_id(1)] +#[derive(Debug, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] +#[borsh(crate = "calimero_sdk::borsh")] pub struct VisitedCities { visited: UnorderedMap>, - #[storage] - storage: Element, } #[app::logic] @@ -23,16 +19,12 @@ impl VisitedCities { #[app::init] pub fn init() -> VisitedCities { VisitedCities { - visited: UnorderedMap::new().unwrap(), - storage: Element::root(), + visited: UnorderedMap::new(), } } pub fn add_person(&mut self, person: String) -> Result { - Ok(self - .visited - .insert(person, UnorderedSet::new().unwrap())? - .is_some()) + Ok(self.visited.insert(person, UnorderedSet::new())?.is_some()) } pub fn add_visited_city(&mut self, person: String, city: String) -> Result { diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 9ad41ce7c..512a0a2be 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -120,6 +120,7 @@ static ROOT_ID: LazyLock = LazyLock::new(|| Id::root()); impl Collection { /// Creates a new collection. + #[expect(clippy::expect_used, reason = "fatal error if it happens")] fn new(id: Option) -> Self { let id = id.unwrap_or_else(|| Id::random()); @@ -204,6 +205,11 @@ impl Collection { Ok(()) } + #[expect( + clippy::unwrap_in_result, + clippy::expect_used, + reason = "fatal error if it happens" + )] fn children_cache(&self) -> StoreResult<&mut IndexSet> { let mut cache = self.children_ids.borrow_mut(); @@ -365,6 +371,7 @@ impl Eq for Collec impl PartialEq for Collection { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn eq(&self, other: &Self) -> bool { let l = self.entries().unwrap().flatten(); let r = other.entries().unwrap().flatten(); @@ -374,6 +381,7 @@ impl Partia } impl Ord for Collection { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn cmp(&self, other: &Self) -> std::cmp::Ordering { let l = self.entries().unwrap().flatten(); let r = other.entries().unwrap().flatten(); @@ -386,8 +394,8 @@ impl Parti for Collection { fn partial_cmp(&self, other: &Self) -> Option { - let l = self.entries().unwrap().flatten(); - let r = other.entries().unwrap().flatten(); + let l = self.entries().ok()?.flatten(); + let r = other.entries().ok()?.flatten(); l.partial_cmp(r) } diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs index f698cb0db..e1a1bb939 100644 --- a/crates/storage/src/collections/root.rs +++ b/crates/storage/src/collections/root.rs @@ -49,6 +49,7 @@ where S: StorageAdaptor, { /// Creates a new root collection with the given value. + #[expect(clippy::unwrap_used, reason = "fatal error if it happens")] pub fn new_internal T>(f: F) -> Self { let mut inner = Collection::new(Some(*ROOT_ID)); @@ -67,6 +68,8 @@ where Id::new([118; 32]) } + #[expect(clippy::mut_from_ref, reason = "'tis fine")] + #[expect(clippy::unwrap_used, reason = "fatal error if it happens")] fn get(&self) -> &mut T { let mut value = self.value.borrow_mut(); @@ -81,6 +84,11 @@ where } /// Fetches the root collection. + #[expect( + clippy::unwrap_used, + clippy::unwrap_in_result, + reason = "fatal error if it happens" + )] pub fn fetch() -> Option { let inner = >::root().unwrap()?; @@ -92,6 +100,7 @@ where } /// Commits the root collection. + #[expect(clippy::unwrap_used, reason = "fatal error if it happens")] pub fn commit(mut self) { if self.dirty { if let Some(value) = self.value.into_inner() { @@ -105,6 +114,7 @@ where } /// Commits the root collection without an instance of the root state. + #[expect(clippy::unwrap_used, reason = "fatal error if it happens")] pub fn commit_headless() { >::commit_root::>(None).unwrap(); } diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index c0ce7a78c..bb9b2c8b9 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -184,8 +184,12 @@ where V: PartialEq + BorshSerialize + BorshDeserialize, S: StorageAdaptor, { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn eq(&self, other: &Self) -> bool { - self.entries().unwrap().eq(other.entries().unwrap()) + let l = self.entries().unwrap(); + let r = other.entries().unwrap(); + + l.eq(r) } } @@ -195,8 +199,12 @@ where V: Ord + BorshSerialize + BorshDeserialize, S: StorageAdaptor, { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.entries().unwrap().cmp(other.entries().unwrap()) + let l = self.entries().unwrap(); + let r = other.entries().unwrap(); + + l.cmp(r) } } @@ -207,9 +215,10 @@ where S: StorageAdaptor, { fn partial_cmp(&self, other: &Self) -> Option { - self.entries() - .unwrap() - .partial_cmp(other.entries().unwrap()) + let l = self.entries().ok()?; + let r = other.entries().ok()?; + + l.partial_cmp(r) } } @@ -219,6 +228,7 @@ where V: fmt::Debug + BorshSerialize + BorshDeserialize, S: StorageAdaptor, { + #[expect(clippy::unwrap_used, clippy::unwrap_in_result, reason = "'tis fine")] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if f.alternate() { f.debug_struct("UnorderedMap") diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index 5bd135cdd..ec7490eda 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -157,8 +157,12 @@ impl PartialEq for UnorderedSet where V: PartialEq + BorshSerialize + BorshDeserialize, { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn eq(&self, other: &Self) -> bool { - self.entries().unwrap().eq(other.entries().unwrap()) + let l = self.entries().unwrap(); + let r = other.entries().unwrap(); + + l.eq(r) } } @@ -166,8 +170,12 @@ impl Ord for UnorderedSet where V: Ord + BorshSerialize + BorshDeserialize, { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.entries().unwrap().cmp(other.entries().unwrap()) + let l = self.entries().unwrap(); + let r = other.entries().unwrap(); + + l.cmp(r) } } @@ -176,9 +184,10 @@ where V: PartialOrd + BorshSerialize + BorshDeserialize, { fn partial_cmp(&self, other: &Self) -> Option { - self.entries() - .unwrap() - .partial_cmp(other.entries().unwrap()) + let l = self.entries().ok()?; + let r = other.entries().ok()?; + + l.partial_cmp(r) } } @@ -186,6 +195,7 @@ impl fmt::Debug for UnorderedSet where V: fmt::Debug + BorshSerialize + BorshDeserialize, { + #[expect(clippy::unwrap_used, clippy::unwrap_in_result, reason = "'tis fine")] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if f.alternate() { f.debug_struct("UnorderedSet") diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index 9582e115a..bc074051d 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -176,8 +176,12 @@ impl PartialEq for Vector where V: PartialEq + BorshSerialize + BorshDeserialize, { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn eq(&self, other: &Self) -> bool { - self.entries().unwrap().eq(other.entries().unwrap()) + let l = self.entries().unwrap(); + let r = other.entries().unwrap(); + + l.eq(r) } } @@ -185,8 +189,12 @@ impl Ord for Vector where V: Ord + BorshSerialize + BorshDeserialize, { + #[expect(clippy::unwrap_used, reason = "'tis fine")] fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.entries().unwrap().cmp(other.entries().unwrap()) + let l = self.entries().unwrap(); + let r = other.entries().unwrap(); + + l.cmp(r) } } @@ -195,9 +203,10 @@ where V: PartialOrd + BorshSerialize + BorshDeserialize, { fn partial_cmp(&self, other: &Self) -> Option { - self.entries() - .unwrap() - .partial_cmp(other.entries().unwrap()) + let l = self.entries().ok()?; + let r = other.entries().ok()?; + + l.partial_cmp(r) } } @@ -205,6 +214,7 @@ impl fmt::Debug for Vector where V: fmt::Debug + BorshSerialize + BorshDeserialize, { + #[expect(clippy::unwrap_used, clippy::unwrap_in_result, reason = "'tis fine")] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if f.alternate() { f.debug_struct("Vector") diff --git a/crates/storage/src/entities.rs b/crates/storage/src/entities.rs index dec94aab7..78e047ad0 100644 --- a/crates/storage/src/entities.rs +++ b/crates/storage/src/entities.rs @@ -383,7 +383,7 @@ pub trait Data: BorshDeserialize + BorshSerialize { /// purpose is to make information such as the Merkle hash trivially available /// and prevent the need for repeated lookups. /// -#[derive(BorshDeserialize, BorshSerialize, Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Copy, Debug, Eq, PartialEq)] #[non_exhaustive] pub struct ChildInfo { /// The unique identifier for the child [`Element`]. @@ -394,6 +394,7 @@ pub struct ChildInfo { /// used to determine whether the data has changed and is valid. pub(crate) merkle_hash: [u8; 32], + /// The metadata for the child [`Element`]. pub(crate) metadata: Metadata, } @@ -543,7 +544,7 @@ impl Display for ChildInfo { /// key-value stores. Therefore, this approach and other similar patterns are /// not suitable for our use case. /// -#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] #[non_exhaustive] pub struct Element { /// The unique identifier for the [`Element`]. @@ -774,17 +775,7 @@ impl Display for Element { /// nanosecond precision. This is more than sufficient for our current needs. /// #[derive( - BorshDeserialize, - BorshSerialize, - Copy, - Clone, - Debug, - Default, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, + BorshDeserialize, BorshSerialize, Copy, Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd, )] #[non_exhaustive] pub struct Metadata { @@ -801,9 +792,7 @@ pub struct Metadata { } /// The timestamp when the [`Element`] was last updated. -#[derive( - BorshDeserialize, BorshSerialize, Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialOrd, -)] +#[derive(BorshDeserialize, BorshSerialize, Copy, Clone, Debug, Default, Eq, Ord, PartialOrd)] pub struct UpdatedAt(u64); impl PartialEq for UpdatedAt { diff --git a/crates/storage/src/index.rs b/crates/storage/src/index.rs index 9fb446e15..dae8a2d01 100644 --- a/crates/storage/src/index.rs +++ b/crates/storage/src/index.rs @@ -16,7 +16,7 @@ use crate::interface::StorageError; use crate::store::{Key, StorageAdaptor}; /// Stored index information for an entity in the storage system. -#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] struct EntityIndex { /// Unique identifier of the entity. id: Id, @@ -246,6 +246,7 @@ impl Index { Ok(ancestors) } + /// Retrieves the metadata of a given entity. pub(crate) fn get_metadata(id: Id) -> Result, StorageError> { Ok(Self::get_index(id)?.map(|index| index.metadata)) } diff --git a/crates/storage/src/integration.rs b/crates/storage/src/integration.rs index aa3a0452e..cd4818224 100644 --- a/crates/storage/src/integration.rs +++ b/crates/storage/src/integration.rs @@ -5,7 +5,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use crate::interface::ComparisonData; /// Comparison data for synchronisation. -#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] #[expect(clippy::exhaustive_structs, reason = "Exhaustive")] pub struct Comparison { /// The serialised data of the entity. diff --git a/crates/storage/src/interface.rs b/crates/storage/src/interface.rs index bce5202d4..295668171 100644 --- a/crates/storage/src/interface.rs +++ b/crates/storage/src/interface.rs @@ -263,7 +263,7 @@ pub type MainInterface = Interface; /// Note: This enum contains the entity type, for passing to the guest for /// processing along with the ID and data. /// -#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] #[expect(clippy::exhaustive_enums, reason = "Exhaustive")] pub enum Action { /// Add an entity with the given ID, type, and data. @@ -317,7 +317,7 @@ pub enum Action { } /// Data that is used for comparison between two nodes. -#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct ComparisonData { /// The unique identifier of the entity being compared. id: Id, diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 363685d6c..14a8e4bcb 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -4,16 +4,11 @@ //! system, as a layer on top of the underlying database store. //! -#![forbid( - unreachable_pub, - unsafe_op_in_unsafe_fn, - clippy::missing_docs_in_private_items -)] +#![forbid(unreachable_pub, unsafe_op_in_unsafe_fn)] #![deny( unsafe_code, clippy::expect_used, clippy::missing_errors_doc, - clippy::missing_panics_doc, clippy::panic, clippy::unwrap_in_result, clippy::unwrap_used From c92a5b616d37945d6920da13d88d464518bb2a5b Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 22 Nov 2024 00:34:58 +0300 Subject: [PATCH 16/19] incorporate sync into macro & root --- apps/kv-store/src/__private.rs | 62 -------------------------- apps/kv-store/src/lib.rs | 2 - crates/sdk/macros/src/logic/method.rs | 10 +++++ crates/storage/src/collections/root.rs | 53 +++++++++++++++++++++- 4 files changed, 61 insertions(+), 66 deletions(-) delete mode 100644 apps/kv-store/src/__private.rs diff --git a/apps/kv-store/src/__private.rs b/apps/kv-store/src/__private.rs deleted file mode 100644 index 0dc76417b..000000000 --- a/apps/kv-store/src/__private.rs +++ /dev/null @@ -1,62 +0,0 @@ -use calimero_sdk::borsh::from_slice; -use calimero_sdk::{app, env}; -use calimero_storage::address::Id; -use calimero_storage::collections::Root; -use calimero_storage::integration::Comparison; -use calimero_storage::interface::{Action, MainInterface, StorageError}; -use calimero_storage::sync::{self, SyncArtifact}; - -use crate::KvStore; - -#[app::logic] -impl KvStore { - pub fn __calimero_sync_next() { - Self::___calimero_sync_next().expect("fatal: sync failed"); - } - - fn ___calimero_sync_next() -> Result<(), StorageError> { - let args = env::input().expect("fatal: missing input"); - - let artifact = - from_slice::(&args).map_err(StorageError::DeserializationError)?; - - match artifact { - SyncArtifact::Actions(actions) => { - for action in actions { - let _ignored = match action { - Action::Compare { id } => { - sync::push_comparison(Comparison { - data: MainInterface::find_by_id_raw(id), - comparison_data: MainInterface::generate_comparison_data(Some(id)) - .unwrap(), - }); - } - Action::Add { .. } | Action::Update { .. } | Action::Delete { .. } => { - MainInterface::apply_action(action).unwrap(); - } - }; - } - } - SyncArtifact::Comparisons(comparisons) => { - if comparisons.is_empty() { - sync::push_comparison(Comparison { - data: MainInterface::find_by_id_raw(Id::root()), - comparison_data: MainInterface::generate_comparison_data(None)?, - }); - } - - for Comparison { - data, - comparison_data, - } in comparisons - { - MainInterface::compare_affective(data, comparison_data)?; - } - } - } - - Root::::commit_headless(); - - Ok(()) - } -} diff --git a/apps/kv-store/src/lib.rs b/apps/kv-store/src/lib.rs index 3cc210b56..807b0b521 100644 --- a/apps/kv-store/src/lib.rs +++ b/apps/kv-store/src/lib.rs @@ -7,8 +7,6 @@ use calimero_sdk::types::Error; use calimero_sdk::{app, env}; use calimero_storage::collections::UnorderedMap; -mod __private; - #[app::state(emits = for<'a> Event<'a>)] #[derive(Debug, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] #[borsh(crate = "calimero_sdk::borsh")] diff --git a/crates/sdk/macros/src/logic/method.rs b/crates/sdk/macros/src/logic/method.rs index 7d6fc7eee..fedf1fd8b 100644 --- a/crates/sdk/macros/src/logic/method.rs +++ b/crates/sdk/macros/src/logic/method.rs @@ -164,6 +164,16 @@ impl ToTokens for PublicLogicMethod<'_> { }; quote_spanned! {name.span()=> + #[cfg(target_arch = "wasm32")] + #[no_mangle] + pub extern "C" fn __calimero_sync_next() { + let Some(args) = ::calimero_sdk::env::input() else { + ::calimero_sdk::env::panic_str("Expected payload to sync method.") + }; + + ::calimero_storage::collections::Root::<#self_>::sync(&args).expect("fatal: sync failed"); + } + impl ::calimero_sdk::state::AppStateInit for #self_ { type Return = #ret; } diff --git a/crates/storage/src/collections/root.rs b/crates/storage/src/collections/root.rs index e1a1bb939..4de3ee951 100644 --- a/crates/storage/src/collections/root.rs +++ b/crates/storage/src/collections/root.rs @@ -5,12 +5,14 @@ use std::cell::RefCell; use std::ops::{Deref, DerefMut}; use std::ptr; -use borsh::{BorshDeserialize, BorshSerialize}; +use borsh::{from_slice, BorshDeserialize, BorshSerialize}; use super::{Collection, ROOT_ID}; use crate::address::Id; -use crate::interface::Interface; +use crate::integration::Comparison; +use crate::interface::{Action, Interface, StorageError}; use crate::store::{MainStorage, StorageAdaptor}; +use crate::sync::{self, SyncArtifact}; /// A set collection that stores unqiue values once. pub struct Root { @@ -118,6 +120,53 @@ where pub fn commit_headless() { >::commit_root::>(None).unwrap(); } + + /// Syncs the root collection. + #[expect(clippy::missing_errors_doc, reason = "NO")] + pub fn sync(args: &[u8]) -> Result<(), StorageError> { + let artifact = + from_slice::(args).map_err(StorageError::DeserializationError)?; + + match artifact { + SyncArtifact::Actions(actions) => { + for action in actions { + let _ignored = match action { + Action::Compare { id } => { + sync::push_comparison(Comparison { + data: >::find_by_id_raw(id), + comparison_data: >::generate_comparison_data(Some( + id, + ))?, + }); + } + Action::Add { .. } | Action::Update { .. } | Action::Delete { .. } => { + >::apply_action(action)?; + } + }; + } + } + SyncArtifact::Comparisons(comparisons) => { + if comparisons.is_empty() { + sync::push_comparison(Comparison { + data: >::find_by_id_raw(Id::root()), + comparison_data: >::generate_comparison_data(None)?, + }); + } + + for Comparison { + data, + comparison_data, + } in comparisons + { + >::compare_affective(data, comparison_data)?; + } + } + } + + Self::commit_headless(); + + Ok(()) + } } impl Deref for Root From 64b0568fdc0731d312cc00a680f445439f383edc Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 22 Nov 2024 00:42:48 +0300 Subject: [PATCH 17/19] restore apps --- Cargo.lock | 2 ++ apps/gen-ext/Cargo.toml | 3 ++- apps/only-peers/Cargo.toml | 1 + crates/sdk/macros/src/logic/method.rs | 2 +- scripts/build-all-apps.sh | 5 ++--- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab7c0be65..d4aa212de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2589,6 +2589,7 @@ version = "0.1.0" dependencies = [ "calimero-sdk", "calimero-sdk-near", + "calimero-storage", ] [[package]] @@ -5494,6 +5495,7 @@ name = "only-peers" version = "0.1.0" dependencies = [ "calimero-sdk", + "calimero-storage", ] [[package]] diff --git a/apps/gen-ext/Cargo.toml b/apps/gen-ext/Cargo.toml index a29fe22be..8d34c86c6 100644 --- a/apps/gen-ext/Cargo.toml +++ b/apps/gen-ext/Cargo.toml @@ -10,5 +10,6 @@ version = "0.1.0" crate-type = ["cdylib"] [dependencies] -calimero-sdk = {path = "../../crates/sdk"} +calimero-sdk = { path = "../../crates/sdk" } calimero-sdk-near = { path = "../../crates/sdk/libs/near" } +calimero-storage = { path = "../../crates/storage" } diff --git a/apps/only-peers/Cargo.toml b/apps/only-peers/Cargo.toml index cc3360bac..11b352580 100644 --- a/apps/only-peers/Cargo.toml +++ b/apps/only-peers/Cargo.toml @@ -11,3 +11,4 @@ crate-type = ["cdylib"] [dependencies] calimero-sdk = { path = "../../crates/sdk" } +calimero-storage = { path = "../../crates/storage" } diff --git a/crates/sdk/macros/src/logic/method.rs b/crates/sdk/macros/src/logic/method.rs index fedf1fd8b..57b508b16 100644 --- a/crates/sdk/macros/src/logic/method.rs +++ b/crates/sdk/macros/src/logic/method.rs @@ -134,7 +134,7 @@ impl ToTokens for PublicLogicMethod<'_> { call = quote_spanned! {ret.ty.span()=> let output = #call; let output = { - #[expect(unused_imports)] + #[allow(unused_imports)] use ::calimero_sdk::__private::IntoResult; match ::calimero_sdk::__private::WrappedReturn::new(output) .into_result() diff --git a/scripts/build-all-apps.sh b/scripts/build-all-apps.sh index 827c97e05..685d38acc 100755 --- a/scripts/build-all-apps.sh +++ b/scripts/build-all-apps.sh @@ -7,9 +7,8 @@ cd "$(dirname $0)" BUILD_SCRIPTS=( "../apps/kv-store/build.sh" - # todo! update to use CRDTs - # "../apps/gen-ext/build.sh" - # "../apps/only-peers/build.sh" + "../apps/gen-ext/build.sh" + "../apps/only-peers/build.sh" ) run_script() { From eb53bbb2c725c073a6ca00914f3f31b6b5e24dc2 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Fri, 22 Nov 2024 00:55:42 +0300 Subject: [PATCH 18/19] nuke storage-macros tests --- crates/storage-macros/src/lib.rs | 15 +- crates/storage-macros/tests/atomic_unit.rs | 335 ------------------ crates/storage-macros/tests/collection.rs | 184 ---------- .../tests/compile_fail/collection.rs | 37 -- .../tests/compile_fail/collection.stderr | 10 - 5 files changed, 8 insertions(+), 573 deletions(-) delete mode 100644 crates/storage-macros/tests/atomic_unit.rs delete mode 100644 crates/storage-macros/tests/collection.rs delete mode 100644 crates/storage-macros/tests/compile_fail/collection.rs delete mode 100644 crates/storage-macros/tests/compile_fail/collection.stderr diff --git a/crates/storage-macros/src/lib.rs b/crates/storage-macros/src/lib.rs index c6363b858..3b6750bc1 100644 --- a/crates/storage-macros/src/lib.rs +++ b/crates/storage-macros/src/lib.rs @@ -95,8 +95,9 @@ mod integration_tests_package_usage { /// ``` /// use calimero_storage::entities::Element; /// use calimero_storage_macros::AtomicUnit; +/// use borsh::{BorshSerialize, BorshDeserialize}; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// #[type_id(43)] /// struct Page { /// title: String, @@ -110,8 +111,9 @@ mod integration_tests_package_usage { /// ``` /// use calimero_storage::entities::Element; /// use calimero_storage_macros::{AtomicUnit, Collection}; +/// use borsh::{BorshSerialize, BorshDeserialize}; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// #[type_id(44)] /// struct Person { /// name: String, @@ -209,7 +211,7 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { #( collections.insert( stringify!(#collection_fields).to_owned(), - calimero_storage::interface::Interface::child_info_for(self.id(), &self.#collection_fields).unwrap_or_default() + calimero_storage::interface::MainInterface::child_info_for(self.id(), &self.#collection_fields).unwrap_or_default() ); )* collections @@ -273,9 +275,9 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { /// ``` /// use calimero_storage_macros::{AtomicUnit, Collection}; /// use calimero_storage::entities::{Data, Element}; +/// use borsh::{BorshSerialize, BorshDeserialize}; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -/// #[type_id(42)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// struct Book { /// title: String, /// pages: Pages, @@ -287,8 +289,7 @@ pub fn atomic_unit_derive(input: TokenStream) -> TokenStream { /// #[children(Page)] /// struct Pages; /// -/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -/// #[type_id(43)] +/// #[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd, BorshSerialize, BorshDeserialize)] /// struct Page { /// content: String, /// #[storage] diff --git a/crates/storage-macros/tests/atomic_unit.rs b/crates/storage-macros/tests/atomic_unit.rs deleted file mode 100644 index b15ca4eb2..000000000 --- a/crates/storage-macros/tests/atomic_unit.rs +++ /dev/null @@ -1,335 +0,0 @@ -#![allow(unused_crate_dependencies, reason = "Creates a lot of noise")] -// Lints specifically disabled for integration tests -#![allow( - non_snake_case, - unreachable_pub, - clippy::cast_lossless, - clippy::cast_precision_loss, - clippy::cognitive_complexity, - clippy::default_numeric_fallback, - clippy::exhaustive_enums, - clippy::exhaustive_structs, - clippy::expect_used, - clippy::indexing_slicing, - clippy::let_underscore_must_use, - clippy::let_underscore_untyped, - clippy::missing_assert_message, - clippy::missing_panics_doc, - clippy::mod_module_files, - clippy::must_use_candidate, - clippy::panic, - clippy::print_stdout, - clippy::tests_outside_test_module, - clippy::unwrap_in_result, - clippy::unwrap_used, - reason = "Not useful in tests" -)] - -use borsh::{to_vec, BorshDeserialize}; -use calimero_storage::address::Path; -use calimero_storage::entities::{Data, Element}; -use calimero_storage::exports::{Digest, Sha256}; -use calimero_storage::interface::MainInterface; -use calimero_storage_macros::AtomicUnit; - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[root] -#[type_id(1)] -struct Private { - public: String, - #[private] - private: String, - #[storage] - storage: Element, -} - -impl Private { - fn new(path: &Path) -> Self { - Self { - public: String::new(), - private: String::new(), - storage: Element::new(path, None), - } - } -} - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[root] -#[type_id(2)] -struct Simple { - name: String, - value: i32, - #[storage] - storage: Element, -} - -impl Simple { - fn new(path: &Path) -> Self { - Self { - name: String::new(), - value: 0, - storage: Element::new(path, None), - } - } -} - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[root] -#[type_id(3)] -struct Skipped { - included: String, - #[skip] - skipped: String, - #[storage] - storage: Element, -} - -impl Skipped { - fn new(path: &Path) -> Self { - Self { - included: String::new(), - skipped: String::new(), - storage: Element::new(path, None), - } - } -} - -#[cfg(test)] -mod basics { - use super::*; - - #[test] - fn creation() { - let path = Path::new("::root::node").unwrap(); - let unit = Simple::new(&path); - - assert_eq!(unit.path(), path); - assert_eq!(unit.element().path(), path); - assert!(unit.element().is_dirty()); - } - - #[test] - fn getters() { - let path = Path::new("::root::node").unwrap(); - let unit = Simple::new(&path); - - assert_eq!(unit.name, ""); - assert_eq!(unit.value, 0); - } - - #[test] - fn setters__set() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - - _ = unit.set_name("Test Name".to_owned()); - _ = unit.set_value(42); - - assert_eq!(unit.name, "Test Name"); - assert_eq!(unit.value, 42); - } - - #[test] - fn setters__confirm_set() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - assert_ne!(unit.name, "Test Name"); - assert_ne!(unit.value, 42); - - assert!(unit.set_name("Test Name".to_owned())); - assert!(unit.set_value(42)); - assert_eq!(unit.name, "Test Name"); - assert_eq!(unit.value, 42); - } - - #[test] - fn setters__confirm_not_set() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - assert_ne!(unit.name, "Test Name"); - assert_ne!(unit.value, 42); - - assert!(unit.set_name("Test Name".to_owned())); - assert!(unit.set_value(42)); - assert_eq!(unit.name, "Test Name"); - assert_eq!(unit.value, 42); - assert!(!unit.set_name("Test Name".to_owned())); - assert!(!unit.set_value(42)); - } - - #[test] - fn setters__confirm_set_dirty() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - assert!(MainInterface::save(&mut unit).unwrap()); - assert!(!unit.element().is_dirty()); - - assert!(unit.set_name("Test Name".to_owned())); - assert!(unit.element().is_dirty()); - } - - #[test] - fn setters__confirm_not_set_not_dirty() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - assert!(MainInterface::save(&mut unit).unwrap()); - assert!(!unit.element().is_dirty()); - - assert!(unit.set_name("Test Name".to_owned())); - assert!(unit.element().is_dirty()); - assert!(MainInterface::save(&mut unit).unwrap()); - assert!(!unit.set_name("Test Name".to_owned())); - assert!(!unit.element().is_dirty()); - } -} - -#[cfg(test)] -mod visibility { - use super::*; - - #[test] - fn private_field() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Private::new(&path); - - _ = unit.set_public("Public".to_owned()); - _ = unit.set_private("Private".to_owned()); - - let serialized = to_vec(&unit).unwrap(); - let deserialized = Private::try_from_slice(&serialized).unwrap(); - - assert_eq!(unit.public, deserialized.public); - assert_ne!(unit.private, deserialized.private); - assert_eq!(deserialized.private, ""); - } - - #[test] - fn public_field() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - - _ = unit.set_name("Public".to_owned()); - - let serialized = to_vec(&unit).unwrap(); - let deserialized = Simple::try_from_slice(&serialized).unwrap(); - - assert_eq!(unit.name, deserialized.name); - } - - #[test] - fn skipped_field() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Skipped::new(&path); - - _ = unit.set_included("Public".to_owned()); - // Skipping fields also skips the setters - // _ = unit.set_skipped("Skipped".to_owned()); - unit.skipped = "Skipped".to_owned(); - - let serialized = to_vec(&unit).unwrap(); - let deserialized = Skipped::try_from_slice(&serialized).unwrap(); - - assert_eq!(unit.included, deserialized.included); - // Skipping fields also skips the getters - // assert_ne!(unit.skipped(), deserialized.skipped()); - assert_ne!(unit.skipped, deserialized.skipped); - assert_eq!(deserialized.skipped, ""); - } -} - -#[cfg(test)] -mod hashing { - use super::*; - - #[test] - fn private_field() { - let path = Path::new("::root::node::leaf").unwrap(); - let mut unit = Private::new(&path); - - _ = unit.set_public("Public".to_owned()); - _ = unit.set_private("Private".to_owned()); - - let mut hasher = Sha256::new(); - hasher.update(unit.id().as_bytes()); - hasher.update(to_vec(&unit.public).unwrap()); - hasher.update(to_vec(&unit.element().metadata()).unwrap()); - let expected_hash: [u8; 32] = hasher.finalize().into(); - - assert_eq!(unit.calculate_merkle_hash().unwrap(), expected_hash); - - _ = unit.set_private("Test 1".to_owned()); - assert_eq!(unit.calculate_merkle_hash().unwrap(), expected_hash); - - _ = unit.set_public("Test 2".to_owned()); - assert_ne!(unit.calculate_merkle_hash().unwrap(), expected_hash); - } - - #[test] - fn public_field() { - let path = Path::new("::root::node::leaf").unwrap(); - let mut unit = Simple::new(&path); - - _ = unit.set_name("Public".to_owned()); - _ = unit.set_value(42); - - let mut hasher = Sha256::new(); - hasher.update(unit.id().as_bytes()); - hasher.update(to_vec(&unit.name).unwrap()); - hasher.update(to_vec(&unit.value).unwrap()); - hasher.update(to_vec(&unit.element().metadata()).unwrap()); - let expected_hash: [u8; 32] = hasher.finalize().into(); - - assert_eq!(unit.calculate_merkle_hash().unwrap(), expected_hash); - } - - #[test] - fn skipped_field() { - let path = Path::new("::root::node::leaf").unwrap(); - let mut unit = Skipped::new(&path); - - _ = unit.set_included("Public".to_owned()); - // Skipping fields also skips the setters - // _ = unit.set_skipped("Skipped".to_owned()); - unit.skipped = "Skipped".to_owned(); - - let mut hasher = Sha256::new(); - hasher.update(unit.id().as_bytes()); - hasher.update(to_vec(&unit.included).unwrap()); - hasher.update(to_vec(&unit.element().metadata()).unwrap()); - let expected_hash: [u8; 32] = hasher.finalize().into(); - - assert_eq!(unit.calculate_merkle_hash().unwrap(), expected_hash); - - unit.skipped = "Test 1".to_owned(); - assert_eq!(unit.calculate_merkle_hash().unwrap(), expected_hash); - - _ = unit.set_included("Test 2".to_owned()); - assert_ne!(unit.calculate_merkle_hash().unwrap(), expected_hash); - } -} - -#[cfg(test)] -mod traits { - use super::*; - - #[test] - fn serialization_and_deserialization() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - - _ = unit.set_name("Test Name".to_owned()); - _ = unit.set_value(42); - - let serialized = to_vec(&unit).unwrap(); - let deserialized = Simple::try_from_slice(&serialized).unwrap(); - - assert_eq!(unit, deserialized); - assert_eq!(unit.id(), deserialized.id()); - assert_eq!(unit.name, deserialized.name); - assert_eq!(unit.path(), deserialized.path()); - assert_eq!(unit.value, deserialized.value); - assert_eq!(unit.element().id(), deserialized.element().id()); - assert_eq!(unit.element().path(), deserialized.element().path()); - assert_eq!(unit.element().metadata(), deserialized.element().metadata()); - } -} diff --git a/crates/storage-macros/tests/collection.rs b/crates/storage-macros/tests/collection.rs deleted file mode 100644 index 7145ef22a..000000000 --- a/crates/storage-macros/tests/collection.rs +++ /dev/null @@ -1,184 +0,0 @@ -#![allow(unused_crate_dependencies, reason = "Creates a lot of noise")] -// Lints specifically disabled for integration tests -#![allow( - non_snake_case, - unreachable_pub, - clippy::cast_lossless, - clippy::cast_precision_loss, - clippy::cognitive_complexity, - clippy::default_numeric_fallback, - clippy::exhaustive_enums, - clippy::exhaustive_structs, - clippy::expect_used, - clippy::indexing_slicing, - clippy::let_underscore_must_use, - clippy::let_underscore_untyped, - clippy::missing_assert_message, - clippy::missing_panics_doc, - clippy::mod_module_files, - clippy::must_use_candidate, - clippy::panic, - clippy::print_stdout, - clippy::tests_outside_test_module, - clippy::unwrap_in_result, - clippy::unwrap_used, - reason = "Not useful in tests" -)] - -use borsh::{to_vec, BorshDeserialize}; -use calimero_storage::address::Path; -use calimero_storage::entities::{Data, Element}; -use calimero_storage::exports::{Digest, Sha256}; -use calimero_storage_macros::{AtomicUnit, Collection}; - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(22)] -struct Child { - content: String, - #[storage] - storage: Element, -} - -impl Child { - fn new(path: &Path) -> Self { - Self { - content: String::new(), - storage: Element::new(path, None), - } - } -} - -#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[children(Child)] -struct Group; - -impl Group { - const fn new() -> Self { - Self {} - } -} - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(21)] -#[root] -struct Parent { - title: String, - #[collection] - children: Group, - #[storage] - storage: Element, -} - -impl Parent { - fn new(path: &Path) -> Self { - Self { - title: String::new(), - children: Group::new(), - storage: Element::new(path, None), - } - } -} - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(23)] -#[root] -struct Simple { - name: String, - value: i32, - #[storage] - storage: Element, -} - -impl Simple { - fn new(path: &Path) -> Self { - Self { - name: String::new(), - value: 0, - storage: Element::new(path, None), - } - } -} - -#[cfg(test)] -mod hierarchy { - use super::*; - - #[test] - fn parent_child() { - let parent_path = Path::new("::root::node").unwrap(); - let mut parent = Parent::new(&parent_path); - _ = parent.set_title("Parent Title".to_owned()); - - let child_path = Path::new("::root::node::leaf").unwrap(); - let mut child = Child::new(&child_path); - _ = child.set_content("Child Content".to_owned()); - - assert_eq!(parent.title, "Parent Title"); - - // TODO: Add in tests for loading and checking children - } - - #[test] - fn compile_fail() { - trybuild::TestCases::new().compile_fail("tests/compile_fail/collection.rs"); - } -} - -#[cfg(test)] -mod hashing { - use super::*; - - #[test] - fn calculate_merkle_hash__child() { - let mut child = Child::new(&Path::new("::root::node::leaf").unwrap()); - _ = child.set_content("Child Content".to_owned()); - - let mut hasher = Sha256::new(); - hasher.update(child.id().as_bytes()); - hasher.update(to_vec(&child.content).unwrap()); - hasher.update(to_vec(&child.element().metadata()).unwrap()); - let expected_hash: [u8; 32] = hasher.finalize().into(); - - assert_eq!(child.calculate_merkle_hash().unwrap(), expected_hash); - } - - #[test] - fn calculate_merkle_hash__parent() { - let mut parent = Parent::new(&Path::new("::root::node").unwrap()); - _ = parent.set_title("Parent Title".to_owned()); - - let mut hasher = Sha256::new(); - hasher.update(parent.id().as_bytes()); - hasher.update(to_vec(&parent.title).unwrap()); - hasher.update(to_vec(&parent.element().metadata()).unwrap()); - let expected_hash: [u8; 32] = hasher.finalize().into(); - - assert_eq!(parent.calculate_merkle_hash().unwrap(), expected_hash); - } -} - -#[cfg(test)] -mod traits { - use super::*; - - #[test] - fn serialization_and_deserialization() { - let path = Path::new("::root::node").unwrap(); - let mut unit = Simple::new(&path); - - _ = unit.set_name("Test Name".to_owned()); - _ = unit.set_value(42); - - let serialized = to_vec(&unit).unwrap(); - let deserialized = Simple::try_from_slice(&serialized).unwrap(); - - assert_eq!(unit, deserialized); - assert_eq!(unit.id(), deserialized.id()); - assert_eq!(unit.name, deserialized.name); - assert_eq!(unit.path(), deserialized.path()); - assert_eq!(unit.value, deserialized.value); - assert_eq!(unit.element().id(), deserialized.element().id()); - assert_eq!(unit.element().path(), deserialized.element().path()); - assert_eq!(unit.element().metadata(), deserialized.element().metadata()); - } -} diff --git a/crates/storage-macros/tests/compile_fail/collection.rs b/crates/storage-macros/tests/compile_fail/collection.rs deleted file mode 100644 index 2c1ecc34a..000000000 --- a/crates/storage-macros/tests/compile_fail/collection.rs +++ /dev/null @@ -1,37 +0,0 @@ -use calimero_storage::address::Path; -use calimero_storage::entities::{Data, Element}; -use calimero_storage::interface::Interface; -use calimero_storage_macros::{AtomicUnit, Collection}; - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[type_id(2)] -struct Child { - #[storage] - storage: Element, -} - -#[derive(Collection, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[children(Child)] -struct Group; - -#[derive(AtomicUnit, Clone, Debug, Eq, PartialEq, PartialOrd)] -#[root] -#[type_id(1)] -struct Parent { - group: Group, - #[storage] - storage: Element, -} - -fn main() { - fn child_type_specification() { - let parent: Parent = Parent { - group: Group {}, - storage: Element::new(&Path::new("::root::node").unwrap(), None), - }; - let _: Vec = Interface::children_of(parent.id(), &parent.group).unwrap(); - - // This should fail to compile if the child type is incorrect - let _: Vec = Interface::children_of(parent.id(), &parent.group).unwrap(); - } -} diff --git a/crates/storage-macros/tests/compile_fail/collection.stderr b/crates/storage-macros/tests/compile_fail/collection.stderr deleted file mode 100644 index eeb500f05..000000000 --- a/crates/storage-macros/tests/compile_fail/collection.stderr +++ /dev/null @@ -1,10 +0,0 @@ -error[E0308]: mismatched types - --> tests/compile_fail/collection.rs:35:30 - | -35 | let _: Vec = Interface::children_of(parent.id(), &parent.group).unwrap(); - | ----------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Vec`, found `Vec` - | | - | expected due to this - | - = note: expected struct `Vec` - found struct `Vec` From 140adf08e95f0e5488b06e88ecc4cc7eb76f1cc1 Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Tue, 26 Nov 2024 13:39:52 +0100 Subject: [PATCH 19/19] update only peers to use crdt collections --- apps/only-peers/src/lib.rs | 44 ++++++++++------- crates/storage/src/collections.rs | 10 ++++ .../storage/src/collections/unordered_map.rs | 35 +++++++++++++ .../storage/src/collections/unordered_set.rs | 32 ++++++++++++ crates/storage/src/collections/vector.rs | 49 +++++++++++++++++++ 5 files changed, 152 insertions(+), 18 deletions(-) diff --git a/apps/only-peers/src/lib.rs b/apps/only-peers/src/lib.rs index ded6f74b4..e8fa3d978 100644 --- a/apps/only-peers/src/lib.rs +++ b/apps/only-peers/src/lib.rs @@ -1,12 +1,14 @@ use calimero_sdk::borsh::{BorshDeserialize, BorshSerialize}; use calimero_sdk::serde::Serialize; +use calimero_sdk::types::Error; use calimero_sdk::{app, env}; +use calimero_storage::collections::Vector; #[app::state(emits = for<'a> Event<'a>)] #[derive(BorshDeserialize, BorshSerialize, Default)] #[borsh(crate = "calimero_sdk::borsh")] pub struct OnlyPeers { - posts: Vec, + posts: Vector, } #[derive(BorshDeserialize, BorshSerialize, Default, Serialize)] @@ -16,10 +18,10 @@ pub struct Post { id: usize, title: String, content: String, - comments: Vec, + comments: Vector, } -#[derive(BorshDeserialize, BorshSerialize, Default, Serialize)] +#[derive(BorshDeserialize, BorshSerialize, Clone, Default, Serialize)] #[borsh(crate = "calimero_sdk::borsh")] #[serde(crate = "calimero_sdk::serde")] pub struct Comment { @@ -48,40 +50,40 @@ impl OnlyPeers { OnlyPeers::default() } - pub fn post(&self, id: usize) -> Option<&Post> { + pub fn post(&self, id: usize) -> Result, Error> { env::log(&format!("Getting post with id: {:?}", id)); - self.posts.get(id) + Ok(self.posts.get(id)?) } - pub fn posts(&self) -> &[Post] { + pub fn posts(&self) -> Result, Error> { env::log("Getting all posts"); - &self.posts + Ok(self.posts.entries()?.collect()) } - pub fn create_post(&mut self, title: String, content: String) -> &Post { + pub fn create_post(&mut self, title: String, content: String) -> Result { env::log(&format!( "Creating post with title: {:?} and content: {:?}", title, content )); app::emit!(Event::PostCreated { - id: self.posts.len(), + id: self.posts.len()?, // todo! should we maybe only emit an ID, and let notified clients fetch the post? title: &title, content: &content, }); self.posts.push(Post { - id: self.posts.len(), + id: self.posts.len()?, title, content, - comments: Vec::new(), - }); + comments: Vector::new(), + })?; - match self.posts.last() { - Some(post) => post, + match self.posts.last()? { + Some(post) => Ok(post), None => env::unreachable(), } } @@ -91,13 +93,15 @@ impl OnlyPeers { post_id: usize, user: String, // todo! expose executor identity to app context text: String, - ) -> Option<&Comment> { + ) -> Result, Error> { env::log(&format!( "Creating comment under post with id: {:?} as user: {:?} with text: {:?}", post_id, user, text )); - let post = self.posts.get_mut(post_id)?; + let Some(mut post) = self.posts.get(post_id)? else { + return Ok(None); + }; app::emit!(Event::CommentCreated { post_id, @@ -106,8 +110,12 @@ impl OnlyPeers { text: &text, }); - post.comments.push(Comment { user, text }); + let comment = Comment { user, text }; + + post.comments.push(comment.clone())?; + + self.posts.update(post_id, post)?; - post.comments.last() + Ok(Some(comment)) } } diff --git a/crates/storage/src/collections.rs b/crates/storage/src/collections.rs index 512a0a2be..4b79e22a3 100644 --- a/crates/storage/src/collections.rs +++ b/crates/storage/src/collections.rs @@ -366,6 +366,16 @@ impl fmt::Debug for Collection { } } +impl Default for Collection +where + T: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, +{ + fn default() -> Self { + Self::new(None) + } +} + impl Eq for Collection {} impl PartialEq diff --git a/crates/storage/src/collections/unordered_map.rs b/crates/storage/src/collections/unordered_map.rs index bb9b2c8b9..d103f0e1b 100644 --- a/crates/storage/src/collections/unordered_map.rs +++ b/crates/storage/src/collections/unordered_map.rs @@ -5,6 +5,8 @@ use core::fmt; use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; +use serde::ser::SerializeMap; +use serde::Serialize; use sha2::{Digest, Sha256}; use super::{Collection, StorageAdaptor}; @@ -240,6 +242,39 @@ where } } +impl Default for UnorderedMap +where + K: Default + BorshSerialize + BorshDeserialize, + V: Default + BorshSerialize + BorshDeserialize, + S: StorageAdaptor, +{ + fn default() -> Self { + Self::new_internal() + } +} + +impl Serialize for UnorderedMap +where + K: BorshSerialize + BorshDeserialize + Serialize, + V: BorshSerialize + BorshDeserialize + Serialize, + S: StorageAdaptor, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: serde::Serializer, + { + let len = self.len().map_err(serde::ser::Error::custom)?; + + let mut seq = serializer.serialize_map(Some(len))?; + + for (k, v) in self.entries().map_err(serde::ser::Error::custom)? { + seq.serialize_entry(&k, &v)?; + } + + seq.end() + } +} + #[cfg(test)] mod tests { use crate::collections::{Root, UnorderedMap}; diff --git a/crates/storage/src/collections/unordered_set.rs b/crates/storage/src/collections/unordered_set.rs index ec7490eda..adb92f517 100644 --- a/crates/storage/src/collections/unordered_set.rs +++ b/crates/storage/src/collections/unordered_set.rs @@ -4,6 +4,8 @@ use core::borrow::Borrow; use core::fmt; use borsh::{BorshDeserialize, BorshSerialize}; +use serde::ser::SerializeSeq; +use serde::Serialize; use sha2::{Digest, Sha256}; use super::Collection; @@ -207,6 +209,36 @@ where } } +impl Default for UnorderedSet +where + V: BorshSerialize + BorshDeserialize, +{ + fn default() -> Self { + Self::new_internal() + } +} + +impl Serialize for UnorderedSet +where + V: BorshSerialize + BorshDeserialize + Serialize, + S: StorageAdaptor, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: serde::Serializer, + { + let len = self.len().map_err(serde::ser::Error::custom)?; + + let mut seq = serializer.serialize_seq(Some(len))?; + + for v in self.entries().map_err(serde::ser::Error::custom)? { + seq.serialize_element(&v)?; + } + + seq.end() + } +} + #[cfg(test)] mod tests { use crate::collections::{Root, UnorderedSet}; diff --git a/crates/storage/src/collections/vector.rs b/crates/storage/src/collections/vector.rs index bc074051d..a1d57ae74 100644 --- a/crates/storage/src/collections/vector.rs +++ b/crates/storage/src/collections/vector.rs @@ -5,6 +5,8 @@ use core::fmt; use std::mem; use borsh::{BorshDeserialize, BorshSerialize}; +use serde::ser::SerializeSeq; +use serde::Serialize; use super::Collection; use crate::collections::error::StoreError; @@ -122,6 +124,22 @@ where Ok(self.inner.entries()?.flatten().fuse()) } + /// Get the last value in the vector. + /// + /// # Errors + /// + /// If an error occurs when interacting with the storage system, or a child + /// [`Element`](crate::entities::Element) cannot be found, an error will be + /// returned. + /// + pub fn last(&self) -> Result, StoreError> { + let Some(last) = self.inner.last()? else { + return Ok(None); + }; + + self.inner.get(last) + } + /// Get the number of entries in the vector. /// /// # Errors @@ -226,6 +244,37 @@ where } } +impl Default for Vector +where + V: BorshSerialize + BorshDeserialize, + S: StorageAdaptor, +{ + fn default() -> Self { + Self::new_internal() + } +} + +impl Serialize for Vector +where + V: BorshSerialize + BorshDeserialize + Serialize, + S: StorageAdaptor, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: serde::Serializer, + { + let len = self.len().map_err(serde::ser::Error::custom)?; + + let mut seq = serializer.serialize_seq(Some(len))?; + + for entry in self.entries().map_err(serde::ser::Error::custom)? { + seq.serialize_element(&entry)?; + } + + seq.end() + } +} + #[cfg(test)] mod tests { use crate::collections::{Root, Vector};