diff --git a/foundationdb-bindingtester/src/main.rs b/foundationdb-bindingtester/src/main.rs index 07a6b4da..a8aa3918 100644 --- a/foundationdb-bindingtester/src/main.rs +++ b/foundationdb-bindingtester/src/main.rs @@ -39,6 +39,8 @@ use foundationdb::directory::directory_layer::DirectoryLayer; use foundationdb::directory::directory_subspace::DirectorySubspace; use foundationdb::directory::error::DirectoryError; use foundationdb::directory::Directory; +use foundationdb::tuple::{PackResult, TupleUnpack}; + use tuple::VersionstampOffset; fn mutation_from_str(s: &str) -> MutationType { @@ -644,11 +646,12 @@ impl StackMachine { let element = self.pop_element().await; match element { Element::Bytes(v) => v, + Element::Nil => Bytes::from(vec![]), _ => panic!("bytes were expected, found {:?}", element), } } - async fn pop_tuple(&mut self, count: usize) -> Vec> { + async fn pop_string_tuple(&mut self, count: usize) -> Vec> { let mut result = vec![]; if count == 0 { @@ -1636,7 +1639,7 @@ impl StackMachine { // raw_prefix. Append it to the directory list. DirectoryCreateSubspace => { debug!("CreateSubspace stack: {:?}", self.stack); - let tuple_prefix = self.pop_tuple(1).await; + let tuple_prefix = self.pop_string_tuple(1).await; let raw_prefix = self.pop_bytes().await; let subspace = Subspace::from_bytes(&raw_prefix).subspace(tuple_prefix.get(0).unwrap()); @@ -1711,7 +1714,7 @@ impl StackMachine { DirectoryCreate => { debug!("Create stack: {:?}", self.stack); - let path = self.pop_tuple(1).await; + let path = self.pop_string_tuple(1).await; let bytes_layer = self.pop_bytes().await; let bytes_prefix = self.pop_bytes().await; @@ -1728,7 +1731,7 @@ impl StackMachine { }; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -1765,7 +1768,7 @@ impl StackMachine { DirectoryOpen => { debug!("DirectoryOpen stack: {:?}", self.stack); - let path = self.pop_tuple(1).await; + let path = self.pop_string_tuple(1).await; let bytes_layer = self.pop_bytes().await; let layer = if bytes_layer.is_empty() { @@ -1775,7 +1778,7 @@ impl StackMachine { }; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -1812,7 +1815,7 @@ impl StackMachine { DirectoryCreateOrOpen => { debug!("CreateOrOpen stack: {:?}", self.stack); - let path = self.pop_tuple(1).await; + let path = self.pop_string_tuple(1).await; let bytes_layer = self.pop_bytes().await; let layer = if bytes_layer.is_empty() { @@ -1822,7 +1825,7 @@ impl StackMachine { }; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -1900,10 +1903,10 @@ impl StackMachine { DirectoryMove => { debug!("Move stack: {:?}", self.stack); - let paths = self.pop_tuple(2).await; + let paths = self.pop_string_tuple(2).await; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -1913,6 +1916,8 @@ impl StackMachine { } }; + debug!("starting"); + match directory .move_to( txn, @@ -1934,6 +1939,7 @@ impl StackMachine { self.push_directory_err(&instr.code, number, e); } }; + debug!("finished"); } // Use the current directory for this operation. @@ -1943,10 +1949,10 @@ impl StackMachine { DirectoryMoveTo => { debug!("MoveTo stack: {:?}", self.stack); - let paths = self.pop_tuple(1).await; + let paths = self.pop_string_tuple(1).await; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -1983,10 +1989,10 @@ impl StackMachine { DirectoryRemove => { debug!("Remove stack: {:?}", self.stack); let count = self.pop_usize().await; - let paths = self.pop_tuple(count).await; + let paths = self.pop_string_tuple(count).await; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -2011,10 +2017,10 @@ impl StackMachine { DirectoryRemoveIfExists => { debug!("RemoveIfExists stack: {:?}", self.stack); let count = self.pop_usize().await; - let paths = self.pop_tuple(count).await; + let paths = self.pop_string_tuple(count).await; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -2030,7 +2036,10 @@ impl StackMachine { .await .expect("could not check of existence") { - directory.remove(txn, paths.to_owned()).await; + directory + .remove(txn, paths.to_owned()) + .await + .expect("could not remove path"); } } @@ -2043,10 +2052,10 @@ impl StackMachine { DirectoryList => { debug!("List stack: {:?}", self.stack); let count = self.pop_usize().await; - let paths = self.pop_tuple(count).await; + let paths = self.pop_string_tuple(count).await; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -2083,10 +2092,10 @@ impl StackMachine { debug!("Exists stack: {:?}", self.stack); let count = self.pop_usize().await; - let paths = self.pop_tuple(count).await; + let paths = self.pop_string_tuple(count).await; let directory = self - .get_current_directory() + .get_current_directory_layer() .expect("could not find a directory"); let txn = match trx { @@ -2111,19 +2120,58 @@ impl StackMachine { // // Pop 1 tuple off the stack as [key_tuple]. Pack key_tuple and push the result // onto the stack. - DirectoryPackKey => unimplemented!(), + DirectoryPackKey => { + let n: usize = self.pop_usize().await; + debug!("DirectoryPackKey {}", n); + let mut buf = Vec::new(); + for _ in 0..n { + let element: Element = self.pop_element().await; + debug!(" - {:?}", element); + buf.push(element); + } + + let tuple = Element::Tuple(buf); + self.push( + number, + Element::Bytes(self.pack_with_current_subspace(&tuple).unwrap().into()), + ); + } // Use the current directory for this operation. // // Pop 1 item off the stack as [key]. Unpack key and push the resulting tuple // onto the stack one item at a time. - DirectoryUnpackKey => unimplemented!(), + DirectoryUnpackKey => { + let data = self.pop_bytes().await; + debug!("directory_unpack {:?}", data); + let data: Vec = self.unpack_with_current_subspace(&data).unwrap().unwrap(); + for element in data { + debug!(" - {:?}", element); + self.push(number, Element::Bytes(pack(&(element,)).into())); + } + } // Use the current directory for this operation. // // Pop 1 tuple off the stack as [tuple]. Create a range using tuple and push // range.begin and range.end onto the stack. - DirectoryRange => unimplemented!(), + DirectoryRange => { + debug!("directory_range stack: {:?}", self.stack); + let n: usize = self.pop_usize().await; + debug!("DirectoryRange {}", n); + let mut buf = Vec::new(); + for _ in 0..n { + let element: Element = self.pop_element().await; + debug!(" - {:?}", element); + buf.push(element); + } + + let tuple = Element::Tuple(buf); + let subspace = self.subspace_with_current(&tuple).unwrap(); + let (begin_range, end_range) = subspace.range(); + self.push(number, Element::Bytes(begin_range.into())); + self.push(number, Element::Bytes(end_range.into())); + } // Use the current directory for this operation. // @@ -2142,7 +2190,21 @@ impl StackMachine { // Pop 1 item off the stack as [prefix]. Let key equal // prefix + tuple.pack([dir_index]). Set key to be the result of calling // directory.key() in the current transaction. - DirectoryLogSubspace => unimplemented!(), + DirectoryLogSubspace => { + debug!("directory_log_subspace stack: {:?}", self.stack); + let raw_prefix = self.pop_bytes().await; + let txn = match trx { + TransactionState::Transaction(ref t) => t, + _ => { + panic!("could not find an active transaction"); + } + }; + + let directory = self.get_current_directory_subspace().unwrap(); + let key = pack(&(self.directory_index, &raw_prefix)); + let value = directory.bytes(); + txn.set(&key, &value); + } // Use the current directory for this operation. // @@ -2161,7 +2223,45 @@ impl StackMachine { // // Where log_subspace[u] is the subspace packed tuple containing only the // single specified unicode string . - DirectoryLogDirectory => unimplemented!(), + DirectoryLogDirectory => { + debug!("directory_log_directory stack: {:?}", self.stack); + let directory = self + .get_current_directory_layer() + .expect("could not find a directory"); + + let txn = match trx { + TransactionState::Transaction(ref t) => t, + _ => { + panic!("could not find an active transaction"); + } + }; + + let raw_prefix = self.pop_bytes().await; + let subspace = Subspace::from_bytes(&*raw_prefix).subspace(&(self.directory_index)); + + let key_path = subspace.pack(&(String::from("path"))); + let value_path = pack(&directory.get_path()); + + let key_layer = subspace.pack(&(String::from("layer"))); + let value_layer = pack(&directory.get_layer()); + + let exists = directory.exists(&txn, vec![]).await.unwrap(); + let key_exists = subspace.pack(&(String::from("exists"))); + let value_exists = pack(&(exists as i32)); + + let children = if exists { + directory.list(txn, vec![]).await.unwrap() + } else { + vec![] + }; + let key_children = subspace.pack(&(String::from("children"))); + let value_children = pack(&children); + + txn.set(&key_path, &value_path); + txn.set(&key_layer, &value_layer); + txn.set(&key_exists, &value_exists); + txn.set(&key_children, &value_children); + } // Use the current directory for this operation. // @@ -2210,7 +2310,7 @@ impl StackMachine { handle.join().expect("joined thread to not panic"); } } - fn get_current_directory(&self) -> Option> { + fn get_current_directory_layer(&self) -> Option> { debug!("current directory is at index {}", self.directory_index); match self.directory_stack.get(self.directory_index) { None => None, @@ -2221,6 +2321,53 @@ impl StackMachine { }, } } + + fn pack_with_current_subspace(&self, v: &T) -> Option> { + match self.directory_stack.get(self.directory_index) { + None => None, + Some(directory_or_subspace) => match directory_or_subspace { + DirectoryStackItem::DirectorySubspace(d) => Some(d.pack(v)), + DirectoryStackItem::Subspace(d) => Some(d.pack(v)), + _ => None, + }, + } + } + + fn unpack_with_current_subspace<'de, T: TupleUnpack<'de>>( + &self, + key: &'de [u8], + ) -> Option> { + match self.directory_stack.get(self.directory_index) { + None => None, + Some(directory_or_subspace) => match directory_or_subspace { + DirectoryStackItem::DirectorySubspace(d) => Some(d.unpack(key)), + DirectoryStackItem::Subspace(d) => Some(d.unpack(key)), + _ => None, + }, + } + } + + fn subspace_with_current(&self, t: &T) -> Option { + match self.directory_stack.get(self.directory_index) { + None => None, + Some(directory_or_subspace) => match directory_or_subspace { + DirectoryStackItem::DirectorySubspace(d) => Some(d.subspace(t)), + DirectoryStackItem::Subspace(d) => Some(d.subspace(t)), + _ => None, + }, + } + } + + fn get_current_directory_subspace(&self) -> Option { + debug!("current directory is at index {}", self.directory_index); + match self.directory_stack.get(self.directory_index) { + None => None, + Some(directory_or_subspace) => match directory_or_subspace { + DirectoryStackItem::DirectorySubspace(d) => Some(d.clone()), + _ => None, + }, + } + } } fn main() { diff --git a/foundationdb/src/directory/directory_layer.rs b/foundationdb/src/directory/directory_layer.rs index ce83f277..25448b5f 100644 --- a/foundationdb/src/directory/directory_layer.rs +++ b/foundationdb/src/directory/directory_layer.rs @@ -2,7 +2,7 @@ use crate::directory::error::DirectoryError; use crate::directory::node::Node; use crate::future::FdbSlice; use crate::tuple::hca::HighContentionAllocator; -use crate::tuple::{Subspace, TuplePack, TupleUnpack}; +use crate::tuple::{Subspace, TuplePack}; use crate::{FdbResult, Transaction}; use crate::directory::DirectorySubspace; @@ -11,7 +11,6 @@ use async_trait::async_trait; use byteorder::{LittleEndian, WriteBytesExt}; use std::cmp::Ordering; - // TODO: useful? const _LAYER_VERSION: (u8, u8, u8) = (1, 0, 0); const MAJOR_VERSION: u32 = 1; @@ -30,41 +29,6 @@ pub(crate) const DEFAULT_SUB_DIRS: i64 = 0; /// A path is represented as a List of strings. Each directory has an associated subspace used to store its content. /// The layer maps each path to a short prefix used for the corresponding subspace. /// In effect, directories provide a level of indirection for access to subspaces. -/// ## How-to use the Directory -/// -/// ```rust -/// use futures::prelude::*; -/// -/// async fn async_main() -> foundationdb::FdbResult<()> { -/// let db = foundationdb::Database::default()?; -/// -/// // creates a transaction -/// let trx = db.create_trx()?; -/// -/// // creates a directory -/// let directory = foundationdb::directory::DirectoryLayer::default(); -/// -/// // use the directory to create a subspace to use -/// let content_subspace = directory.create_or_open( -/// // the transaction used to read/write the directory. -/// &trx, -/// // the path used, which can view as a UNIX path like `/app/my-app`. -/// vec![String::from("app"), String::from("my-app")], -/// None, None, -/// ).await; -/// assert_eq!(true, content_subspace.is_ok()); -/// -/// // Don't forget to commit your transaction to persist the subspace -/// trx.commit().await?; -/// -/// Ok(()) -/// } -/// -/// // Safe because drop is called before the program exits -/// let network = unsafe { foundationdb::boot() }; -/// futures::executor::block_on(async_main()).expect("failed to run"); -/// drop(network); -/// ``` #[derive(Debug, Clone)] pub struct DirectoryLayer { /// the subspace used to store the hierarchy of paths. Each path is composed of Nodes. @@ -145,10 +109,20 @@ impl Directory for DirectoryLayer { /// `exists` returns true if the directory at path (relative to the default root directory) exists, and false otherwise. async fn exists(&self, trx: &Transaction, path: Vec) -> Result { - match dbg!( - self.find_or_create_node(trx, path.to_owned(), false, None) - .await - ) { + match self.check_version(trx, false).await { + Ok(()) => {} + Err(e) => { + return match e { + DirectoryError::MissingDirectory => Ok(false), + _ => Err(e), + } + } + } + + match self + .find_or_create_node(trx, path.to_owned(), false, None) + .await + { Ok(_node) => Ok(true), Err(err) => match err { DirectoryError::PathDoesNotExists => Ok(false), @@ -220,7 +194,7 @@ impl Directory for DirectoryLayer { .await?; let child_name = old_path.last().unwrap().to_owned(); - new_node_parent.remove_child(&trx, child_name).await; + new_node_parent.remove_child(&trx, child_name).await?; return self .contents_of_node(old_node.content_subspace, new_path, old_node.layer) @@ -244,11 +218,20 @@ impl Directory for DirectoryLayer { trx: &Transaction, path: Vec, ) -> Result, DirectoryError> { + dbg!(&path); let node = self .find_or_create_node(trx, path.to_owned(), false, None) .await?; node.list(&trx).await } + + fn get_path(&self) -> Vec { + self.path.clone() + } + + fn get_layer(&self) -> Vec { + vec![] + } } impl DirectoryLayer { diff --git a/foundationdb/src/directory/directory_subspace.rs b/foundationdb/src/directory/directory_subspace.rs index c9bf867f..984d1022 100644 --- a/foundationdb/src/directory/directory_subspace.rs +++ b/foundationdb/src/directory/directory_subspace.rs @@ -16,7 +16,6 @@ pub struct DirectorySubspace { layer: Vec, } -// directory related func impl DirectorySubspace { pub fn new( subspace: Subspace, @@ -31,6 +30,9 @@ impl DirectorySubspace { layer, } } + pub fn subspace(&self, t: &T) -> Subspace { + self.subspace.subspace(t) + } } #[async_trait] @@ -130,22 +132,24 @@ impl Directory for DirectorySubspace { old_path: Vec, new_path: Vec, ) -> Result { - self.move_to( - trx, - self.directory - .partition_subpath(self.path.clone(), old_path.clone()), - self.directory - .partition_subpath(self.path.clone(), new_path), - ) - .await + self.directory + .move_to( + trx, + self.directory + .partition_subpath(self.path.clone(), old_path.clone()), + self.directory + .partition_subpath(self.path.clone(), new_path), + ) + .await } async fn remove(&self, trx: &Transaction, path: Vec) -> Result { - self.remove( - trx, - self.directory.partition_subpath(self.path.clone(), path), - ) - .await + self.directory + .remove( + trx, + self.directory.partition_subpath(self.path.clone(), path), + ) + .await } async fn list( @@ -153,11 +157,21 @@ impl Directory for DirectorySubspace { trx: &Transaction, path: Vec, ) -> Result, DirectoryError> { - self.list( - trx, - self.directory.partition_subpath(self.path.clone(), path), - ) - .await + dbg!(&path); + self.directory + .list( + trx, + self.directory.partition_subpath(self.path.clone(), path), + ) + .await + } + + fn get_path(&self) -> Vec { + self.path.clone() + } + + fn get_layer(&self) -> Vec { + self.layer.clone() } } diff --git a/foundationdb/src/directory/mod.rs b/foundationdb/src/directory/mod.rs index 046ae7ea..03620b4d 100644 --- a/foundationdb/src/directory/mod.rs +++ b/foundationdb/src/directory/mod.rs @@ -18,7 +18,6 @@ mod node; use crate::directory::error::DirectoryError; - use crate::Transaction; use crate::directory::directory_subspace::DirectorySubspace; @@ -66,6 +65,9 @@ pub trait Directory { trx: &Transaction, path: Vec, ) -> Result, DirectoryError>; + + fn get_path(&self) -> Vec; + fn get_layer(&self) -> Vec; } pub fn compare_slice_string(a: &[String], b: &[String]) -> cmp::Ordering { diff --git a/foundationdb/src/directory/node.rs b/foundationdb/src/directory/node.rs index c62cc35a..94e22dfd 100644 --- a/foundationdb/src/directory/node.rs +++ b/foundationdb/src/directory/node.rs @@ -74,7 +74,8 @@ impl Node { pub(crate) async fn list(&self, trx: &Transaction) -> Result, DirectoryError> { let mut results = vec![]; - let range_option = RangeOption::from(&self.node_subspace.to_owned()); + let range_option = + RangeOption::from(&self.node_subspace.to_owned().subspace(&(DEFAULT_SUB_DIRS))); let fdb_values = trx.get_range(&range_option, 1_024, false).await?; @@ -100,9 +101,7 @@ impl Node { trx: &Transaction, child: String, ) -> Result<(), DirectoryError> { - let subspace = self - .node_subspace - .subspace(&(DEFAULT_SUB_DIRS, child)); + let subspace = self.node_subspace.subspace(&(DEFAULT_SUB_DIRS, child)); trx.clear_subspace_range(&subspace); diff --git a/foundationdb/tests/directory.rs b/foundationdb/tests/directory.rs index 686b8bd1..c3604222 100644 --- a/foundationdb/tests/directory.rs +++ b/foundationdb/tests/directory.rs @@ -9,9 +9,7 @@ use foundationdb::directory::directory_layer::DirectoryLayer; use foundationdb::directory::directory_subspace::DirectorySubspace; use foundationdb::directory::error::DirectoryError; use foundationdb::directory::Directory; - -use foundationdb::*; - +use foundationdb::{Database, FdbResult}; mod common; diff --git a/scripts/run_bindingtester.sh b/scripts/run_bindingtester.sh index 2c561ec2..6cfdb74c 100755 --- a/scripts/run_bindingtester.sh +++ b/scripts/run_bindingtester.sh @@ -32,8 +32,8 @@ esac ## Run the test echo "testers['rust'] = Tester('rust', '${bindingtester}', 2040, 23, MAX_API_VERSION, types=ALL_TYPES) " >> ./bindings/bindingtester/known_testers.py - ./bindings/bindingtester/bindingtester.py --test-name scripted rust - ./bindings/bindingtester/bindingtester.py --num-ops 1000 --api-version 610 --test-name api --compare python rust - ./bindings/bindingtester/bindingtester.py --num-ops 1000 --api-version 610 --test-name api --concurrency 5 rust - ./bindings/bindingtester/bindingtester.py --num-ops 1000 --api-version 610 --test-name directory --concurrency 1 rust --seed 1330929912 + python2 ./bindings/bindingtester/bindingtester.py --test-name scripted rust + python2 ./bindings/bindingtester/bindingtester.py --num-ops 1000 --api-version 610 --test-name api --compare python rust + python2 ./bindings/bindingtester/bindingtester.py --num-ops 1000 --api-version 610 --test-name api --concurrency 5 rust + python2 ./bindings/bindingtester/bindingtester.py --num-ops 10 --api-version 610 --test-name directory --concurrency 1 rust --seed 1330929912 --no-directory-snapshot-ops )