diff --git a/Cargo.lock b/Cargo.lock index b44ac7c..9a9b93e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,12 +8,28 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro2" version = "1.0.85" @@ -73,6 +89,7 @@ dependencies = [ name = "slkvs" version = "0.0.1" dependencies = [ + "pretty_assertions", "serde", "serde_json", "wit-bindgen", @@ -112,3 +129,9 @@ checksum = "29c7526379ace8709ee9ab9f2bb50f112d95581063a59ef3097d9c10153886c9" dependencies = [ "bitflags", ] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" diff --git a/Cargo.toml b/Cargo.toml index 5c1277e..ba30c28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ opt-level = 's' wit-bindgen = { version = "0.26.0", default-features = false, features = ["realloc"] } serde = "*" serde_json = "*" +pretty_assertions = "1.4.0" [package.metadata.component.target] path = "wit" - diff --git a/cmds.fish b/cmds.fish index f21c0e3..dfc3745 100644 --- a/cmds.fish +++ b/cmds.fish @@ -1,36 +1,111 @@ -alias gli=golem-cli +# +# This contains fish shell commands to: +# - build and deploy the component +# - interact with the component once it's deployed + +alias gli golem-cli alias build="cargo component build" +################################ +# building the KV store function add_component golem-cli component add --component-name slkvs target/wasm32-wasi/release/slkvs.wasm end +# first-time only +function deploy --description "for the first time deploy of a component with its worker" + cargo component build --release || return 1 + golem-cli worker delete --worker-name fst --component-name slkvs + golem-cli component update --component-name slkvs target/wasm32-wasi/release/slkvs.wasm + golem-cli worker add --worker-name fst --component-name slkvs +end + +# for updates use this +function redeploy -a version --description "redeploy to a version" + cargo component build --release || return 1 + + # Use `golem-cli component update` to figure out which version to use. + # Normal output is something like + # Updated component with ID 3825415a-f2f9-42dc-99d3-715ff89690a0. New version: 1. Component size is 168531 bytes. + set result_msg (golem-cli component update --component-name slkvs target/wasm32-wasi/release/slkvs.wasm) + echo vs: (count $result_msg) + + # extract version + set captures (string match --regex -g 'Updated component with ID (.*?) New version: (\d+). Component size is (\d+) bytes.*' $result_msg) + if test $status -ne 0 + echo Failed to match output from `golem-cli component update`: "\n$result_msg" + return 1 + else + echo -e (string join \\n $result_msg) + end + set target_version $captures[2] + + # do the update + echo -n "Updating to component version $target_version... " + golem-cli worker update --worker-name fst --target-version $target_version --mode auto --component-name slkvs +end + +function worker_restart + golem-cli worker delete --worker-name fst --component-name slkvs + golem-cli worker add --worker-name fst --component-name slkvs +end + +################################ +# talking to the KV store +function gli_quote + for v in $argv + echo -n "$v" | jq -Rs + end +end + +function gli_parameters + # create an array in fish, since all variables are arrays. + set -l quoted (gli_quote $argv) + set joined (string join , $quoted) + echo "[$joined]" +end + +function gli_noquote_parameters + set joined (string join , $argv) + echo "[$joined]" +end + function get golem-cli worker invoke-and-await --component-name=slkvs \ --worker-name=fst \ --function=golem:component/api/get \ - --parameters="[\"$argv[1]\"]" + --parameters=(gli_parameters $argv[1]) +end + +function hgettree + set component_id (gli_component_id) + set worker_name fst + set function_name golem:component/api/gettree + + set params "{\"params\": $(gli_parameters $argv[1])}" + set url "http://localhost:9881/v2/components/$component_id/workers/$worker_name/invoke-and-await?function=$function_name&calling-convention=Component" + echo -e (curl --silent --json $params $url) | jq .result[0] | string unescape | jq . end function gettree golem-cli worker invoke-and-await --component-name=slkvs \ --worker-name=fst \ --function=golem:component/api/gettree \ - --parameters="[\"$argv[1]\"]" + --parameters=(gli_parameters $argv[1]) end function delete golem-cli worker invoke-and-await --component-name=slkvs \ --worker-name=fst \ --function=golem:component/api/delete \ - --parameters="[\"$argv[1]\"]" + --parameters=(gli_parameters $argv[1]) end function add golem-cli worker invoke-and-await --component-name=slkvs \ --worker-name=fst \ --function=golem:component/api/add \ - --parameters="[\"$argv[1]\", \"$argv[2]\"]" + --parameters=(gli_parameters $argv[1] $argv[2]) end function listpaths @@ -38,7 +113,31 @@ function listpaths --component-name=slkvs \ --worker-name=fst \ --function=golem:component/api/listpaths \ - --parameters=$argv + --parameters=(gli_parameters $argv[1]) +end + +function gli_component_id + set result_msg (gli component get --component-name slkvs) + set captures (string match --regex -g 'Component with ID (.*?). Version: (\d+). Component size is (\d+) bytes.*' $result_msg) + if test $status -ne 0 + echo Failed to match output from `golem-cli component update`: "\n$result_msg" + return 1 + end + + set component_id $captures[1] + echo $component_id +end + +function hlistpaths --description "invoke listpaths via http api" + set component_id (gli_component_id) + set worker_name fst + set function_name golem:component/api/listpaths + # curl -d '{"function": "listpaths"}' "http://localhost:9881/v2/components/$component_id/workers/$worker_name/invoke-and-await?function=$function_name&calling-convention=Component" + # Content-Type: multipart/form-data + # curl -F '{"params": null}' "http://localhost:9881/v2/components/$component_id/workers/$worker_name/invoke-and-await?function=$function_name&calling-convention=Component" + # curl -F "function=$function_name" -F "calling-convention=Component" "http://localhost:9881/v2/components/$component_id/workers/$worker_name/invoke-and-await" + set json_rsp (curl --silent --json '{"params": []}' "http://localhost:9881/v2/components/$component_id/workers/$worker_name/invoke-and-await?function=$function_name&calling-convention=Component") + echo $json_rsp | jq .result[0] end function addtree @@ -58,7 +157,7 @@ function addtree --component-name=slkvs \ --worker-name=fst \ --function=golem:component/api/addtree \ - --parameters="[\"$argv[1]\", $escaped_tree]" + --parameters=(gli_noquote_parameters (gli_quote $argv[1]) $escaped_tree) end function drop @@ -67,21 +166,3 @@ function drop --worker-name=fst \ --function=golem:component/api/drop end - -function deploy - cargo component build --release || return 1 - gli worker delete --worker-name fst --component-name slkvs - gli component update --component-name slkvs target/wasm32-wasi/release/slkvs.wasm - gli worker add --worker-name fst --component-name slkvs -end - -function redeploy -a version --description "redeploy to a version" - cargo component build --release || return 1 - gli component update --component-name slkvs target/wasm32-wasi/release/slkvs.wasm - gli worker update --worker-name fst --target-version $argv[1] --mode auto --component-name slkvs -end - -function worker_restart - gli worker delete --worker-name fst --component-name slkvs - gli worker add --worker-name fst --component-name slkvs -end diff --git a/src/lib.rs b/src/lib.rs index 52317b0..2b360f5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,7 +37,16 @@ impl Guest for Component { } fn gettree(path: String) -> Option { - STATE.with_borrow(|state| state.gettree(path) ) + STATE.with_borrow(|state| { + let subtree = state.gettree(path); + // TODO This is a hack and not entirely correct. There could be exactly one + // value at `path`, and it could indeed by null. + if subtree == tree::Collector::Empty { + None + } else { + Some(subtree.to_json().to_string()) + } + }) } fn drop() { diff --git a/src/tree.rs b/src/tree.rs index 19b7920..c0b133e 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -69,7 +69,7 @@ impl From> for SchemaPath { } } -use std::ops::Add; +use std::{collections::HashMap, ops::Add}; impl Add for SchemaPath { @@ -188,6 +188,7 @@ where } } +// The storage for the keys and the values. type PathMap = std::collections::BTreeMap; pub struct LeafPaths(pub PathMap>); @@ -207,6 +208,124 @@ impl From for DingString { } } +use std::collections::BTreeMap; + +/** + Need this to collect the results of a traverse_tree + + Needs to be different to serde_json::Value because: + + 1) it can be empty (rather than null) + + 2) the array must be sparse so that indexes coming across from a + serde_json::Value can be matched. What happens it that a path into a + serde_json::Value might skip indexes, so the output is a sparse collection + of indexes, and the easiest way to model that is a HashMap. +*/ +#[derive(Debug,PartialEq,Eq)] +pub enum Collector { + Empty, + + // Now have to duplicate some of the serde_json::Value members + Null, + Bool(bool), + Number(String), + String(String), + + // instead of serde_json::Value::Array + Sparse(BTreeMap), + + Object(HashMap), +} + +impl From<&Leaf> for Collector { + fn from(leaf: &Leaf) -> Self { + match leaf { + Leaf::String(v) => Collector::String(v.clone()), + + // If this parse fails we should've chosen arbitrary precision at compile + // time. So not much we can do about that here, short of not using + // serde_json::Value as the enum. Which is not worthwhile at this point, + // in this codebase. + Leaf::Number(v) => Collector::Number(v.parse().unwrap()), + + Leaf::Boolean(v) => Collector::Bool(*v), + Leaf::Null => Collector::Null, + } + } +} + +impl Collector { + pub fn to_json(&self) -> serde_json::Value { + use serde_json::Value; + match self { + // TODO hmmmmm :-s + Self::Empty => Value::Null, + Self::Null => Value::Null, + Self::Bool(v) => Value::Bool(*v), + // TODO hmmm. But look, if it's a number then we should be fine here, unless its a "NaN" or "Inf" + Self::Number(v) => Value::Number(v.parse::().unwrap()), + Self::String(v) => Value::String(v.to_string()), + Self::Sparse(v) => { + if let Some((min_index,_)) = v.first_key_value() { + // safe to unwrap here because there is at least one item - see min_index above + let (max_index,_) = v.last_key_value().unwrap(); + let mut values : Vec = Vec::with_capacity(max_index - min_index + 1); + + // order from index least m to greatest n and then assign indexes m-m .. n-m + for (_i,v) in v.iter() { + values.push(v.into()); + } + Value::Array(values) + } else { + Value::Array(vec![]) + } + } + Self::Object(v) => { + let mut values : serde_json::Map = serde_json::Map::new(); + for (key,val) in v.iter() { + values.insert(key.into(),val.into()); + } + Value::Object(values) + } + } + } +} + +impl From<&serde_json::Value> for Collector { + fn from(val: &serde_json::Value) -> Self { + match val { + serde_json::Value::Null => Self::Null, + serde_json::Value::Bool(v) => Self::Bool(*v), + // TODO this conversion must be optimised. + serde_json::Value::Number(v) => Self::Number(v.to_string()), + serde_json::Value::String(v) => Self::String(v.to_string()), + serde_json::Value::Array(v) => { + let mut values : BTreeMap = BTreeMap::new(); + for (i,v) in v.iter().enumerate() { + values.insert(i,v.into()); + } + Self::Sparse(values) + } + serde_json::Value::Object(v) => { + let mut values : HashMap = HashMap::new(); + for (i,v) in v.iter() { + values.insert(i.to_string(),v.into()); + } + Self::Object(values) + } + } + } +} + +impl Into for &Collector { + fn into(self) -> serde_json::Value {self.to_json() } +} + +impl Into for Collector { + fn into(self) -> serde_json::Value { (&self).into() } +} + // This provides a thin wrapper around the BTree/Hash map and implements // function calls coming in from the component. Because it's easier to write // tests this way. @@ -303,60 +422,55 @@ impl LeafPaths { // either a new collection (ie array or map), or an individual value. // // rcp is "recipient", which is kinda like an io, except tree-structured. - fn traverse_tree<'a,'b>(path: &'a [Step], value: &'a Leaf, rcp : &'b mut serde_json::Value) { - use serde_json::Value; - + fn traverse_tree<'a,'b>(path: &'a [Step], value: &'a Leaf, rcp : &'b mut Collector) { // Essentially, a path step is either a key or an index; and a value is a collection or a naked value. match (path, rcp) { // last step, therefore we can insert value - ([Step::Key(k)], Value::Object(ref mut map)) => { + ([Step::Key(k)], Collector::Object(ref mut map)) => { map.insert(k.into(),value.into()); }, - ([Step::Index(i)], Value::Array(ref mut ary)) => { - if *i != ary.len() { panic!("index {i} unexpected compared to length {}", ary.len()) }; - ary.push(value.into()); + ([Step::Index(i)], Collector::Sparse(ref mut ary)) => { + // TODO looks like this check makes no sense with a sparse array? + // if *i != ary.len() { panic!("index {i} unexpected compared to length {}", ary.len()) }; + ary.insert(*i, value.into()); }, - // not the last step, so contruct intermediate and keep going - ([Step::Key(k), rst @ .. ], Value::Object(ref mut map)) => { + // not the last step, so construct intermediate and keep going + ([Step::Key(k), rst @ .. ], Collector::Object(ref mut map)) => { if let Some(intermediate) = map.get_mut(k) { // we already have an object at this key, so reuse it Self::traverse_tree(rst, &value, intermediate); } else { - // Dunno what kind of object it's going to be yet - let mut intermediate = serde_json::Value::Null; + // Dunno yet what kind of object it's going to be + let mut intermediate = Collector::Empty; Self::traverse_tree(rst, &value, &mut intermediate); map.insert(k.into(),intermediate); } } - ([Step::Index(i), rst @ ..], Value::Array(ref mut ary)) => { - if let Some(intermediate) = ary.get_mut(*i) { + // FIXME i is not necessarily an index into ary. Because the tree path may have skipped lower i values. + ([Step::Index(i), rst @ ..], Collector::Sparse(ref mut ary)) => { + if let Some(intermediate) = ary.get_mut(i) { // we already have an object at this index, so reuse it Self::traverse_tree(rst, &value, intermediate); } else { - // Dunno what kind of object it's going to be yet - let mut intermediate = serde_json::Value::Null; + // Dunno yet what kind of object it's going to be + let mut intermediate = Collector::Empty; Self::traverse_tree(rst, &value, &mut intermediate); - ary.push(intermediate); + ary.insert(*i, intermediate); } } - // The cases where rcp is Null, ie we haven't yet figured out what it is. + // The cases where rcp is Empty, ie we haven't yet figured out what it is. // So assign the correct kind of collection, and try again, which will // hit one of the above matches. - // - // (More precisely it's Option == None, - // but that extra level of indirection doesn't really achieve anything, - // so we might as well fold it into serde_json::Value - ([Step::Key(_), ..], rcp @ Value::Null) => { + ([Step::Key(_), ..], rcp @ Collector::Empty) => { // create a new map and try again - let map = serde_json::Map::new(); - *rcp = serde_json::Value::Object(map); + *rcp = Collector::Object(HashMap::new()); Self::traverse_tree(path, value, rcp) } - ([Step::Index(_), ..], rcp @ Value::Null) => { + ([Step::Index(_), ..], rcp @ Collector::Empty) => { // create a new ary and try again - *rcp = serde_json::Value::Array(vec![]); + *rcp = Collector::Sparse(BTreeMap::new()); Self::traverse_tree(path, value, rcp) } (path, rcp) => todo!("oopsies with {:?} {:?}", path, rcp), @@ -364,24 +478,17 @@ impl LeafPaths { } /// Fetch an entire subtree, as a string representation of the json rooted at that path. - /// TODO this conflates Null with None - pub fn gettree(&self, path: String) -> Option { + pub fn gettree(&self, path: String) -> Collector { // fetch all subtree paths with their values let path = SchemaPath::from(path); let subtree_path_values = self.subtree_paths(path); // ok build the object - let mut obj = serde_json::Value::Null; + let mut obj = Collector::Empty; for (schema_path,value) in subtree_path_values { LeafPaths::traverse_tree(&schema_path.0, &value, &mut obj); } - // TODO This is a hack and not entirely correct. There could be exactly one - // value at `path`, and it could indeed by null. - if obj == serde_json::Value::Null { - None - } else { - Some(obj.to_string()) - } + obj } pub fn delete(&mut self, path: String) { @@ -408,6 +515,8 @@ where #[cfg(test)] mod t { use super::*; + #[allow(unused_imports)] + use pretty_assertions::{assert_eq, assert_ne}; macro_rules! step_of { ($x:ident) => (Step::Key("$x".into())); @@ -609,26 +718,25 @@ mod t { #[test] fn subtree_traverse() { - let json = r#"{ + let json = serde_json::json!({ "top": "this", "next": { "inner": "some value" }, "wut": null, "stuff": [9,8,7,6,5] - }"#; + }); let mut leaf_paths = LeafPaths::new(); - leaf_paths.addtree("root".into(), json.into()).unwrap(); + leaf_paths.addtree("root".into(), json.to_string()).unwrap(); let paths = leaf_paths.subtree_paths(path_of_strs!["root"]); - let mut obj = serde_json::Value::Null; + let mut subtree = Collector::Empty; for (schema_path, value) in paths { - LeafPaths::traverse_tree(&schema_path.0, &value, &mut obj); + LeafPaths::traverse_tree(&schema_path.0, &value, &mut subtree); } - let json : serde_json::Value = serde_json::from_str(json).expect("json not parseable"); let json = serde_json::json!({ "root": json }); - assert_eq!(obj,json); + assert_eq!(subtree.to_json(),json); } #[test] @@ -649,16 +757,75 @@ mod t { let mut leaf_paths = LeafPaths::new(); leaf_paths.addtree("root".into(), json.into()).unwrap(); - let json = leaf_paths.gettree("root".into()).unwrap(); - assert_eq!(json, r#"{"root":{"next":{"inner":"some value"},"stuff":[9,8,7,6,5],"things":[{"name":"one"},{"name":"two"},{"name":"tre"}],"top":"this","wut":null}}"#); + let subtree = leaf_paths.gettree("root".into()); + let json : serde_json::Value = (&subtree).into(); + assert_eq!(json.to_string(), r#"{"root":{"next":{"inner":"some value"},"stuff":[9,8,7,6,5],"things":[{"name":"one"},{"name":"two"},{"name":"tre"}],"top":"this","wut":null}}"#); + + let subtree = leaf_paths.gettree("root/things".into()); + assert_eq!(subtree.to_json(), serde_json::json!({"root":{"things":[{"name":"one"},{"name":"two"},{"name":"tre"}]}})); + + let subtree = leaf_paths.gettree("root/things/1".into()); + assert_eq!(subtree.to_json(), serde_json::json!({"root":{"things":[{"name":"two"}]}})); + + let subtree = leaf_paths.gettree("does/not/exist/5/really".into()); + assert_eq!(subtree, Collector::Empty); + } + + #[test] + fn big_gettree() { + let json = r#"{ + "top": "this", + "next": [{ + "inner": "some value", + "tweede": "'n ander waarde", + "third": "stone from the sun" + }], + "wut": null, + "stuff": [9,8,7,6,5], + "things": [ + {"name": "one"}, + {"name": "two"}, + {"name": "tre"} + ] + }"#; + + let mut leaf_paths = LeafPaths::new(); + leaf_paths.addtree("root".into(), json.into()).unwrap(); + + let subtree = leaf_paths.gettree("root/next".into()); + let expected = serde_json::json!({"root":{"next":[{"inner":"some value","third":"stone from the sun","tweede":"'n ander waarde"}]}}); + assert_eq!(subtree.to_json(), expected); - let json = leaf_paths.gettree("root/things".into()).unwrap(); - assert_eq!(json, r#"{"root":{"things":[{"name":"one"},{"name":"two"},{"name":"tre"}]}}"#); + let subtree = leaf_paths.gettree("root/things".into()); + assert_eq!(subtree.to_json().to_string(), r#"{"root":{"things":[{"name":"one"},{"name":"two"},{"name":"tre"}]}}"#); - let json = leaf_paths.gettree("root/things/1".into()).unwrap(); - assert_eq!(json, r#"{"root":{"things":[{"name":"two"}]}}"#); + let subtree = leaf_paths.gettree("root/things/1".into()); + assert_eq!(subtree.to_json().to_string(), r#"{"root":{"things":[{"name":"two"}]}}"#); + + let subtree = leaf_paths.gettree("does/not/exist/5/really".into()); + assert_eq!(subtree.to_json(), serde_json::Value::Null); + } - let json = leaf_paths.gettree("does/not/exist/5/really".into()); - assert!(json.is_none()); + #[test] + // This exercises the construction of the sparse array of the Collector + fn sample_gettree() { + let sample_json_str = include_str!("../sample.json"); + let mut leaf_paths = LeafPaths::new(); + leaf_paths.addtree("root".into(), sample_json_str.into()).unwrap(); + + let subtree = leaf_paths.gettree("root/web-app/servlet/2".into()); + let expected = serde_json::json!({ + "root": { + "web-app": { + "servlet": [ + { + "servlet-class": "org.cofax.cds.AdminServlet", + "servlet-name": "cofaxAdmin", + } + ] + } + } + }); + assert_eq!(subtree.to_json(), expected); } } diff --git a/wit/slkvs.wit b/wit/slkvs.wit index c63664e..1268e48 100644 --- a/wit/slkvs.wit +++ b/wit/slkvs.wit @@ -3,7 +3,7 @@ package golem:component; // For more details about the WIT syntax, see // https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md -// naming is a little odd, because these map directly in cli commands, +// naming is a little odd, because these map directly to cli commands, // and there, it's a PITA to type unnecessary - and _ interface api { add: func(path: string, value: string);