From de04df8129f3370387c1b22263c9021781babd7e Mon Sep 17 00:00:00 2001 From: Trevor McMaster Date: Tue, 26 Sep 2023 14:04:58 -0600 Subject: [PATCH] Config updater improvements (#151) * Initial improvements to keep some of the code parsing over - Can now parse epoch and encode as well as basic response.body.x * Finished converting all functions except collect over to the config-updater * Fixed formatting * Cleaned up the Display Code for Path * Simplified and fixed the converting of Json object queries * Fixed the stack overflow bug - If there was an comparison operators in an expression, it caused a constant stack climb and it repeatedly called display - Also cleaned up some of the log messages * Added code to handle parsing of expressions with right hand side * config-updater updates - Added fix for empty strings that are parsed as templates with no pieces - Added initial code to allow parsing of Segments with rest into multiple segments * Added fix for sting values * Fixed the complex expressions to parse expressions with vars/providers in them * Fixed Clippy warnings * comments and fixes - Fixed PathSegements that were array references that were quoting the number - Renamed to_convert() to convert_to_v2() to better describe the functionality - Added some comments to the parsing Uniform functions * cargo fmt * Changed the default order of loggers * Added code to handle base expresssions with right hand side operations * Removed ignore that we've removed the dependency * Changed npm install in build-guide to npm ci * Updated test for change to logger order * Updated README * Updated bugs in the README * Added an additional bug to README with workaround --- README.md | 9 +- deny.toml | 2 +- guide/build-guide.sh | 2 +- guide/serve-guide.sh | 2 +- lib/config-gen/tests/test.js | 4 +- lib/config/src/configv1/convert_helper.rs | 59 ++- .../src/configv1/expression_functions.rs | 205 ++++++++++ lib/config/src/configv1/select_parser.rs | 369 ++++++++++++++++-- lib/config/src/configv2/loggers.rs | 2 +- lib/config/src/configv2/query.rs | 18 +- lib/config/src/configv2/templating.rs | 38 ++ lib/config/src/shared.rs | 11 + lib/config/src/shared/encode.rs | 14 + pewpew-config-updater/README.md | 5 + 14 files changed, 683 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 3e7a1797..a634df10 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,15 @@ C:\vcpkg> set VCPKGRS_DYNAMIC=1 (or simply set it as your environment variable) Changes: - Major changes: Javascript scripting! - Updated config-wasm to parse legacy and scripting yaml files +- New binary pewpew-config-updater will attempt to convert legacy config yamls to the new version. If it can't convert the code it will leave in PLACEHOLDERS and TODO + - Known issues: + - Expressions in vars will not wrap environment variables in the expected `${e:VAR}` + - vars in `logs` and `provides` will not have the prepended `_v.` before the var name. Bugs: -- Collect returns an array of strings regardless of input type -- auto-converter removes code templates. Leave in and TODO +- Collect returns an array of strings regardless of input type. Workaround, use scripting to `.map(parseInt)`. +- Declare expressions that create strings will escape out any json/quotes. No workaround currently. +- Vars cannot be decimal point values. Ex `peakLoad: 0.87`. Workaround: `peakLoad: ${x:0.87}` - global loggers may not be running in try script Bug fixes: diff --git a/deny.toml b/deny.toml index 360514cc..d68bd04b 100644 --- a/deny.toml +++ b/deny.toml @@ -14,7 +14,7 @@ unmaintained = "warn" yanked = "warn" notice = "warn" ignore = [ - "RUSTSEC-2020-0071", + # "RUSTSEC-2020-0071", ] [licenses] diff --git a/guide/build-guide.sh b/guide/build-guide.sh index c4e601aa..330a879b 100755 --- a/guide/build-guide.sh +++ b/guide/build-guide.sh @@ -42,7 +42,7 @@ wasm-pack build --release -t bundler -d $CFG_GEN_OUTPUT_REACT_DIR --scope fs # build the results viewer (which includes putting the output into the book's src) cd $RESULTS_VIEWER_REACT_DIR -npm install +npm ci npm run build # build the book diff --git a/guide/serve-guide.sh b/guide/serve-guide.sh index e6d1cb58..83e8a542 100755 --- a/guide/serve-guide.sh +++ b/guide/serve-guide.sh @@ -42,7 +42,7 @@ wasm-pack build --release -t bundler -d $CFG_GEN_OUTPUT_REACT_DIR --scope fs # build the results viewer (which includes putting the output into the book's src) cd $RESULTS_VIEWER_REACT_DIR -npm install +npm ci npm run build # build the book diff --git a/lib/config-gen/tests/test.js b/lib/config-gen/tests/test.js index 4ab2f378..7f570987 100644 --- a/lib/config-gen/tests/test.js +++ b/lib/config-gen/tests/test.js @@ -66,15 +66,15 @@ loggers: timestamp: epoch("ms") for_each: [] where: null + limit: null to: stdout pretty: false - limit: null kill: false test2: query: null + limit: null to: !file out.txt pretty: false - limit: null kill: false providers: sequence: !list diff --git a/lib/config/src/configv1/convert_helper.rs b/lib/config/src/configv1/convert_helper.rs index 421d2ca3..ded96caf 100644 --- a/lib/config/src/configv1/convert_helper.rs +++ b/lib/config/src/configv1/convert_helper.rs @@ -374,17 +374,60 @@ fn map_query( for_each: Vec>, where_clause: Option>, ) -> Option> { - if let Some(w) = where_clause { + // Fallback query if we can't parse anything + let empty_query = Query::simple("PLEASE_UPDATE_MANUALLY".to_owned(), vec![], None).unwrap(); + + // Attempt to parse the where_clause + let where_clause = if let Some(w) = where_clause { let w = w.destruct().0; - log::warn!("query `where` item {w:?} must be updated manually"); + Some(w) + } else { + None + }; + // Attempt to parse the for_each + let for_each: Vec = for_each + .iter() + .map(|fe| (fe.inner.to_string(), fe.marker).0) + .collect(); + + // See if we can create a fallback with the where and for_each but without the select + let manual_query = match Query::simple( + "PLEASE_UPDATE_MANUALLY".to_owned(), + for_each.clone(), + where_clause.clone(), + ) { + Ok(q) => q, + Err(e) => { + log::warn!("query `where` or `for_each` item must be updated manually: {e:?}"); + empty_query + } }; - for_each.into_iter().for_each(|fe| { - let fe = fe.destruct().0; - log::warn!("query `for_each` item {fe:?} must be updated manually"); - }); + + // Finally attempt to parse the select but fallback to the manual_query or empty_query select.map(|s| { - log::warn!("query `select` item {s:?} must be updated manually"); - Query::simple("PLEASE_UPDATE_MANUALLY".to_owned(), vec![], None).unwrap() + let select_temp = s.inner(); + log::debug!("select_temp query: {select_temp}"); + let query = match select_temp { + json::Value::Object(_) => { + log::info!("Object query: {select_temp}"); + Query::::complex_json( + format!("{select_temp}").as_str(), + for_each, + where_clause, + ) + } + json::Value::String(s) => Query::simple(s.to_string(), for_each, where_clause), + _ => Query::simple(format!("{select_temp}"), for_each, where_clause), + }; + let query = match query { + Ok(q) => q, + Err(e) => { + log::warn!("query `select` {select_temp} must be updated manually: {e:?}"); + manual_query + } + }; + log::debug!("new query: {query:?}"); + query }) } diff --git a/lib/config/src/configv1/expression_functions.rs b/lib/config/src/configv1/expression_functions.rs index e113b29c..85ac5e12 100644 --- a/lib/config/src/configv1/expression_functions.rs +++ b/lib/config/src/configv1/expression_functions.rs @@ -17,6 +17,48 @@ use zip_all::zip_all; use std::{borrow::Cow, cmp::Ordering, collections::BTreeMap, fmt, iter, sync::Arc, task::Poll}; +/// Helper function for converting to new v2 config where we use a Uniform Range. +/// Takes a format!("{range:?}") and finds the low an high and returns them as strings +/// +/// # Panics +/// +/// Panics if . +fn get_low_high(uniform: String) -> (String, String) { + // Will be something like Uniform(UniformInt { low: 0, range: 10, z: 6 }) + // Or Uniform(UniformFloat { low: 1.1, scale: 9.200000000000001 }) + let re_int = Regex::new(r"low: (\d+), range: (\d+),").unwrap(); + let re_float = Regex::new(r"low: ([0-9\.]+), scale: ([0-9\.]+) ").unwrap(); + match re_int.captures(uniform.as_str()) { + Some(caps) => { + let low = (caps[1]).parse::(); + let range = (caps[2]).parse::(); + if let (Ok(low), Ok(range)) = (low, range) { + let high = range + low; + (format!("{low:.0}"), format!("{high:.0}")) + } else { + ((caps[1]).to_owned(), (caps[2]).to_owned()) + } + } + None => match re_float.captures(uniform.as_str()) { + Some(caps) => { + let low = (caps[1]).parse::(); + let range = (caps[2]).parse::(); + if let (Ok(low), Ok(range)) = (low, range) { + let high = range + low; + if low.fract() == 0.0 && high.fract() == 0.0 { + (format!("{low:.0}"), format!("{high:.0}")) + } else { + (format!("{low:.3}"), format!("{high:.3}")) + } + } else { + ((caps[1]).to_owned(), (caps[2]).to_owned()) + } + } + None => (uniform, "unknown".to_string()), + }, + } +} + #[derive(Clone, Debug)] pub(super) struct Collect { arg: ValueOrExpression, @@ -149,6 +191,12 @@ pub(super) struct Encode { encoding: Encoding, } +impl std::fmt::Display for Encode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "encode({}, \"{}\")", self.arg, self.encoding) + } +} + impl Encode { pub(super) fn new( mut args: Vec, @@ -219,6 +267,12 @@ pub struct Entries { arg: ValueOrExpression, } +impl std::fmt::Display for Entries { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "entries({})", self.arg) + } +} + impl Entries { pub(super) fn new( mut args: Vec, @@ -381,6 +435,12 @@ pub(super) struct If { third: ValueOrExpression, } +impl std::fmt::Display for If { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "if({}, {}, {})", self.first, self.second, self.third) + } +} + impl If { pub(super) fn new( mut args: Vec, @@ -498,6 +558,10 @@ impl If { } }) } + + pub(super) fn convert_to_v2(&self) -> String { + format!("({}) ? {} : {}", self.first, self.second, self.third) + } } #[derive(Clone, Debug)] @@ -507,6 +571,16 @@ pub(super) struct Join { sep2: Option, } +impl std::fmt::Display for Join { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(sep2) = &self.sep2 { + write!(f, "join({}, \"{}\", \"{}\")", self.arg, self.sep, sep2) + } else { + write!(f, "join({}, \"{}\")", self.arg, self.sep) + } + } +} + impl Join { pub(super) fn new( mut args: Vec, @@ -620,6 +694,7 @@ pub(super) struct JsonPath { provider: String, selector: Arc, marker: Marker, + args: Vec, // Save for display } impl fmt::Debug for JsonPath { @@ -634,6 +709,18 @@ impl fmt::Debug for JsonPath { } } +impl std::fmt::Display for JsonPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let args: Vec = self + .args + .clone() + .into_iter() + .map(|arg| format!("{}", arg)) + .collect(); + write!(f, "json_path({})", args.join(",")) + } +} + impl JsonPath { pub(super) fn new( args: Vec, @@ -668,6 +755,7 @@ impl JsonPath { provider: provider.into(), selector: json_path.into(), marker, + args: args.clone(), }; let v = static_vars.get(provider).cloned(); if let Some(v) = v { @@ -733,6 +821,12 @@ pub(super) struct Match { regex: Regex, } +impl std::fmt::Display for Match { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "match({}, \"{}\")", self.arg, self.regex) + } +} + impl Match { pub(super) fn new( args: Vec, @@ -839,6 +933,22 @@ pub(super) struct MinMax { min: bool, } +impl std::fmt::Display for MinMax { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let args: Vec = self + .args + .clone() + .into_iter() + .map(|arg| format!("{arg}")) + .collect(); + if self.min { + write!(f, "min({})", args.join(", ")) + } else { + write!(f, "max({})", args.join(", ")) + } + } +} + impl MinMax { pub(super) fn new( min: bool, @@ -949,6 +1059,24 @@ pub(super) struct Pad { padding: String, } +impl std::fmt::Display for Pad { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.start { + write!( + f, + "start_pad({}, {}, \"{}\")", + self.arg, self.min_length, self.padding + ) + } else { + write!( + f, + "end_pad({}, {}, \"{}\")", + self.arg, self.min_length, self.padding + ) + } + } +} + impl Pad { pub(super) fn new( start: bool, @@ -1066,6 +1194,13 @@ pub enum Random { Float(Uniform), } +impl std::fmt::Display for Random { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (low, high) = self.get_low_high(); + write!(f, "random({}, {})", low, high) + } +} + impl Random { pub(super) fn new( args: Vec, @@ -1111,6 +1246,25 @@ impl Random { Ok((self.evaluate().into_owned(), Vec::new())) })) } + + /// Helper function for converting to new v2 config where we use a Uniform Range. + /// Takes a format!("{range:?}") and finds the low an high and returns them as strings + /// + /// # Panics + /// + /// Panics if . + pub(super) fn get_low_high(&self) -> (String, String) { + let uniform = match self { + Random::Integer(r) => format!("{:?}", r), + Random::Float(r) => format!("{:?}", r), + }; + get_low_high(uniform) + } + + pub(super) fn convert_to_v2(&self) -> String { + let (low, high) = self.get_low_high(); + format!("random({}, {}, ${{p:null}})", low, high) + } } #[derive(Clone, Debug)] @@ -1149,6 +1303,26 @@ pub(super) enum Range { Range(ReversibleRange), } +impl std::fmt::Display for Range { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Range::Args(args) => write!(f, "range({}, {})", args.0, args.1), + Range::Range(range) => { + if range.reverse { + write!( + f, + "range({}, {})", + range.range.end - 1, + range.range.start - 1 + ) + } else { + write!(f, "range({}, {})", range.range.start, range.range.end) + } + } + } + } +} + impl Range { pub(super) fn new( mut args: Vec, @@ -1269,6 +1443,17 @@ pub(super) struct Repeat { random: Option>, } +impl std::fmt::Display for Repeat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(uniform) = self.random { + let (_, high) = get_low_high(format!("{uniform:?}")); + write!(f, "repeat({}, {})", self.min, high) + } else { + write!(f, "repeat({})", self.min) + } + } +} + impl Repeat { pub(super) fn new( mut args: Vec, @@ -1345,6 +1530,16 @@ pub(super) struct Replace { replacer: ValueOrExpression, } +impl std::fmt::Display for Replace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "replace({}, {}, {})", + self.needle, self.haystack, self.replacer + ) + } +} + impl Replace { pub(super) fn new( mut args: Vec, @@ -1467,6 +1662,16 @@ pub(super) struct ParseNum { is_float: bool, } +impl std::fmt::Display for ParseNum { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.is_float { + write!(f, "parseFloat({})", self.arg) + } else { + write!(f, "parseInt({})", self.arg) + } + } +} + impl ParseNum { pub(super) fn new( float: bool, diff --git a/lib/config/src/configv1/select_parser.rs b/lib/config/src/configv1/select_parser.rs index 7cfab8a8..3110b205 100644 --- a/lib/config/src/configv1/select_parser.rs +++ b/lib/config/src/configv1/select_parser.rs @@ -204,6 +204,27 @@ impl FunctionCall { Ok(r) } + fn convert_to_v2(&self) -> String { + debug!("FunctionCall::evaluate function=\"{:?}\"", self); + match self { + FunctionCall::Collect(c) => format!("{c:?}"), + FunctionCall::Encode(e) => format!("{e}"), + FunctionCall::Entries(e) => format!("{e}"), + FunctionCall::Epoch(e) => format!("{e}"), + FunctionCall::If(i) => i.convert_to_v2(), + FunctionCall::Join(j) => format!("{j}"), + FunctionCall::JsonPath(j) => format!("{j}"), + FunctionCall::Match(m) => format!("{m}"), + FunctionCall::MinMax(m) => format!("{m}"), + FunctionCall::Pad(p) => format!("{p}"), + FunctionCall::Range(r) => format!("{r}"), + FunctionCall::Random(r) => r.convert_to_v2(), + FunctionCall::Repeat(r) => format!("{r}"), + FunctionCall::Replace(r) => format!("{r}"), + FunctionCall::ParseNum(p) => format!("{p}"), + } + } + fn evaluate<'a, 'b: 'a>( &'b self, d: Cow<'a, json::Value>, @@ -450,6 +471,23 @@ pub struct Path { pub(super) marker: Marker, } +impl std::fmt::Display for Path { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.rest.is_empty() { + write!(f, "{}", self.start) + } else { + let rest: Vec = self + .rest + .clone() + .into_iter() + .map(|piece| format!("{piece}")) + .collect(); + let rest = rest.join("."); + write!(f, "{}.{}", self.start, rest) + } + } +} + impl Path { fn evaluate<'a, 'b: 'a>( &'b self, @@ -626,6 +664,15 @@ pub enum ValueOrExpression { Expression(Expression), } +impl std::fmt::Display for ValueOrExpression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Value(v) => write!(f, "{}", v), + Self::Expression(x) => write!(f, "{}", x), + } + } +} + impl ValueOrExpression { pub fn new( expr: &str, @@ -708,6 +755,39 @@ pub enum Value { Template(Template), } +impl std::fmt::Display for Value { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Path(p) => { + if p.rest.is_empty() { + match &p.start { + PathStart::Ident(i) => write!(f, "{i}"), + PathStart::FunctionCall(func) => write!(f, "{}", &func.convert_to_v2()), + PathStart::Value(v) => write!(f, "{v}"), + } + } else { + let rest: Vec = p + .rest + .clone() + .into_iter() + .map(|piece| format!("{piece}")) + .collect(); + let rest = rest.join("."); + match &p.start { + PathStart::Ident(i) => write!(f, "{i}.{rest}"), + PathStart::FunctionCall(func) => { + write!(f, "{}.{}", &func.convert_to_v2(), rest) + } + PathStart::Value(v) => write!(f, "{v}.{rest}"), + } + } + } + Self::Json(j) => write!(f, "{j}"), + Self::Template(t) => write!(f, "{}", t), + } + } +} + impl Value { fn evaluate<'a, 'b: 'a>( &'b self, @@ -787,11 +867,21 @@ impl Value { #[derive(Clone, Debug)] pub(super) enum PathSegment { - Number(usize), - String(String), + Number(usize), // [0] + String(String), // like .body on response.body Template(Arc