Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

aw-query: Add preprocessing and refcounting #183

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

aw-query: Add preprocessing and refcounting #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 20 additions & 20 deletions aw-query/src/functions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,97 +7,97 @@ pub type QueryFn =
fn(args: Vec<DataType>, env: &VarEnv, ds: &Datastore) -> Result<DataType, QueryError>;

pub fn fill_env(env: &mut VarEnv) {
env.insert(
env.declare_static(
"print".to_string(),
DataType::Function("print".to_string(), qfunctions::print),
);
env.insert(
env.declare_static(
"query_bucket".to_string(),
DataType::Function("query_bucket".to_string(), qfunctions::query_bucket),
);
env.insert(
env.declare_static(
"query_bucket_names".to_string(),
DataType::Function(
"query_bucket_names".to_string(),
qfunctions::query_bucket_names,
),
);
env.insert(
env.declare_static(
"sort_by_duration".to_string(),
DataType::Function("sort_by_duration".to_string(), qfunctions::sort_by_duration),
);
env.insert(
env.declare_static(
"sort_by_timestamp".to_string(),
DataType::Function(
"sort_by_timestamp".to_string(),
qfunctions::sort_by_timestamp,
),
);
env.insert(
env.declare_static(
"sum_durations".to_string(),
DataType::Function("sum_durations".to_string(), qfunctions::sum_durations),
);
env.insert(
env.declare_static(
"limit_events".to_string(),
DataType::Function("limit_events".to_string(), qfunctions::limit_events),
);
env.insert(
env.declare_static(
"contains".to_string(),
DataType::Function("contains".to_string(), qfunctions::contains),
);
env.insert(
env.declare_static(
"flood".to_string(),
DataType::Function("flood".to_string(), qfunctions::flood),
);
env.insert(
env.declare_static(
"find_bucket".to_string(),
DataType::Function("find_bucket".to_string(), qfunctions::find_bucket),
);
env.insert(
env.declare_static(
"merge_events_by_keys".to_string(),
DataType::Function(
"merge_events_by_keys".to_string(),
qfunctions::merge_events_by_keys,
),
);
env.insert(
env.declare_static(
"chunk_events_by_key".to_string(),
DataType::Function(
"chunk_events_by_key".to_string(),
qfunctions::chunk_events_by_key,
),
);
env.insert(
env.declare_static(
"filter_keyvals".to_string(),
DataType::Function("filter_keyvals".to_string(), qfunctions::filter_keyvals),
);
env.insert(
env.declare_static(
"filter_keyvals_regex".to_string(),
DataType::Function(
"filter_keyvals_regex".to_string(),
qfunctions::filter_keyvals_regex,
),
);
env.insert(
env.declare_static(
"filter_period_intersect".to_string(),
DataType::Function(
"filter_period_intersect".to_string(),
qfunctions::filter_period_intersect,
),
);
env.insert(
env.declare_static(
"split_url_events".to_string(),
DataType::Function("split_url_events".to_string(), qfunctions::split_url_events),
);
env.insert(
env.declare_static(
"concat".to_string(),
DataType::Function("concat".to_string(), qfunctions::concat),
);
env.insert(
env.declare_static(
"categorize".to_string(),
DataType::Function("categorize".into(), qfunctions::categorize),
);
env.insert(
env.declare_static(
"tag".to_string(),
DataType::Function("tag".into(), qfunctions::tag),
);
Expand Down Expand Up @@ -521,7 +521,7 @@ mod validate {
}

pub fn get_timeinterval(env: &VarEnv) -> Result<TimeInterval, QueryError> {
let interval_str = match env.get("TIMEINTERVAL") {
let interval_str = match env.deprecated_get("TIMEINTERVAL") {
Some(data_ti) => match data_ti {
DataType::String(ti_str) => ti_str,
_ => {
Expand Down
34 changes: 9 additions & 25 deletions aw-query/src/interpret.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,43 +1,27 @@
use std::collections::HashMap;

use crate::functions;

use aw_datastore::Datastore;
use aw_models::TimeInterval;

use crate::ast::*;
use crate::DataType;
use crate::QueryError;

pub type VarEnv = HashMap<String, DataType>;

fn init_env(ti: &TimeInterval) -> VarEnv {
let mut env = HashMap::new();
env.insert("TIMEINTERVAL".to_string(), DataType::String(ti.to_string()));
functions::fill_env(&mut env);
env
}
use crate::VarEnv;

pub fn interpret_prog(
p: Program,
ti: &TimeInterval,
env: &mut VarEnv,
ds: &Datastore,
) -> Result<DataType, QueryError> {
let mut env = init_env(ti);
for expr in p.stmts {
interpret_expr(&mut env, ds, expr)?;
interpret_expr(env, ds, expr)?;
}
match env.remove("RETURN") {
match env.take("RETURN") {
Some(ret) => Ok(ret),
None => Err(QueryError::EmptyQuery()),
}
}

fn interpret_expr(
env: &mut HashMap<String, DataType>,
ds: &Datastore,
expr: Expr,
) -> Result<DataType, QueryError> {
fn interpret_expr(env: &mut VarEnv, ds: &Datastore, expr: Expr) -> Result<DataType, QueryError> {
use crate::ast::Expr_::*;
match expr.node {
Add(a, b) => {
Expand Down Expand Up @@ -184,9 +168,8 @@ fn interpret_expr(
env.insert(var, val);
Ok(DataType::None())
}
// FIXME: avoid clone, it's slow
Var(var) => match env.get(&var) {
Some(v) => Ok(v.clone()),
Var(var) => match env.take(&var) {
Some(v) => Ok(v),
None => Err(QueryError::VariableNotDefined(var.to_string())),
},
Bool(lit) => Ok(DataType::Bool(lit)),
Expand All @@ -195,6 +178,7 @@ fn interpret_expr(
Return(e) => {
let val = interpret_expr(env, ds, *e)?;
// TODO: Once RETURN is deprecated we can fix this
env.declare("RETURN".to_string());
env.insert("RETURN".to_string(), val);
Ok(DataType::None())
}
Expand All @@ -215,7 +199,7 @@ fn interpret_expr(
DataType::List(l) => l,
_ => unreachable!(),
};
let var = match env.get(&fname[..]) {
let var = match env.take(&fname[..]) {
Some(v) => v,
None => return Err(QueryError::VariableNotDefined(fname.clone())),
};
Expand Down
15 changes: 13 additions & 2 deletions aw-query/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ mod lexer;
unused_braces
)]
mod parser;
mod preprocess;
mod varenv;

pub use crate::datatype::DataType;
pub use crate::interpret::VarEnv;
pub use crate::varenv::VarEnv;

// TODO: add line numbers to errors
// (works during lexing, but not during parsing I believe)
Expand All @@ -50,6 +52,13 @@ impl fmt::Display for QueryError {
}
}

fn init_env(ti: &TimeInterval) -> VarEnv {
let mut env = VarEnv::new();
env.declare_static("TIMEINTERVAL".to_string(), DataType::String(ti.to_string()));
functions::fill_env(&mut env);
env
}

pub fn query(code: &str, ti: &TimeInterval, ds: &Datastore) -> Result<DataType, QueryError> {
let lexer = lexer::Lexer::new(code);
let program = match parser::parse(lexer) {
Expand All @@ -60,5 +69,7 @@ pub fn query(code: &str, ti: &TimeInterval, ds: &Datastore) -> Result<DataType,
return Err(QueryError::ParsingError(format!("{:?}", e)));
}
};
interpret::interpret_prog(program, ti, ds)
let mut env = init_env(ti);
preprocess::preprocess_prog(&program, &mut env, ds)?;
interpret::interpret_prog(program, &mut env, ds)
}
77 changes: 77 additions & 0 deletions aw-query/src/preprocess.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
use aw_datastore::Datastore;

use crate::ast::*;
use crate::QueryError;
use crate::VarEnv;

pub fn preprocess_prog(p: &Program, env: &mut VarEnv, ds: &Datastore) -> Result<(), QueryError> {
for expr in &p.stmts {
preprocess_expr(env, ds, expr)?;
}
Ok(())
}

fn preprocess_expr(env: &mut VarEnv, ds: &Datastore, expr: &Expr) -> Result<(), QueryError> {
use crate::ast::Expr_::*;
match &expr.node {
Var(ref var) => env.add_ref(var)?,
Add(ref a, ref b) => {
preprocess_expr(env, ds, a)?;
preprocess_expr(env, ds, b)?;
}
Sub(ref a, ref b) => {
preprocess_expr(env, ds, a)?;
preprocess_expr(env, ds, b)?;
}
Mul(ref a, ref b) => {
preprocess_expr(env, ds, a)?;
preprocess_expr(env, ds, b)?;
}
Div(ref a, ref b) => {
preprocess_expr(env, ds, a)?;
preprocess_expr(env, ds, b)?;
}
Mod(ref a, ref b) => {
preprocess_expr(env, ds, a)?;
preprocess_expr(env, ds, b)?;
}
Equal(ref a, ref b) => {
preprocess_expr(env, ds, a)?;
preprocess_expr(env, ds, b)?;
}
Assign(ref var, ref b) => {
preprocess_expr(env, ds, b)?;
env.declare(var.to_string());
}
Function(ref fname, ref args) => {
env.add_ref(fname)?;
preprocess_expr(env, ds, args)?;
}
If(ref ifs) => {
for (cond, block) in ifs {
// TODO: could be optimized?
preprocess_expr(env, ds, cond)?;
for expr in block {
preprocess_expr(env, ds, expr)?;
}
}
}
List(list) => {
for entry in list {
preprocess_expr(env, ds, entry)?;
}
}
Dict(d) => {
for (_key, val_uninterpreted) in d {
preprocess_expr(env, ds, val_uninterpreted)?;
}
}
Return(e) => {
preprocess_expr(env, ds, e)?;
}
Bool(_lit) => (),
Number(_lit) => (),
String(_lit) => (),
};
Ok(())
}
Loading