Skip to content

Commit

Permalink
fix: json path query and value functions (#2060)
Browse files Browse the repository at this point in the history 
* feat: udf onnx feature expression

* feat: udf onnx feature expression

* feat: udf onnx feature expression
  • Loading branch information
Chloe Kim authored Sep 20, 2023
1 parent 70ef6ae commit baf5bb8
Show file tree
Hide file tree
Showing 3 changed files with 254 additions and 6 deletions.
26 changes: 24 additions & 2 deletions dozer-sql/expression/src/json_functions.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,18 @@ impl JsonFunctionType {
let json_input = args[0].evaluate(record, schema)?;
let path = args[1].evaluate(record, schema)?.to_string();

Ok(Field::Json(self.evaluate_json(json_input, path)?))
if let Ok(json_value) = self.evaluate_json(json_input, path) {
match json_value {
JsonValue::Object(_) => Ok(Field::Null),
JsonValue::Array(_) => Ok(Field::Null),
JsonValue::String(val) => Ok(Field::Json(JsonValue::String(val))),
JsonValue::Bool(val) => Ok(Field::Json(JsonValue::Bool(val))),
JsonValue::Number(val) => Ok(Field::Json(JsonValue::Number(val))),
JsonValue::Null => Ok(Field::Null),
}
} else {
Ok(Field::Null)
}
}

pub(crate) fn evaluate_json_query(
Expand All @@ -74,7 +85,18 @@ impl JsonFunctionType {
let json_input = args[0].evaluate(record, schema)?;
let path = args[1].evaluate(record, schema)?.to_string();

Ok(Field::Json(self.evaluate_json(json_input, path)?))
if let Ok(json_value) = self.evaluate_json(json_input, path) {
match json_value {
JsonValue::Object(val) => Ok(Field::Json(JsonValue::Object(val))),
JsonValue::Array(val) => Ok(Field::Json(JsonValue::Array(val))),
JsonValue::String(_) => Ok(Field::Null),
JsonValue::Bool(_) => Ok(Field::Null),
JsonValue::Number(_) => Ok(Field::Null),
JsonValue::Null => Ok(Field::Null),
}
} else {
Ok(Field::Null)
}
}
}

Expand Down
232 changes: 229 additions & 3 deletions dozer-sql/src/expression/tests/json_functions.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ fn test_json_value_null() {
vec![Field::Json(json_val)],
);

assert_eq!(f, Field::Json(JsonValue::Null));
assert_eq!(f, Field::Null);
}

#[test]
Expand Down Expand Up @@ -165,7 +165,7 @@ fn test_json_query_null() {
.clone(),
vec![Field::Json(json_val)],
);
assert_eq!(f, Field::Json(JsonValue::String("Basic".to_string())));
assert_eq!(f, Field::Null);
}

#[test]
Expand All @@ -187,7 +187,7 @@ fn test_json_query_len_one_array() {
.unwrap();

let f = run_fct(
"SELECT JSON_VALUE(jsonInfo,'$.info.tags') FROM users",
"SELECT JSON_QUERY(jsonInfo,'$.info.tags') FROM users",
Schema::default()
.field(
FieldDefinition::new(
Expand Down Expand Up @@ -645,3 +645,229 @@ fn test_json_value_cast() {

assert_eq!(f, Field::Boolean(true));
}

#[test]
fn test_json_value_diff_1() {
let json_val = serde_json_to_json_value(json!(
{ "x": [0,1], "y": "[0,1]", "z": "Monty" }
))
.unwrap();

let mut f = run_fct(
"SELECT JSON_QUERY(jsonInfo,'$') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val.clone())],
);

assert_eq!(f, Field::Json(json_val.clone()));

f = run_fct(
"SELECT JSON_VALUE(jsonInfo,'$') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val)],
);

assert_eq!(f, Field::Null);
}

#[test]
fn test_json_value_diff_2() {
let json_val = serde_json_to_json_value(json!(
{ "x": [0,1], "y": "[0,1]", "z": "Monty" }
))
.unwrap();

let mut f = run_fct(
"SELECT JSON_QUERY(jsonInfo,'$.x') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val.clone())],
);

assert_eq!(
f,
Field::Json(JsonValue::Array(vec![
JsonValue::Number(OrderedFloat(0_f64)),
JsonValue::Number(OrderedFloat(1_f64))
]))
);

f = run_fct(
"SELECT JSON_VALUE(jsonInfo,'$.x') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val)],
);

assert_eq!(f, Field::Null);
}

#[test]
fn test_json_value_diff_3() {
let json_val = serde_json_to_json_value(json!(
{ "x": [0,1], "y": "[0,1]", "z": "Monty" }
))
.unwrap();

let mut f = run_fct(
"SELECT JSON_QUERY(jsonInfo,'$.y') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val.clone())],
);

assert_eq!(f, Field::Null);

f = run_fct(
"SELECT JSON_VALUE(jsonInfo,'$.y') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val)],
);

assert_eq!(f, Field::Json(JsonValue::String("[0,1]".to_string())));
}

#[test]
fn test_json_value_diff_4() {
let json_val = serde_json_to_json_value(json!(
{ "x": [0,1], "y": "[0,1]", "z": "Monty" }
))
.unwrap();

let mut f = run_fct(
"SELECT JSON_QUERY(jsonInfo,'$.z') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val.clone())],
);

assert_eq!(f, Field::Null);

f = run_fct(
"SELECT JSON_VALUE(jsonInfo,'$.z') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val)],
);

assert_eq!(f, Field::Json(JsonValue::String("Monty".to_string())));
}

#[test]
fn test_json_value_diff_5() {
let json_val = serde_json_to_json_value(json!(
{ "x": [0,1], "y": "[0,1]", "z": "Monty" }
))
.unwrap();

let mut f = run_fct(
"SELECT JSON_QUERY(jsonInfo,'$.x[0]') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val.clone())],
);

assert_eq!(f, Field::Null);

f = run_fct(
"SELECT JSON_VALUE(jsonInfo,'$.x[0]') FROM users",
Schema::default()
.field(
FieldDefinition::new(
String::from("jsonInfo"),
FieldType::Json,
false,
SourceDefinition::Dynamic,
),
false,
)
.clone(),
vec![Field::Json(json_val)],
);

assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(0_f64))));
}
2 changes: 1 addition & 1 deletion dozer-tests/src/e2e_tests/cases/mongodb/dozer-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ sources:
table_name: movies
connection: mflix
sql: |
SELECT "_id", JSON_VALUE(data, '$.title') as title, JSON_QUERY(data, '$.genres') as genres
SELECT "_id", JSON_QUERY(data, '$.title') as title, JSON_QUERY(data, '$.genres') as genres
INTO movie_data
FROM movies
endpoints:
Expand Down

0 comments on commit baf5bb8

Please sign in to comment.