From baf5bb81b77861d8988cdecca0542ead1653d4e6 Mon Sep 17 00:00:00 2001 From: Chloe Kim Date: Thu, 21 Sep 2023 00:06:06 +0800 Subject: [PATCH] fix: json path query and value functions (#2060) * feat: udf onnx feature expression * feat: udf onnx feature expression * feat: udf onnx feature expression --- dozer-sql/expression/src/json_functions.rs | 26 +- .../src/expression/tests/json_functions.rs | 232 +++++++++++++++++- .../e2e_tests/cases/mongodb/dozer-config.yaml | 2 +- 3 files changed, 254 insertions(+), 6 deletions(-) diff --git a/dozer-sql/expression/src/json_functions.rs b/dozer-sql/expression/src/json_functions.rs index c4b6872ca1..5c4a0734ce 100644 --- a/dozer-sql/expression/src/json_functions.rs +++ b/dozer-sql/expression/src/json_functions.rs @@ -55,7 +55,18 @@ impl JsonFunctionType { let json_input = args[0].evaluate(record, schema)?; let path = args[1].evaluate(record, schema)?.to_string(); - Ok(Field::Json(self.evaluate_json(json_input, path)?)) + if let Ok(json_value) = self.evaluate_json(json_input, path) { + match json_value { + JsonValue::Object(_) => Ok(Field::Null), + JsonValue::Array(_) => Ok(Field::Null), + JsonValue::String(val) => Ok(Field::Json(JsonValue::String(val))), + JsonValue::Bool(val) => Ok(Field::Json(JsonValue::Bool(val))), + JsonValue::Number(val) => Ok(Field::Json(JsonValue::Number(val))), + JsonValue::Null => Ok(Field::Null), + } + } else { + Ok(Field::Null) + } } pub(crate) fn evaluate_json_query( @@ -74,7 +85,18 @@ impl JsonFunctionType { let json_input = args[0].evaluate(record, schema)?; let path = args[1].evaluate(record, schema)?.to_string(); - Ok(Field::Json(self.evaluate_json(json_input, path)?)) + if let Ok(json_value) = self.evaluate_json(json_input, path) { + match json_value { + JsonValue::Object(val) => Ok(Field::Json(JsonValue::Object(val))), + JsonValue::Array(val) => Ok(Field::Json(JsonValue::Array(val))), + JsonValue::String(_) => Ok(Field::Null), + JsonValue::Bool(_) => Ok(Field::Null), + JsonValue::Number(_) => Ok(Field::Null), + JsonValue::Null => Ok(Field::Null), + } + } else { + Ok(Field::Null) + } } } diff --git a/dozer-sql/src/expression/tests/json_functions.rs b/dozer-sql/src/expression/tests/json_functions.rs index e4ead464b8..5ed41ccd08 100644 --- a/dozer-sql/src/expression/tests/json_functions.rs +++ b/dozer-sql/src/expression/tests/json_functions.rs @@ -76,7 +76,7 @@ fn test_json_value_null() { vec![Field::Json(json_val)], ); - assert_eq!(f, Field::Json(JsonValue::Null)); + assert_eq!(f, Field::Null); } #[test] @@ -165,7 +165,7 @@ fn test_json_query_null() { .clone(), vec![Field::Json(json_val)], ); - assert_eq!(f, Field::Json(JsonValue::String("Basic".to_string()))); + assert_eq!(f, Field::Null); } #[test] @@ -187,7 +187,7 @@ fn test_json_query_len_one_array() { .unwrap(); let f = run_fct( - "SELECT JSON_VALUE(jsonInfo,'$.info.tags') FROM users", + "SELECT JSON_QUERY(jsonInfo,'$.info.tags') FROM users", Schema::default() .field( FieldDefinition::new( @@ -645,3 +645,229 @@ fn test_json_value_cast() { assert_eq!(f, Field::Boolean(true)); } + +#[test] +fn test_json_value_diff_1() { + let json_val = serde_json_to_json_value(json!( + { "x": [0,1], "y": "[0,1]", "z": "Monty" } + )) + .unwrap(); + + let mut f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Json(json_val.clone())); + + f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Null); +} + +#[test] +fn test_json_value_diff_2() { + let json_val = serde_json_to_json_value(json!( + { "x": [0,1], "y": "[0,1]", "z": "Monty" } + )) + .unwrap(); + + let mut f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.x') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!( + f, + Field::Json(JsonValue::Array(vec![ + JsonValue::Number(OrderedFloat(0_f64)), + JsonValue::Number(OrderedFloat(1_f64)) + ])) + ); + + f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.x') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Null); +} + +#[test] +fn test_json_value_diff_3() { + let json_val = serde_json_to_json_value(json!( + { "x": [0,1], "y": "[0,1]", "z": "Monty" } + )) + .unwrap(); + + let mut f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.y') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Null); + + f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.y') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Json(JsonValue::String("[0,1]".to_string()))); +} + +#[test] +fn test_json_value_diff_4() { + let json_val = serde_json_to_json_value(json!( + { "x": [0,1], "y": "[0,1]", "z": "Monty" } + )) + .unwrap(); + + let mut f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.z') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Null); + + f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.z') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Json(JsonValue::String("Monty".to_string()))); +} + +#[test] +fn test_json_value_diff_5() { + let json_val = serde_json_to_json_value(json!( + { "x": [0,1], "y": "[0,1]", "z": "Monty" } + )) + .unwrap(); + + let mut f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.x[0]') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Null); + + f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.x[0]') FROM users", + Schema::default() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(0_f64)))); +} diff --git a/dozer-tests/src/e2e_tests/cases/mongodb/dozer-config.yaml b/dozer-tests/src/e2e_tests/cases/mongodb/dozer-config.yaml index 787461dc3d..41f3b4b798 100644 --- a/dozer-tests/src/e2e_tests/cases/mongodb/dozer-config.yaml +++ b/dozer-tests/src/e2e_tests/cases/mongodb/dozer-config.yaml @@ -9,7 +9,7 @@ sources: table_name: movies connection: mflix sql: | - SELECT "_id", JSON_VALUE(data, '$.title') as title, JSON_QUERY(data, '$.genres') as genres + SELECT "_id", JSON_QUERY(data, '$.title') as title, JSON_QUERY(data, '$.genres') as genres INTO movie_data FROM movies endpoints: