diff --git a/dozer-ingestion/aerospike/src/connector.rs b/dozer-ingestion/aerospike/src/connector.rs index 33f36d3a7f..c87fdef157 100644 --- a/dozer-ingestion/aerospike/src/connector.rs +++ b/dozer-ingestion/aerospike/src/connector.rs @@ -811,7 +811,6 @@ pub(crate) fn map_value_to_field( check_type("int8")?; let string = value.as_str().ok_or_else(unsupported_type)?; Ok(Field::Int8(string.parse()?)) - } FieldType::U128 => { check_type("str")?; diff --git a/dozer-ingestion/tests/test_suite/basic.rs b/dozer-ingestion/tests/test_suite/basic.rs index c86e975421..2f1e30f883 100644 --- a/dozer-ingestion/tests/test_suite/basic.rs +++ b/dozer-ingestion/tests/test_suite/basic.rs @@ -293,6 +293,9 @@ fn assert_record_matches_schema(record: &Record, schema: &Schema, only_match_pk: FieldType::Int => { assert!(value.as_int().is_some()) } + FieldType::Int8 => { + assert!(value.as_int().is_some()) + } FieldType::I128 => { assert!(value.as_i128().is_some()) } diff --git a/dozer-ingestion/tests/test_suite/connectors/object_store/arrow.rs b/dozer-ingestion/tests/test_suite/connectors/object_store/arrow.rs index c3bc3ec4f1..5376a15df7 100644 --- a/dozer-ingestion/tests/test_suite/connectors/object_store/arrow.rs +++ b/dozer-ingestion/tests/test_suite/connectors/object_store/arrow.rs @@ -278,6 +278,7 @@ fn field_type_to_arrow(field_type: FieldType) -> Option Some(arrow::datatypes::DataType::UInt64), FieldType::U128 => None, FieldType::Int => Some(arrow::datatypes::DataType::Int64), + FieldType::Int8 => Some(arrow::datatypes::DataType::Int64), FieldType::I128 => None, FieldType::Float => Some(arrow::datatypes::DataType::Float64), FieldType::Boolean => Some(arrow::datatypes::DataType::Boolean), @@ -349,6 +350,18 @@ fn fields_to_arrow<'a, F: IntoIterator>( } Arc::new(builder.finish()) } + FieldType::Int8 => { + let mut builder = arrow::array::Int64Array::builder(count); + for field in fields { + match field { + Field::Int(value) => builder.append_value(*value), + Field::Int8(value) => builder.append_value(*value as i64), + Field::Null => builder.append_null(), + _ => panic!("Unexpected field type"), + } + } + Arc::new(builder.finish()) + } FieldType::I128 => panic!("Unexpected field type"), FieldType::Float => { let mut builder = arrow::array::Float64Array::builder(count); diff --git a/dozer-ingestion/tests/test_suite/connectors/sql.rs b/dozer-ingestion/tests/test_suite/connectors/sql.rs index 6268456915..3608f1214c 100644 --- a/dozer-ingestion/tests/test_suite/connectors/sql.rs +++ b/dozer-ingestion/tests/test_suite/connectors/sql.rs @@ -158,6 +158,7 @@ fn field_type_to_sql(field_type: FieldType) -> Option { FieldType::UInt => None, FieldType::U128 => None, FieldType::Int => Some("INT8".to_string()), + FieldType::Int8 => Some("INT8".to_string()), FieldType::I128 => None, FieldType::Float => Some("FLOAT8".to_string()), FieldType::Boolean => Some("BOOLEAN".to_string()), @@ -229,6 +230,7 @@ fn field_to_sql(field: &Field) -> String { Field::UInt(i) => i.to_string(), Field::U128(i) => i.to_string(), Field::Int(i) => i.to_string(), + Field::Int8(i) => i.to_string(), Field::I128(i) => i.to_string(), Field::Float(f) => f.to_string(), Field::Boolean(b) => b.to_string(), diff --git a/dozer-sink-aerospike/src/aerospike.rs b/dozer-sink-aerospike/src/aerospike.rs index e7ded8552a..d6d59d9a1e 100644 --- a/dozer-sink-aerospike/src/aerospike.rs +++ b/dozer-sink-aerospike/src/aerospike.rs @@ -558,7 +558,11 @@ pub(crate) unsafe fn new_record_map( as_orderedmap_set(map, key, check_alloc(as_integer_new(*v)) as *const as_val); } Field::Int8(v) => { - as_orderedmap_set(map, key, check_alloc(as_integer_new((*v).into())) as *const as_val); + as_orderedmap_set( + map, + key, + check_alloc(as_integer_new((*v).into())) as *const as_val, + ); } Field::I128(v) => { map_set_str(map, key, v, allocated_strings); diff --git a/dozer-sink-clickhouse/src/sink.rs b/dozer-sink-clickhouse/src/sink.rs index 26f2a09eb3..06dbd8e1a2 100644 --- a/dozer-sink-clickhouse/src/sink.rs +++ b/dozer-sink-clickhouse/src/sink.rs @@ -156,7 +156,7 @@ impl ClickhouseSink { ) -> Self { let mut schema = schema.clone(); - if table.engine == "CollapsingMergeTree" && schema.fields.len() > 0 { + if table.engine == "CollapsingMergeTree" && !schema.fields.is_empty() { // get source from any field in schema let source = schema.fields[0].source.clone(); schema.fields.push(FieldDefinition { diff --git a/dozer-sql/expression/src/comparison/mod.rs b/dozer-sql/expression/src/comparison/mod.rs index 861464a88f..644815288d 100644 --- a/dozer-sql/expression/src/comparison/mod.rs +++ b/dozer-sql/expression/src/comparison/mod.rs @@ -862,10 +862,9 @@ pub fn evaluate_lt( Field::Float(right_v) => Ok(Field::Boolean((left_v as f64) < *right_v)), // left: Int, right: Decimal Field::Decimal(right_v) => { - let left_v_d = Decimal::from_i64(left_v as i64).ok_or(PipelineError::UnableToCast( - format!("{}", left_v), - "Decimal".to_string(), - ))?; + let left_v_d = Decimal::from_i64(left_v as i64).ok_or( + PipelineError::UnableToCast(format!("{}", left_v), "Decimal".to_string()), + )?; Ok(Field::Boolean(left_v_d < right_v)) } // left: Int, right: String or Text @@ -903,7 +902,7 @@ pub fn evaluate_lt( Field::I128(right_v) => Ok(Field::Boolean(left_v < right_v)), // left: I128, right: UInt Field::UInt(right_v) => Ok(Field::Boolean(left_v < (right_v as i128))), - + // left: I128, right: U128 Field::U128(right_v) => Ok(Field::Boolean(left_v < (right_v as i128))), // left: I128, right: Float diff --git a/dozer-sql/expression/src/mathematical/mod.rs b/dozer-sql/expression/src/mathematical/mod.rs index b0e62caf3a..fa084be15e 100644 --- a/dozer-sql/expression/src/mathematical/mod.rs +++ b/dozer-sql/expression/src/mathematical/mod.rs @@ -205,7 +205,7 @@ macro_rules! define_math_operator { ))), } } - + Field::Int8(v) => { let right_v = v as i64; return match $op { @@ -235,11 +235,9 @@ macro_rules! define_math_operator { ), )?, ))), - } + }; } - - - + // left: Float, right: I128 Field::I128(right_v) => { return match $op { @@ -702,294 +700,294 @@ macro_rules! define_math_operator { Field::Int8(v) => { let left_v = v as i64; return match right_p { - - // left: Int, right: Int - Field::Int(right_v) => { - return match $op { - // When Int / Int division happens - "/" => { - if right_v == 0_i64 { - Err(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - )) - } else { - Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - OrderedFloat::::from_i64(right_v).ok_or( - PipelineError::UnableToCast( - format!("{}", right_v), - "f64".to_string(), - ), - )?, - ))) + // left: Int, right: Int + Field::Int(right_v) => { + return match $op { + // When Int / Int division happens + "/" => { + if right_v == 0_i64 { + Err(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + )) + } else { + Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + OrderedFloat::::from_i64(right_v).ok_or( + PipelineError::UnableToCast( + format!("{}", right_v), + "f64".to_string(), + ), + )?, + ))) + } } - } - // When it's not division operation - "+" | "-" | "*" | "%" => { - Ok(Field::Int($fct(Wrapping(left_v), Wrapping(right_v)).0)) - } - &_ => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), - }; - } + // When it's not division operation + "+" | "-" | "*" | "%" => { + Ok(Field::Int($fct(Wrapping(left_v), Wrapping(right_v)).0)) + } + &_ => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + }; + } - Field::Int8(v) => { - let right_v = v as i64; - return match $op { - // When Int / Int division happens - "/" => { - if right_v == 0_i64 { - Err(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - )) - } else { - Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - OrderedFloat::::from_i64(right_v).ok_or( - PipelineError::UnableToCast( - format!("{}", right_v), - "f64".to_string(), - ), - )?, - ))) + Field::Int8(v) => { + let right_v = v as i64; + return match $op { + // When Int / Int division happens + "/" => { + if right_v == 0_i64 { + Err(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + )) + } else { + Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + OrderedFloat::::from_i64(right_v).ok_or( + PipelineError::UnableToCast( + format!("{}", right_v), + "f64".to_string(), + ), + )?, + ))) + } } - } - // When it's not division operation - "+" | "-" | "*" | "%" => { - Ok(Field::Int($fct(Wrapping(left_v), Wrapping(right_v)).0)) - } - &_ => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), - }; - } + // When it's not division operation + "+" | "-" | "*" | "%" => { + Ok(Field::Int($fct(Wrapping(left_v), Wrapping(right_v)).0)) + } + &_ => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + }; + } - // left: Int, right: I128 - Field::I128(right_v) => { - return match $op { - // When Int / I128 division happens - "/" => { - if right_v == 0_i128 { - Err(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - )) - } else { - Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - OrderedFloat::::from_i128(right_v).ok_or( - PipelineError::UnableToCast( - format!("{}", right_v), - "f64".to_string(), - ), - )?, - ))) + // left: Int, right: I128 + Field::I128(right_v) => { + return match $op { + // When Int / I128 division happens + "/" => { + if right_v == 0_i128 { + Err(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + )) + } else { + Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + OrderedFloat::::from_i128(right_v).ok_or( + PipelineError::UnableToCast( + format!("{}", right_v), + "f64".to_string(), + ), + )?, + ))) + } } - } - // When it's not division operation - "+" | "-" | "*" | "%" => Ok(Field::I128( - $fct(Wrapping(left_v as i128), Wrapping(right_v)).0, - )), - &_ => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), - }; - } - // left: Int, right: UInt - Field::UInt(right_v) => { - return match $op { - // When Int / UInt division happens - "/" => { - if right_v == 0_u64 { - Err(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - )) - } else { - Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - OrderedFloat::::from_u64(right_v).ok_or( - PipelineError::UnableToCast( - format!("{}", right_v), - "f64".to_string(), - ), - )?, - ))) + // When it's not division operation + "+" | "-" | "*" | "%" => Ok(Field::I128( + $fct(Wrapping(left_v as i128), Wrapping(right_v)).0, + )), + &_ => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + }; + } + // left: Int, right: UInt + Field::UInt(right_v) => { + return match $op { + // When Int / UInt division happens + "/" => { + if right_v == 0_u64 { + Err(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + )) + } else { + Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + OrderedFloat::::from_u64(right_v).ok_or( + PipelineError::UnableToCast( + format!("{}", right_v), + "f64".to_string(), + ), + )?, + ))) + } } - } - // When it's not division operation - "+" | "-" | "*" | "%" => Ok(Field::Int( - $fct(Wrapping(left_v), Wrapping(right_v as i64)).0, - )), - &_ => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), - }; - } - // left: Int, right: U128 - Field::U128(right_v) => { - return match $op { - // When Int / U128 division happens - "/" => { - if right_v == 0_u128 { - Err(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - )) - } else { - Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - OrderedFloat::::from_u128(right_v).ok_or( - PipelineError::UnableToCast( - format!("{}", right_v), - "f64".to_string(), - ), - )?, - ))) + // When it's not division operation + "+" | "-" | "*" | "%" => Ok(Field::Int( + $fct(Wrapping(left_v), Wrapping(right_v as i64)).0, + )), + &_ => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + }; + } + // left: Int, right: U128 + Field::U128(right_v) => { + return match $op { + // When Int / U128 division happens + "/" => { + if right_v == 0_u128 { + Err(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + )) + } else { + Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + OrderedFloat::::from_u128(right_v).ok_or( + PipelineError::UnableToCast( + format!("{}", right_v), + "f64".to_string(), + ), + )?, + ))) + } } - } - // When it's not division operation - "+" | "-" | "*" | "%" => Ok(Field::I128( - $fct(Wrapping(left_v as i128), Wrapping(right_v as i128)).0, - )), - &_ => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), - }; - } - // left: Int, right: Float - Field::Float(right_v) => { - return match $op { - "/" | "%" => { - if right_v == 0_f64 { - Err(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - )) - } else { - Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - right_v, - ))) + // When it's not division operation + "+" | "-" | "*" | "%" => Ok(Field::I128( + $fct(Wrapping(left_v as i128), Wrapping(right_v as i128)).0, + )), + &_ => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + }; + } + // left: Int, right: Float + Field::Float(right_v) => { + return match $op { + "/" | "%" => { + if right_v == 0_f64 { + Err(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + )) + } else { + Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + right_v, + ))) + } } + &_ => Ok(Field::Float($fct( + OrderedFloat::::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "f64".to_string(), + ), + )?, + right_v, + ))), } - &_ => Ok(Field::Float($fct( - OrderedFloat::::from_i64(left_v).ok_or( - PipelineError::UnableToCast( - format!("{}", left_v), - "f64".to_string(), - ), - )?, - right_v, - ))), } - } - // left: Int, right: Decimal - Field::Decimal(right_v) => { - return match $op { - "/" => Ok(Field::Decimal( - Decimal::from_i64(left_v) - .ok_or(PipelineError::UnableToCast( - format!("{}", left_v), - "Decimal".to_string(), - ))? - .checked_div(right_v) - .ok_or(PipelineError::SqlError( - OperationError::DivisionByZeroOrOverflow, - ))?, - )), - "%" => Ok(Field::Decimal( - Decimal::from_i64(left_v) - .ok_or(PipelineError::UnableToCast( - format!("{}", left_v), - "Decimal".to_string(), - ))? - .checked_rem(right_v) - .ok_or(PipelineError::SqlError( - OperationError::ModuloByZeroOrOverflow, - ))?, - )), - "*" => Ok(Field::Decimal( - Decimal::from_i64(left_v) - .ok_or(PipelineError::UnableToCast( - format!("{}", left_v), - "Decimal".to_string(), - ))? - .checked_mul(right_v) - .ok_or(PipelineError::SqlError( - OperationError::MultiplicationOverflow, - ))?, - )), - "+" | "-" => Ok(Field::Decimal($fct( - Decimal::from_i64(left_v).ok_or(PipelineError::UnableToCast( - format!("{}", left_v), - "Decimal".to_string(), - ))?, - right_v, - ))), - &_ => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), + // left: Int, right: Decimal + Field::Decimal(right_v) => { + return match $op { + "/" => Ok(Field::Decimal( + Decimal::from_i64(left_v) + .ok_or(PipelineError::UnableToCast( + format!("{}", left_v), + "Decimal".to_string(), + ))? + .checked_div(right_v) + .ok_or(PipelineError::SqlError( + OperationError::DivisionByZeroOrOverflow, + ))?, + )), + "%" => Ok(Field::Decimal( + Decimal::from_i64(left_v) + .ok_or(PipelineError::UnableToCast( + format!("{}", left_v), + "Decimal".to_string(), + ))? + .checked_rem(right_v) + .ok_or(PipelineError::SqlError( + OperationError::ModuloByZeroOrOverflow, + ))?, + )), + "*" => Ok(Field::Decimal( + Decimal::from_i64(left_v) + .ok_or(PipelineError::UnableToCast( + format!("{}", left_v), + "Decimal".to_string(), + ))? + .checked_mul(right_v) + .ok_or(PipelineError::SqlError( + OperationError::MultiplicationOverflow, + ))?, + )), + "+" | "-" => Ok(Field::Decimal($fct( + Decimal::from_i64(left_v).ok_or( + PipelineError::UnableToCast( + format!("{}", left_v), + "Decimal".to_string(), + ), + )?, + right_v, + ))), + &_ => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + } } - } - // left: Int, right: Null - Field::Null => Ok(Field::Null), - Field::Boolean(_) - | Field::String(_) - | Field::Text(_) - | Field::Binary(_) - | Field::Timestamp(_) - | Field::Date(_) - | Field::Json(_) - | Field::Point(_) - | Field::Duration(_) => Err(PipelineError::InvalidTypeComparison( - left_p, - right_p, - $op.to_string(), - )), - - - }}, + // left: Int, right: Null + Field::Null => Ok(Field::Null), + Field::Boolean(_) + | Field::String(_) + | Field::Text(_) + | Field::Binary(_) + | Field::Timestamp(_) + | Field::Date(_) + | Field::Json(_) + | Field::Point(_) + | Field::Duration(_) => Err(PipelineError::InvalidTypeComparison( + left_p, + right_p, + $op.to_string(), + )), + }; + } Field::I128(left_v) => match right_p { // left: I128, right: Int Field::Int(right_v) => { @@ -1325,7 +1323,7 @@ macro_rules! define_math_operator { )), }; } - + // left: UInt, right: I128 Field::I128(right_v) => { return match $op { @@ -1596,7 +1594,7 @@ macro_rules! define_math_operator { )), }; } - + Field::Int8(right_v) => { return match $op { // When U128 / Int division happens @@ -1633,7 +1631,7 @@ macro_rules! define_math_operator { )), }; } - + // left: U128, right: I128 Field::I128(right_v) => { return match $op { @@ -1883,7 +1881,7 @@ macro_rules! define_math_operator { )) } } - + // left: Decimal, right: I128 Field::I128(right_v) => { if right_v == 0_i128 { diff --git a/dozer-sql/expression/src/python_udf.rs b/dozer-sql/expression/src/python_udf.rs index 011dc68772..25f5394f04 100644 --- a/dozer-sql/expression/src/python_udf.rs +++ b/dozer-sql/expression/src/python_udf.rs @@ -62,6 +62,7 @@ pub fn evaluate_py_udf( FieldType::UInt => Field::UInt(res.extract::()?), FieldType::U128 => Field::U128(res.extract::()?), FieldType::Int => Field::Int(res.extract::()?), + FieldType::Int8 => Field::Int8(res.extract::()?), FieldType::I128 => Field::I128(res.extract::()?), FieldType::Float => Field::Float(OrderedFloat::from(res.extract::()?)), FieldType::Boolean => Field::Boolean(res.extract::()?), diff --git a/dozer-sql/expression/src/scalar/string.rs b/dozer-sql/expression/src/scalar/string.rs index 452416abe1..7e557fe63c 100644 --- a/dozer-sql/expression/src/scalar/string.rs +++ b/dozer-sql/expression/src/scalar/string.rs @@ -302,9 +302,7 @@ pub(crate) fn evaluate_chr( }) } } - Field::Int8(i) => { - Ok(Field::String(((i as u8) as char).to_string())) - } + Field::Int8(i) => Ok(Field::String(((i as u8) as char).to_string())), Field::I128(i) => { if i >= 0 { Ok(Field::String((((i % 256) as u8) as char).to_string())) diff --git a/dozer-types/src/types/field.rs b/dozer-types/src/types/field.rs index 6de4938475..af8d9671ad 100644 --- a/dozer-types/src/types/field.rs +++ b/dozer-types/src/types/field.rs @@ -1207,6 +1207,7 @@ impl pyo3::ToPyObject for Field { Field::UInt(val) => val.to_object(py), Field::U128(val) => val.to_object(py), Field::Int(val) => val.to_object(py), + Field::Int8(val) => val.to_object(py), Field::I128(val) => val.to_object(py), Field::Float(val) => val.0.to_object(py), Field::Decimal(val) => val.to_f64().unwrap().to_object(py),