Skip to content

Commit

Permalink
Improve array_contains
Browse files Browse the repository at this point in the history
  • Loading branch information
andygrove committed Jan 22, 2025
1 parent 44170d7 commit 16c4287
Show file tree
Hide file tree
Showing 13 changed files with 115 additions and 402 deletions.
9 changes: 9 additions & 0 deletions docs/source/user-guide/expressions.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,15 @@ The following Spark expressions are currently available. Any known compatibility
| VariancePop | |
| VarianceSamp | |

## Arrays

| Expression | Notes |
|-----------------| ----------- |
| array_append | |
| array_contains | |
| array_intersect | |
| array_remove | |

## Complex Types

| Expression | Notes |
Expand Down
2 changes: 1 addition & 1 deletion native/core/src/execution/serde.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ pub fn to_arrow_datatype(dt_value: &DataType) -> ArrowDataType {
{
DatatypeStruct::List(info) => {
let field = Field::new(
"item",
"element",
to_arrow_datatype(info.element_type.as_ref().unwrap()),
info.contains_null,
);
Expand Down
3 changes: 3 additions & 0 deletions native/core/src/parquet/schema_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,9 @@ fn cast_supported(from_type: &DataType, to_type: &DataType, options: &SparkParqu
}
true
}
(List(from_element), List(to_element)) => {
cast_supported(from_element.data_type(), to_element.data_type(), options)
}
_ => false,
}
}
Expand Down
5 changes: 5 additions & 0 deletions native/spark-expr/src/conversion_funcs/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ pub fn cast_supported(
) -> bool {
use DataType::*;

println!("cast_supported: {from_type} -> {to_type}");

let from_type = if let Dictionary(_, dt) = from_type {
dt
} else {
Expand Down Expand Up @@ -200,6 +202,9 @@ pub fn cast_supported(
.iter()
.zip(to_fields.iter())
.all(|(a, b)| cast_supported(a.data_type(), b.data_type(), options)),
(List(from_type), List(to_type)) => {
cast_supported(from_type.data_type(), to_type.data_type(), options)
}
_ => false,
}
}
Expand Down
2 changes: 0 additions & 2 deletions native/spark-expr/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@
mod error;

mod kernels;
mod schema_adapter;
mod static_invoke;
pub use schema_adapter::SparkSchemaAdapterFactory;
pub use static_invoke::*;

mod struct_funcs;
Expand Down
Loading

0 comments on commit 16c4287

Please sign in to comment.