diff --git a/crates/build/re_types_builder/src/codegen/cpp/mod.rs b/crates/build/re_types_builder/src/codegen/cpp/mod.rs index a588e17151d6..24f91b5d9bf5 100644 --- a/crates/build/re_types_builder/src/codegen/cpp/mod.rs +++ b/crates/build/re_types_builder/src/codegen/cpp/mod.rs @@ -1729,12 +1729,12 @@ fn quote_fill_arrow_array_builder( // C-style enum, encoded as a sparse arrow union ObjectClass::Enum => { quote! { - #parameter_check - ARROW_RETURN_NOT_OK(#builder->Reserve(static_cast(num_elements))); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - const auto variant = elements[elem_idx]; - ARROW_RETURN_NOT_OK(#builder->Append(static_cast(variant))); - } + #parameter_check + ARROW_RETURN_NOT_OK(#builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto variant = elements[elem_idx]; + ARROW_RETURN_NOT_OK(#builder->Append(static_cast(variant))); + } } } @@ -2482,6 +2482,7 @@ fn quote_arrow_field_type( let name = &field.name; let datatype = quote_arrow_datatype(&field.typ, objects, includes, false); let is_nullable = field.is_nullable || field.typ == Type::Unit; // null type is always nullable + let is_nullable = is_nullable || field.typ.is_union(objects); // Rerun unions always has a `_null_marker: null` variant, so they are always nullable quote! { arrow::field(#name, #datatype, #is_nullable) @@ -2496,6 +2497,7 @@ fn quote_arrow_elem_type( let typ: Type = elem_type.clone().into(); let datatype = quote_arrow_datatype(&typ, objects, includes, false); let is_nullable = typ == Type::Unit; // null type must be nullable + let is_nullable = is_nullable || elem_type.is_union(objects); // Rerun unions always has a `_null_marker: null` variant, so they are always nullable quote! { arrow::field("item", #datatype, #is_nullable) } diff --git a/crates/build/re_types_builder/src/codegen/python/mod.rs b/crates/build/re_types_builder/src/codegen/python/mod.rs index a183b6681ae3..15adb1f36d71 100644 --- a/crates/build/re_types_builder/src/codegen/python/mod.rs +++ b/crates/build/re_types_builder/src/codegen/python/mod.rs @@ -2550,7 +2550,8 @@ fn quote_arrow_field(field: &Field) -> String { } = field; let datatype = quote_arrow_datatype(data_type); - let is_nullable = if *is_nullable { "True" } else { "False" }; + let is_nullable = *is_nullable || matches!(data_type.to_logical_type(), DataType::Union { .. }); // Rerun unions always has a `_null_marker: null` variant, so they are always nullable + let is_nullable = if is_nullable { "True" } else { "False" }; let metadata = quote_metadata_map(metadata); format!(r#"pa.field("{name}", {datatype}, nullable={is_nullable}, metadata={metadata})"#) diff --git a/crates/build/re_types_builder/src/codegen/rust/api.rs b/crates/build/re_types_builder/src/codegen/rust/api.rs index beefef0528f9..3c10c6260b76 100644 --- a/crates/build/re_types_builder/src/codegen/rust/api.rs +++ b/crates/build/re_types_builder/src/codegen/rust/api.rs @@ -958,13 +958,13 @@ fn quote_trait_impls_for_datatype_or_component( let quoted_serializer = if let Some(forwarded_type) = forwarded_type.as_ref() { quote! { - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - #forwarded_type::to_arrow2_opt(data.into_iter().map(|datum| { + #forwarded_type::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), @@ -978,9 +978,9 @@ fn quote_trait_impls_for_datatype_or_component( quote! { // NOTE: Don't inline this, this gets _huge_. - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a { @@ -989,10 +989,14 @@ fn quote_trait_impls_for_datatype_or_component( #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; use ::re_types_core::{Loggable as _, ResultExt as _}; + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } + Ok(#quoted_serializer) } } diff --git a/crates/build/re_types_builder/src/codegen/rust/arrow.rs b/crates/build/re_types_builder/src/codegen/rust/arrow.rs index 3d5e5ed978cc..890f9d95004b 100644 --- a/crates/build/re_types_builder/src/codegen/rust/arrow.rs +++ b/crates/build/re_types_builder/src/codegen/rust/arrow.rs @@ -5,6 +5,10 @@ use quote::quote; // --- /// `(Datatype, is_recursive)` +/// +/// If `is_recursive` is set to `true`, +/// then the generate code will often be shorter, as it it will +/// defer to calling `arrow_datatype()` on the inner type. pub struct ArrowDataTypeTokenizer<'a>(pub &'a ::arrow2::datatypes::DataType, pub bool); impl quote::ToTokens for ArrowDataTypeTokenizer<'_> { @@ -31,18 +35,18 @@ impl quote::ToTokens for ArrowDataTypeTokenizer<'_> { DataType::LargeUtf8 => quote!(DataType::LargeUtf8), DataType::List(field) => { - let field = ArrowFieldTokenizer(field); + let field = ArrowFieldTokenizer::new(field); quote!(DataType::List(std::sync::Arc::new(#field))) } DataType::FixedSizeList(field, length) => { - let field = ArrowFieldTokenizer(field); + let field = ArrowFieldTokenizer::new(field); let length = Literal::usize_unsuffixed(*length); quote!(DataType::FixedSizeList(std::sync::Arc::new(#field), #length)) } DataType::Union(fields, types, mode) => { - let fields = fields.iter().map(ArrowFieldTokenizer); + let fields = fields.iter().map(ArrowFieldTokenizer::new); let mode = match mode { UnionMode::Dense => quote!(UnionMode::Dense), UnionMode::Sparse => quote!(UnionMode::Sparse), @@ -66,18 +70,20 @@ impl quote::ToTokens for ArrowDataTypeTokenizer<'_> { } DataType::Struct(fields) => { - let fields = fields.iter().map(ArrowFieldTokenizer); + let fields = fields.iter().map(ArrowFieldTokenizer::new); quote!(DataType::Struct(Fields::from(vec![ #(#fields,)* ]))) } DataType::Extension(fqname, datatype, _metadata) => { if *recursive { + // TODO(emilk): if the logical datatype is a primitive, then we can just use it directly + // so we get shorter generated code. let fqname_use = quote_fqname_as_type_path(fqname); quote!(<#fqname_use>::arrow_datatype()) } else { let datatype = ArrowDataTypeTokenizer(datatype.to_logical_type(), false); quote!(#datatype) - // TODO(cmc): Bring back extensions once we've fully replaced `arrow2-convert`! + // TODO(#3741): Bring back extensions once we've fully replaced `arrow2-convert`! // let datatype = ArrowDataTypeTokenizer(datatype, false); // let metadata = OptionTokenizer(metadata.as_ref()); // quote!(DataType::Extension(#fqname.to_owned(), Box::new(#datatype), #metadata)) @@ -90,16 +96,30 @@ impl quote::ToTokens for ArrowDataTypeTokenizer<'_> { } } -pub struct ArrowFieldTokenizer<'a>(pub &'a ::arrow2::datatypes::Field); +pub struct ArrowFieldTokenizer<'a> { + field: &'a ::arrow2::datatypes::Field, +} + +impl<'a> ArrowFieldTokenizer<'a> { + pub fn new(field: &'a ::arrow2::datatypes::Field) -> Self { + Self { field } + } +} impl quote::ToTokens for ArrowFieldTokenizer<'_> { fn to_tokens(&self, tokens: &mut TokenStream) { + let Self { field } = self; let arrow2::datatypes::Field { name, data_type, is_nullable, metadata, - } = &self.0; + } = field; + + // Unions in Rerun always has a `_null_markers` arm, so all unions are nullable, + // whether they are specified as such or not. + let is_nullable = + *is_nullable || matches!(field.data_type.to_logical_type(), DataType::Union { .. }); let datatype = ArrowDataTypeTokenizer(data_type, true); @@ -164,3 +184,22 @@ pub fn is_backed_by_arrow_buffer(typ: &DataType) -> bool { | DataType::Float64 ) } + +pub fn quoted_arrow_primitive_type(datatype: &DataType) -> TokenStream { + match datatype { + DataType::Null => quote!(NullType), + DataType::Boolean => quote!(BooleanType), + DataType::Int8 => quote!(Int8Type), + DataType::Int16 => quote!(Int16Type), + DataType::Int32 => quote!(Int32Type), + DataType::Int64 => quote!(Int64Type), + DataType::UInt8 => quote!(UInt8Type), + DataType::UInt16 => quote!(UInt16Type), + DataType::UInt32 => quote!(UInt32Type), + DataType::UInt64 => quote!(UInt64Type), + DataType::Float16 => quote!(Float16Type), + DataType::Float32 => quote!(Float32Type), + DataType::Float64 => quote!(Float64Type), + _ => unimplemented!("Not a primitive type: {datatype:#?}"), + } +} diff --git a/crates/build/re_types_builder/src/codegen/rust/serializer.rs b/crates/build/re_types_builder/src/codegen/rust/serializer.rs index 31d28f71b95f..7dcec6703b64 100644 --- a/crates/build/re_types_builder/src/codegen/rust/serializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/serializer.rs @@ -1,11 +1,14 @@ use arrow2::datatypes::DataType; -use proc_macro2::TokenStream; +use proc_macro2::{Literal, TokenStream}; use quote::{format_ident, quote}; use crate::{ArrowRegistry, Object, Objects}; use super::{ - arrow::{is_backed_by_arrow_buffer, quote_fqname_as_type_path}, + arrow::{ + is_backed_by_arrow_buffer, quote_fqname_as_type_path, quoted_arrow_primitive_type, + ArrowFieldTokenizer, + }, util::{is_tuple_struct_from_obj, quote_comment}, }; @@ -19,8 +22,6 @@ pub fn quote_arrow_serializer( ) -> TokenStream { let datatype = &arrow_registry.get(&obj.fqname); - let quoted_datatype = quote! { Self::arrow_datatype() }; - let is_enum = obj.is_enum(); let is_arrow_transparent = obj.datatype.is_none(); let is_tuple_struct = is_tuple_struct_from_obj(obj); @@ -35,10 +36,10 @@ pub fn quote_arrow_serializer( } }; - let quoted_bitmap = |var| { + let quoted_validity = |var| { quote! { - let #var: Option = { - // NOTE: Don't compute a bitmap if there isn't at least one null element. + let #var: Option = { + // NOTE: Don't compute a validity if there isn't at least one null element. let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) } @@ -48,10 +49,10 @@ pub fn quote_arrow_serializer( if is_enum { let quoted_data_src = data_src.clone(); let quoted_data_dst = format_ident!("data0"); - let bitmap_dst = format_ident!("{quoted_data_dst}_bitmap"); + let validity_dst = format_ident!("{quoted_data_dst}_validity"); // The choice of true or false for `elements_are_nullable` here is a bit confusing. - // This code-gen path forms the basis of `to_arrow2_opt`, which implies that we + // This code-gen path forms the basis of `to_arrow_opt`, which implies that we // support nullable elements. Additionally, this MAY be used as a recursive code // path when using an enum within a struct, and that struct within the field may // be null, as such the elements are always handled as nullable. @@ -61,14 +62,13 @@ pub fn quote_arrow_serializer( let quoted_serializer = quote_arrow_field_serializer( objects, datatype.to_logical_type(), - "ed_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, "ed_data_dst, InnerRepr::NativeIterable, ); - let quoted_bitmap = quoted_bitmap(bitmap_dst); + let quoted_validity = quoted_validity(validity_dst); quote! {{ let (somes, #quoted_data_dst): (Vec<_>, Vec<_>) = #quoted_data_src @@ -84,7 +84,7 @@ pub fn quote_arrow_serializer( .unzip(); - #quoted_bitmap; + #quoted_validity; #quoted_serializer }} @@ -102,7 +102,7 @@ pub fn quote_arrow_serializer( obj_field.name.as_str() } ); - let bitmap_dst = format_ident!("{quoted_data_dst}_bitmap"); + let validity_dst = format_ident!("{quoted_data_dst}_validity"); let quoted_member_accessor = if is_tuple_struct { quote!(0) @@ -116,14 +116,13 @@ pub fn quote_arrow_serializer( let quoted_serializer = quote_arrow_field_serializer( objects, datatype, - "ed_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, "ed_data_dst, InnerRepr::NativeIterable, ); - let quoted_bitmap = quoted_bitmap(bitmap_dst); + let quoted_validity = quoted_validity(validity_dst); let quoted_flatten = quoted_flatten(obj_field.is_nullable); @@ -144,28 +143,27 @@ pub fn quote_arrow_serializer( .unzip(); - #quoted_bitmap; + #quoted_validity; #quoted_serializer }} } else { // NOTE: This can only be struct or union/enum at this point. match datatype.to_logical_type() { - DataType::Struct(_) => { + DataType::Struct(fields) => { + let quoted_fields = fields.iter().map(ArrowFieldTokenizer::new); + let quoted_field_serializers = obj.fields.iter().map(|obj_field| { let data_dst = format_ident!("{}", obj_field.name); - let bitmap_dst = format_ident!("{data_dst}_bitmap"); + let validity_dst = format_ident!("{data_dst}_validity"); let inner_datatype = &arrow_registry.get(&obj_field.fqname); - let quoted_inner_datatype = - super::arrow::ArrowDataTypeTokenizer(inner_datatype, false); let elements_are_nullable = true; let quoted_serializer = quote_arrow_field_serializer( objects, inner_datatype, - "ed_inner_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, &data_dst, InnerRepr::NativeIterable, @@ -173,7 +171,7 @@ pub fn quote_arrow_serializer( let quoted_flatten = quoted_flatten(obj_field.is_nullable); - let quoted_bitmap = quoted_bitmap(bitmap_dst); + let quoted_validity = quoted_validity(validity_dst); quote! {{ let (somes, #data_dst): (Vec<_>, Vec<_>) = #data_src @@ -191,15 +189,17 @@ pub fn quote_arrow_serializer( .unzip(); - #quoted_bitmap; + #quoted_validity; #quoted_serializer }} }); - let quoted_bitmap = quoted_bitmap(format_ident!("bitmap")); + let quoted_declare_validity = quoted_validity(format_ident!("validity")); quote! {{ + let fields = Fields::from(vec![#(#quoted_fields,)*]); + let (somes, #data_src): (Vec<_>, Vec<_>) = #data_src .into_iter() .map(|datum| { @@ -208,20 +208,20 @@ pub fn quote_arrow_serializer( }) .unzip(); - #quoted_bitmap; + #quoted_declare_validity; - StructArray::new( - #quoted_datatype.into(), + as_array_ref(StructArray::new( + fields, vec![#(#quoted_field_serializers,)*], - bitmap, - ).boxed() + validity, + )) }} } - DataType::Union(_, _, arrow2::datatypes::UnionMode::Sparse) => { + DataType::Union(fields, _, arrow2::datatypes::UnionMode::Sparse) => { // We use sparse unions for enums, which means only 8 bits is required for each field, // and nulls are encoded with a special 0-index `_null_markers` variant. - + let quoted_fields = fields.iter().map(ArrowFieldTokenizer::new); let quoted_data_collect = quote! { let #data_src: Vec<_> = #data_src .into_iter() @@ -232,7 +232,7 @@ pub fn quote_arrow_serializer( .collect(); }; - let quoted_types = quote! { + let quoted_type_ids = quote! { #data_src .iter() .map(|a| match a.as_deref() { @@ -251,28 +251,32 @@ pub fn quote_arrow_serializer( #quoted_data_collect + let fields = vec![#(#quoted_fields,)*]; + let type_ids: Vec = #quoted_type_ids; let num_variants = #num_variants; - let types = #quoted_types; - - let fields: Vec<_> = std::iter::repeat( - NullArray::new( - arrow2::datatypes::DataType::Null, + let children: Vec<_> = std::iter::repeat( + as_array_ref(NullArray::new( #data_src.len(), - ).boxed() + )) ).take(1 + num_variants) // +1 for the virtual `nulls` arm .collect(); - UnionArray::new( - #quoted_datatype.into(), - types, - fields, + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), None, - ).boxed() + children, + )?) }} } - DataType::Union(_, _, arrow2::datatypes::UnionMode::Dense) => { + DataType::Union(fields, _, arrow2::datatypes::UnionMode::Dense) => { + let quoted_field_type_ids = (0..fields.len()).map(Literal::usize_unsuffixed); + let quoted_fields = fields.iter().map(ArrowFieldTokenizer::new); let quoted_data_collect = quote! { let #data_src: Vec<_> = #data_src .into_iter() @@ -289,13 +293,12 @@ pub fn quote_arrow_serializer( // Short circuit for empty variants since they're trivial to solve at this level: if obj_field.typ == crate::Type::Unit { return quote! { - NullArray::new( - arrow2::datatypes::DataType::Null, + as_array_ref(NullArray::new( #data_src .iter() .filter(|datum| matches!(datum.as_deref(), Some(Self::#quoted_obj_field_name))) .count(), - ).boxed() + )) }; } @@ -303,17 +306,14 @@ pub fn quote_arrow_serializer( // We handle nullability with a special null variant that is always present. let elements_are_nullable = false; - let bitmap_dst = format_ident!("{}_bitmap", data_dst); + let validity_dst = format_ident!("{}_validity", data_dst); let inner_datatype = &arrow_registry.get(&obj_field.fqname); - let quoted_inner_datatype = - super::arrow::ArrowDataTypeTokenizer(inner_datatype, false); let quoted_serializer = quote_arrow_field_serializer( objects, inner_datatype, - "ed_inner_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, &data_dst, InnerRepr::NativeIterable, @@ -328,18 +328,19 @@ pub fn quote_arrow_serializer( }) .collect(); - let #bitmap_dst: Option = None; + let #validity_dst: Option = None; #quoted_serializer }} }); - let quoted_fields = quote! { + let quoted_children_serializers = quoted_field_serializers.clone(); + + let quoted_children = quote! { vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, + as_array_ref(NullArray::new( #data_src.iter().filter(|v| v.is_none()).count(), - ).boxed(), - #(#quoted_field_serializers,)* + )), + #(#quoted_children_serializers,)* ] }; @@ -351,7 +352,7 @@ pub fn quote_arrow_serializer( } }; - let quoted_types = { + let quoted_type_ids = { let quoted_branches = obj.fields.iter().enumerate().map(|(i, obj_field)| { let i = 1 + i as i8; // NOTE: +1 to account for `nulls` virtual arm let quoted_obj_field_name = @@ -421,16 +422,21 @@ pub fn quote_arrow_serializer( #quoted_data_collect - let types = #quoted_types; - let fields = #quoted_fields; - let offsets = Some(#quoted_offsets); - - UnionArray::new( - #quoted_datatype.into(), - types, - fields, - offsets, - ).boxed() + let field_type_ids = [#(#quoted_field_type_ids,)*]; + let fields = vec![#(#quoted_fields,)*]; + let type_ids: Vec = #quoted_type_ids; + let offsets = #quoted_offsets; + let children = #quoted_children; + + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + )?) }} } @@ -456,14 +462,13 @@ enum InnerRepr { /// * unions: nullability is encoded as a separate variant /// * lists inside of fields that are lists: we don't support intra-list nullability /// -/// TODO(#2993): However, we still emit a validity bitmap for lists inside lists -/// to `validity_bitmap_ident`since Python and Rust do so. +/// TODO(#2993): However, we still emit a validity/null bitmaps for lists inside lists +/// since Python and Rust do so. #[allow(clippy::too_many_arguments)] fn quote_arrow_field_serializer( objects: &Objects, datatype: &DataType, - quoted_datatype: &dyn quote::ToTokens, - bitmap_src: &proc_macro2::Ident, + validity_src: &proc_macro2::Ident, elements_are_nullable: bool, data_src: &proc_macro2::Ident, inner_repr: InnerRepr, @@ -485,8 +490,8 @@ fn quote_arrow_field_serializer( }; return quote! {{ - _ = #bitmap_src; - #fqname_use::to_arrow2_opt(#data_src #option_wrapper)? + _ = #validity_src; + #fqname_use::to_arrow_opt(#data_src #option_wrapper)? }}; } } @@ -506,7 +511,7 @@ fn quote_arrow_field_serializer( | DataType::Float16 | DataType::Float32 | DataType::Float64 => { - // NOTE: We need values for all slots, regardless of what the bitmap says, + // NOTE: We need values for all slots, regardless of what the validity says, // hence `unwrap_or_default` (unless elements_are_nullable is false) let quoted_transparent_mapping = if inner_is_arrow_transparent { let inner_obj = inner_obj.as_ref().unwrap(); @@ -546,29 +551,28 @@ fn quote_arrow_field_serializer( if datatype.to_logical_type() == &DataType::Boolean { quote! { - BooleanArray::new( - #quoted_datatype.into(), - #data_src.into_iter() #quoted_transparent_mapping .collect(), - #bitmap_src, - ).boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from(#data_src.into_iter() #quoted_transparent_mapping .collect::>()), + #validity_src, + )) } } else { + let arrow_primitive_type = quoted_arrow_primitive_type(datatype.to_logical_type()); + match inner_repr { // A primitive that's an inner element of a list will already have been mapped // to a buffer type. InnerRepr::ArrowBuffer => quote! { - PrimitiveArray::new( - #quoted_datatype.into(), + as_array_ref(PrimitiveArray::<#arrow_primitive_type>::new( #data_src, - #bitmap_src, - ).boxed() + #validity_src, + )) }, InnerRepr::NativeIterable => quote! { - PrimitiveArray::new( - #quoted_datatype.into(), - #data_src.into_iter() #quoted_transparent_mapping .collect(), - #bitmap_src, - ).boxed() + as_array_ref(PrimitiveArray::<#arrow_primitive_type>::new( + ScalarBuffer::from(#data_src.into_iter() #quoted_transparent_mapping .collect::>()), + #validity_src, + )) }, } } @@ -579,7 +583,7 @@ fn quote_arrow_field_serializer( } DataType::Utf8 => { - // NOTE: We need values for all slots, regardless of what the bitmap says, + // NOTE: We need values for all slots, regardless of what the validity says, // hence `unwrap_or_default`. let (quoted_transparent_mapping, quoted_transparent_length) = if inner_is_arrow_transparent { @@ -624,24 +628,22 @@ fn quote_arrow_field_serializer( let inner_data_and_offsets = if elements_are_nullable { quote! { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter().map(|opt| opt.as_ref() #quoted_transparent_length .unwrap_or_default()) - )? - .into(); + ); // NOTE: Flattening to remove the guaranteed layer of nullability: we don't care // about it while building the backing buffer since it's all offsets driven. - let inner_data: arrow2::buffer::Buffer = + let inner_data: arrow::buffer::Buffer = #data_src.into_iter().flatten() #quoted_transparent_mapping.collect(); } } else { quote! { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter() #quoted_transparent_length - )? - .into(); + ); - let inner_data: arrow2::buffer::Buffer = + let inner_data: arrow::buffer::Buffer = #data_src.into_iter() #quoted_transparent_mapping.collect(); } }; @@ -654,13 +656,12 @@ fn quote_arrow_field_serializer( // It would be nice to use quote_comment here and put this safety notice in the generated code, // but that seems to push us over some complexity limit causing rustfmt to fail. #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { Utf8Array::::new_unchecked(#quoted_datatype.into(), offsets, inner_data, #bitmap_src) }.boxed() + as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, #validity_src) }) }} } - DataType::List(inner) | DataType::FixedSizeList(inner, _) => { - let inner_datatype = inner.data_type(); - let quoted_inner_datatype = super::arrow::ArrowDataTypeTokenizer(inner_datatype, false); + DataType::List(inner_field) | DataType::FixedSizeList(inner_field, _) => { + let inner_datatype = inner_field.data_type(); // Note: We only use the ArrowBuffer optimization for `Lists` but not `FixedSizeList`. // This is because the `ArrowBuffer` has a dynamic length, which would add more overhead @@ -668,7 +669,7 @@ fn quote_arrow_field_serializer( // // TODO(jleibs): If we need to support large FixedSizeList types where the `ArrowBuffer` // optimization would be significant, we can introduce a new attribute to force this. - let inner_repr = if is_backed_by_arrow_buffer(inner.data_type()) + let inner_repr = if is_backed_by_arrow_buffer(inner_field.data_type()) && matches!(datatype, DataType::List(_)) { InnerRepr::ArrowBuffer @@ -677,14 +678,13 @@ fn quote_arrow_field_serializer( }; let quoted_inner_data = format_ident!("{data_src}_inner_data"); - let inner_bitmap_ident = format_ident!("{data_src}_inner_bitmap"); + let inner_validity_ident = format_ident!("{data_src}_inner_validity"); let inner_elements_are_nullable = false; // We don't support intra-list nullability. let quoted_inner = quote_arrow_field_serializer( objects, inner_datatype, - "ed_inner_datatype, - &inner_bitmap_ident, + &inner_validity_ident, inner_elements_are_nullable, "ed_inner_data, inner_repr, @@ -798,31 +798,40 @@ fn quote_arrow_field_serializer( }; quote! { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter(). #map_to_length - )?.into(); + ); } } else { quote! {} }; - let quoted_create = if let DataType::List(_) = datatype { - quote! { - ListArray::try_new( - #quoted_datatype.into(), - offsets, - #quoted_inner, - #bitmap_src, - )?.boxed() + let field = ArrowFieldTokenizer::new(inner_field); + let quoted_field = quote!(std::sync::Arc::new(#field)); + + let quoted_create = match datatype.to_logical_type() { + DataType::List(_) => { + quote! { + as_array_ref(ListArray::try_new( + #quoted_field, + offsets, + #quoted_inner, + #validity_src, + )?) + } } - } else { - quote! { - FixedSizeListArray::new( - #quoted_datatype.into(), - #quoted_inner, - #bitmap_src, - ).boxed() + DataType::FixedSizeList(_, size) => { + let quoted_size = Literal::usize_unsuffixed(*size); + quote! { + as_array_ref(FixedSizeListArray::new( + #quoted_field, + #quoted_size, + #quoted_inner, + #validity_src, + )) + } } + _ => unreachable!(), }; // TODO(#2993): The inner @@ -834,15 +843,15 @@ fn quote_arrow_field_serializer( // if we make the C++ and Python codegen match the rust behavior or // make our comparison tests more lenient. // - // This workaround does not apply if we don't have any validity bitmap on the outer type. + // This workaround does not apply if we don't have any validity validity on the outer type. // (as it is always the case with unions where the nullability is encoded as a separate variant) - let quoted_inner_bitmap = if let (true, DataType::FixedSizeList(_, count)) = + let quoted_inner_validity = if let (true, DataType::FixedSizeList(_, count)) = (elements_are_nullable, datatype.to_logical_type()) { quote! { - let #inner_bitmap_ident: Option = - #bitmap_src.as_ref().map(|bitmap| { - bitmap + let #inner_validity_ident: Option = + #validity_src.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(#count)) .flatten() @@ -853,20 +862,18 @@ fn quote_arrow_field_serializer( } else { // TODO(cmc): We don't support intra-list nullability in our IDL at the moment. quote! { - let #inner_bitmap_ident: Option = None; + let #inner_validity_ident: Option = None; } }; match inner_repr { InnerRepr::ArrowBuffer => { quote! {{ - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - #quoted_declare_offsets - let #quoted_inner_data: Buffer<_> = #quoted_transparent_mapping; + let #quoted_inner_data: ScalarBuffer<_> = #quoted_transparent_mapping; - #quoted_inner_bitmap + #quoted_inner_validity #quoted_create }} @@ -874,15 +881,13 @@ fn quote_arrow_field_serializer( InnerRepr::NativeIterable => { quote! {{ - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - #quoted_declare_offsets let #quoted_inner_data: Vec<_> = #quoted_transparent_mapping .collect(); - #quoted_inner_bitmap + #quoted_inner_validity #quoted_create }} @@ -903,8 +908,8 @@ fn quote_arrow_field_serializer( }; quote! {{ - _ = #bitmap_src; - #fqname_use::to_arrow2_opt(#data_src #option_wrapper)? + _ = #validity_src; + #fqname_use::to_arrow_opt(#data_src #option_wrapper)? }} } diff --git a/crates/build/re_types_builder/src/objects.rs b/crates/build/re_types_builder/src/objects.rs index 1f529a4e9152..87512f2e0463 100644 --- a/crates/build/re_types_builder/src/objects.rs +++ b/crates/build/re_types_builder/src/objects.rs @@ -1134,6 +1134,19 @@ impl Type { Self::Object(fqname) => objects[fqname].has_default_destructor(objects), } } + + pub fn is_union(&self, objects: &Objects) -> bool { + if let Self::Object(fqname) = self { + let obj = &objects[fqname]; + if obj.is_arrow_transparent() { + obj.fields[0].typ.is_union(objects) + } else { + obj.class == ObjectClass::Union + } + } else { + false + } + } } /// The underlying element type for arrays/vectors/maps. @@ -1258,6 +1271,19 @@ impl ElementType { Self::Bool | Self::Object(_) | Self::String => false, } } + + pub fn is_union(&self, objects: &Objects) -> bool { + if let Self::Object(fqname) = self { + let obj = &objects[fqname]; + if obj.is_arrow_transparent() { + obj.fields[0].typ.is_union(objects) + } else { + obj.class == ObjectClass::Union + } + } else { + false + } + } } // --- Common --- diff --git a/crates/store/re_types/src/blueprint/components/active_tab.rs b/crates/store/re_types/src/blueprint/components/active_tab.rs index d372ddddb791..144cde6a2651 100644 --- a/crates/store/re_types/src/blueprint/components/active_tab.rs +++ b/crates/store/re_types/src/blueprint/components/active_tab.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for ActiveTab { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/apply_latest_at.rs b/crates/store/re_types/src/blueprint/components/apply_latest_at.rs index 440068bdec6d..8afa9418e09f 100644 --- a/crates/store/re_types/src/blueprint/components/apply_latest_at.rs +++ b/crates/store/re_types/src/blueprint/components/apply_latest_at.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for ApplyLatestAt { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/background_kind.rs b/crates/store/re_types/src/blueprint/components/background_kind.rs index c993bcbd6f35..dd78ace0f99a 100644 --- a/crates/store/re_types/src/blueprint/components/background_kind.rs +++ b/crates/store/re_types/src/blueprint/components/background_kind.rs @@ -90,17 +90,21 @@ impl ::re_types_core::Loggable for BackgroundKind { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -110,16 +114,19 @@ impl ::re_types_core::Loggable for BackgroundKind { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/column_share.rs b/crates/store/re_types/src/blueprint/components/column_share.rs index e23a7d39cc3e..130b7eab8eea 100644 --- a/crates/store/re_types/src/blueprint/components/column_share.rs +++ b/crates/store/re_types/src/blueprint/components/column_share.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ColumnShare { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/component_column_selector.rs b/crates/store/re_types/src/blueprint/components/component_column_selector.rs index 39675fda0703..a7e9de8c3668 100644 --- a/crates/store/re_types/src/blueprint/components/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/components/component_column_selector.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { crate::blueprint::datatypes::ComponentColumnSelector::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::ComponentColumnSelector::to_arrow2_opt(data.into_iter().map( + crate::blueprint::datatypes::ComponentColumnSelector::to_arrow_opt(data.into_iter().map( |datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), diff --git a/crates/store/re_types/src/blueprint/components/corner2d.rs b/crates/store/re_types/src/blueprint/components/corner2d.rs index 62d9e96351c6..27e3fe2f0838 100644 --- a/crates/store/re_types/src/blueprint/components/corner2d.rs +++ b/crates/store/re_types/src/blueprint/components/corner2d.rs @@ -92,17 +92,21 @@ impl ::re_types_core::Loggable for Corner2D { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -112,16 +116,19 @@ impl ::re_types_core::Loggable for Corner2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/filter_by_range.rs b/crates/store/re_types/src/blueprint/components/filter_by_range.rs index 4538c20fd6ac..f48e3f65fcc5 100644 --- a/crates/store/re_types/src/blueprint/components/filter_by_range.rs +++ b/crates/store/re_types/src/blueprint/components/filter_by_range.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for FilterByRange { crate::blueprint::datatypes::FilterByRange::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::FilterByRange::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::FilterByRange::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs b/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs index fa8a3c20851a..47da6800b958 100644 --- a/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs +++ b/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for FilterIsNotNull { crate::blueprint::datatypes::FilterIsNotNull::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::FilterIsNotNull::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::FilterIsNotNull::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/included_content.rs b/crates/store/re_types/src/blueprint/components/included_content.rs index fd5ca2170ae2..590900ece0ff 100644 --- a/crates/store/re_types/src/blueprint/components/included_content.rs +++ b/crates/store/re_types/src/blueprint/components/included_content.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for IncludedContent { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/interactive.rs b/crates/store/re_types/src/blueprint/components/interactive.rs index 4fbec8d71dc6..a99f7505c0af 100644 --- a/crates/store/re_types/src/blueprint/components/interactive.rs +++ b/crates/store/re_types/src/blueprint/components/interactive.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for Interactive { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs b/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs index 13280b5e1e2a..a0d60c1e2444 100644 --- a/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs +++ b/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for LockRangeDuringZoom { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/map_provider.rs b/crates/store/re_types/src/blueprint/components/map_provider.rs index 0de6496dcf70..ada45a3f9463 100644 --- a/crates/store/re_types/src/blueprint/components/map_provider.rs +++ b/crates/store/re_types/src/blueprint/components/map_provider.rs @@ -92,17 +92,21 @@ impl ::re_types_core::Loggable for MapProvider { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -112,16 +116,19 @@ impl ::re_types_core::Loggable for MapProvider { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/panel_state.rs b/crates/store/re_types/src/blueprint/components/panel_state.rs index 2b9decb690b8..7b19e5ec31f5 100644 --- a/crates/store/re_types/src/blueprint/components/panel_state.rs +++ b/crates/store/re_types/src/blueprint/components/panel_state.rs @@ -82,17 +82,21 @@ impl ::re_types_core::Loggable for PanelState { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -102,16 +106,19 @@ impl ::re_types_core::Loggable for PanelState { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/query_expression.rs b/crates/store/re_types/src/blueprint/components/query_expression.rs index 41549031ce0a..ec60f3a0a29d 100644 --- a/crates/store/re_types/src/blueprint/components/query_expression.rs +++ b/crates/store/re_types/src/blueprint/components/query_expression.rs @@ -81,13 +81,13 @@ impl ::re_types_core::Loggable for QueryExpression { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/row_share.rs b/crates/store/re_types/src/blueprint/components/row_share.rs index cc75f807a264..2b70f7fe32bf 100644 --- a/crates/store/re_types/src/blueprint/components/row_share.rs +++ b/crates/store/re_types/src/blueprint/components/row_share.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for RowShare { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/selected_columns.rs b/crates/store/re_types/src/blueprint/components/selected_columns.rs index 1290721b396f..83b30d1acc1b 100644 --- a/crates/store/re_types/src/blueprint/components/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/components/selected_columns.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SelectedColumns { crate::blueprint::datatypes::SelectedColumns::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::SelectedColumns::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::SelectedColumns::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/space_view_class.rs b/crates/store/re_types/src/blueprint/components/space_view_class.rs index 7e17ee0bd137..6e0c0cf363c2 100644 --- a/crates/store/re_types/src/blueprint/components/space_view_class.rs +++ b/crates/store/re_types/src/blueprint/components/space_view_class.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SpaceViewClass { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/space_view_origin.rs b/crates/store/re_types/src/blueprint/components/space_view_origin.rs index b3e01469cad5..82e0ecf6e155 100644 --- a/crates/store/re_types/src/blueprint/components/space_view_origin.rs +++ b/crates/store/re_types/src/blueprint/components/space_view_origin.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SpaceViewOrigin { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs b/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs index db8295ed7625..350db3ae3dff 100644 --- a/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs +++ b/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs @@ -76,20 +76,20 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { crate::blueprint::datatypes::TensorDimensionIndexSlider::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::TensorDimensionIndexSlider::to_arrow2_opt( - data.into_iter().map(|datum| { + crate::blueprint::datatypes::TensorDimensionIndexSlider::to_arrow_opt(data.into_iter().map( + |datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), }) - }), - ) + }, + )) } fn from_arrow2_opt( diff --git a/crates/store/re_types/src/blueprint/components/timeline_name.rs b/crates/store/re_types/src/blueprint/components/timeline_name.rs index b8efd276ac94..fd7ebbe0398f 100644 --- a/crates/store/re_types/src/blueprint/components/timeline_name.rs +++ b/crates/store/re_types/src/blueprint/components/timeline_name.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TimelineName { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/view_fit.rs b/crates/store/re_types/src/blueprint/components/view_fit.rs index c0a6f9e10f19..90711dc0d391 100644 --- a/crates/store/re_types/src/blueprint/components/view_fit.rs +++ b/crates/store/re_types/src/blueprint/components/view_fit.rs @@ -88,17 +88,21 @@ impl ::re_types_core::Loggable for ViewFit { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -108,16 +112,19 @@ impl ::re_types_core::Loggable for ViewFit { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs b/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs index ea8a3ec28da2..c52a0f6137ca 100644 --- a/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs +++ b/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ViewerRecommendationHash { crate::datatypes::UInt64::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::UInt64::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::UInt64::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/visible.rs b/crates/store/re_types/src/blueprint/components/visible.rs index 0abcf7cc4dda..372c85ca7cfe 100644 --- a/crates/store/re_types/src/blueprint/components/visible.rs +++ b/crates/store/re_types/src/blueprint/components/visible.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Visible { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/visible_time_range.rs b/crates/store/re_types/src/blueprint/components/visible_time_range.rs index 462bfae52d23..2875854faf72 100644 --- a/crates/store/re_types/src/blueprint/components/visible_time_range.rs +++ b/crates/store/re_types/src/blueprint/components/visible_time_range.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for VisibleTimeRange { crate::datatypes::VisibleTimeRange::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::VisibleTimeRange::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::VisibleTimeRange::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs b/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs index 2db4a32793fa..e5355ecb9d5c 100644 --- a/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs +++ b/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for VisualBounds2D { crate::datatypes::Range2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Range2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Range2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/zoom_level.rs b/crates/store/re_types/src/blueprint/components/zoom_level.rs index 4d1b883f30f4..1708693a90b5 100644 --- a/crates/store/re_types/src/blueprint/components/zoom_level.rs +++ b/crates/store/re_types/src/blueprint/components/zoom_level.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ZoomLevel { crate::datatypes::Float64::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float64::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float64::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs index 580c81ac7be7..aa2f8ff6408f 100644 --- a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs @@ -61,18 +61,34 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "entity_path", + ::arrow_datatype(), + false, + ), + Field::new( + "component", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +96,12 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, entity_path): (Vec<_>, Vec<_>) = data @@ -95,33 +111,29 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { (datum.is_some(), datum) }) .unzip(); - let entity_path_bitmap: Option = { + let entity_path_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( entity_path.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = entity_path + ); + let inner_data: arrow::buffer::Buffer = entity_path .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - entity_path_bitmap, + entity_path_validity, ) - } - .boxed() + }) } }, { @@ -132,39 +144,31 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { (datum.is_some(), datum) }) .unzip(); - let component_bitmap: Option = { + let component_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( component.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = component + ); + let inner_data: arrow::buffer::Buffer = component .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - component_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, component_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs b/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs index 56548ef92a32..5387426ba146 100644 --- a/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs +++ b/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs @@ -57,18 +57,30 @@ impl ::re_types_core::Loggable for FilterByRange { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "start", + ::arrow_datatype(), + false, + ), + Field::new("end", ::arrow_datatype(), false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -76,12 +88,12 @@ impl ::re_types_core::Loggable for FilterByRange { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, start): (Vec<_>, Vec<_>) = data @@ -91,19 +103,19 @@ impl ::re_types_core::Loggable for FilterByRange { (datum.is_some(), datum) }) .unzip(); - let start_bitmap: Option = { + let start_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Int64.into(), - start - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - start_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + start + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + start_validity, + )) }, { let (somes, end): (Vec<_>, Vec<_>) = data @@ -113,23 +125,22 @@ impl ::re_types_core::Loggable for FilterByRange { (datum.is_some(), datum) }) .unzip(); - let end_bitmap: Option = { + let end_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Int64.into(), - end.into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - end_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + end.into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + end_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs b/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs index 1f1d20ebf0d1..5a481196232a 100644 --- a/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs +++ b/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs @@ -58,18 +58,30 @@ impl ::re_types_core::Loggable for FilterIsNotNull { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("active", ::arrow_datatype(), false), + Field::new( + "column", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -77,12 +89,12 @@ impl ::re_types_core::Loggable for FilterIsNotNull { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, active): (Vec<_>, Vec<_>) = data @@ -92,19 +104,19 @@ impl ::re_types_core::Loggable for FilterIsNotNull { (datum.is_some(), datum) }) .unzip(); - let active_bitmap: Option = { + let active_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - DataType::Boolean.into(), - active - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - active_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + active + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + active_validity, + )) }, { let (somes, column): (Vec<_>, Vec<_>) = data @@ -114,21 +126,20 @@ impl ::re_types_core::Loggable for FilterIsNotNull { (datum.is_some(), datum) }) .unzip(); - let column_bitmap: Option = { + let column_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = column_bitmap; - crate::blueprint::datatypes::ComponentColumnSelector::to_arrow2_opt( + _ = column_validity; + crate::blueprint::datatypes::ComponentColumnSelector::to_arrow_opt( column, )? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs index e2282cbe6a72..cff91ae89b29 100644 --- a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs @@ -70,18 +70,42 @@ impl ::re_types_core::Loggable for SelectedColumns { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "time_columns", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "component_columns", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -89,63 +113,98 @@ impl ::re_types_core::Loggable for SelectedColumns { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), - vec![ - { let (somes, time_columns) : (Vec < _ >, Vec < _ >) = data - .iter().map(| datum | { let datum = datum.as_ref().map(| datum | - { datum.time_columns.clone() }); (datum.is_some(), datum) }) - .unzip(); let time_columns_bitmap : Option < - arrow2::bitmap::Bitmap > = { let any_nones = somes.iter().any(| - some | ! * some); any_nones.then(|| somes.into()) }; { use - arrow2:: { buffer::Buffer, offset::OffsetsBuffer }; let offsets = - arrow2::offset::Offsets:: < i32 > ::try_from_lengths(time_columns - .iter().map(| opt | opt.as_ref().map_or(0, | datum | datum - .len()))) ? .into(); let time_columns_inner_data : Vec < _ > = - time_columns.into_iter().flatten().flatten().collect(); let - time_columns_inner_bitmap : Option < arrow2::bitmap::Bitmap > = - None; - ListArray::try_new(DataType::List(std::sync::Arc::new(Field::new("item", - < crate ::datatypes::Utf8 > ::arrow_datatype(), false))).into(), - offsets, { let offsets = arrow2::offset::Offsets:: < i32 > - ::try_from_lengths(time_columns_inner_data.iter().map(| datum | { - datum.0.len() })) ? .into(); let inner_data : - arrow2::buffer::Buffer < u8 > = time_columns_inner_data - .into_iter().flat_map(| datum | { datum.0.0 }).collect(); - - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] unsafe - { Utf8Array:: < i32 > ::new_unchecked(DataType::Utf8.into(), - offsets, inner_data, time_columns_inner_bitmap) } .boxed() }, - time_columns_bitmap,) ? .boxed() } }, { let (somes, - component_columns) : (Vec < _ >, Vec < _ >) = data.iter().map(| - datum | { let datum = datum.as_ref().map(| datum | { datum - .component_columns.clone() }); (datum.is_some(), datum) }) - .unzip(); let component_columns_bitmap : Option < - arrow2::bitmap::Bitmap > = { let any_nones = somes.iter().any(| - some | ! * some); any_nones.then(|| somes.into()) }; { use - arrow2:: { buffer::Buffer, offset::OffsetsBuffer }; let offsets = - arrow2::offset::Offsets:: < i32 > - ::try_from_lengths(component_columns.iter().map(| opt | opt - .as_ref().map_or(0, | datum | datum.len()))) ? .into(); let - component_columns_inner_data : Vec < _ > = component_columns - .into_iter().flatten().flatten().collect(); let - component_columns_inner_bitmap : Option < arrow2::bitmap::Bitmap - > = None; - ListArray::try_new(DataType::List(std::sync::Arc::new(Field::new("item", + as_array_ref(StructArray::new( + fields, + vec![ + { + let (somes, time_columns): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| datum.time_columns.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let time_columns_validity: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( + time_columns + .iter() + .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), + ); + let time_columns_inner_data: Vec<_> = + time_columns.into_iter().flatten().flatten().collect(); + let time_columns_inner_validity: Option = + None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), + offsets, + { + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( + time_columns_inner_data.iter().map(|datum| datum.0.len()), + ); + let inner_data: arrow::buffer::Buffer = time_columns_inner_data + .into_iter() + .flat_map(|datum| datum.0 .0) + .collect(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + as_array_ref(unsafe { + StringArray::new_unchecked( + offsets, + inner_data, + time_columns_inner_validity, + ) + }) + }, + time_columns_validity, + )?) + } + }, + { + let (somes, component_columns): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = + datum.as_ref().map(|datum| datum.component_columns.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let component_columns_validity: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( + component_columns + .iter() + .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), + ); + let component_columns_inner_data: Vec<_> = + component_columns.into_iter().flatten().flatten().collect(); + let component_columns_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new(std::sync::Arc::new(Field::new("item", < crate ::blueprint::datatypes::ComponentColumnSelector > - ::arrow_datatype(), false))).into(), offsets, { _ = - component_columns_inner_bitmap; crate - ::blueprint::datatypes::ComponentColumnSelector::to_arrow2_opt(component_columns_inner_data - .into_iter().map(Some)) ? }, component_columns_bitmap,) ? - .boxed() } }, - ], - bitmap, - ) - .boxed() + ::arrow_datatype(), false)), offsets, { _ = + component_columns_inner_validity; crate + ::blueprint::datatypes::ComponentColumnSelector::to_arrow_opt(component_columns_inner_data + .into_iter().map(Some)) ? }, component_columns_validity,) ?) + } + }, + ], + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs b/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs index 4c3c4c1e0d0b..0a8bf5693d07 100644 --- a/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs +++ b/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs @@ -65,18 +65,23 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new("dimension", DataType::UInt32, false)]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -84,12 +89,12 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, dimension): (Vec<_>, Vec<_>) = data .iter() @@ -98,23 +103,22 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { (datum.is_some(), datum) }) .unzip(); - let dimension_bitmap: Option = { + let dimension_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - dimension - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - dimension_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + dimension + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + dimension_validity, + )) }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/components/aggregation_policy.rs b/crates/store/re_types/src/components/aggregation_policy.rs index c76c4317ec92..17a4f8fc6b96 100644 --- a/crates/store/re_types/src/components/aggregation_policy.rs +++ b/crates/store/re_types/src/components/aggregation_policy.rs @@ -114,17 +114,21 @@ impl ::re_types_core::Loggable for AggregationPolicy { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -134,16 +138,19 @@ impl ::re_types_core::Loggable for AggregationPolicy { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/albedo_factor.rs b/crates/store/re_types/src/components/albedo_factor.rs index d5ebcc94b5e7..a11f0a6d9ee6 100644 --- a/crates/store/re_types/src/components/albedo_factor.rs +++ b/crates/store/re_types/src/components/albedo_factor.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for AlbedoFactor { crate::datatypes::Rgba32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Rgba32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Rgba32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/annotation_context.rs b/crates/store/re_types/src/components/annotation_context.rs index 04a17d807543..0d433c1ab4ac 100644 --- a/crates/store/re_types/src/components/annotation_context.rs +++ b/crates/store/re_types/src/components/annotation_context.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for AnnotationContext { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,32 +89,33 @@ impl ::re_types_core::Loggable for AnnotationContext { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::datatypes::ClassDescriptionMapElem::to_arrow2_opt( + _ = data0_inner_validity; + crate::datatypes::ClassDescriptionMapElem::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/axis_length.rs b/crates/store/re_types/src/components/axis_length.rs index 52c80dc16e84..0d81cb23a7d1 100644 --- a/crates/store/re_types/src/components/axis_length.rs +++ b/crates/store/re_types/src/components/axis_length.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for AxisLength { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/blob.rs b/crates/store/re_types/src/components/blob.rs index 561a6ac5529b..78dd2967aa50 100644 --- a/crates/store/re_types/src/components/blob.rs +++ b/crates/store/re_types/src/components/blob.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for Blob { crate::datatypes::Blob::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Blob::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Blob::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/class_id.rs b/crates/store/re_types/src/components/class_id.rs index 43d038b2be57..7df2edaa9226 100644 --- a/crates/store/re_types/src/components/class_id.rs +++ b/crates/store/re_types/src/components/class_id.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for ClassId { crate::datatypes::ClassId::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::ClassId::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::ClassId::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/color.rs b/crates/store/re_types/src/components/color.rs index de259fe82e2d..731c1ba07f04 100644 --- a/crates/store/re_types/src/components/color.rs +++ b/crates/store/re_types/src/components/color.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for Color { crate::datatypes::Rgba32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Rgba32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Rgba32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/colormap.rs b/crates/store/re_types/src/components/colormap.rs index 3f01fa78fd88..0ae6dd35456b 100644 --- a/crates/store/re_types/src/components/colormap.rs +++ b/crates/store/re_types/src/components/colormap.rs @@ -151,17 +151,21 @@ impl ::re_types_core::Loggable for Colormap { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -171,16 +175,19 @@ impl ::re_types_core::Loggable for Colormap { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/depth_meter.rs b/crates/store/re_types/src/components/depth_meter.rs index a9ffd0b01ead..6f9c37f3a046 100644 --- a/crates/store/re_types/src/components/depth_meter.rs +++ b/crates/store/re_types/src/components/depth_meter.rs @@ -79,13 +79,13 @@ impl ::re_types_core::Loggable for DepthMeter { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/disconnected_space.rs b/crates/store/re_types/src/components/disconnected_space.rs index 2d3f6068a756..ed96d1dd7406 100644 --- a/crates/store/re_types/src/components/disconnected_space.rs +++ b/crates/store/re_types/src/components/disconnected_space.rs @@ -83,13 +83,13 @@ impl ::re_types_core::Loggable for DisconnectedSpace { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/draw_order.rs b/crates/store/re_types/src/components/draw_order.rs index c7ab0a195754..a3ae26ecbc25 100644 --- a/crates/store/re_types/src/components/draw_order.rs +++ b/crates/store/re_types/src/components/draw_order.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for DrawOrder { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/entity_path.rs b/crates/store/re_types/src/components/entity_path.rs index d2894dd7813c..b9b6c23c4ada 100644 --- a/crates/store/re_types/src/components/entity_path.rs +++ b/crates/store/re_types/src/components/entity_path.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for EntityPath { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/fill_mode.rs b/crates/store/re_types/src/components/fill_mode.rs index 8931e51b40c2..f96e05e7ceb3 100644 --- a/crates/store/re_types/src/components/fill_mode.rs +++ b/crates/store/re_types/src/components/fill_mode.rs @@ -100,17 +100,21 @@ impl ::re_types_core::Loggable for FillMode { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -120,16 +124,19 @@ impl ::re_types_core::Loggable for FillMode { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/fill_ratio.rs b/crates/store/re_types/src/components/fill_ratio.rs index d8863dc39964..53f23b6af66c 100644 --- a/crates/store/re_types/src/components/fill_ratio.rs +++ b/crates/store/re_types/src/components/fill_ratio.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for FillRatio { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/gamma_correction.rs b/crates/store/re_types/src/components/gamma_correction.rs index 75c2cd3bedda..456903d5b6e4 100644 --- a/crates/store/re_types/src/components/gamma_correction.rs +++ b/crates/store/re_types/src/components/gamma_correction.rs @@ -78,13 +78,13 @@ impl ::re_types_core::Loggable for GammaCorrection { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/geo_line_string.rs b/crates/store/re_types/src/components/geo_line_string.rs index ca171c2adcdb..48580a0b01d4 100644 --- a/crates/store/re_types/src/components/geo_line_string.rs +++ b/crates/store/re_types/src/components/geo_line_string.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for GeoLineString { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,50 +79,47 @@ impl ::re_types_core::Loggable for GeoLineString { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data_inner_data: Vec<_> = data0_inner_data .into_iter() .map(|datum| datum.0) .flatten() .collect(); - let data0_inner_data_inner_bitmap: Option = None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float64, false)), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float64.into(), - data0_inner_data_inner_data.into_iter().collect(), - data0_inner_data_inner_bitmap, - ) - .boxed(), - data0_inner_bitmap, - ) - .boxed() + let data0_inner_data_inner_validity: Option = + None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0_inner_data_inner_data.into_iter().collect::>(), + ), + data0_inner_data_inner_validity, + )), + data0_inner_validity, + )) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/graph_edge.rs b/crates/store/re_types/src/components/graph_edge.rs index 64dc08db7e52..e85b7c37a5dd 100644 --- a/crates/store/re_types/src/components/graph_edge.rs +++ b/crates/store/re_types/src/components/graph_edge.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for GraphEdge { crate::datatypes::Utf8Pair::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8Pair::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8Pair::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/graph_node.rs b/crates/store/re_types/src/components/graph_node.rs index 5e42187423bc..daa820248817 100644 --- a/crates/store/re_types/src/components/graph_node.rs +++ b/crates/store/re_types/src/components/graph_node.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for GraphNode { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/graph_type.rs b/crates/store/re_types/src/components/graph_type.rs index 38d5dd830701..87979d997457 100644 --- a/crates/store/re_types/src/components/graph_type.rs +++ b/crates/store/re_types/src/components/graph_type.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for GraphType { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,16 +101,19 @@ impl ::re_types_core::Loggable for GraphType { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/half_size2d.rs b/crates/store/re_types/src/components/half_size2d.rs index 183c4a2cc54c..d6bf57e23794 100644 --- a/crates/store/re_types/src/components/half_size2d.rs +++ b/crates/store/re_types/src/components/half_size2d.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for HalfSize2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/half_size3d.rs b/crates/store/re_types/src/components/half_size3d.rs index fb4c6917e9de..d4ba44268531 100644 --- a/crates/store/re_types/src/components/half_size3d.rs +++ b/crates/store/re_types/src/components/half_size3d.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for HalfSize3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/image_buffer.rs b/crates/store/re_types/src/components/image_buffer.rs index e9a643de855d..79bafe36e5e3 100644 --- a/crates/store/re_types/src/components/image_buffer.rs +++ b/crates/store/re_types/src/components/image_buffer.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ImageBuffer { crate::datatypes::Blob::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Blob::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Blob::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/image_format.rs b/crates/store/re_types/src/components/image_format.rs index d816d249afe1..74afc7ba4df4 100644 --- a/crates/store/re_types/src/components/image_format.rs +++ b/crates/store/re_types/src/components/image_format.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for ImageFormat { crate::datatypes::ImageFormat::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::ImageFormat::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::ImageFormat::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/image_plane_distance.rs b/crates/store/re_types/src/components/image_plane_distance.rs index 5e297e9b198b..45935f9f1c45 100644 --- a/crates/store/re_types/src/components/image_plane_distance.rs +++ b/crates/store/re_types/src/components/image_plane_distance.rs @@ -73,13 +73,13 @@ impl ::re_types_core::Loggable for ImagePlaneDistance { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/keypoint_id.rs b/crates/store/re_types/src/components/keypoint_id.rs index bdb8aae1063b..ecc81d62c883 100644 --- a/crates/store/re_types/src/components/keypoint_id.rs +++ b/crates/store/re_types/src/components/keypoint_id.rs @@ -89,13 +89,13 @@ impl ::re_types_core::Loggable for KeypointId { crate::datatypes::KeypointId::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::KeypointId::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::KeypointId::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/lat_lon.rs b/crates/store/re_types/src/components/lat_lon.rs index 3350cfbaa9fc..a6e64499ab02 100644 --- a/crates/store/re_types/src/components/lat_lon.rs +++ b/crates/store/re_types/src/components/lat_lon.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for LatLon { crate::datatypes::DVec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::DVec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::DVec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/length.rs b/crates/store/re_types/src/components/length.rs index a576a8827fd9..26127dce80ae 100644 --- a/crates/store/re_types/src/components/length.rs +++ b/crates/store/re_types/src/components/length.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for Length { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/line_strip2d.rs b/crates/store/re_types/src/components/line_strip2d.rs index e4cc494c81e2..c4df3ea366d6 100644 --- a/crates/store/re_types/src/components/line_strip2d.rs +++ b/crates/store/re_types/src/components/line_strip2d.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for LineStrip2D { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,50 +89,47 @@ impl ::re_types_core::Loggable for LineStrip2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data_inner_data: Vec<_> = data0_inner_data .into_iter() .map(|datum| datum.0) .flatten() .collect(); - let data0_inner_data_inner_bitmap: Option = None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float32, false)), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data_inner_data.into_iter().collect(), - data0_inner_data_inner_bitmap, - ) - .boxed(), - data0_inner_bitmap, - ) - .boxed() + let data0_inner_data_inner_validity: Option = + None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0_inner_data_inner_data.into_iter().collect::>(), + ), + data0_inner_data_inner_validity, + )), + data0_inner_validity, + )) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/line_strip3d.rs b/crates/store/re_types/src/components/line_strip3d.rs index 241ab65e8130..0bbbc236b2b8 100644 --- a/crates/store/re_types/src/components/line_strip3d.rs +++ b/crates/store/re_types/src/components/line_strip3d.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for LineStrip3D { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,50 +89,47 @@ impl ::re_types_core::Loggable for LineStrip3D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data_inner_data: Vec<_> = data0_inner_data .into_iter() .map(|datum| datum.0) .flatten() .collect(); - let data0_inner_data_inner_bitmap: Option = None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float32, false)), - 3, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data_inner_data.into_iter().collect(), - data0_inner_data_inner_bitmap, - ) - .boxed(), - data0_inner_bitmap, - ) - .boxed() + let data0_inner_data_inner_validity: Option = + None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0_inner_data_inner_data.into_iter().collect::>(), + ), + data0_inner_data_inner_validity, + )), + data0_inner_validity, + )) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/magnification_filter.rs b/crates/store/re_types/src/components/magnification_filter.rs index b1bd2c038cc7..77faef980612 100644 --- a/crates/store/re_types/src/components/magnification_filter.rs +++ b/crates/store/re_types/src/components/magnification_filter.rs @@ -86,17 +86,21 @@ impl ::re_types_core::Loggable for MagnificationFilter { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -106,16 +110,19 @@ impl ::re_types_core::Loggable for MagnificationFilter { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/marker_shape.rs b/crates/store/re_types/src/components/marker_shape.rs index 388ee5ad5e94..20ef6f9610c9 100644 --- a/crates/store/re_types/src/components/marker_shape.rs +++ b/crates/store/re_types/src/components/marker_shape.rs @@ -128,17 +128,21 @@ impl ::re_types_core::Loggable for MarkerShape { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -148,16 +152,19 @@ impl ::re_types_core::Loggable for MarkerShape { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/marker_size.rs b/crates/store/re_types/src/components/marker_size.rs index e1927f83ddee..206022bf21f5 100644 --- a/crates/store/re_types/src/components/marker_size.rs +++ b/crates/store/re_types/src/components/marker_size.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for MarkerSize { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/media_type.rs b/crates/store/re_types/src/components/media_type.rs index cf29f6909bd4..2eacec30e60e 100644 --- a/crates/store/re_types/src/components/media_type.rs +++ b/crates/store/re_types/src/components/media_type.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for MediaType { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/name.rs b/crates/store/re_types/src/components/name.rs index 94e94354b0cf..06f9fc3bea2c 100644 --- a/crates/store/re_types/src/components/name.rs +++ b/crates/store/re_types/src/components/name.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Name { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/opacity.rs b/crates/store/re_types/src/components/opacity.rs index 146e6cf15654..a8c7048b63b9 100644 --- a/crates/store/re_types/src/components/opacity.rs +++ b/crates/store/re_types/src/components/opacity.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for Opacity { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pinhole_projection.rs b/crates/store/re_types/src/components/pinhole_projection.rs index 355e268fc373..847249910dde 100644 --- a/crates/store/re_types/src/components/pinhole_projection.rs +++ b/crates/store/re_types/src/components/pinhole_projection.rs @@ -81,13 +81,13 @@ impl ::re_types_core::Loggable for PinholeProjection { crate::datatypes::Mat3x3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Mat3x3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Mat3x3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_rotation_axis_angle.rs b/crates/store/re_types/src/components/pose_rotation_axis_angle.rs index d41ea1b0a22c..c36f231806c9 100644 --- a/crates/store/re_types/src/components/pose_rotation_axis_angle.rs +++ b/crates/store/re_types/src/components/pose_rotation_axis_angle.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for PoseRotationAxisAngle { crate::datatypes::RotationAxisAngle::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::RotationAxisAngle::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::RotationAxisAngle::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_rotation_quat.rs b/crates/store/re_types/src/components/pose_rotation_quat.rs index 05541e505641..fa2abc7d118d 100644 --- a/crates/store/re_types/src/components/pose_rotation_quat.rs +++ b/crates/store/re_types/src/components/pose_rotation_quat.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for PoseRotationQuat { crate::datatypes::Quaternion::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Quaternion::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Quaternion::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_scale3d.rs b/crates/store/re_types/src/components/pose_scale3d.rs index b4378b6c23b4..ec58d7e8c816 100644 --- a/crates/store/re_types/src/components/pose_scale3d.rs +++ b/crates/store/re_types/src/components/pose_scale3d.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for PoseScale3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_transform_mat3x3.rs b/crates/store/re_types/src/components/pose_transform_mat3x3.rs index 4ddcd78a1b71..83d0da756fe7 100644 --- a/crates/store/re_types/src/components/pose_transform_mat3x3.rs +++ b/crates/store/re_types/src/components/pose_transform_mat3x3.rs @@ -84,13 +84,13 @@ impl ::re_types_core::Loggable for PoseTransformMat3x3 { crate::datatypes::Mat3x3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Mat3x3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Mat3x3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_translation3d.rs b/crates/store/re_types/src/components/pose_translation3d.rs index a4d4e6424676..2fba1c14cfd3 100644 --- a/crates/store/re_types/src/components/pose_translation3d.rs +++ b/crates/store/re_types/src/components/pose_translation3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for PoseTranslation3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/position2d.rs b/crates/store/re_types/src/components/position2d.rs index 145668f91d1f..a4bd51d415cb 100644 --- a/crates/store/re_types/src/components/position2d.rs +++ b/crates/store/re_types/src/components/position2d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Position2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/position3d.rs b/crates/store/re_types/src/components/position3d.rs index dc03a9b250f6..959582b1e83a 100644 --- a/crates/store/re_types/src/components/position3d.rs +++ b/crates/store/re_types/src/components/position3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Position3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/radius.rs b/crates/store/re_types/src/components/radius.rs index 15bd75ee62c6..a835ec36faed 100644 --- a/crates/store/re_types/src/components/radius.rs +++ b/crates/store/re_types/src/components/radius.rs @@ -79,13 +79,13 @@ impl ::re_types_core::Loggable for Radius { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/range1d.rs b/crates/store/re_types/src/components/range1d.rs index 90404d0457cf..8126bd996ea4 100644 --- a/crates/store/re_types/src/components/range1d.rs +++ b/crates/store/re_types/src/components/range1d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Range1D { crate::datatypes::Range1D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Range1D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Range1D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/resolution.rs b/crates/store/re_types/src/components/resolution.rs index a5395a62b4e7..391bf37f9a0c 100644 --- a/crates/store/re_types/src/components/resolution.rs +++ b/crates/store/re_types/src/components/resolution.rs @@ -73,13 +73,13 @@ impl ::re_types_core::Loggable for Resolution { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/rotation_axis_angle.rs b/crates/store/re_types/src/components/rotation_axis_angle.rs index be681335c48b..494c8c726f0a 100644 --- a/crates/store/re_types/src/components/rotation_axis_angle.rs +++ b/crates/store/re_types/src/components/rotation_axis_angle.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for RotationAxisAngle { crate::datatypes::RotationAxisAngle::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::RotationAxisAngle::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::RotationAxisAngle::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/rotation_quat.rs b/crates/store/re_types/src/components/rotation_quat.rs index ff8ded9db655..796fb844cb59 100644 --- a/crates/store/re_types/src/components/rotation_quat.rs +++ b/crates/store/re_types/src/components/rotation_quat.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for RotationQuat { crate::datatypes::Quaternion::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Quaternion::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Quaternion::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/scalar.rs b/crates/store/re_types/src/components/scalar.rs index 9521649b7603..1058e369e2d6 100644 --- a/crates/store/re_types/src/components/scalar.rs +++ b/crates/store/re_types/src/components/scalar.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for Scalar { crate::datatypes::Float64::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float64::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float64::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/scale3d.rs b/crates/store/re_types/src/components/scale3d.rs index 33beb6bd70c9..3ff9af13e107 100644 --- a/crates/store/re_types/src/components/scale3d.rs +++ b/crates/store/re_types/src/components/scale3d.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for Scale3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/show_labels.rs b/crates/store/re_types/src/components/show_labels.rs index 9c72df4a18bb..b20c59d4b517 100644 --- a/crates/store/re_types/src/components/show_labels.rs +++ b/crates/store/re_types/src/components/show_labels.rs @@ -78,13 +78,13 @@ impl ::re_types_core::Loggable for ShowLabels { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/stroke_width.rs b/crates/store/re_types/src/components/stroke_width.rs index ae14c912af90..5a6f522cc994 100644 --- a/crates/store/re_types/src/components/stroke_width.rs +++ b/crates/store/re_types/src/components/stroke_width.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for StrokeWidth { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/tensor_data.rs b/crates/store/re_types/src/components/tensor_data.rs index 5bc6b96067ea..2faca36bb69e 100644 --- a/crates/store/re_types/src/components/tensor_data.rs +++ b/crates/store/re_types/src/components/tensor_data.rs @@ -79,13 +79,13 @@ impl ::re_types_core::Loggable for TensorData { crate::datatypes::TensorData::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorData::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::TensorData::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/tensor_dimension_index_selection.rs b/crates/store/re_types/src/components/tensor_dimension_index_selection.rs index 405b26b96193..01ede1b23e6c 100644 --- a/crates/store/re_types/src/components/tensor_dimension_index_selection.rs +++ b/crates/store/re_types/src/components/tensor_dimension_index_selection.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { crate::datatypes::TensorDimensionIndexSelection::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorDimensionIndexSelection::to_arrow2_opt(data.into_iter().map( + crate::datatypes::TensorDimensionIndexSelection::to_arrow_opt(data.into_iter().map( |datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), diff --git a/crates/store/re_types/src/components/tensor_height_dimension.rs b/crates/store/re_types/src/components/tensor_height_dimension.rs index 26f053539042..119c5933267f 100644 --- a/crates/store/re_types/src/components/tensor_height_dimension.rs +++ b/crates/store/re_types/src/components/tensor_height_dimension.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TensorHeightDimension { crate::datatypes::TensorDimensionSelection::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorDimensionSelection::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::TensorDimensionSelection::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/tensor_width_dimension.rs b/crates/store/re_types/src/components/tensor_width_dimension.rs index 6877c4b1474d..fc7da7124b9b 100644 --- a/crates/store/re_types/src/components/tensor_width_dimension.rs +++ b/crates/store/re_types/src/components/tensor_width_dimension.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TensorWidthDimension { crate::datatypes::TensorDimensionSelection::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorDimensionSelection::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::TensorDimensionSelection::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/texcoord2d.rs b/crates/store/re_types/src/components/texcoord2d.rs index c9c5a724232f..ff9ea66ed5fe 100644 --- a/crates/store/re_types/src/components/texcoord2d.rs +++ b/crates/store/re_types/src/components/texcoord2d.rs @@ -87,13 +87,13 @@ impl ::re_types_core::Loggable for Texcoord2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/text.rs b/crates/store/re_types/src/components/text.rs index 5d5afa486884..14829ca69e11 100644 --- a/crates/store/re_types/src/components/text.rs +++ b/crates/store/re_types/src/components/text.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Text { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/text_log_level.rs b/crates/store/re_types/src/components/text_log_level.rs index a64239622976..8cb71910bb95 100644 --- a/crates/store/re_types/src/components/text_log_level.rs +++ b/crates/store/re_types/src/components/text_log_level.rs @@ -80,13 +80,13 @@ impl ::re_types_core::Loggable for TextLogLevel { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/transform_mat3x3.rs b/crates/store/re_types/src/components/transform_mat3x3.rs index 8b9f349d1f2b..c60364a8cb21 100644 --- a/crates/store/re_types/src/components/transform_mat3x3.rs +++ b/crates/store/re_types/src/components/transform_mat3x3.rs @@ -84,13 +84,13 @@ impl ::re_types_core::Loggable for TransformMat3x3 { crate::datatypes::Mat3x3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Mat3x3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Mat3x3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/transform_relation.rs b/crates/store/re_types/src/components/transform_relation.rs index 0acb7a41088b..16f317bf4c5b 100644 --- a/crates/store/re_types/src/components/transform_relation.rs +++ b/crates/store/re_types/src/components/transform_relation.rs @@ -89,17 +89,21 @@ impl ::re_types_core::Loggable for TransformRelation { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -109,16 +113,19 @@ impl ::re_types_core::Loggable for TransformRelation { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/translation3d.rs b/crates/store/re_types/src/components/translation3d.rs index f3b0ea611f9c..8c7bbe48e03a 100644 --- a/crates/store/re_types/src/components/translation3d.rs +++ b/crates/store/re_types/src/components/translation3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Translation3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/triangle_indices.rs b/crates/store/re_types/src/components/triangle_indices.rs index 7253c0499b19..f22283f0c1a7 100644 --- a/crates/store/re_types/src/components/triangle_indices.rs +++ b/crates/store/re_types/src/components/triangle_indices.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TriangleIndices { crate::datatypes::UVec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::UVec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::UVec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/value_range.rs b/crates/store/re_types/src/components/value_range.rs index 4fdaa772d331..d4dc347e9e86 100644 --- a/crates/store/re_types/src/components/value_range.rs +++ b/crates/store/re_types/src/components/value_range.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for ValueRange { crate::datatypes::Range1D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Range1D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Range1D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/vector2d.rs b/crates/store/re_types/src/components/vector2d.rs index 8500bd3aabc8..6769f40a04ac 100644 --- a/crates/store/re_types/src/components/vector2d.rs +++ b/crates/store/re_types/src/components/vector2d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Vector2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/vector3d.rs b/crates/store/re_types/src/components/vector3d.rs index 795985b9fb80..42ca53ba7c37 100644 --- a/crates/store/re_types/src/components/vector3d.rs +++ b/crates/store/re_types/src/components/vector3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Vector3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/video_timestamp.rs b/crates/store/re_types/src/components/video_timestamp.rs index b9a75cb1023e..96b6d1cc4d2a 100644 --- a/crates/store/re_types/src/components/video_timestamp.rs +++ b/crates/store/re_types/src/components/video_timestamp.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for VideoTimestamp { crate::datatypes::VideoTimestamp::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::VideoTimestamp::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::VideoTimestamp::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/view_coordinates.rs b/crates/store/re_types/src/components/view_coordinates.rs index ae377dfa1fa1..4160466dc856 100644 --- a/crates/store/re_types/src/components/view_coordinates.rs +++ b/crates/store/re_types/src/components/view_coordinates.rs @@ -92,13 +92,13 @@ impl ::re_types_core::Loggable for ViewCoordinates { crate::datatypes::ViewCoordinates::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::ViewCoordinates::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::ViewCoordinates::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/datatypes/angle.rs b/crates/store/re_types/src/datatypes/angle.rs index e00e35c0a074..a75e60af9a22 100644 --- a/crates/store/re_types/src/datatypes/angle.rs +++ b/crates/store/re_types/src/datatypes/angle.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Angle { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, radians): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,16 +86,19 @@ impl ::re_types_core::Loggable for Angle { (datum.is_some(), datum) }) .unzip(); - let radians_bitmap: Option = { + let radians_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - radians.into_iter().map(|v| v.unwrap_or_default()).collect(), - radians_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + radians + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + radians_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/annotation_info.rs b/crates/store/re_types/src/datatypes/annotation_info.rs index 3fbe50c49993..110da98b4dc1 100644 --- a/crates/store/re_types/src/datatypes/annotation_info.rs +++ b/crates/store/re_types/src/datatypes/annotation_info.rs @@ -62,18 +62,27 @@ impl ::re_types_core::Loggable for AnnotationInfo { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("id", DataType::UInt16, false), + Field::new("label", ::arrow_datatype(), true), + Field::new("color", ::arrow_datatype(), true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -81,12 +90,12 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, id): (Vec<_>, Vec<_>) = data @@ -96,16 +105,18 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let id_bitmap: Option = { + let id_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - id.into_iter().map(|v| v.unwrap_or_default()).collect(), - id_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + id.into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + id_validity, + )) }, { let (somes, label): (Vec<_>, Vec<_>) = data @@ -116,33 +127,25 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let label_bitmap: Option = { + let label_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( label.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = label + ); + let inner_data: arrow::buffer::Buffer = label .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - label_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, label_validity) + }) } }, { @@ -154,24 +157,23 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let color_bitmap: Option = { + let color_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - color - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - color_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + color + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + color_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/blob.rs b/crates/store/re_types/src/datatypes/blob.rs index 6501ddfe5194..b102046ddfcf 100644 --- a/crates/store/re_types/src/datatypes/blob.rs +++ b/crates/store/re_types/src/datatypes/blob.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for Blob { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,38 +89,33 @@ impl ::re_types_core::Loggable for Blob { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.num_instances())), - )? - .into(); - let data0_inner_data: Buffer<_> = data0 + ); + let data0_inner_data: ScalarBuffer<_> = data0 .iter() .flatten() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), offsets, - PrimitiveArray::new( - DataType::UInt8.into(), + as_array_ref(PrimitiveArray::::new( data0_inner_data, - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - )? - .boxed() + data0_inner_validity, + )), + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/datatypes/channel_datatype.rs b/crates/store/re_types/src/datatypes/channel_datatype.rs index 204d22bf0c2b..c3f52fe3a00e 100644 --- a/crates/store/re_types/src/datatypes/channel_datatype.rs +++ b/crates/store/re_types/src/datatypes/channel_datatype.rs @@ -136,17 +136,21 @@ impl ::re_types_core::Loggable for ChannelDatatype { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -156,16 +160,19 @@ impl ::re_types_core::Loggable for ChannelDatatype { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/class_description.rs b/crates/store/re_types/src/datatypes/class_description.rs index dfbdf480eccd..0bc823d1b5b9 100644 --- a/crates/store/re_types/src/datatypes/class_description.rs +++ b/crates/store/re_types/src/datatypes/class_description.rs @@ -94,18 +94,47 @@ impl ::re_types_core::Loggable for ClassDescription { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "info", + ::arrow_datatype(), + false, + ), + Field::new( + "keypoint_annotations", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "keypoint_connections", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -113,12 +142,12 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, info): (Vec<_>, Vec<_>) = data @@ -128,13 +157,13 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let info_bitmap: Option = { + let info_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = info_bitmap; - crate::datatypes::AnnotationInfo::to_arrow2_opt(info)? + _ = info_validity; + crate::datatypes::AnnotationInfo::to_arrow_opt(info)? } }, { @@ -147,42 +176,39 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let keypoint_annotations_bitmap: Option = { + let keypoint_annotations_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( keypoint_annotations .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let keypoint_annotations_inner_data: Vec<_> = keypoint_annotations .into_iter() .flatten() .flatten() .collect(); - let keypoint_annotations_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let keypoint_annotations_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = keypoint_annotations_inner_bitmap; - crate::datatypes::AnnotationInfo::to_arrow2_opt( + _ = keypoint_annotations_inner_validity; + crate::datatypes::AnnotationInfo::to_arrow_opt( keypoint_annotations_inner_data.into_iter().map(Some), )? }, - keypoint_annotations_bitmap, - )? - .boxed() + keypoint_annotations_validity, + )?) } }, { @@ -195,48 +221,44 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let keypoint_connections_bitmap: Option = { + let keypoint_connections_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( keypoint_connections .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let keypoint_connections_inner_data: Vec<_> = keypoint_connections .into_iter() .flatten() .flatten() .collect(); - let keypoint_connections_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let keypoint_connections_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = keypoint_connections_inner_bitmap; - crate::datatypes::KeypointPair::to_arrow2_opt( + _ = keypoint_connections_inner_validity; + crate::datatypes::KeypointPair::to_arrow_opt( keypoint_connections_inner_data.into_iter().map(Some), )? }, - keypoint_connections_bitmap, - )? - .boxed() + keypoint_connections_validity, + )?) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/class_description_map_elem.rs b/crates/store/re_types/src/datatypes/class_description_map_elem.rs index 6edbc87efc3d..fabdbe23187f 100644 --- a/crates/store/re_types/src/datatypes/class_description_map_elem.rs +++ b/crates/store/re_types/src/datatypes/class_description_map_elem.rs @@ -63,18 +63,34 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "class_id", + ::arrow_datatype(), + false, + ), + Field::new( + "class_description", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -82,12 +98,12 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, class_id): (Vec<_>, Vec<_>) = data @@ -97,19 +113,19 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { (datum.is_some(), datum) }) .unzip(); - let class_id_bitmap: Option = { + let class_id_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - class_id - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - class_id_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + class_id + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + class_id_validity, + )) }, { let (somes, class_description): (Vec<_>, Vec<_>) = data @@ -120,19 +136,18 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { (datum.is_some(), datum) }) .unzip(); - let class_description_bitmap: Option = { + let class_description_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = class_description_bitmap; - crate::datatypes::ClassDescription::to_arrow2_opt(class_description)? + _ = class_description_validity; + crate::datatypes::ClassDescription::to_arrow_opt(class_description)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/class_id.rs b/crates/store/re_types/src/datatypes/class_id.rs index cfaec84e2b23..f96ec8c23076 100644 --- a/crates/store/re_types/src/datatypes/class_id.rs +++ b/crates/store/re_types/src/datatypes/class_id.rs @@ -74,17 +74,21 @@ impl ::re_types_core::Loggable for ClassId { DataType::UInt16 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -94,16 +98,19 @@ impl ::re_types_core::Loggable for ClassId { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/color_model.rs b/crates/store/re_types/src/datatypes/color_model.rs index 17595e694f15..bbb4ab094b6c 100644 --- a/crates/store/re_types/src/datatypes/color_model.rs +++ b/crates/store/re_types/src/datatypes/color_model.rs @@ -98,17 +98,21 @@ impl ::re_types_core::Loggable for ColorModel { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -118,16 +122,19 @@ impl ::re_types_core::Loggable for ColorModel { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/dvec2d.rs b/crates/store/re_types/src/datatypes/dvec2d.rs index 1001799afe77..98af2bc51c97 100644 --- a/crates/store/re_types/src/datatypes/dvec2d.rs +++ b/crates/store/re_types/src/datatypes/dvec2d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for DVec2D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for DVec2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for DVec2D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float64.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/image_format.rs b/crates/store/re_types/src/datatypes/image_format.rs index f28ef43b48b7..4980d67bde5b 100644 --- a/crates/store/re_types/src/datatypes/image_format.rs +++ b/crates/store/re_types/src/datatypes/image_format.rs @@ -91,18 +91,41 @@ impl ::re_types_core::Loggable for ImageFormat { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("width", DataType::UInt32, false), + Field::new("height", DataType::UInt32, false), + Field::new( + "pixel_format", + ::arrow_datatype(), + true, + ), + Field::new( + "color_model", + ::arrow_datatype(), + true, + ), + Field::new( + "channel_datatype", + ::arrow_datatype(), + true, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -110,12 +133,12 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, width): (Vec<_>, Vec<_>) = data @@ -125,16 +148,19 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let width_bitmap: Option = { + let width_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - width.into_iter().map(|v| v.unwrap_or_default()).collect(), - width_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + width + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + width_validity, + )) }, { let (somes, height): (Vec<_>, Vec<_>) = data @@ -144,16 +170,19 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let height_bitmap: Option = { + let height_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - height.into_iter().map(|v| v.unwrap_or_default()).collect(), - height_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + height + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + height_validity, + )) }, { let (somes, pixel_format): (Vec<_>, Vec<_>) = data @@ -166,13 +195,13 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let pixel_format_bitmap: Option = { + let pixel_format_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = pixel_format_bitmap; - crate::datatypes::PixelFormat::to_arrow2_opt(pixel_format)? + _ = pixel_format_validity; + crate::datatypes::PixelFormat::to_arrow_opt(pixel_format)? } }, { @@ -186,13 +215,13 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let color_model_bitmap: Option = { + let color_model_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = color_model_bitmap; - crate::datatypes::ColorModel::to_arrow2_opt(color_model)? + _ = color_model_validity; + crate::datatypes::ColorModel::to_arrow_opt(color_model)? } }, { @@ -206,19 +235,18 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let channel_datatype_bitmap: Option = { + let channel_datatype_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = channel_datatype_bitmap; - crate::datatypes::ChannelDatatype::to_arrow2_opt(channel_datatype)? + _ = channel_datatype_validity; + crate::datatypes::ChannelDatatype::to_arrow_opt(channel_datatype)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/keypoint_id.rs b/crates/store/re_types/src/datatypes/keypoint_id.rs index 363f740ceb49..d01fe3439c7e 100644 --- a/crates/store/re_types/src/datatypes/keypoint_id.rs +++ b/crates/store/re_types/src/datatypes/keypoint_id.rs @@ -76,17 +76,21 @@ impl ::re_types_core::Loggable for KeypointId { DataType::UInt16 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -96,16 +100,19 @@ impl ::re_types_core::Loggable for KeypointId { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/keypoint_pair.rs b/crates/store/re_types/src/datatypes/keypoint_pair.rs index 0eb42c747454..ed172c617203 100644 --- a/crates/store/re_types/src/datatypes/keypoint_pair.rs +++ b/crates/store/re_types/src/datatypes/keypoint_pair.rs @@ -61,18 +61,34 @@ impl ::re_types_core::Loggable for KeypointPair { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "keypoint0", + ::arrow_datatype(), + false, + ), + Field::new( + "keypoint1", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +96,12 @@ impl ::re_types_core::Loggable for KeypointPair { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, keypoint0): (Vec<_>, Vec<_>) = data @@ -95,19 +111,19 @@ impl ::re_types_core::Loggable for KeypointPair { (datum.is_some(), datum) }) .unzip(); - let keypoint0_bitmap: Option = { + let keypoint0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - keypoint0 - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - keypoint0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + keypoint0 + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + keypoint0_validity, + )) }, { let (somes, keypoint1): (Vec<_>, Vec<_>) = data @@ -117,24 +133,23 @@ impl ::re_types_core::Loggable for KeypointPair { (datum.is_some(), datum) }) .unzip(); - let keypoint1_bitmap: Option = { + let keypoint1_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - keypoint1 - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - keypoint1_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + keypoint1 + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + keypoint1_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/mat3x3.rs b/crates/store/re_types/src/datatypes/mat3x3.rs index 303f14de9774..9a5c33b32af3 100644 --- a/crates/store/re_types/src/datatypes/mat3x3.rs +++ b/crates/store/re_types/src/datatypes/mat3x3.rs @@ -74,17 +74,21 @@ impl ::re_types_core::Loggable for Mat3x3 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -94,12 +98,11 @@ impl ::re_types_core::Loggable for Mat3x3 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -109,26 +112,24 @@ impl ::re_types_core::Loggable for Mat3x3 { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(9usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 9, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/mat4x4.rs b/crates/store/re_types/src/datatypes/mat4x4.rs index 4a9b9081fae3..e877dbc87c1d 100644 --- a/crates/store/re_types/src/datatypes/mat4x4.rs +++ b/crates/store/re_types/src/datatypes/mat4x4.rs @@ -74,17 +74,21 @@ impl ::re_types_core::Loggable for Mat4x4 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -94,12 +98,11 @@ impl ::re_types_core::Loggable for Mat4x4 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -109,26 +112,24 @@ impl ::re_types_core::Loggable for Mat4x4 { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(16usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 16, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/pixel_format.rs b/crates/store/re_types/src/datatypes/pixel_format.rs index 13c80f4c84f3..85b2aa02dc17 100644 --- a/crates/store/re_types/src/datatypes/pixel_format.rs +++ b/crates/store/re_types/src/datatypes/pixel_format.rs @@ -218,17 +218,21 @@ impl ::re_types_core::Loggable for PixelFormat { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -238,16 +242,19 @@ impl ::re_types_core::Loggable for PixelFormat { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/quaternion.rs b/crates/store/re_types/src/datatypes/quaternion.rs index b45e589dc61c..26a69d7baeca 100644 --- a/crates/store/re_types/src/datatypes/quaternion.rs +++ b/crates/store/re_types/src/datatypes/quaternion.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for Quaternion { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,12 +89,11 @@ impl ::re_types_core::Loggable for Quaternion { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -100,26 +103,24 @@ impl ::re_types_core::Loggable for Quaternion { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/range1d.rs b/crates/store/re_types/src/datatypes/range1d.rs index 24fc4122531c..e003c5ab2f04 100644 --- a/crates/store/re_types/src/datatypes/range1d.rs +++ b/crates/store/re_types/src/datatypes/range1d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Range1D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Range1D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Range1D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float64.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/range2d.rs b/crates/store/re_types/src/datatypes/range2d.rs index d38ca45ee68f..201b3406d280 100644 --- a/crates/store/re_types/src/datatypes/range2d.rs +++ b/crates/store/re_types/src/datatypes/range2d.rs @@ -62,18 +62,34 @@ impl ::re_types_core::Loggable for Range2D { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "x_range", + ::arrow_datatype(), + false, + ), + Field::new( + "y_range", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -81,12 +97,12 @@ impl ::re_types_core::Loggable for Range2D { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, x_range): (Vec<_>, Vec<_>) = data @@ -96,45 +112,36 @@ impl ::re_types_core::Loggable for Range2D { (datum.is_some(), datum) }) .unzip(); - let x_range_bitmap: Option = { + let x_range_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let x_range_inner_data: Vec<_> = x_range .into_iter() .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) .flatten() .collect(); - let x_range_inner_bitmap: Option = - x_range_bitmap.as_ref().map(|bitmap| { - bitmap + let x_range_inner_validity: Option = + x_range_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new( - "item", - DataType::Float64, - false, - )), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float64.into(), - x_range_inner_data.into_iter().collect(), - x_range_inner_bitmap, - ) - .boxed(), - x_range_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + x_range_inner_data.into_iter().collect::>(), + ), + x_range_inner_validity, + )), + x_range_validity, + )) } }, { @@ -145,51 +152,41 @@ impl ::re_types_core::Loggable for Range2D { (datum.is_some(), datum) }) .unzip(); - let y_range_bitmap: Option = { + let y_range_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let y_range_inner_data: Vec<_> = y_range .into_iter() .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) .flatten() .collect(); - let y_range_inner_bitmap: Option = - y_range_bitmap.as_ref().map(|bitmap| { - bitmap + let y_range_inner_validity: Option = + y_range_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new( - "item", - DataType::Float64, - false, - )), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float64.into(), - y_range_inner_data.into_iter().collect(), - y_range_inner_bitmap, - ) - .boxed(), - y_range_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + y_range_inner_data.into_iter().collect::>(), + ), + y_range_inner_validity, + )), + y_range_validity, + )) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/rgba32.rs b/crates/store/re_types/src/datatypes/rgba32.rs index 3d6862855ddb..af704b5aa2f1 100644 --- a/crates/store/re_types/src/datatypes/rgba32.rs +++ b/crates/store/re_types/src/datatypes/rgba32.rs @@ -64,17 +64,21 @@ impl ::re_types_core::Loggable for Rgba32 { DataType::UInt32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -84,16 +88,19 @@ impl ::re_types_core::Loggable for Rgba32 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/rotation_axis_angle.rs b/crates/store/re_types/src/datatypes/rotation_axis_angle.rs index 9b74207cd837..403af5cb9dce 100644 --- a/crates/store/re_types/src/datatypes/rotation_axis_angle.rs +++ b/crates/store/re_types/src/datatypes/rotation_axis_angle.rs @@ -57,18 +57,26 @@ impl ::re_types_core::Loggable for RotationAxisAngle { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("axis", ::arrow_datatype(), false), + Field::new("angle", ::arrow_datatype(), false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -76,12 +84,12 @@ impl ::re_types_core::Loggable for RotationAxisAngle { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, axis): (Vec<_>, Vec<_>) = data @@ -91,45 +99,36 @@ impl ::re_types_core::Loggable for RotationAxisAngle { (datum.is_some(), datum) }) .unzip(); - let axis_bitmap: Option = { + let axis_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let axis_inner_data: Vec<_> = axis .into_iter() .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) .flatten() .collect(); - let axis_inner_bitmap: Option = - axis_bitmap.as_ref().map(|bitmap| { - bitmap + let axis_inner_validity: Option = + axis_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new( - "item", - DataType::Float32, - false, - )), - 3, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - axis_inner_data.into_iter().collect(), - axis_inner_bitmap, - ) - .boxed(), - axis_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + axis_inner_data.into_iter().collect::>(), + ), + axis_inner_validity, + )), + axis_validity, + )) } }, { @@ -140,24 +139,25 @@ impl ::re_types_core::Loggable for RotationAxisAngle { (datum.is_some(), datum) }) .unzip(); - let angle_bitmap: Option = { + let angle_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - angle - .into_iter() - .map(|datum| datum.map(|datum| datum.radians).unwrap_or_default()) - .collect(), - angle_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + angle + .into_iter() + .map(|datum| { + datum.map(|datum| datum.radians).unwrap_or_default() + }) + .collect::>(), + ), + angle_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_buffer.rs b/crates/store/re_types/src/datatypes/tensor_buffer.rs index 7e5055659005..d95b11b68421 100644 --- a/crates/store/re_types/src/datatypes/tensor_buffer.rs +++ b/crates/store/re_types/src/datatypes/tensor_buffer.rs @@ -209,17 +209,21 @@ impl ::re_types_core::Loggable for TensorBuffer { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -229,7 +233,110 @@ impl ::re_types_core::Loggable for TensorBuffer { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "U8", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt8, + false, + ))), + false, + ), + Field::new( + "U16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt16, + false, + ))), + false, + ), + Field::new( + "U32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt32, + false, + ))), + false, + ), + Field::new( + "U64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt64, + false, + ))), + false, + ), + Field::new( + "I8", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int8, + false, + ))), + false, + ), + Field::new( + "I16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int16, + false, + ))), + false, + ), + Field::new( + "I32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + false, + ))), + false, + ), + Field::new( + "I64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int64, + false, + ))), + false, + ), + Field::new( + "F16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float16, + false, + ))), + false, + ), + Field::new( + "F32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float32, + false, + ))), + false, + ), + Field::new( + "F64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float64, + false, + ))), + false, + ), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -246,12 +353,86 @@ impl ::re_types_core::Loggable for TensorBuffer { Some(Self::F64(_)) => 11i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), + let offsets = { + let mut u8_offset = 0; + let mut u16_offset = 0; + let mut u32_offset = 0; + let mut u64_offset = 0; + let mut i8_offset = 0; + let mut i16_offset = 0; + let mut i32_offset = 0; + let mut i64_offset = 0; + let mut f16_offset = 0; + let mut f32_offset = 0; + let mut f64_offset = 0; + let mut nulls_offset = 0; + data.iter() + .map(|v| match v.as_deref() { + None => { + let offset = nulls_offset; + nulls_offset += 1; + offset + } + Some(Self::U8(_)) => { + let offset = u8_offset; + u8_offset += 1; + offset + } + Some(Self::U16(_)) => { + let offset = u16_offset; + u16_offset += 1; + offset + } + Some(Self::U32(_)) => { + let offset = u32_offset; + u32_offset += 1; + offset + } + Some(Self::U64(_)) => { + let offset = u64_offset; + u64_offset += 1; + offset + } + Some(Self::I8(_)) => { + let offset = i8_offset; + i8_offset += 1; + offset + } + Some(Self::I16(_)) => { + let offset = i16_offset; + i16_offset += 1; + offset + } + Some(Self::I32(_)) => { + let offset = i32_offset; + i32_offset += 1; + offset + } + Some(Self::I64(_)) => { + let offset = i64_offset; + i64_offset += 1; + offset + } + Some(Self::F16(_)) => { + let offset = f16_offset; + f16_offset += 1; + offset + } + Some(Self::F32(_)) => { + let offset = f32_offset; + f32_offset += 1; + offset + } + Some(Self::F64(_)) => { + let offset = f64_offset; + f64_offset += 1; + offset + } + }) + .collect() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), { let u8: Vec<_> = data .iter() @@ -260,37 +441,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u8_bitmap: Option = None; + let u8_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u8.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u8_inner_data: Buffer<_> = u8 + ); + let u8_inner_data: ScalarBuffer<_> = u8 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u8_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt8, - false, - ))) - .into(), + let u8_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), offsets, - PrimitiveArray::new( - DataType::UInt8.into(), + as_array_ref(PrimitiveArray::::new( u8_inner_data, - u8_inner_bitmap, - ) - .boxed(), - u8_bitmap, - )? - .boxed() + u8_inner_validity, + )), + u8_validity, + )?) } }, { @@ -301,37 +472,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u16_bitmap: Option = None; + let u16_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u16.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u16_inner_data: Buffer<_> = u16 + ); + let u16_inner_data: ScalarBuffer<_> = u16 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u16_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt16, - false, - ))) - .into(), + let u16_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt16, false)), offsets, - PrimitiveArray::new( - DataType::UInt16.into(), + as_array_ref(PrimitiveArray::::new( u16_inner_data, - u16_inner_bitmap, - ) - .boxed(), - u16_bitmap, - )? - .boxed() + u16_inner_validity, + )), + u16_validity, + )?) } }, { @@ -342,37 +503,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u32_bitmap: Option = None; + let u32_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u32.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u32_inner_data: Buffer<_> = u32 + ); + let u32_inner_data: ScalarBuffer<_> = u32 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u32_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt32, - false, - ))) - .into(), + let u32_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), offsets, - PrimitiveArray::new( - DataType::UInt32.into(), + as_array_ref(PrimitiveArray::::new( u32_inner_data, - u32_inner_bitmap, - ) - .boxed(), - u32_bitmap, - )? - .boxed() + u32_inner_validity, + )), + u32_validity, + )?) } }, { @@ -383,37 +534,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u64_bitmap: Option = None; + let u64_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u64.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u64_inner_data: Buffer<_> = u64 + ); + let u64_inner_data: ScalarBuffer<_> = u64 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u64_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt64, - false, - ))) - .into(), + let u64_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt64, false)), offsets, - PrimitiveArray::new( - DataType::UInt64.into(), + as_array_ref(PrimitiveArray::::new( u64_inner_data, - u64_inner_bitmap, - ) - .boxed(), - u64_bitmap, - )? - .boxed() + u64_inner_validity, + )), + u64_validity, + )?) } }, { @@ -424,37 +565,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i8_bitmap: Option = None; + let i8_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i8.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i8_inner_data: Buffer<_> = i8 + ); + let i8_inner_data: ScalarBuffer<_> = i8 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i8_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int8, - false, - ))) - .into(), + let i8_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int8, false)), offsets, - PrimitiveArray::new( - DataType::Int8.into(), + as_array_ref(PrimitiveArray::::new( i8_inner_data, - i8_inner_bitmap, - ) - .boxed(), - i8_bitmap, - )? - .boxed() + i8_inner_validity, + )), + i8_validity, + )?) } }, { @@ -465,37 +596,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i16_bitmap: Option = None; + let i16_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i16.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i16_inner_data: Buffer<_> = i16 + ); + let i16_inner_data: ScalarBuffer<_> = i16 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i16_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int16, - false, - ))) - .into(), + let i16_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int16, false)), offsets, - PrimitiveArray::new( - DataType::Int16.into(), + as_array_ref(PrimitiveArray::::new( i16_inner_data, - i16_inner_bitmap, - ) - .boxed(), - i16_bitmap, - )? - .boxed() + i16_inner_validity, + )), + i16_validity, + )?) } }, { @@ -506,37 +627,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i32_bitmap: Option = None; + let i32_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i32.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i32_inner_data: Buffer<_> = i32 + ); + let i32_inner_data: ScalarBuffer<_> = i32 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i32_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int32, - false, - ))) - .into(), + let i32_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int32, false)), offsets, - PrimitiveArray::new( - DataType::Int32.into(), + as_array_ref(PrimitiveArray::::new( i32_inner_data, - i32_inner_bitmap, - ) - .boxed(), - i32_bitmap, - )? - .boxed() + i32_inner_validity, + )), + i32_validity, + )?) } }, { @@ -547,37 +658,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i64_bitmap: Option = None; + let i64_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i64.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i64_inner_data: Buffer<_> = i64 + ); + let i64_inner_data: ScalarBuffer<_> = i64 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i64_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int64, - false, - ))) - .into(), + let i64_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int64, false)), offsets, - PrimitiveArray::new( - DataType::Int64.into(), + as_array_ref(PrimitiveArray::::new( i64_inner_data, - i64_inner_bitmap, - ) - .boxed(), - i64_bitmap, - )? - .boxed() + i64_inner_validity, + )), + i64_validity, + )?) } }, { @@ -588,37 +689,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let f16_bitmap: Option = None; + let f16_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( f16.iter().map(|datum| datum.num_instances()), - )? - .into(); - let f16_inner_data: Buffer<_> = f16 + ); + let f16_inner_data: ScalarBuffer<_> = f16 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let f16_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float16, - false, - ))) - .into(), + let f16_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float16, false)), offsets, - PrimitiveArray::new( - DataType::Float16.into(), + as_array_ref(PrimitiveArray::::new( f16_inner_data, - f16_inner_bitmap, - ) - .boxed(), - f16_bitmap, - )? - .boxed() + f16_inner_validity, + )), + f16_validity, + )?) } }, { @@ -629,37 +720,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let f32_bitmap: Option = None; + let f32_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( f32.iter().map(|datum| datum.num_instances()), - )? - .into(); - let f32_inner_data: Buffer<_> = f32 + ); + let f32_inner_data: ScalarBuffer<_> = f32 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let f32_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float32, - false, - ))) - .into(), + let f32_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), offsets, - PrimitiveArray::new( - DataType::Float32.into(), + as_array_ref(PrimitiveArray::::new( f32_inner_data, - f32_inner_bitmap, - ) - .boxed(), - f32_bitmap, - )? - .boxed() + f32_inner_validity, + )), + f32_validity, + )?) } }, { @@ -670,119 +751,38 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let f64_bitmap: Option = None; + let f64_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( f64.iter().map(|datum| datum.num_instances()), - )? - .into(); - let f64_inner_data: Buffer<_> = f64 + ); + let f64_inner_data: ScalarBuffer<_> = f64 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let f64_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float64, - false, - ))) - .into(), + let f64_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), offsets, - PrimitiveArray::new( - DataType::Float64.into(), + as_array_ref(PrimitiveArray::::new( f64_inner_data, - f64_inner_bitmap, - ) - .boxed(), - f64_bitmap, - )? - .boxed() + f64_inner_validity, + )), + f64_validity, + )?) } }, ]; - let offsets = Some({ - let mut u8_offset = 0; - let mut u16_offset = 0; - let mut u32_offset = 0; - let mut u64_offset = 0; - let mut i8_offset = 0; - let mut i16_offset = 0; - let mut i32_offset = 0; - let mut i64_offset = 0; - let mut f16_offset = 0; - let mut f32_offset = 0; - let mut f64_offset = 0; - let mut nulls_offset = 0; - data.iter() - .map(|v| match v.as_deref() { - None => { - let offset = nulls_offset; - nulls_offset += 1; - offset - } - Some(Self::U8(_)) => { - let offset = u8_offset; - u8_offset += 1; - offset - } - Some(Self::U16(_)) => { - let offset = u16_offset; - u16_offset += 1; - offset - } - Some(Self::U32(_)) => { - let offset = u32_offset; - u32_offset += 1; - offset - } - Some(Self::U64(_)) => { - let offset = u64_offset; - u64_offset += 1; - offset - } - Some(Self::I8(_)) => { - let offset = i8_offset; - i8_offset += 1; - offset - } - Some(Self::I16(_)) => { - let offset = i16_offset; - i16_offset += 1; - offset - } - Some(Self::I32(_)) => { - let offset = i32_offset; - i32_offset += 1; - offset - } - Some(Self::I64(_)) => { - let offset = i64_offset; - i64_offset += 1; - offset - } - Some(Self::F16(_)) => { - let offset = f16_offset; - f16_offset += 1; - offset - } - Some(Self::F32(_)) => { - let offset = f32_offset; - f32_offset += 1; - offset - } - Some(Self::F64(_)) => { - let offset = f64_offset; - f64_offset += 1; - offset - } - }) - .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + )?) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_data.rs b/crates/store/re_types/src/datatypes/tensor_data.rs index 644a0b8c69ee..6ab60d8ca0d9 100644 --- a/crates/store/re_types/src/datatypes/tensor_data.rs +++ b/crates/store/re_types/src/datatypes/tensor_data.rs @@ -68,23 +68,43 @@ impl ::re_types_core::Loggable for TensorData { Field::new( "buffer", ::arrow_datatype(), - false, + true, ), ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "shape", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "buffer", + ::arrow_datatype(), + true, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -92,12 +112,12 @@ impl ::re_types_core::Loggable for TensorData { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, shape): (Vec<_>, Vec<_>) = data @@ -107,38 +127,34 @@ impl ::re_types_core::Loggable for TensorData { (datum.is_some(), datum) }) .unzip(); - let shape_bitmap: Option = { + let shape_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( shape .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let shape_inner_data: Vec<_> = shape.into_iter().flatten().flatten().collect(); - let shape_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let shape_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = shape_inner_bitmap; - crate::datatypes::TensorDimension::to_arrow2_opt( + _ = shape_inner_validity; + crate::datatypes::TensorDimension::to_arrow_opt( shape_inner_data.into_iter().map(Some), )? }, - shape_bitmap, - )? - .boxed() + shape_validity, + )?) } }, { @@ -149,19 +165,18 @@ impl ::re_types_core::Loggable for TensorData { (datum.is_some(), datum) }) .unzip(); - let buffer_bitmap: Option = { + let buffer_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = buffer_bitmap; - crate::datatypes::TensorBuffer::to_arrow2_opt(buffer)? + _ = buffer_validity; + crate::datatypes::TensorBuffer::to_arrow_opt(buffer)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_dimension.rs b/crates/store/re_types/src/datatypes/tensor_dimension.rs index f0c30b88529a..d066945ff359 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension.rs @@ -53,18 +53,26 @@ impl ::re_types_core::Loggable for TensorDimension { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("size", DataType::UInt64, false), + Field::new("name", DataType::Utf8, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -72,12 +80,12 @@ impl ::re_types_core::Loggable for TensorDimension { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, size): (Vec<_>, Vec<_>) = data @@ -87,16 +95,18 @@ impl ::re_types_core::Loggable for TensorDimension { (datum.is_some(), datum) }) .unzip(); - let size_bitmap: Option = { + let size_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt64.into(), - size.into_iter().map(|v| v.unwrap_or_default()).collect(), - size_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + size.into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + size_validity, + )) }, { let (somes, name): (Vec<_>, Vec<_>) = data @@ -107,34 +117,26 @@ impl ::re_types_core::Loggable for TensorDimension { (datum.is_some(), datum) }) .unzip(); - let name_bitmap: Option = { + let name_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { let offsets = - arrow2::offset::Offsets::::try_from_lengths(name.iter().map( + arrow::buffer::OffsetBuffer::::from_lengths(name.iter().map( |opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default(), - ))? - .into(); - let inner_data: arrow2::buffer::Buffer = + )); + let inner_data: arrow::buffer::Buffer = name.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - name_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, name_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs b/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs index b2d55eabfca3..7a85be295e0d 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs @@ -55,18 +55,26 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("dimension", DataType::UInt32, false), + Field::new("index", DataType::UInt64, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -74,12 +82,12 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, dimension): (Vec<_>, Vec<_>) = data @@ -89,19 +97,19 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { (datum.is_some(), datum) }) .unzip(); - let dimension_bitmap: Option = { + let dimension_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - dimension - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - dimension_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + dimension + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + dimension_validity, + )) }, { let (somes, index): (Vec<_>, Vec<_>) = data @@ -111,21 +119,23 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { (datum.is_some(), datum) }) .unzip(); - let index_bitmap: Option = { + let index_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt64.into(), - index.into_iter().map(|v| v.unwrap_or_default()).collect(), - index_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + index + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + index_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs b/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs index 74fe871edc8c..70baf2180964 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs @@ -53,18 +53,26 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("dimension", DataType::UInt32, false), + Field::new("invert", DataType::Boolean, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -72,12 +80,12 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, dimension): (Vec<_>, Vec<_>) = data @@ -87,19 +95,19 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { (datum.is_some(), datum) }) .unzip(); - let dimension_bitmap: Option = { + let dimension_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - dimension - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - dimension_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + dimension + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + dimension_validity, + )) }, { let (somes, invert): (Vec<_>, Vec<_>) = data @@ -109,21 +117,23 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { (datum.is_some(), datum) }) .unzip(); - let invert_bitmap: Option = { + let invert_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - DataType::Boolean.into(), - invert.into_iter().map(|v| v.unwrap_or_default()).collect(), - invert_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + invert + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + invert_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/utf8pair.rs b/crates/store/re_types/src/datatypes/utf8pair.rs index 73105741330c..cc9b292a0328 100644 --- a/crates/store/re_types/src/datatypes/utf8pair.rs +++ b/crates/store/re_types/src/datatypes/utf8pair.rs @@ -53,18 +53,26 @@ impl ::re_types_core::Loggable for Utf8Pair { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("first", ::arrow_datatype(), false), + Field::new("second", ::arrow_datatype(), false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -72,12 +80,12 @@ impl ::re_types_core::Loggable for Utf8Pair { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, first): (Vec<_>, Vec<_>) = data @@ -87,33 +95,25 @@ impl ::re_types_core::Loggable for Utf8Pair { (datum.is_some(), datum) }) .unzip(); - let first_bitmap: Option = { + let first_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( first.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = first + ); + let inner_data: arrow::buffer::Buffer = first .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - first_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, first_validity) + }) } }, { @@ -124,39 +124,30 @@ impl ::re_types_core::Loggable for Utf8Pair { (datum.is_some(), datum) }) .unzip(); - let second_bitmap: Option = { + let second_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( second.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = second + ); + let inner_data: arrow::buffer::Buffer = second .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - second_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, second_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/uuid.rs b/crates/store/re_types/src/datatypes/uuid.rs index 61f6aef39274..646c303d9bc5 100644 --- a/crates/store/re_types/src/datatypes/uuid.rs +++ b/crates/store/re_types/src/datatypes/uuid.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for Uuid { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, bytes): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,12 +89,11 @@ impl ::re_types_core::Loggable for Uuid { (datum.is_some(), datum) }) .unzip(); - let bytes_bitmap: Option = { + let bytes_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let bytes_inner_data: Vec<_> = bytes .into_iter() .flat_map(|v| match v { @@ -100,26 +103,24 @@ impl ::re_types_core::Loggable for Uuid { ), }) .collect(); - let bytes_inner_bitmap: Option = - bytes_bitmap.as_ref().map(|bitmap| { - bitmap + let bytes_inner_validity: Option = + bytes_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(16usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt8.into(), - bytes_inner_data.into_iter().collect(), - bytes_inner_bitmap, - ) - .boxed(), - bytes_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 16, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(bytes_inner_data.into_iter().collect::>()), + bytes_inner_validity, + )), + bytes_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/uvec2d.rs b/crates/store/re_types/src/datatypes/uvec2d.rs index 8a3220a543e0..fb6f3ed5b354 100644 --- a/crates/store/re_types/src/datatypes/uvec2d.rs +++ b/crates/store/re_types/src/datatypes/uvec2d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for UVec2D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for UVec2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for UVec2D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/uvec3d.rs b/crates/store/re_types/src/datatypes/uvec3d.rs index 3d22d04e5070..c92cb64ca1af 100644 --- a/crates/store/re_types/src/datatypes/uvec3d.rs +++ b/crates/store/re_types/src/datatypes/uvec3d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for UVec3D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for UVec3D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for UVec3D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/uvec4d.rs b/crates/store/re_types/src/datatypes/uvec4d.rs index da202b7ae2f4..32b4b56ac522 100644 --- a/crates/store/re_types/src/datatypes/uvec4d.rs +++ b/crates/store/re_types/src/datatypes/uvec4d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for UVec4D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for UVec4D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for UVec4D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/vec2d.rs b/crates/store/re_types/src/datatypes/vec2d.rs index 85ad4879b70e..6bef61a0eae3 100644 --- a/crates/store/re_types/src/datatypes/vec2d.rs +++ b/crates/store/re_types/src/datatypes/vec2d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Vec2D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Vec2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Vec2D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/vec3d.rs b/crates/store/re_types/src/datatypes/vec3d.rs index ff1dcc07fe86..9087ef708793 100644 --- a/crates/store/re_types/src/datatypes/vec3d.rs +++ b/crates/store/re_types/src/datatypes/vec3d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Vec3D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Vec3D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Vec3D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/vec4d.rs b/crates/store/re_types/src/datatypes/vec4d.rs index 6f6fb956fb89..b5f1c179985b 100644 --- a/crates/store/re_types/src/datatypes/vec4d.rs +++ b/crates/store/re_types/src/datatypes/vec4d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Vec4D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Vec4D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Vec4D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/video_timestamp.rs b/crates/store/re_types/src/datatypes/video_timestamp.rs index 14bf4065b8af..8f690dae7f96 100644 --- a/crates/store/re_types/src/datatypes/video_timestamp.rs +++ b/crates/store/re_types/src/datatypes/video_timestamp.rs @@ -64,17 +64,21 @@ impl ::re_types_core::Loggable for VideoTimestamp { DataType::Int64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -84,16 +88,19 @@ impl ::re_types_core::Loggable for VideoTimestamp { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/view_coordinates.rs b/crates/store/re_types/src/datatypes/view_coordinates.rs index f7de8aeb8d8a..7294717aa306 100644 --- a/crates/store/re_types/src/datatypes/view_coordinates.rs +++ b/crates/store/re_types/src/datatypes/view_coordinates.rs @@ -82,17 +82,21 @@ impl ::re_types_core::Loggable for ViewCoordinates { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -102,12 +106,11 @@ impl ::re_types_core::Loggable for ViewCoordinates { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -117,26 +120,24 @@ impl ::re_types_core::Loggable for ViewCoordinates { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt8.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer1.rs b/crates/store/re_types/src/testing/components/affix_fuzzer1.rs index 23fc88eb7ef5..929831427396 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer1.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { crate::testing::datatypes::AffixFuzzer1::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs index ca252cc37b6d..b73afe90515b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -93,30 +97,23 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs index 24a487f42cf3..d3e65fae1d35 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for AffixFuzzer11 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,38 +101,33 @@ impl ::re_types_core::Loggable for AffixFuzzer11 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.num_instances())), - )? - .into(); - let data0_inner_data: Buffer<_> = data0 + ); + let data0_inner_data: ScalarBuffer<_> = data0 .iter() .flatten() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), offsets, - PrimitiveArray::new( - DataType::Float32.into(), + as_array_ref(PrimitiveArray::::new( data0_inner_data, - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - )? - .boxed() + data0_inner_validity, + )), + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs index baf938edb179..956c0138a3c5 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,45 +101,35 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0_inner_data.into_iter().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - data0_inner_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_inner_validity) + }) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs index c2e4aea264f2..6f1383be46fd 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,45 +101,35 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0_inner_data.into_iter().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - data0_inner_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_inner_validity) + }) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer14.rs b/crates/store/re_types/src/testing/components/affix_fuzzer14.rs index 28c697c5aace..54ad6a98da8a 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer14.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer14.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer14 { crate::testing::datatypes::AffixFuzzer3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer15.rs b/crates/store/re_types/src/testing/components/affix_fuzzer15.rs index 97940644f3c7..bf4d7ada4791 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer15.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer15.rs @@ -99,17 +99,21 @@ impl ::re_types_core::Loggable for AffixFuzzer15 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -119,13 +123,13 @@ impl ::re_types_core::Loggable for AffixFuzzer15 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs index 9a6715a98a42..783e0f901b5a 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs @@ -51,21 +51,25 @@ impl ::re_types_core::Loggable for AffixFuzzer16 { DataType::List(std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), - false, + true, ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer16 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + true, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs index 848fefbf891f..1a63cb86585e 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs @@ -51,21 +51,25 @@ impl ::re_types_core::Loggable for AffixFuzzer17 { DataType::List(std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), - false, + true, ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer17 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + true, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs index 69fdddd48a7e..231b40067c0b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs @@ -51,21 +51,25 @@ impl ::re_types_core::Loggable for AffixFuzzer18 { DataType::List(std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), - false, + true, ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer18 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + true, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer4::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer4::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer19.rs b/crates/store/re_types/src/testing/components/affix_fuzzer19.rs index d2388aa54a9a..5e14eb383e73 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer19.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer19.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer19 { crate::testing::datatypes::AffixFuzzer5::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer5::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer5::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer2.rs b/crates/store/re_types/src/testing/components/affix_fuzzer2.rs index 6198ffce321b..f8d0d3f36646 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer2.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer2.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer2 { crate::testing::datatypes::AffixFuzzer1::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer20.rs b/crates/store/re_types/src/testing/components/affix_fuzzer20.rs index 0b051434900f..e856fac201c0 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer20.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { crate::testing::datatypes::AffixFuzzer20::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer20::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer20::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer21.rs b/crates/store/re_types/src/testing/components/affix_fuzzer21.rs index b16f44aba56e..c0e0e19df83b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer21.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer21.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { crate::testing::datatypes::AffixFuzzer21::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer21::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer21::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer22.rs b/crates/store/re_types/src/testing/components/affix_fuzzer22.rs index 6f3e7bed2cec..5e7ca35ae2ef 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer22.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer22.rs @@ -79,17 +79,21 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -99,13 +103,13 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer22::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer22::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer23.rs b/crates/store/re_types/src/testing/components/affix_fuzzer23.rs index 5a3629e8eefa..c35246cbc236 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer23.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer23.rs @@ -83,17 +83,21 @@ impl ::re_types_core::Loggable for AffixFuzzer23 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -103,13 +107,13 @@ impl ::re_types_core::Loggable for AffixFuzzer23 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::MultiEnum::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::MultiEnum::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer3.rs b/crates/store/re_types/src/testing/components/affix_fuzzer3.rs index a6161782c1bd..ba87966d50a9 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer3.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { crate::testing::datatypes::AffixFuzzer1::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer4.rs b/crates/store/re_types/src/testing/components/affix_fuzzer4.rs index 31631ff304e3..4f161e22f277 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer4.rs @@ -110,17 +110,21 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -130,13 +134,13 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer5.rs b/crates/store/re_types/src/testing/components/affix_fuzzer5.rs index 2a1e60c30ec2..aedec46b86cc 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer5.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer5.rs @@ -110,17 +110,21 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -130,13 +134,13 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer6.rs b/crates/store/re_types/src/testing/components/affix_fuzzer6.rs index 7201c7e465b4..f682a2f627ea 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer6.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer6.rs @@ -110,17 +110,21 @@ impl ::re_types_core::Loggable for AffixFuzzer6 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -130,13 +134,13 @@ impl ::re_types_core::Loggable for AffixFuzzer6 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs index a28471b432a4..7bfb69f49e2c 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for AffixFuzzer7 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer7 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer8.rs b/crates/store/re_types/src/testing/components/affix_fuzzer8.rs index 6584bb83cb90..18c136749626 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer8.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer8.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer8 { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -93,16 +97,19 @@ impl ::re_types_core::Loggable for AffixFuzzer8 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs index d7a58f1781cb..fe012f10e97d 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -93,30 +97,23 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs index 40f874c2c97a..ba2d2c431c38 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs @@ -107,18 +107,61 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("single_float_optional", DataType::Float32, true), + Field::new("single_string_required", DataType::Utf8, false), + Field::new("single_string_optional", DataType::Utf8, true), + Field::new( + "many_floats_optional", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float32, + false, + ))), + true, + ), + Field::new( + "many_strings_required", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Utf8, + false, + ))), + false, + ), + Field::new( + "many_strings_optional", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Utf8, + false, + ))), + true, + ), + Field::new("flattened_scalar", DataType::Float32, false), + Field::new( + "almost_flattened_scalar", + ::arrow_datatype(), + false, + ), + Field::new("from_parent", DataType::Boolean, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -126,12 +169,12 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, single_float_optional): (Vec<_>, Vec<_>) = data @@ -144,19 +187,19 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let single_float_optional_bitmap: Option = { + let single_float_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - single_float_optional - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - single_float_optional_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + single_float_optional + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + single_float_optional_validity, + )) }, { let (somes, single_string_required): (Vec<_>, Vec<_>) = data @@ -168,32 +211,30 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let single_string_required_bitmap: Option = { + let single_string_required_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( single_string_required.iter().map(|opt| { opt.as_ref().map(|datum| datum.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = single_string_required + ); + let inner_data: arrow::buffer::Buffer = single_string_required .into_iter() .flatten() .flat_map(|s| s.0) .collect(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - single_string_required_bitmap, + single_string_required_validity, ) - } - .boxed() + }) } }, { @@ -207,32 +248,30 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let single_string_optional_bitmap: Option = { + let single_string_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( single_string_optional.iter().map(|opt| { opt.as_ref().map(|datum| datum.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = single_string_optional + ); + let inner_data: arrow::buffer::Buffer = single_string_optional .into_iter() .flatten() .flat_map(|s| s.0) .collect(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - single_string_optional_bitmap, + single_string_optional_validity, ) - } - .boxed() + }) } }, { @@ -246,44 +285,36 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let many_floats_optional_bitmap: Option = { + let many_floats_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_floats_optional.iter().map(|opt| { opt.as_ref().map_or(0, |datum| datum.num_instances()) }), - )? - .into(); - let many_floats_optional_inner_data: Buffer<_> = many_floats_optional - .iter() - .flatten() - .map(|b| b.as_slice()) - .collect::>() - .concat() - .into(); - let many_floats_optional_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float32, - false, - ))) - .into(), + ); + let many_floats_optional_inner_data: ScalarBuffer<_> = + many_floats_optional + .iter() + .flatten() + .map(|b| b.as_slice()) + .collect::>() + .concat() + .into(); + let many_floats_optional_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), offsets, - PrimitiveArray::new( - DataType::Float32.into(), + as_array_ref(PrimitiveArray::::new( many_floats_optional_inner_data, - many_floats_optional_inner_bitmap, - ) - .boxed(), - many_floats_optional_bitmap, - )? - .boxed() + many_floats_optional_inner_validity, + )), + many_floats_optional_validity, + )?) } }, { @@ -296,59 +327,49 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let many_strings_required_bitmap: Option = { + let many_strings_required_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_required .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let many_strings_required_inner_data: Vec<_> = many_strings_required .into_iter() .flatten() .flatten() .collect(); - let many_strings_required_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Utf8, - false, - ))) - .into(), + let many_strings_required_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_required_inner_data .iter() .map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = many_strings_required_inner_data .into_iter() .flat_map(|s| s.0) .collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - many_strings_required_inner_bitmap, + many_strings_required_inner_validity, ) - } - .boxed() + }) }, - many_strings_required_bitmap, - )? - .boxed() + many_strings_required_validity, + )?) } }, { @@ -362,59 +383,49 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let many_strings_optional_bitmap: Option = { + let many_strings_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_optional .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let many_strings_optional_inner_data: Vec<_> = many_strings_optional .into_iter() .flatten() .flatten() .collect(); - let many_strings_optional_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Utf8, - false, - ))) - .into(), + let many_strings_optional_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_optional_inner_data .iter() .map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = many_strings_optional_inner_data .into_iter() .flat_map(|s| s.0) .collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - many_strings_optional_inner_bitmap, + many_strings_optional_inner_validity, ) - } - .boxed() + }) }, - many_strings_optional_bitmap, - )? - .boxed() + many_strings_optional_validity, + )?) } }, { @@ -426,19 +437,19 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let flattened_scalar_bitmap: Option = { + let flattened_scalar_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - flattened_scalar - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - flattened_scalar_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + flattened_scalar + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + flattened_scalar_validity, + )) }, { let (somes, almost_flattened_scalar): (Vec<_>, Vec<_>) = data @@ -450,13 +461,13 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let almost_flattened_scalar_bitmap: Option = { + let almost_flattened_scalar_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = almost_flattened_scalar_bitmap; - crate::testing::datatypes::FlattenedScalar::to_arrow2_opt( + _ = almost_flattened_scalar_validity; + crate::testing::datatypes::FlattenedScalar::to_arrow_opt( almost_flattened_scalar, )? } @@ -472,24 +483,23 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let from_parent_bitmap: Option = { + let from_parent_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - DataType::Boolean.into(), - from_parent - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - from_parent_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + from_parent + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + from_parent_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs index 070ae5cf38ae..01d1dab25abc 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs @@ -57,17 +57,21 @@ impl ::re_types_core::Loggable for AffixFuzzer2 { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -77,16 +81,19 @@ impl ::re_types_core::Loggable for AffixFuzzer2 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs index 663d5160ef6f..487260e2bc0b 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs @@ -58,18 +58,34 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "p", + ::arrow_datatype(), + false, + ), + Field::new( + "s", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -77,12 +93,12 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, p): (Vec<_>, Vec<_>) = data @@ -92,18 +108,18 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { (datum.is_some(), datum) }) .unzip(); - let p_bitmap: Option = { + let p_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - p.into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - p_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + p.into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + p_validity, + )) }, { let (somes, s): (Vec<_>, Vec<_>) = data @@ -113,39 +129,30 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { (datum.is_some(), datum) }) .unzip(); - let s_bitmap: Option = { + let s_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( s.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = s + ); + let inner_data: arrow::buffer::Buffer = s .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - s_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, s_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs index 3aef143f90d8..e24e91edb5f6 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs @@ -57,18 +57,34 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("single_half", DataType::Float16, false), + Field::new( + "many_halves", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float16, + false, + ))), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -76,12 +92,12 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, single_half): (Vec<_>, Vec<_>) = data @@ -91,19 +107,19 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { (datum.is_some(), datum) }) .unzip(); - let single_half_bitmap: Option = { + let single_half_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float16.into(), - single_half - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - single_half_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + single_half + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + single_half_validity, + )) }, { let (somes, many_halves): (Vec<_>, Vec<_>) = data @@ -113,49 +129,39 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { (datum.is_some(), datum) }) .unzip(); - let many_halves_bitmap: Option = { + let many_halves_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_halves.iter().map(|opt| { opt.as_ref().map_or(0, |datum| datum.num_instances()) }), - )? - .into(); - let many_halves_inner_data: Buffer<_> = many_halves + ); + let many_halves_inner_data: ScalarBuffer<_> = many_halves .iter() .flatten() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let many_halves_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float16, - false, - ))) - .into(), + let many_halves_inner_validity: Option = + None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float16, false)), offsets, - PrimitiveArray::new( - DataType::Float16.into(), + as_array_ref(PrimitiveArray::::new( many_halves_inner_data, - many_halves_inner_bitmap, - ) - .boxed(), - many_halves_bitmap, - )? - .boxed() + many_halves_inner_validity, + )), + many_halves_validity, + )?) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs index 9fc33457b8b9..213207901dfd 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs @@ -66,18 +66,30 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new( + "fixed_sized_native", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 4, + ), + false, + )]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -85,12 +97,12 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, fixed_sized_native): (Vec<_>, Vec<_>) = data .iter() @@ -100,12 +112,11 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { (datum.is_some(), datum) }) .unzip(); - let fixed_sized_native_bitmap: Option = { + let fixed_sized_native_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let fixed_sized_native_inner_data: Vec<_> = fixed_sized_native .into_iter() .flat_map(|v| match v { @@ -115,35 +126,32 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { ), }) .collect(); - let fixed_sized_native_inner_bitmap: Option = - fixed_sized_native_bitmap.as_ref().map(|bitmap| { - bitmap + let fixed_sized_native_inner_validity: Option = + fixed_sized_native_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), - 4, - ) - .into(), - PrimitiveArray::new( - DataType::UInt8.into(), - fixed_sized_native_inner_data.into_iter().collect(), - fixed_sized_native_inner_bitmap, - ) - .boxed(), - fixed_sized_native_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + fixed_sized_native_inner_data + .into_iter() + .collect::>(), + ), + fixed_sized_native_inner_validity, + )), + fixed_sized_native_validity, + )) } }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs index f32c9e182238..cfd03c3220f8 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs @@ -83,17 +83,21 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -103,7 +107,30 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2, 3, 4]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new("degrees", DataType::Float32, false), + Field::new( + "craziness", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "fixed_size_shenanigans", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + ), + false, + ), + Field::new("empty_variant", DataType::Null, true), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -113,12 +140,44 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { Some(Self::EmptyVariant) => 4i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), + let offsets = { + let mut degrees_offset = 0; + let mut craziness_offset = 0; + let mut fixed_size_shenanigans_offset = 0; + let mut empty_variant_offset = 0; + let mut nulls_offset = 0; + data.iter() + .map(|v| match v.as_deref() { + None => { + let offset = nulls_offset; + nulls_offset += 1; + offset + } + Some(Self::Degrees(_)) => { + let offset = degrees_offset; + degrees_offset += 1; + offset + } + Some(Self::Craziness(_)) => { + let offset = craziness_offset; + craziness_offset += 1; + offset + } + Some(Self::FixedSizeShenanigans(_)) => { + let offset = fixed_size_shenanigans_offset; + fixed_size_shenanigans_offset += 1; + offset + } + Some(Self::EmptyVariant) => { + let offset = empty_variant_offset; + empty_variant_offset += 1; + offset + } + }) + .collect() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), { let degrees: Vec<_> = data .iter() @@ -127,13 +186,11 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { _ => None, }) .collect(); - let degrees_bitmap: Option = None; - PrimitiveArray::new( - DataType::Float32.into(), - degrees.into_iter().collect(), - degrees_bitmap, - ) - .boxed() + let degrees_validity: Option = None; + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(degrees.into_iter().collect::>()), + degrees_validity, + )) }, { let craziness: Vec<_> = data @@ -143,33 +200,29 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { _ => None, }) .collect(); - let craziness_bitmap: Option = None; + let craziness_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( craziness.iter().map(|datum| datum.len()), - )? - .into(); + ); let craziness_inner_data: Vec<_> = craziness.into_iter().flatten().collect(); - let craziness_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let craziness_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = craziness_inner_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt( + _ = craziness_inner_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt( craziness_inner_data.into_iter().map(Some), )? }, - craziness_bitmap, - )? - .boxed() + craziness_validity, + )?) } }, { @@ -180,75 +233,42 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { _ => None, }) .collect(); - let fixed_size_shenanigans_bitmap: Option = None; + let fixed_size_shenanigans_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let fixed_size_shenanigans_inner_data: Vec<_> = fixed_size_shenanigans.into_iter().flatten().collect(); - let fixed_size_shenanigans_inner_bitmap: Option = - None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float32, false)), - 3, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - fixed_size_shenanigans_inner_data.into_iter().collect(), - fixed_size_shenanigans_inner_bitmap, - ) - .boxed(), - fixed_size_shenanigans_bitmap, - ) - .boxed() + let fixed_size_shenanigans_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + fixed_size_shenanigans_inner_data + .into_iter() + .collect::>(), + ), + fixed_size_shenanigans_inner_validity, + )), + fixed_size_shenanigans_validity, + )) } }, - NullArray::new( - arrow2::datatypes::DataType::Null, + as_array_ref(NullArray::new( data.iter() .filter(|datum| matches!(datum.as_deref(), Some(Self::EmptyVariant))) .count(), - ) - .boxed(), + )), ]; - let offsets = Some({ - let mut degrees_offset = 0; - let mut craziness_offset = 0; - let mut fixed_size_shenanigans_offset = 0; - let mut empty_variant_offset = 0; - let mut nulls_offset = 0; - data.iter() - .map(|v| match v.as_deref() { - None => { - let offset = nulls_offset; - nulls_offset += 1; - offset - } - Some(Self::Degrees(_)) => { - let offset = degrees_offset; - degrees_offset += 1; - offset - } - Some(Self::Craziness(_)) => { - let offset = craziness_offset; - craziness_offset += 1; - offset - } - Some(Self::FixedSizeShenanigans(_)) => { - let offset = fixed_size_shenanigans_offset; - fixed_size_shenanigans_offset += 1; - offset - } - Some(Self::EmptyVariant) => { - let offset = empty_variant_offset; - empty_variant_offset += 1; - offset - } - }) - .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + )?) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs index cb52d1e578ea..fc7b470f5272 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs @@ -56,14 +56,14 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { Field::new( "single_required", ::arrow_datatype(), - false, + true, ), Field::new( "many_required", DataType::List(std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), - false, + true, ))), false, ), @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -93,7 +97,25 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "single_required", + ::arrow_datatype(), + true, + ), + Field::new( + "many_required", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + true, + ))), + false, + ), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -101,12 +123,32 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { Some(Self::ManyRequired(_)) => 2i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), + let offsets = { + let mut single_required_offset = 0; + let mut many_required_offset = 0; + let mut nulls_offset = 0; + data.iter() + .map(|v| match v.as_deref() { + None => { + let offset = nulls_offset; + nulls_offset += 1; + offset + } + Some(Self::SingleRequired(_)) => { + let offset = single_required_offset; + single_required_offset += 1; + offset + } + Some(Self::ManyRequired(_)) => { + let offset = many_required_offset; + many_required_offset += 1; + offset + } + }) + .collect() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), { let single_required: Vec<_> = data .iter() @@ -115,10 +157,10 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { _ => None, }) .collect(); - let single_required_bitmap: Option = None; + let single_required_validity: Option = None; { - _ = single_required_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = single_required_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( single_required.into_iter().map(Some), )? } @@ -131,61 +173,40 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { _ => None, }) .collect(); - let many_required_bitmap: Option = None; + let many_required_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_required.iter().map(|datum| datum.len()), - )? - .into(); + ); let many_required_inner_data: Vec<_> = many_required.into_iter().flatten().collect(); - let many_required_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let many_required_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), - false, - ))) - .into(), + true, + )), offsets, { - _ = many_required_inner_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = many_required_inner_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( many_required_inner_data.into_iter().map(Some), )? }, - many_required_bitmap, - )? - .boxed() + many_required_validity, + )?) } }, ]; - let offsets = Some({ - let mut single_required_offset = 0; - let mut many_required_offset = 0; - let mut nulls_offset = 0; - data.iter() - .map(|v| match v.as_deref() { - None => { - let offset = nulls_offset; - nulls_offset += 1; - offset - } - Some(Self::SingleRequired(_)) => { - let offset = single_required_offset; - single_required_offset += 1; - offset - } - Some(Self::ManyRequired(_)) => { - let offset = many_required_offset; - many_required_offset += 1; - offset - } - }) - .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + )?) }) } @@ -252,7 +273,7 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { let expected = DataType::List(std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), - false, + true, ))); let actual = arrow_data.data_type().clone(); DeserializationError::datatype_mismatch(expected, actual) diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs index 468ab9becfc6..5c040392f301 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs @@ -80,18 +80,27 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new( + "single_optional_union", + ::arrow_datatype(), + true, + )]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -99,12 +108,12 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, single_optional_union): (Vec<_>, Vec<_>) = data .iter() @@ -116,20 +125,19 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { (datum.is_some(), datum) }) .unzip(); - let single_optional_union_bitmap: Option = { + let single_optional_union_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = single_optional_union_bitmap; - crate::testing::datatypes::AffixFuzzer4::to_arrow2_opt( + _ = single_optional_union_validity; + crate::testing::datatypes::AffixFuzzer4::to_arrow_opt( single_optional_union, )? } }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/enum_test.rs b/crates/store/re_types/src/testing/datatypes/enum_test.rs index 18bd4f253eca..8f30d669d996 100644 --- a/crates/store/re_types/src/testing/datatypes/enum_test.rs +++ b/crates/store/re_types/src/testing/datatypes/enum_test.rs @@ -104,17 +104,21 @@ impl ::re_types_core::Loggable for EnumTest { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -124,16 +128,19 @@ impl ::re_types_core::Loggable for EnumTest { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs b/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs index 5d703e045abd..8ab89f869d30 100644 --- a/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs +++ b/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs @@ -63,18 +63,23 @@ impl ::re_types_core::Loggable for FlattenedScalar { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new("value", DataType::Float32, false)]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -82,12 +87,12 @@ impl ::re_types_core::Loggable for FlattenedScalar { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, value): (Vec<_>, Vec<_>) = data .iter() @@ -96,20 +101,22 @@ impl ::re_types_core::Loggable for FlattenedScalar { (datum.is_some(), datum) }) .unzip(); - let value_bitmap: Option = { + let value_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - value.into_iter().map(|v| v.unwrap_or_default()).collect(), - value_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + value + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + value_validity, + )) }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/multi_enum.rs b/crates/store/re_types/src/testing/datatypes/multi_enum.rs index ee199b47177f..7140e76ec8a4 100644 --- a/crates/store/re_types/src/testing/datatypes/multi_enum.rs +++ b/crates/store/re_types/src/testing/datatypes/multi_enum.rs @@ -61,18 +61,34 @@ impl ::re_types_core::Loggable for MultiEnum { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "value1", + ::arrow_datatype(), + false, + ), + Field::new( + "value2", + ::arrow_datatype(), + true, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +96,12 @@ impl ::re_types_core::Loggable for MultiEnum { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, value1): (Vec<_>, Vec<_>) = data @@ -95,13 +111,13 @@ impl ::re_types_core::Loggable for MultiEnum { (datum.is_some(), datum) }) .unzip(); - let value1_bitmap: Option = { + let value1_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = value1_bitmap; - crate::testing::datatypes::EnumTest::to_arrow2_opt(value1)? + _ = value1_validity; + crate::testing::datatypes::EnumTest::to_arrow_opt(value1)? } }, { @@ -113,19 +129,18 @@ impl ::re_types_core::Loggable for MultiEnum { (datum.is_some(), datum) }) .unzip(); - let value2_bitmap: Option = { + let value2_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = value2_bitmap; - crate::testing::datatypes::ValuedEnum::to_arrow2_opt(value2)? + _ = value2_validity; + crate::testing::datatypes::ValuedEnum::to_arrow_opt(value2)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/primitive_component.rs b/crates/store/re_types/src/testing/datatypes/primitive_component.rs index ce330f882d93..885db4cb2b1d 100644 --- a/crates/store/re_types/src/testing/datatypes/primitive_component.rs +++ b/crates/store/re_types/src/testing/datatypes/primitive_component.rs @@ -58,17 +58,21 @@ impl ::re_types_core::Loggable for PrimitiveComponent { DataType::UInt32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl ::re_types_core::Loggable for PrimitiveComponent { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/string_component.rs b/crates/store/re_types/src/testing/datatypes/string_component.rs index 49d4e3e9cb15..b001198a7d2e 100644 --- a/crates/store/re_types/src/testing/datatypes/string_component.rs +++ b/crates/store/re_types/src/testing/datatypes/string_component.rs @@ -58,17 +58,21 @@ impl ::re_types_core::Loggable for StringComponent { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,30 +82,23 @@ impl ::re_types_core::Loggable for StringComponent { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types/src/testing/datatypes/valued_enum.rs b/crates/store/re_types/src/testing/datatypes/valued_enum.rs index 33cd899e675a..3761179f8bcf 100644 --- a/crates/store/re_types/src/testing/datatypes/valued_enum.rs +++ b/crates/store/re_types/src/testing/datatypes/valued_enum.rs @@ -86,17 +86,21 @@ impl ::re_types_core::Loggable for ValuedEnum { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -106,16 +110,19 @@ impl ::re_types_core::Loggable for ValuedEnum { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/tests/types/fuzzy.rs b/crates/store/re_types/tests/types/fuzzy.rs index ae2fac91930e..9ceceadb12d0 100644 --- a/crates/store/re_types/tests/types/fuzzy.rs +++ b/crates/store/re_types/tests/types/fuzzy.rs @@ -212,7 +212,7 @@ fn roundtrip() { { let arch = AffixFuzzer1::new( - fuzzy1.clone(), // + fuzzy1.clone(), fuzzy2.clone(), fuzzy3.clone(), fuzzy4.clone(), @@ -237,15 +237,15 @@ fn roundtrip() { ); #[rustfmt::skip] - let expected_extensions: HashMap<_, _> = [ - ("fuzz1001", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1002", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1003", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1004", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1005", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1006", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ] - .into(); + let expected_extensions: HashMap<_, _> = [ + ("fuzz1001", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1002", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1003", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1004", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1005", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1006", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), + ] + .into(); eprintln!("arch = {arch:#?}"); let serialized = arch.to_arrow2().unwrap(); @@ -254,14 +254,14 @@ fn roundtrip() { // eprintln!("field = {field:#?}"); // eprintln!("array = {array:#?}"); if field.name == "rerun.testing.components.AffixFuzzer21" { - // TODO(jleibs): Fields that contain Float16 apparently don't supported fmt + // TODO(#3741): Fields that contain Float16 apparently don't supported fmt // https://github.com/rerun-io/re_arrow2/blob/33a32000001df800e4840d92c33b03e7007311e1/src/array/primitive/fmt.rs#L39 eprintln!("{} = Can't be printed (float16 not supported)", field.name); } else { eprintln!("{} = {array:#?}", field.name); } - // TODO(cmc): Re-enable extensions and these assertions once `arrow2-convert` + // TODO(#3741): Re-enable extensions and these assertions once `arrow2-convert` // has been fully replaced. if false { util::assert_extensions( @@ -299,15 +299,15 @@ fn roundtrip() { ); #[rustfmt::skip] - let expected_extensions: HashMap<_, _> = [ - ("fuzz1101", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1102", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1103", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1104", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1105", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz1106", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ] - .into(); + let expected_extensions: HashMap<_, _> = [ + ("fuzz1101", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1102", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1103", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1104", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1105", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz1106", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), + ] + .into(); eprintln!("arch = {arch:#?}"); let serialized = arch.to_arrow2().unwrap(); @@ -316,14 +316,14 @@ fn roundtrip() { // eprintln!("field = {field:#?}"); // eprintln!("array = {array:#?}"); if field.name == "rerun.testing.components.AffixFuzzer21" { - // TODO(jleibs): Fields that contain Float16 apparently don't supported fmt + // TODO(#3741): Fields that contain Float16 apparently don't supported fmt // https://github.com/rerun-io/re_arrow2/blob/33a32000001df800e4840d92c33b03e7007311e1/src/array/primitive/fmt.rs#L39 eprintln!("{} = Can't be printed (float16 not supported)", field.name); } else { eprintln!("{} = {array:#?}", field.name); } - // TODO(cmc): Re-enable extensions and these assertions once `arrow2-convert` + // TODO(#3741): Re-enable extensions and these assertions once `arrow2-convert` // has been fully replaced. if false { util::assert_extensions( @@ -352,15 +352,15 @@ fn roundtrip() { .with_fuzz2018(fuzzy18_1.clone()); #[rustfmt::skip] - let expected_extensions: HashMap<_, _> = [ - ("fuzz2001", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2002", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2003", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2004", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2005", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2006", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ] - .into(); + let expected_extensions: HashMap<_, _> = [ + ("fuzz2001", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2002", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2003", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2004", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2005", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2006", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), + ] + .into(); eprintln!("arch = {arch:#?}"); let serialized = arch.to_arrow2().unwrap(); @@ -369,14 +369,14 @@ fn roundtrip() { // eprintln!("field = {field:#?}"); // eprintln!("array = {array:#?}"); if field.name == "rerun.testing.components.AffixFuzzer21" { - // TODO(jleibs): Fields that contain Float16 apparently don't supported fmt + // TODO(#3741): Fields that contain Float16 apparently don't supported fmt // https://github.com/rerun-io/re_arrow2/blob/33a32000001df800e4840d92c33b03e7007311e1/src/array/primitive/fmt.rs#L39 eprintln!("{} = Can't be printed (float16 not supported)", field.name); } else { eprintln!("{} = {array:#?}", field.name); } - // TODO(cmc): Re-enable extensions and these assertions once `arrow2-convert` + // TODO(#3741): Re-enable extensions and these assertions once `arrow2-convert` // has been fully replaced. if false { util::assert_extensions( @@ -405,15 +405,15 @@ fn roundtrip() { .with_fuzz2118([fuzzy18_1.clone(), fuzzy18_2.clone(), fuzzy18_3.clone()]); #[rustfmt::skip] - let expected_extensions: HashMap<_, _> = [ - ("fuzz2101", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2102", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2103", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2104", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2105", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), - ("fuzz2106", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), - ] - .into(); + let expected_extensions: HashMap<_, _> = [ + ("fuzz2101", vec!["rerun.testing.components.AffixFuzzer1", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2102", vec!["rerun.testing.components.AffixFuzzer2", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2103", vec!["rerun.testing.components.AffixFuzzer3", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2104", vec!["rerun.testing.components.AffixFuzzer4", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2105", vec!["rerun.testing.components.AffixFuzzer5", "rerun.testing.datatypes.AffixFuzzer1"]), + ("fuzz2106", vec!["rerun.testing.components.AffixFuzzer6", "rerun.testing.datatypes.AffixFuzzer1"]), + ] + .into(); eprintln!("arch = {arch:#?}"); let serialized = arch.to_arrow2().unwrap(); @@ -422,14 +422,14 @@ fn roundtrip() { // eprintln!("field = {field:#?}"); // eprintln!("array = {array:#?}"); if field.name == "rerun.testing.components.AffixFuzzer21" { - // TODO(jleibs): Fields that contain Float16 apparently don't supported fmt + // TODO(#3741): Fields that contain Float16 apparently don't supported fmt // https://github.com/rerun-io/re_arrow2/blob/33a32000001df800e4840d92c33b03e7007311e1/src/array/primitive/fmt.rs#L39 eprintln!("{} = Can't be printed (float16 not supported)", field.name); } else { eprintln!("{} = {array:#?}", field.name); } - // TODO(cmc): Re-enable extensions and these assertions once `arrow2-convert` + // TODO(#3741): Re-enable extensions and these assertions once `arrow2-convert` // has been fully replaced. if false { util::assert_extensions( diff --git a/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs b/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs index c976534556d0..e76dc9edd155 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for AutoLayout { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs b/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs index 797a9f14876d..b23a9816b680 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for AutoSpaceViews { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs b/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs index af6b6fd6d5b3..a870a7b3184e 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs @@ -87,17 +87,21 @@ impl ::re_types_core::Loggable for ContainerKind { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -107,16 +111,19 @@ impl ::re_types_core::Loggable for ContainerKind { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs b/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs index 468eb395f924..6d5c2cd36447 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for GridColumns { crate::datatypes::UInt32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::UInt32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::UInt32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs b/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs index 4e2538c66fb9..020fbd69bbdb 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for IncludedSpaceView { crate::datatypes::Uuid::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Uuid::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Uuid::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs b/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs index da8acada3789..0c754d3403ce 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for RootContainer { crate::datatypes::Uuid::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Uuid::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Uuid::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs b/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs index 062f4e5d3bc0..482494005730 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SpaceViewMaximized { crate::datatypes::Uuid::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Uuid::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Uuid::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs b/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs index 82bc233e67e1..a5c7020c1cd1 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs @@ -107,13 +107,13 @@ impl ::re_types_core::Loggable for VisualizerOverrides { crate::blueprint::datatypes::Utf8List::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::Utf8List::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::Utf8List::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs index 7b079ff28f06..0dd4a91fef63 100644 --- a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs +++ b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs @@ -63,17 +63,21 @@ impl ::re_types_core::Loggable for Utf8List { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -83,45 +87,35 @@ impl ::re_types_core::Loggable for Utf8List { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0_inner_data.into_iter().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - data0_inner_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_inner_validity) + }) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types_core/src/components/clear_is_recursive.rs b/crates/store/re_types_core/src/components/clear_is_recursive.rs index f45db2e65b45..04d68f0ab05c 100644 --- a/crates/store/re_types_core/src/components/clear_is_recursive.rs +++ b/crates/store/re_types_core/src/components/clear_is_recursive.rs @@ -74,13 +74,13 @@ impl crate::Loggable for ClearIsRecursive { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_core/src/datatypes/bool.rs b/crates/store/re_types_core/src/datatypes/bool.rs index afcd6aa88995..4ff9c81191f2 100644 --- a/crates/store/re_types_core/src/datatypes/bool.rs +++ b/crates/store/re_types_core/src/datatypes/bool.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Bool { DataType::Boolean } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,16 +83,19 @@ impl crate::Loggable for Bool { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/entity_path.rs b/crates/store/re_types_core/src/datatypes/entity_path.rs index 57714f3eeb01..6c276a78e811 100644 --- a/crates/store/re_types_core/src/datatypes/entity_path.rs +++ b/crates/store/re_types_core/src/datatypes/entity_path.rs @@ -59,17 +59,21 @@ impl crate::Loggable for EntityPath { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,30 +83,23 @@ impl crate::Loggable for EntityPath { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types_core/src/datatypes/float32.rs b/crates/store/re_types_core/src/datatypes/float32.rs index af808a56775c..cde48a656d39 100644 --- a/crates/store/re_types_core/src/datatypes/float32.rs +++ b/crates/store/re_types_core/src/datatypes/float32.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Float32 { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,16 +83,19 @@ impl crate::Loggable for Float32 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/float64.rs b/crates/store/re_types_core/src/datatypes/float64.rs index b7395356b011..d89edcdd0603 100644 --- a/crates/store/re_types_core/src/datatypes/float64.rs +++ b/crates/store/re_types_core/src/datatypes/float64.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Float64 { DataType::Float64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,16 +83,19 @@ impl crate::Loggable for Float64 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/time_int.rs b/crates/store/re_types_core/src/datatypes/time_int.rs index d84e9b5aee4a..d40d1b3be4f8 100644 --- a/crates/store/re_types_core/src/datatypes/time_int.rs +++ b/crates/store/re_types_core/src/datatypes/time_int.rs @@ -58,17 +58,21 @@ impl crate::Loggable for TimeInt { DataType::Int64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for TimeInt { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/time_range.rs b/crates/store/re_types_core/src/datatypes/time_range.rs index 7089e285a08d..79fe0c255b9c 100644 --- a/crates/store/re_types_core/src/datatypes/time_range.rs +++ b/crates/store/re_types_core/src/datatypes/time_range.rs @@ -52,28 +52,44 @@ impl crate::Loggable for TimeRange { Field::new( "start", ::arrow_datatype(), - false, + true, ), Field::new( "end", ::arrow_datatype(), - false, + true, ), ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "start", + ::arrow_datatype(), + true, + ), + Field::new( + "end", + ::arrow_datatype(), + true, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -81,12 +97,12 @@ impl crate::Loggable for TimeRange { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, start): (Vec<_>, Vec<_>) = data @@ -96,13 +112,13 @@ impl crate::Loggable for TimeRange { (datum.is_some(), datum) }) .unzip(); - let start_bitmap: Option = { + let start_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = start_bitmap; - crate::datatypes::TimeRangeBoundary::to_arrow2_opt(start)? + _ = start_validity; + crate::datatypes::TimeRangeBoundary::to_arrow_opt(start)? } }, { @@ -113,19 +129,18 @@ impl crate::Loggable for TimeRange { (datum.is_some(), datum) }) .unzip(); - let end_bitmap: Option = { + let end_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = end_bitmap; - crate::datatypes::TimeRangeBoundary::to_arrow2_opt(end)? + _ = end_validity; + crate::datatypes::TimeRangeBoundary::to_arrow_opt(end)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs index aaa3095c2f7b..c4f589b3dc00 100644 --- a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs +++ b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs @@ -77,17 +77,21 @@ impl crate::Loggable for TimeRangeBoundary { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -97,7 +101,22 @@ impl crate::Loggable for TimeRangeBoundary { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2, 3]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "CursorRelative", + ::arrow_datatype(), + false, + ), + Field::new( + "Absolute", + ::arrow_datatype(), + false, + ), + Field::new("Infinite", DataType::Null, true), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -106,53 +125,7 @@ impl crate::Loggable for TimeRangeBoundary { Some(Self::Infinite) => 3i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), - { - let cursor_relative: Vec<_> = data - .iter() - .filter_map(|datum| match datum.as_deref() { - Some(Self::CursorRelative(v)) => Some(v.clone()), - _ => None, - }) - .collect(); - let cursor_relative_bitmap: Option = None; - PrimitiveArray::new( - DataType::Int64.into(), - cursor_relative.into_iter().map(|datum| datum.0).collect(), - cursor_relative_bitmap, - ) - .boxed() - }, - { - let absolute: Vec<_> = data - .iter() - .filter_map(|datum| match datum.as_deref() { - Some(Self::Absolute(v)) => Some(v.clone()), - _ => None, - }) - .collect(); - let absolute_bitmap: Option = None; - PrimitiveArray::new( - DataType::Int64.into(), - absolute.into_iter().map(|datum| datum.0).collect(), - absolute_bitmap, - ) - .boxed() - }, - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter() - .filter(|datum| matches!(datum.as_deref(), Some(Self::Infinite))) - .count(), - ) - .boxed(), - ]; - let offsets = Some({ + let offsets = { let mut cursor_relative_offset = 0; let mut absolute_offset = 0; let mut infinite_offset = 0; @@ -181,8 +154,61 @@ impl crate::Loggable for TimeRangeBoundary { } }) .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), + { + let cursor_relative: Vec<_> = data + .iter() + .filter_map(|datum| match datum.as_deref() { + Some(Self::CursorRelative(v)) => Some(v.clone()), + _ => None, + }) + .collect(); + let cursor_relative_validity: Option = None; + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + cursor_relative + .into_iter() + .map(|datum| datum.0) + .collect::>(), + ), + cursor_relative_validity, + )) + }, + { + let absolute: Vec<_> = data + .iter() + .filter_map(|datum| match datum.as_deref() { + Some(Self::Absolute(v)) => Some(v.clone()), + _ => None, + }) + .collect(); + let absolute_validity: Option = None; + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + absolute + .into_iter() + .map(|datum| datum.0) + .collect::>(), + ), + absolute_validity, + )) + }, + as_array_ref(NullArray::new( + data.iter() + .filter(|datum| matches!(datum.as_deref(), Some(Self::Infinite))) + .count(), + )), + ]; + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + )?) }) } diff --git a/crates/store/re_types_core/src/datatypes/uint16.rs b/crates/store/re_types_core/src/datatypes/uint16.rs index d72659834ab0..2f2a07f52df4 100644 --- a/crates/store/re_types_core/src/datatypes/uint16.rs +++ b/crates/store/re_types_core/src/datatypes/uint16.rs @@ -58,17 +58,21 @@ impl crate::Loggable for UInt16 { DataType::UInt16 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for UInt16 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/uint32.rs b/crates/store/re_types_core/src/datatypes/uint32.rs index 9f64b394645c..81d81d27c9fd 100644 --- a/crates/store/re_types_core/src/datatypes/uint32.rs +++ b/crates/store/re_types_core/src/datatypes/uint32.rs @@ -58,17 +58,21 @@ impl crate::Loggable for UInt32 { DataType::UInt32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for UInt32 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/uint64.rs b/crates/store/re_types_core/src/datatypes/uint64.rs index e3bd7be8887b..bbde9100ee3b 100644 --- a/crates/store/re_types_core/src/datatypes/uint64.rs +++ b/crates/store/re_types_core/src/datatypes/uint64.rs @@ -58,17 +58,21 @@ impl crate::Loggable for UInt64 { DataType::UInt64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for UInt64 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/utf8.rs b/crates/store/re_types_core/src/datatypes/utf8.rs index d835960f3054..1cf49c0065d0 100644 --- a/crates/store/re_types_core/src/datatypes/utf8.rs +++ b/crates/store/re_types_core/src/datatypes/utf8.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Utf8 { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,30 +83,23 @@ impl crate::Loggable for Utf8 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types_core/src/datatypes/visible_time_range.rs b/crates/store/re_types_core/src/datatypes/visible_time_range.rs index c2fcf4ff6902..b2a741147f56 100644 --- a/crates/store/re_types_core/src/datatypes/visible_time_range.rs +++ b/crates/store/re_types_core/src/datatypes/visible_time_range.rs @@ -61,18 +61,34 @@ impl crate::Loggable for VisibleTimeRange { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "timeline", + ::arrow_datatype(), + false, + ), + Field::new( + "range", + ::arrow_datatype(), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +96,12 @@ impl crate::Loggable for VisibleTimeRange { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, timeline): (Vec<_>, Vec<_>) = data @@ -95,33 +111,25 @@ impl crate::Loggable for VisibleTimeRange { (datum.is_some(), datum) }) .unzip(); - let timeline_bitmap: Option = { + let timeline_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( timeline.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = timeline + ); + let inner_data: arrow::buffer::Buffer = timeline .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - timeline_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, timeline_validity) + }) } }, { @@ -132,19 +140,18 @@ impl crate::Loggable for VisibleTimeRange { (datum.is_some(), datum) }) .unzip(); - let range_bitmap: Option = { + let range_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = range_bitmap; - crate::datatypes::TimeRange::to_arrow2_opt(range)? + _ = range_validity; + crate::datatypes::TimeRange::to_arrow_opt(range)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types_core/src/loggable.rs b/crates/store/re_types_core/src/loggable.rs index 3646a3f19720..9912a521ae58 100644 --- a/crates/store/re_types_core/src/loggable.rs +++ b/crates/store/re_types_core/src/loggable.rs @@ -53,7 +53,10 @@ pub trait Loggable: 'static + Send + Sync + Clone + Sized + SizeBytes { data: impl IntoIterator>>>, ) -> SerializationResult> where - Self: 'a; + Self: 'a, + { + Self::to_arrow_opt(data).map(|array| array.into()) + } // --- Optional serialization methods --- diff --git a/rerun_cpp/src/rerun/datatypes/tensor_data.cpp b/rerun_cpp/src/rerun/datatypes/tensor_data.cpp index 393ac018152c..dfaa17f489c0 100644 --- a/rerun_cpp/src/rerun/datatypes/tensor_data.cpp +++ b/rerun_cpp/src/rerun/datatypes/tensor_data.cpp @@ -26,7 +26,7 @@ namespace rerun { arrow::field( "buffer", Loggable::arrow_datatype(), - false + true ), }); return datatype; diff --git a/rerun_cpp/src/rerun/datatypes/time_range.cpp b/rerun_cpp/src/rerun/datatypes/time_range.cpp index 988ca4e4b7ab..18c22a7dc1d4 100644 --- a/rerun_cpp/src/rerun/datatypes/time_range.cpp +++ b/rerun_cpp/src/rerun/datatypes/time_range.cpp @@ -16,12 +16,12 @@ namespace rerun { arrow::field( "start", Loggable::arrow_datatype(), - false + true ), arrow::field( "end", Loggable::arrow_datatype(), - false + true ), }); return datatype; diff --git a/rerun_cpp/tests/generated/components/affix_fuzzer16.cpp b/rerun_cpp/tests/generated/components/affix_fuzzer16.cpp index 5d6f70721eb8..4bbe913a2a38 100644 --- a/rerun_cpp/tests/generated/components/affix_fuzzer16.cpp +++ b/rerun_cpp/tests/generated/components/affix_fuzzer16.cpp @@ -13,7 +13,7 @@ namespace rerun::components {} namespace rerun { const std::shared_ptr& Loggable::arrow_datatype() { static const auto datatype = arrow::list( - arrow::field("item", Loggable::arrow_datatype(), false) + arrow::field("item", Loggable::arrow_datatype(), true) ); return datatype; } diff --git a/rerun_cpp/tests/generated/components/affix_fuzzer17.cpp b/rerun_cpp/tests/generated/components/affix_fuzzer17.cpp index 77c7a7d7d015..7fe3e9dd7746 100644 --- a/rerun_cpp/tests/generated/components/affix_fuzzer17.cpp +++ b/rerun_cpp/tests/generated/components/affix_fuzzer17.cpp @@ -13,7 +13,7 @@ namespace rerun::components {} namespace rerun { const std::shared_ptr& Loggable::arrow_datatype() { static const auto datatype = arrow::list( - arrow::field("item", Loggable::arrow_datatype(), false) + arrow::field("item", Loggable::arrow_datatype(), true) ); return datatype; } diff --git a/rerun_cpp/tests/generated/components/affix_fuzzer18.cpp b/rerun_cpp/tests/generated/components/affix_fuzzer18.cpp index c3d574588b2e..4677e85ee015 100644 --- a/rerun_cpp/tests/generated/components/affix_fuzzer18.cpp +++ b/rerun_cpp/tests/generated/components/affix_fuzzer18.cpp @@ -13,7 +13,7 @@ namespace rerun::components {} namespace rerun { const std::shared_ptr& Loggable::arrow_datatype() { static const auto datatype = arrow::list( - arrow::field("item", Loggable::arrow_datatype(), false) + arrow::field("item", Loggable::arrow_datatype(), true) ); return datatype; } diff --git a/rerun_cpp/tests/generated/datatypes/affix_fuzzer4.cpp b/rerun_cpp/tests/generated/datatypes/affix_fuzzer4.cpp index f0f3d20a7eed..a5d65655006a 100644 --- a/rerun_cpp/tests/generated/datatypes/affix_fuzzer4.cpp +++ b/rerun_cpp/tests/generated/datatypes/affix_fuzzer4.cpp @@ -17,14 +17,14 @@ namespace rerun { arrow::field( "single_required", Loggable::arrow_datatype(), - false + true ), arrow::field( "many_required", arrow::list(arrow::field( "item", Loggable::arrow_datatype(), - false + true )), false ), diff --git a/rerun_py/rerun_sdk/rerun/datatypes/tensor_data.py b/rerun_py/rerun_sdk/rerun/datatypes/tensor_data.py index 1b686305f11e..fa8e1e8364a5 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/tensor_data.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/tensor_data.py @@ -154,7 +154,7 @@ class TensorDataBatch(BaseBatch[TensorDataArrayLike]): metadata={}, ), ]), - nullable=False, + nullable=True, metadata={}, ), ]) diff --git a/rerun_py/rerun_sdk/rerun/datatypes/time_range.py b/rerun_py/rerun_sdk/rerun/datatypes/time_range.py index dd2337cbbdfd..f7193ad98c8d 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/time_range.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/time_range.py @@ -84,7 +84,7 @@ class TimeRangeBatch(BaseBatch[TimeRangeArrayLike]): pa.field("Absolute", pa.int64(), nullable=False, metadata={}), pa.field("Infinite", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), pa.field( @@ -95,7 +95,7 @@ class TimeRangeBatch(BaseBatch[TimeRangeArrayLike]): pa.field("Absolute", pa.int64(), nullable=False, metadata={}), pa.field("Infinite", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), ]) diff --git a/rerun_py/rerun_sdk/rerun/datatypes/visible_time_range.py b/rerun_py/rerun_sdk/rerun/datatypes/visible_time_range.py index 7dc274e8a7f9..ef8e4fa756cb 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/visible_time_range.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/visible_time_range.py @@ -64,7 +64,7 @@ class VisibleTimeRangeBatch(BaseBatch[VisibleTimeRangeArrayLike]): pa.field("Absolute", pa.int64(), nullable=False, metadata={}), pa.field("Infinite", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), pa.field( @@ -75,7 +75,7 @@ class VisibleTimeRangeBatch(BaseBatch[VisibleTimeRangeArrayLike]): pa.field("Absolute", pa.int64(), nullable=False, metadata={}), pa.field("Infinite", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), ]), diff --git a/rerun_py/tests/test_types/components/affix_fuzzer16.py b/rerun_py/tests/test_types/components/affix_fuzzer16.py index ca13f45de439..74679e637bd6 100644 --- a/rerun_py/tests/test_types/components/affix_fuzzer16.py +++ b/rerun_py/tests/test_types/components/affix_fuzzer16.py @@ -98,7 +98,7 @@ class AffixFuzzer16Batch(BaseBatch[AffixFuzzer16ArrayLike], ComponentBatchMixin) ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ) ) diff --git a/rerun_py/tests/test_types/components/affix_fuzzer17.py b/rerun_py/tests/test_types/components/affix_fuzzer17.py index 93701c24f3ed..029191165b1e 100644 --- a/rerun_py/tests/test_types/components/affix_fuzzer17.py +++ b/rerun_py/tests/test_types/components/affix_fuzzer17.py @@ -98,7 +98,7 @@ class AffixFuzzer17Batch(BaseBatch[AffixFuzzer17ArrayLike], ComponentBatchMixin) ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ) ) diff --git a/rerun_py/tests/test_types/components/affix_fuzzer18.py b/rerun_py/tests/test_types/components/affix_fuzzer18.py index 1792ee8dd91c..b6c952e7da11 100644 --- a/rerun_py/tests/test_types/components/affix_fuzzer18.py +++ b/rerun_py/tests/test_types/components/affix_fuzzer18.py @@ -102,7 +102,7 @@ class AffixFuzzer18Batch(BaseBatch[AffixFuzzer18ArrayLike], ComponentBatchMixin) ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), pa.field( @@ -174,7 +174,7 @@ class AffixFuzzer18Batch(BaseBatch[AffixFuzzer18ArrayLike], ComponentBatchMixin) ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ) ), @@ -182,7 +182,7 @@ class AffixFuzzer18Batch(BaseBatch[AffixFuzzer18ArrayLike], ComponentBatchMixin) metadata={}, ), ]), - nullable=False, + nullable=True, metadata={}, ) ) diff --git a/rerun_py/tests/test_types/datatypes/affix_fuzzer4.py b/rerun_py/tests/test_types/datatypes/affix_fuzzer4.py index d099aa6e768c..1a35350f32cf 100644 --- a/rerun_py/tests/test_types/datatypes/affix_fuzzer4.py +++ b/rerun_py/tests/test_types/datatypes/affix_fuzzer4.py @@ -108,7 +108,7 @@ class AffixFuzzer4Batch(BaseBatch[AffixFuzzer4ArrayLike]): ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), pa.field( @@ -170,7 +170,7 @@ class AffixFuzzer4Batch(BaseBatch[AffixFuzzer4ArrayLike]): ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ) ), diff --git a/rerun_py/tests/test_types/datatypes/affix_fuzzer5.py b/rerun_py/tests/test_types/datatypes/affix_fuzzer5.py index 9c1e579ef3e3..e2c8965a57b3 100644 --- a/rerun_py/tests/test_types/datatypes/affix_fuzzer5.py +++ b/rerun_py/tests/test_types/datatypes/affix_fuzzer5.py @@ -111,7 +111,7 @@ class AffixFuzzer5Batch(BaseBatch[AffixFuzzer5ArrayLike]): ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ), pa.field( @@ -183,7 +183,7 @@ class AffixFuzzer5Batch(BaseBatch[AffixFuzzer5ArrayLike]): ), pa.field("empty_variant", pa.null(), nullable=True, metadata={}), ]), - nullable=False, + nullable=True, metadata={}, ) ),