From 44bf4c68b0627d45b6db4a6ab240888668cfa6e9 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Thu, 19 Dec 2024 14:57:29 +0100 Subject: [PATCH] wip --- Cargo.lock | 3 + crates/store/re_chunk/src/iter.rs | 663 ++++++++++-------- crates/store/re_chunk/src/lib.rs | 4 +- crates/store/re_grpc_client/src/lib.rs | 2 +- crates/store/re_query/examples/range.rs | 2 +- crates/store/re_query/tests/range.rs | 2 +- crates/store/re_types_core/src/lib.rs | 2 +- crates/viewer/re_view/src/results_ext.rs | 90 +-- crates/viewer/re_view_graph/Cargo.toml | 1 + .../re_view_graph/src/visualizers/edges.rs | 42 +- .../re_view_graph/src/visualizers/nodes.rs | 16 +- .../src/visualizers/geo_line_strings.rs | 6 +- .../re_view_map/src/visualizers/geo_points.rs | 8 +- crates/viewer/re_view_spatial/Cargo.toml | 1 + .../src/max_image_dimension_subscriber.rs | 4 +- .../src/visualizers/arrows2d.rs | 20 +- .../src/visualizers/arrows3d.rs | 18 +- .../src/visualizers/assets3d.rs | 9 +- .../src/visualizers/boxes2d.rs | 23 +- .../src/visualizers/boxes3d.rs | 23 +- .../src/visualizers/capsules3d.rs | 18 +- .../src/visualizers/depth_images.rs | 12 +- .../src/visualizers/ellipsoids.rs | 23 +- .../src/visualizers/encoded_image.rs | 8 +- .../re_view_spatial/src/visualizers/images.rs | 6 +- .../src/visualizers/lines2d.rs | 22 +- .../src/visualizers/lines3d.rs | 22 +- .../re_view_spatial/src/visualizers/meshes.rs | 25 +- .../src/visualizers/points2d.rs | 23 +- .../src/visualizers/points3d.rs | 23 +- .../src/visualizers/segmentation_images.rs | 6 +- .../visualizers/utilities/entity_iterator.rs | 85 +-- .../re_view_spatial/src/visualizers/videos.rs | 2 +- .../re_view_tensor/src/visualizer_system.rs | 2 +- .../re_view_text_log/src/visualizer_system.rs | 6 +- crates/viewer/re_view_time_series/Cargo.toml | 1 + .../src/line_visualizer_system.rs | 10 +- .../src/point_visualizer_system.rs | 10 +- 38 files changed, 612 insertions(+), 631 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8103ce57341b..5a795dcb9078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6648,6 +6648,7 @@ dependencies = [ "ahash", "egui", "fjadra", + "itertools 0.13.0", "nohash-hasher", "re_chunk", "re_data_ui", @@ -6709,6 +6710,7 @@ dependencies = [ "once_cell", "ordered-float", "re_arrow2", + "re_chunk", "re_chunk_store", "re_data_ui", "re_entity_db", @@ -6800,6 +6802,7 @@ dependencies = [ "egui_plot", "itertools 0.13.0", "rayon", + "re_chunk", "re_chunk_store", "re_format", "re_log", diff --git a/crates/store/re_chunk/src/iter.rs b/crates/store/re_chunk/src/iter.rs index 4bce90046356..fba566278edb 100644 --- a/crates/store/re_chunk/src/iter.rs +++ b/crates/store/re_chunk/src/iter.rs @@ -2,8 +2,9 @@ use std::sync::Arc; use arrow2::{ array::{ - BooleanArray as Arrow2BooleanArray, FixedSizeListArray as Arrow2FixedSizeListArray, - ListArray as Arrow2ListArray, PrimitiveArray as Arrow2PrimitiveArray, + Array as Arrow2Array, BooleanArray as Arrow2BooleanArray, + FixedSizeListArray as Arrow2FixedSizeListArray, ListArray as Arrow2ListArray, + PrimitiveArray as Arrow2PrimitiveArray, StructArray as Arrow2StructArray, Utf8Array as Arrow2Utf8Array, }, bitmap::Bitmap as Arrow2Bitmap, @@ -200,74 +201,57 @@ impl Chunk { } } - /// Returns an iterator over the raw primitive values of a [`Chunk`], for a given component. + /// Returns an iterator over the all the sliced component batches in a [`Chunk`]'s column, for + /// a given component. + /// + /// The generic `S` parameter will decide the type of data returned. It is _very_ permissive. + /// See [`ChunkComponentSlicer`] for all the available implementations. /// /// This is a very fast path: the entire column will be downcasted at once, and then every /// component batch will be a slice reference into that global slice. - /// Use this when working with simple arrow datatypes and performance matters (e.g. scalars, - /// points, etc). /// - /// See also: - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_primitive_array_list`] - /// * [`Self::iter_string`] - /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component`]. + /// See also [`Self::iter_slices_from_struct_field`]. #[inline] - pub fn iter_primitive( - &self, - component_name: &ComponentName, - ) -> impl Iterator + '_ { - let Some(list_array) = self.get_first_component(component_name) else { - return Either::Left(std::iter::empty()); - }; - - let Some(values) = list_array - .values() - .as_any() - .downcast_ref::>() - else { - if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); - } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); - } + pub fn iter_slices<'a, S: 'a + ChunkComponentSlicer>( + &'a self, + component_name: ComponentName, + ) -> impl Iterator> + 'a { + let Some(list_array) = self.get_first_component(&component_name) else { return Either::Left(std::iter::empty()); }; - let values = values.values().as_slice(); - // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. - Either::Right( - self.iter_component_offsets(component_name) - .map(move |(idx, len)| &values[idx..idx + len]), - ) + Either::Right(S::slice( + component_name, + &**list_array.values() as _, + self.iter_component_offsets(&component_name), + )) } - /// Returns an iterator over the raw boolean values of a [`Chunk`], for a given component. + /// Returns an iterator over the all the sliced component batches in a [`Chunk`]'s column, for + /// a specific struct field of given component. + /// + /// The target component must be a `StructArray`. + /// + /// The generic `S` parameter will decide the type of data returned. It is _very_ permissive. + /// See [`ChunkComponentSlicer`] for all the available implementations. /// /// This is a very fast path: the entire column will be downcasted at once, and then every /// component batch will be a slice reference into that global slice. - /// Use this when working with simple arrow datatypes and performance matters. /// - /// See also: - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_primitive_array_list`] - /// * [`Self::iter_string`] - /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component`]. - #[inline] - pub fn iter_bool( - &self, - component_name: &ComponentName, - ) -> impl Iterator + '_ { - let Some(list_array) = self.get_first_component(component_name) else { + /// See also [`Self::iter_slices_from_struct_field`]. + pub fn iter_slices_from_struct_field<'a, S: 'a + ChunkComponentSlicer>( + &'a self, + component_name: ComponentName, + field_name: &'a str, + ) -> impl Iterator> + '_ { + let Some(list_array) = self.get_first_component(&component_name) else { return Either::Left(std::iter::empty()); }; - let Some(values) = list_array + let Some(struct_array) = list_array .values() .as_any() - .downcast_ref::() + .downcast_ref::() else { if cfg!(debug_assertions) { panic!("downcast failed for {component_name}, data discarded"); @@ -276,185 +260,365 @@ impl Chunk { } return Either::Left(std::iter::empty()); }; - let values = values.values().clone(); - - // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. - Either::Right( - self.iter_component_offsets(component_name) - .map(move |(idx, len)| values.clone().sliced(idx, len)), - ) - } - /// Returns an iterator over the raw primitive arrays of a [`Chunk`], for a given component. - /// - /// This is a very fast path: the entire column will be downcasted at once, and then every - /// component batch will be a slice reference into that global slice. - /// Use this when working with simple arrow datatypes and performance matters (e.g. scalars, - /// points, etc). - /// - /// See also: - /// * [`Self::iter_primitive`] - /// * [`Self::iter_string`] - /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component`]. - pub fn iter_primitive_array( - &self, - component_name: &ComponentName, - ) -> impl Iterator + '_ - where - [T; N]: bytemuck::Pod, - { - let Some(list_array) = self.get_first_component(component_name) else { - return Either::Left(std::iter::empty()); - }; - - let Some(fixed_size_list_array) = list_array - .values() - .as_any() - .downcast_ref::() + let Some(field_idx) = struct_array + .fields() + .iter() + .enumerate() + .find_map(|(i, field)| (field.name == field_name).then_some(i)) else { if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); + panic!("field {field_name} not found for {component_name}, data discarded"); } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); + re_log::error_once!( + "field {field_name} not found for {component_name}, data discarded" + ); } return Either::Left(std::iter::empty()); }; - let Some(values) = fixed_size_list_array - .values() - .as_any() - .downcast_ref::>() - else { + let Some(array) = struct_array.values().get(field_idx) else { if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); + panic!("field {field_name} not found for {component_name}, data discarded"); } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); + re_log::error_once!( + "field {field_name} not found for {component_name}, data discarded" + ); } return Either::Left(std::iter::empty()); }; - let size = fixed_size_list_array.size(); - let values = values.values().as_slice(); - - // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. - Either::Right( - self.iter_component_offsets(component_name) - .map(move |(idx, len)| { - bytemuck::cast_slice(&values[idx * size..idx * size + len * size]) - }), - ) + Either::Right(S::slice( + component_name, + &**array, + self.iter_component_offsets(&component_name), + )) } +} - /// Returns an iterator over the raw list of primitive arrays of a [`Chunk`], for a given component. - /// - /// This is a very fast path: the entire column will be downcasted at once, and then every - /// component batch will be a slice reference into that global slice. - /// Use this when working with simple arrow datatypes and performance matters (e.g. strips, etc). - /// - /// See also: - /// * [`Self::iter_primitive`] - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_string`] - /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component`]. - pub fn iter_primitive_array_list( - &self, - component_name: &ComponentName, - ) -> impl Iterator> + '_ - where - [T; N]: bytemuck::Pod, - { - let Some(list_array) = self.get_first_component(component_name) else { - return Either::Left(std::iter::empty()); - }; +// --- - let Some(inner_list_array) = list_array - .values() - .as_any() - .downcast_ref::>() - else { - if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); - } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); - } - return Either::Left(std::iter::empty()); - }; +/// A `ChunkComponentSlicer` knows how to efficiently slice component batches out of a Chunk column. +/// +/// See [`Chunk::iter_slices`] and [`Chunk::iter_slices_from_struct_field`]. +pub trait ChunkComponentSlicer { + type Item<'a>; + + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a; +} - let inner_offsets = inner_list_array.offsets(); - let inner_lengths = inner_list_array.offsets().lengths().collect_vec(); +/// The actual implementation of `impl_native_type!`, so that we don't have to work in a macro. +fn slice_as_native<'a, T: arrow2::types::NativeType + arrow::datatypes::ArrowNativeType>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, +) -> impl Iterator + 'a { + let Some(values) = array.as_any().downcast_ref::>() else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + let values = values.values().as_slice(); - let Some(fixed_size_list_array) = inner_list_array - .values() - .as_any() - .downcast_ref::() - else { - if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); - } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); + // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. + Either::Right(component_offsets.map(move |(idx, len)| &values[idx..idx + len])) +} + +// We use a macro instead of a blanket impl because this violates orphan rules. +macro_rules! impl_native_type { + ($type:ty) => { + impl ChunkComponentSlicer for $type { + type Item<'a> = &'a [$type]; + + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a { + slice_as_native(component_name, array, component_offsets) } - return Either::Left(std::iter::empty()); - }; + } + }; +} - let Some(values) = fixed_size_list_array - .values() - .as_any() - .downcast_ref::>() - else { - if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); - } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); +impl_native_type!(u8); +impl_native_type!(u16); +impl_native_type!(u32); +impl_native_type!(u64); +impl_native_type!(i8); +impl_native_type!(i16); +impl_native_type!(i32); +impl_native_type!(i64); +impl_native_type!(f32); +impl_native_type!(f64); +impl_native_type!(i128); + +/// The actual implementation of `impl_array_native_type!`, so that we don't have to work in a macro. +fn slice_as_array_native< + 'a, + const N: usize, + T: arrow2::types::NativeType + arrow::datatypes::ArrowNativeType, +>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, +) -> impl Iterator + 'a +where + [T; N]: bytemuck::Pod, +{ + let Some(fixed_size_list_array) = array.as_any().downcast_ref::() + else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let Some(values) = fixed_size_list_array + .values() + .as_any() + .downcast_ref::>() + else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let size = fixed_size_list_array.size(); + let values = values.values().as_slice(); + + // NOTE: No need for validity checks here, `component_offsets` already takes care of that. + Either::Right( + component_offsets.map(move |(idx, len)| { + bytemuck::cast_slice(&values[idx * size..idx * size + len * size]) + }), + ) +} + +// We use a macro instead of a blanket impl because this violates orphan rules. +macro_rules! impl_array_native_type { + ($type:ty) => { + impl ChunkComponentSlicer for [$type; N] + where + [$type; N]: bytemuck::Pod, + { + type Item<'a> = &'a [[$type; N]]; + + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a { + slice_as_array_native(component_name, array, component_offsets) } - return Either::Left(std::iter::empty()); - }; + } + }; +} - let size = fixed_size_list_array.size(); - let values = values.values(); +impl_array_native_type!(u8); +impl_array_native_type!(u16); +impl_array_native_type!(u32); +impl_array_native_type!(u64); +impl_array_native_type!(i8); +impl_array_native_type!(i16); +impl_array_native_type!(i32); +impl_array_native_type!(i64); +impl_array_native_type!(f32); +impl_array_native_type!(f64); +impl_array_native_type!(i128); + +/// The actual implementation of `impl_buffer_native_type!`, so that we don't have to work in a macro. +fn slice_as_buffer_native<'a, T: arrow2::types::NativeType + arrow::datatypes::ArrowNativeType>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, +) -> impl Iterator>> + 'a { + let Some(inner_list_array) = array.as_any().downcast_ref::>() else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let Some(values) = inner_list_array + .values() + .as_any() + .downcast_ref::>() + else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let values = values.values(); + let offsets = inner_list_array.offsets(); + let lengths = inner_list_array.offsets().lengths().collect_vec(); + + // NOTE: No need for validity checks here, `component_offsets` already takes care of that. + Either::Right(component_offsets.map(move |(idx, len)| { + let offsets = &offsets.as_slice()[idx..idx + len]; + let lengths = &lengths.as_slice()[idx..idx + len]; + izip!(offsets, lengths) + // NOTE: Not an actual clone, just a refbump of the underlying buffer. + .map(|(&idx, &len)| values.clone().sliced(idx as _, len).into()) + .collect_vec() + })) +} - // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. - Either::Right( - self.iter_component_offsets(component_name) - .map(move |(idx, len)| { - let inner_offsets = &inner_offsets.as_slice()[idx..idx + len]; - let inner_lengths = &inner_lengths.as_slice()[idx..idx + len]; - izip!(inner_offsets, inner_lengths) - .map(|(&idx, &len)| { - let idx = idx as usize; - bytemuck::cast_slice(&values[idx * size..idx * size + len * size]) - }) - .collect_vec() - }), - ) - } +// We use a macro instead of a blanket impl because this violates orphan rules. +macro_rules! impl_buffer_native_type { + ($type:ty) => { + impl ChunkComponentSlicer for &[$type] { + type Item<'a> = Vec>; + + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a { + slice_as_buffer_native(component_name, array, component_offsets) + } + } + }; +} - /// Returns an iterator over the raw strings of a [`Chunk`], for a given component. - /// - /// This is a very fast path: the entire column will be downcasted at once, and then every - /// component batch will be a slice reference into that global slice. - /// Use this when working with simple arrow datatypes and performance matters (e.g. labels, etc). - /// - /// See also: - /// * [`Self::iter_primitive`] - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_primitive_array_list`] - /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component`]. - pub fn iter_string( - &self, - component_name: &ComponentName, - ) -> impl Iterator> + '_ { - let Some(list_array) = self.get_first_component(component_name) else { - return Either::Left(std::iter::empty()); - }; +impl_buffer_native_type!(u8); +impl_buffer_native_type!(u16); +impl_buffer_native_type!(u32); +impl_buffer_native_type!(u64); +impl_buffer_native_type!(i8); +impl_buffer_native_type!(i16); +impl_buffer_native_type!(i32); +impl_buffer_native_type!(i64); +impl_buffer_native_type!(f32); +impl_buffer_native_type!(f64); +impl_buffer_native_type!(i128); + +/// The actual implementation of `impl_array_list_native_type!`, so that we don't have to work in a macro. +fn slice_as_array_list_native< + 'a, + const N: usize, + T: arrow2::types::NativeType + arrow::datatypes::ArrowNativeType, +>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, +) -> impl Iterator> + 'a +where + [T; N]: bytemuck::Pod, +{ + let Some(inner_list_array) = array.as_any().downcast_ref::>() else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let inner_offsets = inner_list_array.offsets(); + let inner_lengths = inner_list_array.offsets().lengths().collect_vec(); + + let Some(fixed_size_list_array) = inner_list_array + .values() + .as_any() + .downcast_ref::() + else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let Some(values) = fixed_size_list_array + .values() + .as_any() + .downcast_ref::>() + else { + if cfg!(debug_assertions) { + panic!("downcast failed for {component_name}, data discarded"); + } else { + re_log::error_once!("downcast failed for {component_name}, data discarded"); + } + return Either::Left(std::iter::empty()); + }; + + let size = fixed_size_list_array.size(); + let values = values.values(); + + // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. + Either::Right(component_offsets.map(move |(idx, len)| { + let inner_offsets = &inner_offsets.as_slice()[idx..idx + len]; + let inner_lengths = &inner_lengths.as_slice()[idx..idx + len]; + izip!(inner_offsets, inner_lengths) + .map(|(&idx, &len)| { + let idx = idx as usize; + bytemuck::cast_slice(&values[idx * size..idx * size + len * size]) + }) + .collect_vec() + })) +} - let Some(utf8_array) = list_array - .values() - .as_any() - .downcast_ref::>() - else { +// We use a macro instead of a blanket impl because this violates orphan rules. +macro_rules! impl_array_list_native_type { + ($type:ty) => { + impl ChunkComponentSlicer for &[[$type; N]] + where + [$type; N]: bytemuck::Pod, + { + type Item<'a> = Vec<&'a [[$type; N]]>; + + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a { + slice_as_array_list_native(component_name, array, component_offsets) + } + } + }; +} + +impl_array_list_native_type!(u8); +impl_array_list_native_type!(u16); +impl_array_list_native_type!(u32); +impl_array_list_native_type!(u64); +impl_array_list_native_type!(i8); +impl_array_list_native_type!(i16); +impl_array_list_native_type!(i32); +impl_array_list_native_type!(i64); +impl_array_list_native_type!(f32); +impl_array_list_native_type!(f64); +impl_array_list_native_type!(i128); + +impl ChunkComponentSlicer for String { + type Item<'a> = Vec; + + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a { + let Some(utf8_array) = array.as_any().downcast_ref::>() else { if cfg!(debug_assertions) { panic!("downcast failed for {component_name}, data discarded"); } else { @@ -467,57 +631,26 @@ impl Chunk { let offsets = utf8_array.offsets(); let lengths = utf8_array.offsets().lengths().collect_vec(); - // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. - Either::Right( - self.iter_component_offsets(component_name) - .map(move |(idx, len)| { - let offsets = &offsets.as_slice()[idx..idx + len]; - let lengths = &lengths.as_slice()[idx..idx + len]; - izip!(offsets, lengths) - .map(|(&idx, &len)| ArrowString::from(values.clone().sliced(idx as _, len))) - .collect_vec() - }), - ) + // NOTE: No need for validity checks here, `component_offsets` already takes care of that. + Either::Right(component_offsets.map(move |(idx, len)| { + let offsets = &offsets.as_slice()[idx..idx + len]; + let lengths = &lengths.as_slice()[idx..idx + len]; + izip!(offsets, lengths) + .map(|(&idx, &len)| ArrowString::from(values.clone().sliced(idx as _, len))) + .collect_vec() + })) } +} - /// Returns an iterator over the raw buffers of a [`Chunk`], for a given component. - /// - /// This is a very fast path: the entire column will be downcasted at once, and then every - /// component batch will be a slice reference into that global slice. - /// Use this when working with simple arrow datatypes and performance matters (e.g. blobs, etc). - /// - /// See also: - /// * [`Self::iter_primitive`] - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_primitive_array_list`] - /// * [`Self::iter_string`]. - /// * [`Self::iter_component`]. - pub fn iter_buffer( - &self, - component_name: &ComponentName, - ) -> impl Iterator>> + '_ { - let Some(list_array) = self.get_first_component(component_name) else { - return Either::Left(std::iter::empty()); - }; - - let Some(inner_list_array) = list_array - .values() - .as_any() - .downcast_ref::>() - else { - if cfg!(debug_assertions) { - panic!("downcast failed for {component_name}, data discarded"); - } else { - re_log::error_once!("downcast failed for {component_name}, data discarded"); - } - return Either::Left(std::iter::empty()); - }; +impl ChunkComponentSlicer for bool { + type Item<'a> = Arrow2Bitmap; - let Some(values) = inner_list_array - .values() - .as_any() - .downcast_ref::>() - else { + fn slice<'a>( + component_name: ComponentName, + array: &'a dyn Arrow2Array, + component_offsets: impl Iterator + 'a, + ) -> impl Iterator> + 'a { + let Some(values) = array.as_any().downcast_ref::() else { if cfg!(debug_assertions) { panic!("downcast failed for {component_name}, data discarded"); } else { @@ -525,23 +658,10 @@ impl Chunk { } return Either::Left(std::iter::empty()); }; + let values = values.values().clone(); - let values = values.values(); - let offsets = inner_list_array.offsets(); - let lengths = inner_list_array.offsets().lengths().collect_vec(); - - // NOTE: No need for validity checks here, `iter_offsets` already takes care of that. - Either::Right( - self.iter_component_offsets(component_name) - .map(move |(idx, len)| { - let offsets = &offsets.as_slice()[idx..idx + len]; - let lengths = &lengths.as_slice()[idx..idx + len]; - izip!(offsets, lengths) - // NOTE: Not an actual clone, just a refbump of the underlying buffer. - .map(|(&idx, &len)| values.clone().sliced(idx as _, len).into()) - .collect_vec() - }), - ) + // NOTE: No need for validity checks here, `component_offsets` already takes care of that. + Either::Right(component_offsets.map(move |(idx, len)| values.clone().sliced(idx, len))) } } @@ -708,11 +828,8 @@ impl Chunk { /// things, and is therefore very slow at the moment. Avoid this on performance critical paths. /// /// See also: - /// * [`Self::iter_primitive`] - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_primitive_array_list`] - /// * [`Self::iter_string`] - /// * [`Self::iter_buffer`]. + /// * [`Self::iter_slices`] + /// * [`Self::iter_slices_from_struct_field`] #[inline] pub fn iter_component( &self, diff --git a/crates/store/re_chunk/src/lib.rs b/crates/store/re_chunk/src/lib.rs index 9ee2ed555e18..6f24bbe8d371 100644 --- a/crates/store/re_chunk/src/lib.rs +++ b/crates/store/re_chunk/src/lib.rs @@ -28,7 +28,9 @@ pub use self::builder::{ChunkBuilder, TimeColumnBuilder}; pub use self::chunk::{Chunk, ChunkComponents, ChunkError, ChunkResult, TimeColumn}; pub use self::helpers::{ChunkShared, UnitChunkShared}; pub use self::id::{ChunkId, RowId}; -pub use self::iter::{ChunkComponentIter, ChunkComponentIterItem, ChunkIndicesIter}; +pub use self::iter::{ + ChunkComponentIter, ChunkComponentIterItem, ChunkComponentSlicer, ChunkIndicesIter, +}; pub use self::latest_at::LatestAtQuery; pub use self::range::{RangeQuery, RangeQueryOptions}; pub use self::transport::TransportChunk; diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 9be46a4b8260..46b4fc33cf0b 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -480,7 +480,7 @@ async fn stream_catalog_async( )))?; let recording_uri_arrays: Vec> = chunk - .iter_string(&"id".into()) + .iter_slices::("id".into()) .map(|id| { let rec_id = &id[0]; // each component batch is of length 1 i.e. single 'id' value diff --git a/crates/store/re_query/examples/range.rs b/crates/store/re_query/examples/range.rs index e90127801706..296cef4c47df 100644 --- a/crates/store/re_query/examples/range.rs +++ b/crates/store/re_query/examples/range.rs @@ -65,7 +65,7 @@ fn main() -> anyhow::Result<()> { .flat_map(|chunk| { izip!( chunk.iter_component_indices(&query.timeline(), &MyColor::name()), - chunk.iter_primitive::(&MyColor::name()), + chunk.iter_slices::(MyColor::name()), ) }); diff --git a/crates/store/re_query/tests/range.rs b/crates/store/re_query/tests/range.rs index 472fe47e8493..1ee4cecbde95 100644 --- a/crates/store/re_query/tests/range.rs +++ b/crates/store/re_query/tests/range.rs @@ -1089,7 +1089,7 @@ fn query_and_compare( .flat_map(|chunk| { itertools::izip!( chunk.iter_component_indices(&query.timeline(), &MyColor::name()), - chunk.iter_primitive::(&MyColor::name()), + chunk.iter_slices::(MyColor::name()), ) }) .collect_vec(); diff --git a/crates/store/re_types_core/src/lib.rs b/crates/store/re_types_core/src/lib.rs index 52fb0659c524..a9446c8d44e5 100644 --- a/crates/store/re_types_core/src/lib.rs +++ b/crates/store/re_types_core/src/lib.rs @@ -345,7 +345,7 @@ pub mod external { /// #[macro_export] macro_rules! static_assert_struct_has_fields { - ($strct:ty, $($field:ident: $field_typ:ty),+) => { + ($strct:ty, $($field:ident: $field_typ:ty),+ $(,)?) => { const _: fn(&$strct) = |s: &$strct| { $(let _: &$field_typ = &s.$field;)+ }; diff --git a/crates/viewer/re_view/src/results_ext.rs b/crates/viewer/re_view/src/results_ext.rs index f51350480e55..12f2907388f9 100644 --- a/crates/viewer/re_view/src/results_ext.rs +++ b/crates/viewer/re_view/src/results_ext.rs @@ -3,9 +3,9 @@ use std::{borrow::Cow, sync::Arc}; use itertools::Itertools as _; use re_chunk_store::{Chunk, LatestAtQuery, RangeQuery, UnitChunkShared}; -use re_log_types::external::arrow2::bitmap::Bitmap as Arrow2Bitmap; use re_log_types::hash::Hash64; use re_query::{LatestAtResults, RangeResults}; +use re_types::ArchetypeFieldName; use re_types_core::ComponentName; use re_viewer_context::{DataResult, QueryContext, ViewContext}; @@ -239,9 +239,8 @@ pub trait RangeResultsExt { /// Returns a zero-copy iterator over all the results for the given `(timeline, component)` pair. /// /// Call one of the following methods on the returned [`HybridResultsChunkIter`]: - /// * [`HybridResultsChunkIter::primitive`] - /// * [`HybridResultsChunkIter::primitive_array`] - /// * [`HybridResultsChunkIter::string`] + /// * [`HybridResultsChunkIter::slice`] + /// * [`HybridResultsChunkIter::slice_from_struct_field`] fn iter_as( &self, timeline: Timeline, @@ -408,7 +407,7 @@ impl RangeResultsExt for HybridResults<'_> { // --- use re_chunk::{ChunkComponentIterItem, RowId, TimeInt, Timeline}; -use re_chunk_store::external::{re_chunk, re_chunk::external::arrow2}; +use re_chunk_store::external::re_chunk; /// The iterator type backing [`HybridResults::iter_as`]. pub struct HybridResultsChunkIter<'a> { @@ -436,90 +435,31 @@ impl<'a> HybridResultsChunkIter<'a> { }) } - /// Iterate as indexed booleans. + /// Iterate as indexed, sliced, deserialized component batches. /// - /// See [`Chunk::iter_bool`] for more information. - pub fn bool(&'a self) -> impl Iterator + 'a { - self.chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&self.timeline, &self.component_name), - chunk.iter_bool(&self.component_name) - ) - }) - } - - /// Iterate as indexed primitives. - /// - /// See [`Chunk::iter_primitive`] for more information. - pub fn primitive( - &'a self, - ) -> impl Iterator + 'a { - self.chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&self.timeline, &self.component_name), - chunk.iter_primitive::(&self.component_name) - ) - }) - } - - /// Iterate as indexed primitive arrays. - /// - /// See [`Chunk::iter_primitive_array`] for more information. - pub fn primitive_array( - &'a self, - ) -> impl Iterator + 'a - where - [T; N]: bytemuck::Pod, - { - self.chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&self.timeline, &self.component_name), - chunk.iter_primitive_array::(&self.component_name) - ) - }) - } - - /// Iterate as indexed list of primitive arrays. - /// - /// See [`Chunk::iter_primitive_array_list`] for more information. - pub fn primitive_array_list( - &'a self, - ) -> impl Iterator)> + 'a - where - [T; N]: bytemuck::Pod, - { - self.chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&self.timeline, &self.component_name), - chunk.iter_primitive_array_list::(&self.component_name) - ) - }) - } - - /// Iterate as indexed UTF-8 strings. - /// - /// See [`Chunk::iter_string`] for more information. - pub fn string( + /// See [`Chunk::iter_slices`] for more information. + pub fn slice( &'a self, - ) -> impl Iterator)> + 'a { + ) -> impl Iterator)> + 'a { self.chunks.iter().flat_map(|chunk| { itertools::izip!( chunk.iter_component_indices(&self.timeline, &self.component_name), - chunk.iter_string(&self.component_name) + chunk.iter_slices::(self.component_name) ) }) } - /// Iterate as indexed buffers. + /// Iterate as indexed, sliced, deserialized component batches for a specific struct field. /// - /// See [`Chunk::iter_buffer`] for more information. - pub fn buffer( + /// See [`Chunk::iter_slices_from_struct_field`] for more information. + pub fn slice_from_struct_field( &'a self, - ) -> impl Iterator>)> + 'a { + field_name: &'a ArchetypeFieldName, + ) -> impl Iterator)> + 'a { self.chunks.iter().flat_map(|chunk| { itertools::izip!( chunk.iter_component_indices(&self.timeline, &self.component_name), - chunk.iter_buffer(&self.component_name) + chunk.iter_slices_from_struct_field::(self.component_name, field_name) ) }) } diff --git a/crates/viewer/re_view_graph/Cargo.toml b/crates/viewer/re_view_graph/Cargo.toml index 87841bba1e15..b1f469349f5b 100644 --- a/crates/viewer/re_view_graph/Cargo.toml +++ b/crates/viewer/re_view_graph/Cargo.toml @@ -37,4 +37,5 @@ re_viewport_blueprint.workspace = true ahash.workspace = true egui.workspace = true fjadra.workspace = true +itertools.workspace = true nohash-hasher.workspace = true diff --git a/crates/viewer/re_view_graph/src/visualizers/edges.rs b/crates/viewer/re_view_graph/src/visualizers/edges.rs index fac36b789bf5..3847a75ed358 100644 --- a/crates/viewer/re_view_graph/src/visualizers/edges.rs +++ b/crates/viewer/re_view_graph/src/visualizers/edges.rs @@ -1,10 +1,6 @@ -use re_chunk::LatestAtQuery; +use re_chunk::{ArchetypeFieldName, LatestAtQuery}; use re_log_types::{EntityPath, Instance}; -use re_types::{ - self, archetypes, - components::{self, GraphEdge, GraphNode}, - Component as _, -}; +use re_types::{self, archetypes, components, datatypes, Component as _}; use re_view::{DataResultQuery, RangeResultsExt}; use re_viewer_context::{ self, IdentifiedViewSystem, ViewContext, ViewContextCollection, ViewQuery, @@ -21,8 +17,8 @@ pub struct EdgesVisualizer { pub struct EdgeInstance { // We will need this in the future, when we want to select individual edges. pub _instance: Instance, - pub source: GraphNode, - pub target: GraphNode, + pub source: components::GraphNode, + pub target: components::GraphNode, pub source_index: NodeId, pub target_index: NodeId, } @@ -52,6 +48,15 @@ impl VisualizerSystem for EdgesVisualizer { ) -> Result, ViewSystemExecutionError> { let timeline_query = LatestAtQuery::new(query.timeline, query.latest_at); + // TODO(cmc): could we (improve and then) use reflection for this? + re_types::static_assert_struct_has_fields!( + datatypes::Utf8Pair, + first: datatypes::Utf8, + second: datatypes::Utf8, + ); + let source: ArchetypeFieldName = "first".into(); + let target: ArchetypeFieldName = "second".into(); + for data_result in query.iter_visible_data_results(ctx, Self::identifier()) { let results = data_result .latest_at_with_blueprint_resolved_data::( @@ -59,17 +64,22 @@ impl VisualizerSystem for EdgesVisualizer { &timeline_query, ); - let all_indexed_edges = results.iter_as(query.timeline, components::GraphEdge::name()); + let all_edges = results.iter_as(query.timeline, components::GraphEdge::name()); let graph_type = results.get_mono_with_fallback::(); - // TODO(cmc): Provide a `iter_struct`. - for (_index, edges) in all_indexed_edges.component_slow::() { - let edges = edges - .iter() + let sources = all_edges + .slice_from_struct_field::(&source) + .map(|(_index, source)| source); + let targets = all_edges + .slice_from_struct_field::(&target) + .map(|(_index, target)| target); + + for (sources, targets) in itertools::izip!(sources, targets) { + let edges = itertools::izip!(sources, targets) .enumerate() - .map(|(i, edge)| { - let source = GraphNode::from(edge.first.clone()); - let target = GraphNode::from(edge.second.clone()); + .map(|(i, (source, target))| { + let source = components::GraphNode(source.into()); + let target = components::GraphNode(target.into()); let entity_path = &data_result.entity_path; let source_index = NodeId::from_entity_node(entity_path, &source); diff --git a/crates/viewer/re_view_graph/src/visualizers/nodes.rs b/crates/viewer/re_view_graph/src/visualizers/nodes.rs index ce0a06fa7200..bdb926e128de 100644 --- a/crates/viewer/re_view_graph/src/visualizers/nodes.rs +++ b/crates/viewer/re_view_graph/src/visualizers/nodes.rs @@ -78,7 +78,7 @@ impl VisualizerSystem for NodeVisualizer { &timeline_query, ); - let all_indexed_nodes = results.iter_as(query.timeline, components::GraphNode::name()); + let all_nodes = results.iter_as(query.timeline, components::GraphNode::name()); let all_colors = results.iter_as(query.timeline, components::Color::name()); let all_positions = results.iter_as(query.timeline, components::Position2D::name()); let all_labels = results.iter_as(query.timeline, components::Text::name()); @@ -88,12 +88,11 @@ impl VisualizerSystem for NodeVisualizer { .map_or(true, bool::from); let data = range_zip_1x4( - // TODO(cmc): Provide a `iter_struct`. - all_indexed_nodes.component_slow::(), - all_colors.primitive::(), - all_positions.primitive_array::<2, f32>(), - all_labels.string(), - all_radii.primitive::(), + all_nodes.slice::(), + all_colors.slice::(), + all_positions.slice::<[f32; 2]>(), + all_labels.slice::(), + all_radii.slice::(), ); for (_index, nodes, colors, positions, labels, radii) in data { @@ -114,6 +113,7 @@ impl VisualizerSystem for NodeVisualizer { Option::::default, ) .map(|(node, instance, color, position, label, radius)| { + let node = components::GraphNode(node.clone().into()); let color = color.map(|&c| egui::Color32::from(Color::new(c))); let label = match (label, show_label) { (Some(label), true) => Label::Text { @@ -133,7 +133,7 @@ impl VisualizerSystem for NodeVisualizer { NodeInstance { instance_index: instance, - id: NodeId::from_entity_node(&data_result.entity_path, node), + id: NodeId::from_entity_node(&data_result.entity_path, &node), position: position.map(|[x, y]| egui::Pos2::new(x, y)), label, } diff --git a/crates/viewer/re_view_map/src/visualizers/geo_line_strings.rs b/crates/viewer/re_view_map/src/visualizers/geo_line_strings.rs index 74550c4f368a..bd1c9341be55 100644 --- a/crates/viewer/re_view_map/src/visualizers/geo_line_strings.rs +++ b/crates/viewer/re_view_map/src/visualizers/geo_line_strings.rs @@ -66,9 +66,9 @@ impl VisualizerSystem for GeoLineStringsVisualizer { // iterate over each chunk and find all relevant component slices for (_index, lines, colors, radii) in re_query::range_zip_1x2( - all_lines.primitive_array_list::<2, f64>(), - all_colors.primitive::(), - all_radii.primitive::(), + all_lines.slice::<&[[f64; 2]]>(), + all_colors.slice::(), + all_radii.slice::(), ) { // required component let lines = lines.as_slice(); diff --git a/crates/viewer/re_view_map/src/visualizers/geo_points.rs b/crates/viewer/re_view_map/src/visualizers/geo_points.rs index f904049f8893..eff0e4aecc56 100644 --- a/crates/viewer/re_view_map/src/visualizers/geo_points.rs +++ b/crates/viewer/re_view_map/src/visualizers/geo_points.rs @@ -69,10 +69,10 @@ impl VisualizerSystem for GeoPointsVisualizer { // iterate over each chunk and find all relevant component slices for (_index, positions, colors, radii, class_ids) in re_query::range_zip_1x3( - all_positions.primitive_array::<2, f64>(), - all_colors.primitive::(), - all_radii.primitive::(), - all_class_ids.primitive::(), + all_positions.slice::<[f64; 2]>(), + all_colors.slice::(), + all_radii.slice::(), + all_class_ids.slice::(), ) { // required component let num_instances = positions.len(); diff --git a/crates/viewer/re_view_spatial/Cargo.toml b/crates/viewer/re_view_spatial/Cargo.toml index 71c69a1082df..dec170042964 100644 --- a/crates/viewer/re_view_spatial/Cargo.toml +++ b/crates/viewer/re_view_spatial/Cargo.toml @@ -19,6 +19,7 @@ workspace = true all-features = true [dependencies] +re_chunk.workspace = true re_chunk_store.workspace = true re_data_ui.workspace = true re_entity_db.workspace = true diff --git a/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs b/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs index 327d24b60db5..02f025bac866 100644 --- a/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs +++ b/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs @@ -84,8 +84,8 @@ impl PerStoreChunkSubscriber for MaxImageDimensionsStoreSubscriber { } // Handle `ImageEncoded`, `AssetVideo`… - let blobs = event.diff.chunk.iter_buffer(&Blob::name()); - let media_types = event.diff.chunk.iter_string(&MediaType::name()); + let blobs = event.diff.chunk.iter_slices::<&[u8]>(Blob::name()); + let media_types = event.diff.chunk.iter_slices::(MediaType::name()); for (blob, media_type) in itertools::izip!(blobs, media_types.map(Some).chain(std::iter::repeat(None))) { diff --git a/crates/viewer/re_view_spatial/src/visualizers/arrows2d.rs b/crates/viewer/re_view_spatial/src/visualizers/arrows2d.rs index 93b2e76457a6..4a8a6c42e771 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/arrows2d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/arrows2d.rs @@ -200,7 +200,7 @@ impl VisualizerSystem for Arrows2DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -214,7 +214,7 @@ impl VisualizerSystem for Arrows2DVisualizer { let num_vectors = all_vector_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<2, f32>(&Vector2D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 2]>(Vector2D::name())) .map(|vectors| vectors.len()) .sum(); @@ -227,7 +227,7 @@ impl VisualizerSystem for Arrows2DVisualizer { let timeline = ctx.query.timeline(); let all_vectors_indexed = - iter_primitive_array::<2, f32>(&all_vector_chunks, timeline, Vector2D::name()); + iter_slices::<[f32; 2]>(&all_vector_chunks, timeline, Vector2D::name()); let all_origins = results.iter_as(timeline, Position2D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); @@ -238,13 +238,13 @@ impl VisualizerSystem for Arrows2DVisualizer { let data = re_query::range_zip_1x7( all_vectors_indexed, - all_origins.primitive_array::<2, f32>(), - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_keypoint_ids.primitive::(), - all_show_labels.bool(), + all_origins.slice::<[f32; 2]>(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_keypoint_ids.slice::(), + all_show_labels.slice::(), ) .map( |( diff --git a/crates/viewer/re_view_spatial/src/visualizers/arrows3d.rs b/crates/viewer/re_view_spatial/src/visualizers/arrows3d.rs index 2e02f969f53d..9739ae8c1bc7 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/arrows3d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/arrows3d.rs @@ -199,7 +199,7 @@ impl VisualizerSystem for Arrows3DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -213,7 +213,7 @@ impl VisualizerSystem for Arrows3DVisualizer { let num_vectors = all_vector_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<3, f32>(&Vector3D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 3]>(Vector3D::name())) .map(|vectors| vectors.len()) .sum(); @@ -226,7 +226,7 @@ impl VisualizerSystem for Arrows3DVisualizer { let timeline = ctx.query.timeline(); let all_vectors_indexed = - iter_primitive_array::<3, f32>(&all_vector_chunks, timeline, Vector3D::name()); + iter_slices::<[f32; 3]>(&all_vector_chunks, timeline, Vector3D::name()); let all_origins = results.iter_as(timeline, Position3D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); @@ -236,12 +236,12 @@ impl VisualizerSystem for Arrows3DVisualizer { let data = re_query::range_zip_1x6( all_vectors_indexed, - all_origins.primitive_array::<3, f32>(), - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_show_labels.bool(), + all_origins.slice::<[f32; 3]>(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_show_labels.slice::(), ) .map( |(_index, vectors, origins, colors, radii, labels, class_ids, show_labels)| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/assets3d.rs b/crates/viewer/re_view_spatial/src/visualizers/assets3d.rs index ab884dade8fc..cf5fa4df0eba 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/assets3d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/assets3d.rs @@ -142,7 +142,7 @@ impl VisualizerSystem for Asset3DVisualizer { let mut instances = Vec::new(); - use super::entity_iterator::{iter_buffer, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -155,7 +155,8 @@ impl VisualizerSystem for Asset3DVisualizer { }; let timeline = ctx.query.timeline(); - let all_blobs_indexed = iter_buffer::(&all_blob_chunks, timeline, Blob::name()); + let all_blobs_indexed = + iter_slices::<&[u8]>(&all_blob_chunks, timeline, Blob::name()); let all_media_types = results.iter_as(timeline, MediaType::name()); let all_albedo_factors = results.iter_as(timeline, AlbedoFactor::name()); @@ -163,8 +164,8 @@ impl VisualizerSystem for Asset3DVisualizer { let data = re_query::range_zip_1x2( all_blobs_indexed, - all_media_types.string(), - all_albedo_factors.primitive::(), + all_media_types.slice::(), + all_albedo_factors.slice::(), ) .filter_map(|(index, blobs, media_types, albedo_factors)| { blobs.first().map(|blob| Asset3DComponentData { diff --git a/crates/viewer/re_view_spatial/src/visualizers/boxes2d.rs b/crates/viewer/re_view_spatial/src/visualizers/boxes2d.rs index 2a236f2cd3e6..75434f6c4638 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/boxes2d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/boxes2d.rs @@ -199,7 +199,7 @@ impl VisualizerSystem for Boxes2DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -214,7 +214,7 @@ impl VisualizerSystem for Boxes2DVisualizer { let num_boxes: usize = all_half_size_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<2, f32>(&HalfSize2D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 2]>(HalfSize2D::name())) .map(|vectors| vectors.len()) .sum(); if num_boxes == 0 { @@ -226,11 +226,8 @@ impl VisualizerSystem for Boxes2DVisualizer { line_builder.reserve_vertices(num_boxes * 5)?; let timeline = ctx.query.timeline(); - let all_half_sizes_indexed = iter_primitive_array::<2, f32>( - &all_half_size_chunks, - timeline, - HalfSize2D::name(), - ); + let all_half_sizes_indexed = + iter_slices::<[f32; 2]>(&all_half_size_chunks, timeline, HalfSize2D::name()); let all_centers = results.iter_as(timeline, Position2D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); @@ -240,12 +237,12 @@ impl VisualizerSystem for Boxes2DVisualizer { let data = re_query::range_zip_1x6( all_half_sizes_indexed, - all_centers.primitive_array::<2, f32>(), - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_show_labels.bool(), + all_centers.slice::<[f32; 2]>(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_show_labels.slice::(), ) .map( |( diff --git a/crates/viewer/re_view_spatial/src/visualizers/boxes3d.rs b/crates/viewer/re_view_spatial/src/visualizers/boxes3d.rs index 2d48e092ed32..b6c90cd65ed6 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/boxes3d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/boxes3d.rs @@ -121,7 +121,7 @@ impl VisualizerSystem for Boxes3DVisualizer { let mut builder = ProcMeshDrawableBuilder::new(&mut self.0, render_ctx, view_query, "boxes3d", &Fallback); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -136,7 +136,7 @@ impl VisualizerSystem for Boxes3DVisualizer { let num_boxes: usize = all_half_size_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<3, f32>(&HalfSize3D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 3]>(HalfSize3D::name())) .map(|vectors| vectors.len()) .sum(); if num_boxes == 0 { @@ -144,11 +144,8 @@ impl VisualizerSystem for Boxes3DVisualizer { } let timeline = ctx.query.timeline(); - let all_half_sizes_indexed = iter_primitive_array::<3, f32>( - &all_half_size_chunks, - timeline, - HalfSize3D::name(), - ); + let all_half_sizes_indexed = + iter_slices::<[f32; 3]>(&all_half_size_chunks, timeline, HalfSize3D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); let all_labels = results.iter_as(timeline, Text::name()); @@ -159,7 +156,7 @@ impl VisualizerSystem for Boxes3DVisualizer { let all_fill_modes = results.iter_as(timeline, FillMode::name()); // fill mode is currently a non-repeated component let fill_mode: FillMode = all_fill_modes - .primitive::() + .slice::() .next() .and_then(|(_, fill_modes)| { fill_modes.first().copied().and_then(FillMode::from_u8) @@ -179,11 +176,11 @@ impl VisualizerSystem for Boxes3DVisualizer { let data = re_query::range_zip_1x5( all_half_sizes_indexed, - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_show_labels.bool(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_show_labels.slice::(), ) .map( |(_index, half_sizes, colors, radii, labels, class_ids, show_labels)| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/capsules3d.rs b/crates/viewer/re_view_spatial/src/visualizers/capsules3d.rs index a2e2c5f1b41c..a85b1d630680 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/capsules3d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/capsules3d.rs @@ -148,7 +148,7 @@ impl VisualizerSystem for Capsules3DVisualizer { &Fallback, ); - use super::entity_iterator::{iter_primitive, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -167,12 +167,12 @@ impl VisualizerSystem for Capsules3DVisualizer { let num_lengths: usize = all_length_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive::(&Length::name())) + .flat_map(|chunk| chunk.iter_slices::(Length::name())) .map(|lengths| lengths.len()) .sum(); let num_radii: usize = all_radius_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive::(&components::Radius::name())) + .flat_map(|chunk| chunk.iter_slices::(components::Radius::name())) .map(|radii| radii.len()) .sum(); let num_instances = num_lengths.max(num_radii); @@ -182,9 +182,9 @@ impl VisualizerSystem for Capsules3DVisualizer { let timeline = ctx.query.timeline(); let all_lengths_indexed = - iter_primitive::(&all_length_chunks, timeline, Length::name()); + iter_slices::(&all_length_chunks, timeline, Length::name()); let all_radii_indexed = - iter_primitive::(&all_radius_chunks, timeline, components::Radius::name()); + iter_slices::(&all_radius_chunks, timeline, components::Radius::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_labels = results.iter_as(timeline, Text::name()); let all_show_labels = results.iter_as(timeline, ShowLabels::name()); @@ -193,10 +193,10 @@ impl VisualizerSystem for Capsules3DVisualizer { let data = re_query::range_zip_2x4( all_lengths_indexed, all_radii_indexed, - all_colors.primitive::(), - all_labels.string(), - all_show_labels.bool(), - all_class_ids.primitive::(), + all_colors.slice::(), + all_labels.slice::(), + all_show_labels.slice::(), + all_class_ids.slice::(), ) .map( |(_index, lengths, radii, colors, labels, show_labels, class_ids)| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/depth_images.rs b/crates/viewer/re_view_spatial/src/visualizers/depth_images.rs index 94c53af9c08b..7c62b385459d 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/depth_images.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/depth_images.rs @@ -259,7 +259,7 @@ impl VisualizerSystem for DepthImageVisualizer { let mut depth_clouds = Vec::new(); - use super::entity_iterator::{iter_buffer, iter_component, process_archetype}; + use super::entity_iterator::{iter_component, iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -278,7 +278,7 @@ impl VisualizerSystem for DepthImageVisualizer { let timeline = ctx.query.timeline(); let all_buffers_indexed = - iter_buffer::(&all_buffer_chunks, timeline, ImageBuffer::name()); + iter_slices::<&[u8]>(&all_buffer_chunks, timeline, ImageBuffer::name()); let all_formats_indexed = iter_component::( &all_format_chunks, timeline, @@ -292,10 +292,10 @@ impl VisualizerSystem for DepthImageVisualizer { let mut data = re_query::range_zip_1x5( all_buffers_indexed, all_formats_indexed, - all_colormaps.primitive::(), - all_value_ranges.primitive_array::<2, f64>(), - all_depth_meters.primitive::(), - all_fill_ratios.primitive::(), + all_colormaps.slice::(), + all_value_ranges.slice::<[f64; 2]>(), + all_depth_meters.slice::(), + all_fill_ratios.slice::(), ) .filter_map( |(index, buffers, format, colormap, value_range, depth_meter, fill_ratio)| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/ellipsoids.rs b/crates/viewer/re_view_spatial/src/visualizers/ellipsoids.rs index fa4f55c57f6c..b4e9b252344e 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/ellipsoids.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/ellipsoids.rs @@ -133,7 +133,7 @@ impl VisualizerSystem for Ellipsoids3DVisualizer { &Fallback, ); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -148,7 +148,7 @@ impl VisualizerSystem for Ellipsoids3DVisualizer { let num_ellipsoids: usize = all_half_size_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<3, f32>(&HalfSize3D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 3]>(HalfSize3D::name())) .map(|vectors| vectors.len()) .sum(); if num_ellipsoids == 0 { @@ -161,11 +161,8 @@ impl VisualizerSystem for Ellipsoids3DVisualizer { // line_builder.reserve_vertices(num_ellipsoids * sphere_mesh.vertex_count)?; let timeline = ctx.query.timeline(); - let all_half_sizes_indexed = iter_primitive_array::<3, f32>( - &all_half_size_chunks, - timeline, - HalfSize3D::name(), - ); + let all_half_sizes_indexed = + iter_slices::<[f32; 3]>(&all_half_size_chunks, timeline, HalfSize3D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_line_radii = results.iter_as(timeline, Radius::name()); // Deserialized because it's a union. @@ -176,12 +173,12 @@ impl VisualizerSystem for Ellipsoids3DVisualizer { let data = re_query::range_zip_1x6( all_half_sizes_indexed, - all_colors.primitive::(), - all_line_radii.primitive::(), - all_fill_modes.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_show_labels.bool(), + all_colors.slice::(), + all_line_radii.slice::(), + all_fill_modes.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_show_labels.slice::(), ) .map( |( diff --git a/crates/viewer/re_view_spatial/src/visualizers/encoded_image.rs b/crates/viewer/re_view_spatial/src/visualizers/encoded_image.rs index 7e8f24a4fcd5..bc720479c1d4 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/encoded_image.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/encoded_image.rs @@ -133,7 +133,7 @@ impl EncodedImageVisualizer { results: &HybridResults<'_>, spatial_ctx: &SpatialSceneEntityContext<'_>, ) { - use super::entity_iterator::iter_buffer; + use super::entity_iterator::iter_slices; use re_view::RangeResultsExt as _; let entity_path = ctx.target_entity_path; @@ -143,14 +143,14 @@ impl EncodedImageVisualizer { }; let timeline = ctx.query.timeline(); - let all_blobs_indexed = iter_buffer::(&all_blob_chunks, timeline, Blob::name()); + let all_blobs_indexed = iter_slices::<&[u8]>(&all_blob_chunks, timeline, Blob::name()); let all_media_types = results.iter_as(timeline, MediaType::name()); let all_opacities = results.iter_as(timeline, Opacity::name()); for ((_time, tensor_data_row_id), blobs, media_types, opacities) in re_query::range_zip_1x2( all_blobs_indexed, - all_media_types.string(), - all_opacities.primitive::(), + all_media_types.slice::(), + all_opacities.slice::(), ) { let Some(blob) = blobs.first() else { continue; diff --git a/crates/viewer/re_view_spatial/src/visualizers/images.rs b/crates/viewer/re_view_spatial/src/visualizers/images.rs index 00fdecf89b37..f65147801287 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/images.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/images.rs @@ -118,7 +118,7 @@ impl ImageVisualizer { results: &HybridResults<'_>, spatial_ctx: &SpatialSceneEntityContext<'_>, ) { - use super::entity_iterator::{iter_buffer, iter_component}; + use super::entity_iterator::{iter_component, iter_slices}; use re_view::RangeResultsExt as _; let entity_path = ctx.target_entity_path; @@ -132,7 +132,7 @@ impl ImageVisualizer { let timeline = ctx.query.timeline(); let all_buffers_indexed = - iter_buffer::(&all_buffer_chunks, timeline, ImageBuffer::name()); + iter_slices::<&[u8]>(&all_buffer_chunks, timeline, ImageBuffer::name()); let all_formats_indexed = iter_component::(&all_formats_chunks, timeline, ImageFormat::name()); let all_opacities = results.iter_as(timeline, Opacity::name()); @@ -140,7 +140,7 @@ impl ImageVisualizer { let data = re_query::range_zip_1x2( all_buffers_indexed, all_formats_indexed, - all_opacities.primitive::(), + all_opacities.slice::(), ) .filter_map(|(index, buffers, formats, opacities)| { let buffer = buffers.first()?; diff --git a/crates/viewer/re_view_spatial/src/visualizers/lines2d.rs b/crates/viewer/re_view_spatial/src/visualizers/lines2d.rs index d5b5db1f7cae..8cb7b1590fe6 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/lines2d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/lines2d.rs @@ -182,7 +182,7 @@ impl VisualizerSystem for Lines2DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array_list, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -199,9 +199,7 @@ impl VisualizerSystem for Lines2DVisualizer { let num_strips = all_strip_chunks .iter() - .flat_map(|chunk| { - chunk.iter_primitive_array_list::<2, f32>(&LineStrip2D::name()) - }) + .flat_map(|chunk| chunk.iter_slices::<[f32; 2]>(LineStrip2D::name())) .map(|strips| strips.len()) .sum(); if num_strips == 0 { @@ -211,15 +209,13 @@ impl VisualizerSystem for Lines2DVisualizer { let num_vertices = all_strip_chunks .iter() - .flat_map(|chunk| { - chunk.iter_primitive_array_list::<2, f32>(&LineStrip2D::name()) - }) + .flat_map(|chunk| chunk.iter_slices::<[f32; 2]>(LineStrip2D::name())) .map(|strips| strips.iter().map(|strip| strip.len()).sum::()) .sum::(); line_builder.reserve_vertices(num_vertices)?; let all_strips_indexed = - iter_primitive_array_list(&all_strip_chunks, timeline, LineStrip2D::name()); + iter_slices::<&[[f32; 2]]>(&all_strip_chunks, timeline, LineStrip2D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); let all_labels = results.iter_as(timeline, Text::name()); @@ -228,11 +224,11 @@ impl VisualizerSystem for Lines2DVisualizer { let data = re_query::range_zip_1x5( all_strips_indexed, - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_show_labels.bool(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_show_labels.slice::(), ) .map( |(_index, strips, colors, radii, labels, class_ids, show_labels)| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/lines3d.rs b/crates/viewer/re_view_spatial/src/visualizers/lines3d.rs index 9ef6d4511fb2..bdced188aa5a 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/lines3d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/lines3d.rs @@ -187,7 +187,7 @@ impl VisualizerSystem for Lines3DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array_list, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -202,9 +202,7 @@ impl VisualizerSystem for Lines3DVisualizer { let num_strips = all_strip_chunks .iter() - .flat_map(|chunk| { - chunk.iter_primitive_array_list::<3, f32>(&LineStrip3D::name()) - }) + .flat_map(|chunk| chunk.iter_slices::<[f32; 3]>(LineStrip3D::name())) .map(|strips| strips.len()) .sum(); if num_strips == 0 { @@ -214,16 +212,14 @@ impl VisualizerSystem for Lines3DVisualizer { let num_vertices = all_strip_chunks .iter() - .flat_map(|chunk| { - chunk.iter_primitive_array_list::<3, f32>(&LineStrip3D::name()) - }) + .flat_map(|chunk| chunk.iter_slices::<[f32; 3]>(LineStrip3D::name())) .map(|strips| strips.iter().map(|strip| strip.len()).sum::()) .sum::(); line_builder.reserve_vertices(num_vertices)?; let timeline = ctx.query.timeline(); let all_strips_indexed = - iter_primitive_array_list(&all_strip_chunks, timeline, LineStrip3D::name()); + iter_slices::<&[[f32; 3]]>(&all_strip_chunks, timeline, LineStrip3D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); let all_labels = results.iter_as(timeline, Text::name()); @@ -232,11 +228,11 @@ impl VisualizerSystem for Lines3DVisualizer { let data = re_query::range_zip_1x5( all_strips_indexed, - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_show_labels.bool(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_show_labels.slice::(), ) .map( |(_index, strips, colors, radii, labels, class_ids, show_labels)| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/meshes.rs b/crates/viewer/re_view_spatial/src/visualizers/meshes.rs index d5e83f60f9d2..aaa6dd9ceb30 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/meshes.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/meshes.rs @@ -176,7 +176,7 @@ impl VisualizerSystem for Mesh3DVisualizer { let mut instances = Vec::new(); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -191,7 +191,7 @@ impl VisualizerSystem for Mesh3DVisualizer { }; let timeline = ctx.query.timeline(); - let all_vertex_positions_indexed = iter_primitive_array::<3, f32>( + let all_vertex_positions_indexed = iter_slices::<[f32; 3]>( &all_vertex_position_chunks, timeline, Position3D::name(), @@ -209,16 +209,15 @@ impl VisualizerSystem for Mesh3DVisualizer { let data = re_query::range_zip_1x8( all_vertex_positions_indexed, - all_vertex_normals.primitive_array::<3, f32>(), - all_vertex_colors.primitive::(), - all_vertex_texcoords.primitive_array::<2, f32>(), - all_triangle_indices.primitive_array::<3, u32>(), - all_albedo_factors.primitive::(), - // TODO(cmc): Provide a `iter_blob`. - all_albedo_buffers.component_slow::(), + all_vertex_normals.slice::<[f32; 3]>(), + all_vertex_colors.slice::(), + all_vertex_texcoords.slice::<[f32; 2]>(), + all_triangle_indices.slice::<[f32; 3]>(), + all_albedo_factors.slice::(), + all_albedo_buffers.slice::<&[u8]>(), // Legit call to `component_slow`, `ImageFormat` is real complicated. all_albedo_formats.component_slow::(), - all_class_ids.primitive::(), + all_class_ids.slice::(), ) .map( |( @@ -250,7 +249,11 @@ impl VisualizerSystem for Mesh3DVisualizer { bytemuck::cast_slice(albedo_factors) }) .first(), - albedo_buffer: albedo_buffers.unwrap_or_default().first().cloned(), // shallow clone + albedo_buffer: albedo_buffers + .unwrap_or_default() + .first() + .cloned() + .map(Into::into), // shallow clone albedo_format: albedo_formats.unwrap_or_default().first().copied(), class_ids: class_ids .map_or(&[], |class_ids| bytemuck::cast_slice(class_ids)), diff --git a/crates/viewer/re_view_spatial/src/visualizers/points2d.rs b/crates/viewer/re_view_spatial/src/visualizers/points2d.rs index 2b54e568c1d7..ecc05c39f821 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/points2d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/points2d.rs @@ -206,7 +206,7 @@ impl VisualizerSystem for Points2DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_POINT_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -221,7 +221,7 @@ impl VisualizerSystem for Points2DVisualizer { let num_positions = all_position_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<2, f32>(&Position2D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 2]>(Position2D::name())) .map(|points| points.len()) .sum(); @@ -232,11 +232,8 @@ impl VisualizerSystem for Points2DVisualizer { point_builder.reserve(num_positions)?; let timeline = ctx.query.timeline(); - let all_positions_indexed = iter_primitive_array::<2, f32>( - &all_position_chunks, - timeline, - Position2D::name(), - ); + let all_positions_indexed = + iter_slices::<[f32; 2]>(&all_position_chunks, timeline, Position2D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); let all_labels = results.iter_as(timeline, Text::name()); @@ -246,12 +243,12 @@ impl VisualizerSystem for Points2DVisualizer { let data = re_query::range_zip_1x6( all_positions_indexed, - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_keypoint_ids.primitive::(), - all_show_labels.bool(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_keypoint_ids.slice::(), + all_show_labels.slice::(), ) .map( |( diff --git a/crates/viewer/re_view_spatial/src/visualizers/points3d.rs b/crates/viewer/re_view_spatial/src/visualizers/points3d.rs index 7928fea07477..55521f3bcd98 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/points3d.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/points3d.rs @@ -196,7 +196,7 @@ impl VisualizerSystem for Points3DVisualizer { re_view::SIZE_BOOST_IN_POINTS_FOR_POINT_OUTLINES, ); - use super::entity_iterator::{iter_primitive_array, process_archetype}; + use super::entity_iterator::{iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -211,7 +211,7 @@ impl VisualizerSystem for Points3DVisualizer { let num_positions = all_position_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive_array::<3, f32>(&Position3D::name())) + .flat_map(|chunk| chunk.iter_slices::<[f32; 3]>(Position3D::name())) .map(|points| points.len()) .sum(); @@ -222,11 +222,8 @@ impl VisualizerSystem for Points3DVisualizer { point_builder.reserve(num_positions)?; let timeline = ctx.query.timeline(); - let all_positions_indexed = iter_primitive_array::<3, f32>( - &all_position_chunks, - timeline, - Position3D::name(), - ); + let all_positions_indexed = + iter_slices::<[f32; 3]>(&all_position_chunks, timeline, Position3D::name()); let all_colors = results.iter_as(timeline, Color::name()); let all_radii = results.iter_as(timeline, Radius::name()); let all_labels = results.iter_as(timeline, Text::name()); @@ -236,12 +233,12 @@ impl VisualizerSystem for Points3DVisualizer { let data = re_query::range_zip_1x6( all_positions_indexed, - all_colors.primitive::(), - all_radii.primitive::(), - all_labels.string(), - all_class_ids.primitive::(), - all_keypoint_ids.primitive::(), - all_show_labels.bool(), + all_colors.slice::(), + all_radii.slice::(), + all_labels.slice::(), + all_class_ids.slice::(), + all_keypoint_ids.slice::(), + all_show_labels.slice::(), ) .map( |( diff --git a/crates/viewer/re_view_spatial/src/visualizers/segmentation_images.rs b/crates/viewer/re_view_spatial/src/visualizers/segmentation_images.rs index 87cf23354ec3..9521bafabb86 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/segmentation_images.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/segmentation_images.rs @@ -67,7 +67,7 @@ impl VisualizerSystem for SegmentationImageVisualizer { return Err(ViewSystemExecutionError::NoRenderContextError); }; - use super::entity_iterator::{iter_buffer, iter_component, process_archetype}; + use super::entity_iterator::{iter_component, iter_slices, process_archetype}; process_archetype::( ctx, view_query, @@ -88,7 +88,7 @@ impl VisualizerSystem for SegmentationImageVisualizer { let timeline = ctx.query.timeline(); let all_buffers_indexed = - iter_buffer::(&all_buffer_chunks, timeline, ImageBuffer::name()); + iter_slices::<&[u8]>(&all_buffer_chunks, timeline, ImageBuffer::name()); let all_formats_indexed = iter_component::( &all_formats_chunks, timeline, @@ -99,7 +99,7 @@ impl VisualizerSystem for SegmentationImageVisualizer { let data = re_query::range_zip_1x2( all_buffers_indexed, all_formats_indexed, - all_opacities.primitive::(), + all_opacities.slice::(), ) .filter_map(|(index, buffers, formats, opacity)| { let buffer = buffers.first()?; diff --git a/crates/viewer/re_view_spatial/src/visualizers/utilities/entity_iterator.rs b/crates/viewer/re_view_spatial/src/visualizers/utilities/entity_iterator.rs index f851f4dc0809..78a7dcb249a9 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/utilities/entity_iterator.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/utilities/entity_iterator.rs @@ -130,7 +130,7 @@ where // --- use re_chunk::{Chunk, ChunkComponentIterItem, ComponentName, RowId}; -use re_chunk_store::external::{re_chunk, re_chunk::external::arrow2}; +use re_chunk_store::external::re_chunk; /// Iterate `chunks` as indexed deserialized batches. /// @@ -151,91 +151,16 @@ pub fn iter_component<'a, C: re_types::Component>( /// Iterate `chunks` as indexed primitives. /// -/// See [`Chunk::iter_primitive`] for more information. -#[allow(unused)] -pub fn iter_primitive<'a, T: arrow2::types::NativeType>( - chunks: &'a std::borrow::Cow<'a, [Chunk]>, - timeline: Timeline, - component_name: ComponentName, -) -> impl Iterator + 'a { - chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&timeline, &component_name), - chunk.iter_primitive::(&component_name) - ) - }) -} - -/// Iterate `chunks` as indexed primitive arrays. -/// -/// See [`Chunk::iter_primitive_array`] for more information. -#[allow(unused)] -pub fn iter_primitive_array<'a, const N: usize, T: arrow2::types::NativeType>( - chunks: &'a std::borrow::Cow<'a, [Chunk]>, - timeline: Timeline, - component_name: ComponentName, -) -> impl Iterator + 'a -where - [T; N]: bytemuck::Pod, -{ - chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&timeline, &component_name), - chunk.iter_primitive_array::(&component_name) - ) - }) -} - -/// Iterate `chunks` as indexed list of primitive arrays. -/// -/// See [`Chunk::iter_primitive_array_list`] for more information. -#[allow(unused)] -pub fn iter_primitive_array_list<'a, const N: usize, T: arrow2::types::NativeType>( - chunks: &'a std::borrow::Cow<'a, [Chunk]>, - timeline: Timeline, - component_name: ComponentName, -) -> impl Iterator)> + 'a -where - [T; N]: bytemuck::Pod, -{ - chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&timeline, &component_name), - chunk.iter_primitive_array_list::(&component_name) - ) - }) -} - -/// Iterate `chunks` as indexed UTF-8 strings. -/// -/// See [`Chunk::iter_string`] for more information. -#[allow(unused)] -pub fn iter_string<'a>( - chunks: &'a std::borrow::Cow<'a, [Chunk]>, - timeline: Timeline, - component_name: ComponentName, -) -> impl Iterator)> + 'a { - chunks.iter().flat_map(move |chunk| { - itertools::izip!( - chunk.iter_component_indices(&timeline, &component_name), - chunk.iter_string(&component_name) - ) - }) -} - -/// Iterate `chunks` as indexed buffers. -/// -/// See [`Chunk::iter_buffer`] for more information. -#[allow(unused)] -pub fn iter_buffer<'a, T: arrow::datatypes::ArrowNativeType + arrow2::types::NativeType>( +/// See [`Chunk::iter_slices`] for more information. +pub fn iter_slices<'a, T: 'a + re_chunk::ChunkComponentSlicer>( chunks: &'a std::borrow::Cow<'a, [Chunk]>, timeline: Timeline, component_name: ComponentName, -) -> impl Iterator>)> + 'a { +) -> impl Iterator)> + 'a { chunks.iter().flat_map(move |chunk| { itertools::izip!( chunk.iter_component_indices(&timeline, &component_name), - chunk.iter_buffer(&component_name) + chunk.iter_slices::(component_name) ) }) } diff --git a/crates/viewer/re_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_view_spatial/src/visualizers/videos.rs index 55e0321fae27..f8362e9ae71d 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/videos.rs @@ -104,7 +104,7 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { timeline, VideoTimestamp::name(), ), - all_video_references.string(), + all_video_references.slice::(), ) { let Some(video_timestamp): Option<&VideoTimestamp> = video_timestamps.first() else { diff --git a/crates/viewer/re_view_tensor/src/visualizer_system.rs b/crates/viewer/re_view_tensor/src/visualizer_system.rs index aa5e0cbd2412..a44193024953 100644 --- a/crates/viewer/re_view_tensor/src/visualizer_system.rs +++ b/crates/viewer/re_view_tensor/src/visualizer_system.rs @@ -69,7 +69,7 @@ impl VisualizerSystem for TensorSystem { let all_ranges = results.iter_as(timeline, ValueRange::name()); for ((_, tensor_row_id), tensors, data_ranges) in - re_query::range_zip_1x1(all_tensors_indexed, all_ranges.primitive_array::<2, f64>()) + re_query::range_zip_1x1(all_tensors_indexed, all_ranges.slice::<[f64; 2]>()) { let Some(tensor) = tensors.first() else { continue; diff --git a/crates/viewer/re_view_text_log/src/visualizer_system.rs b/crates/viewer/re_view_text_log/src/visualizer_system.rs index d6330d8b842d..9f597807b1dd 100644 --- a/crates/viewer/re_view_text_log/src/visualizer_system.rs +++ b/crates/viewer/re_view_text_log/src/visualizer_system.rs @@ -110,9 +110,9 @@ impl TextLogSystem { let all_colors = results.iter_as(timeline, Color::name()); let all_frames = range_zip_1x2( - all_texts.string(), - all_levels.string(), - all_colors.primitive::(), + all_texts.slice::(), + all_levels.slice::(), + all_colors.slice::(), ); let all_frames = izip!(all_timepoints, all_frames); diff --git a/crates/viewer/re_view_time_series/Cargo.toml b/crates/viewer/re_view_time_series/Cargo.toml index aceec9cdbbfe..636b56ba5f65 100644 --- a/crates/viewer/re_view_time_series/Cargo.toml +++ b/crates/viewer/re_view_time_series/Cargo.toml @@ -19,6 +19,7 @@ workspace = true all-features = true [dependencies] +re_chunk.workspace = true re_chunk_store.workspace = true re_format.workspace = true re_log.workspace = true diff --git a/crates/viewer/re_view_time_series/src/line_visualizer_system.rs b/crates/viewer/re_view_time_series/src/line_visualizer_system.rs index 03ee2a16469b..6ea021d08867 100644 --- a/crates/viewer/re_view_time_series/src/line_visualizer_system.rs +++ b/crates/viewer/re_view_time_series/src/line_visualizer_system.rs @@ -260,7 +260,7 @@ impl SeriesLineSystem { debug_assert_eq!(Scalar::arrow_datatype(), ArrowDatatype::Float64); all_scalar_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive::(&Scalar::name())) + .flat_map(|chunk| chunk.iter_slices::(Scalar::name())) .enumerate() .for_each(|(i, values)| { if !values.is_empty() { @@ -301,7 +301,7 @@ impl SeriesLineSystem { re_tracing::profile_scope!("override/default fast path"); let color = all_color_chunks[0] - .iter_primitive::(&Color::name()) + .iter_slices::(Color::name()) .next() .and_then(map_raw_color); @@ -314,7 +314,7 @@ impl SeriesLineSystem { let all_colors = all_color_chunks.iter().flat_map(|chunk| { itertools::izip!( chunk.iter_component_indices(&query.timeline(), &Color::name()), - chunk.iter_primitive::(&Color::name()) + chunk.iter_slices::(Color::name()) ) }); @@ -344,7 +344,7 @@ impl SeriesLineSystem { re_tracing::profile_scope!("override/default fast path"); let stroke_width = all_stroke_width_chunks[0] - .iter_primitive::(&StrokeWidth::name()) + .iter_slices::(StrokeWidth::name()) .next() .and_then(|stroke_widths| stroke_widths.first().copied()); @@ -362,7 +362,7 @@ impl SeriesLineSystem { &query.timeline(), &StrokeWidth::name() ), - chunk.iter_primitive::(&StrokeWidth::name()) + chunk.iter_slices::(StrokeWidth::name()) ) }); diff --git a/crates/viewer/re_view_time_series/src/point_visualizer_system.rs b/crates/viewer/re_view_time_series/src/point_visualizer_system.rs index 912fbd149474..94810b6e1b8a 100644 --- a/crates/viewer/re_view_time_series/src/point_visualizer_system.rs +++ b/crates/viewer/re_view_time_series/src/point_visualizer_system.rs @@ -275,7 +275,7 @@ impl SeriesPointSystem { let mut i = 0; all_scalar_chunks .iter() - .flat_map(|chunk| chunk.iter_primitive::(&Scalar::name())) + .flat_map(|chunk| chunk.iter_slices::(Scalar::name())) .for_each(|values| { if !values.is_empty() { if values.len() > 1 { @@ -318,7 +318,7 @@ impl SeriesPointSystem { re_tracing::profile_scope!("override/default fast path"); let color = all_color_chunks[0] - .iter_primitive::(&Color::name()) + .iter_slices::(Color::name()) .next() .and_then(map_raw_color); @@ -331,7 +331,7 @@ impl SeriesPointSystem { let all_colors = all_color_chunks.iter().flat_map(|chunk| { itertools::izip!( chunk.iter_component_indices(&query.timeline(), &Color::name()), - chunk.iter_primitive::(&Color::name()) + chunk.iter_slices::(Color::name()) ) }); @@ -360,7 +360,7 @@ impl SeriesPointSystem { re_tracing::profile_scope!("override/default fast path"); let marker_size = all_marker_size_chunks[0] - .iter_primitive::(&MarkerSize::name()) + .iter_slices::(MarkerSize::name()) .next() .and_then(|marker_sizes| marker_sizes.first().copied()); @@ -377,7 +377,7 @@ impl SeriesPointSystem { itertools::izip!( chunk .iter_component_indices(&query.timeline(), &MarkerSize::name()), - chunk.iter_primitive::(&MarkerSize::name()) + chunk.iter_slices::(Color::name()) ) });