From 69fdcfc1a7e75b33a085f76eeed466201a420080 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Thu, 19 Dec 2024 16:31:09 +0100 Subject: [PATCH 1/3] remove Chunk::iter_component_arrays --- crates/store/re_chunk/src/iter.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/crates/store/re_chunk/src/iter.rs b/crates/store/re_chunk/src/iter.rs index c417e814c6ad..cbca9de2891d 100644 --- a/crates/store/re_chunk/src/iter.rs +++ b/crates/store/re_chunk/src/iter.rs @@ -200,27 +200,6 @@ impl Chunk { } } - /// Returns an iterator over the raw arrays of a [`Chunk`], for a given component. - /// - /// See also: - /// * [`Self::iter_primitive`] - /// * [`Self::iter_primitive_array`] - /// * [`Self::iter_primitive_array_list`] - /// * [`Self::iter_string`] - /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component`]. - #[inline] - pub fn iter_component_arrays( - &self, - component_name: &ComponentName, - ) -> impl Iterator> + '_ { - let Some(list_array) = self.get_first_component(component_name) else { - return Either::Left(std::iter::empty()); - }; - - Either::Right(list_array.iter().flatten()) - } - /// Returns an iterator over the raw primitive values of a [`Chunk`], for a given component. /// /// This is a very fast path: the entire column will be downcasted at once, and then every From c302bd4bba38e915d5b0b2e57ed4a1534e06bdde Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Thu, 19 Dec 2024 16:44:15 +0100 Subject: [PATCH 2/3] propagate changes --- crates/store/re_chunk/src/iter.rs | 6 +-- crates/store/re_grpc_client/src/lib.rs | 10 +---- .../src/max_image_dimension_subscriber.rs | 40 +++++++++---------- 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/crates/store/re_chunk/src/iter.rs b/crates/store/re_chunk/src/iter.rs index cbca9de2891d..89e118f8d9a0 100644 --- a/crates/store/re_chunk/src/iter.rs +++ b/crates/store/re_chunk/src/iter.rs @@ -2,9 +2,9 @@ use std::sync::Arc; use arrow2::{ array::{ - Array as Arrow2Array, BooleanArray as Arrow2BooleanArray, - FixedSizeListArray as Arrow2FixedSizeListArray, ListArray as Arrow2ListArray, - PrimitiveArray as Arrow2PrimitiveArray, Utf8Array as Arrow2Utf8Array, + BooleanArray as Arrow2BooleanArray, FixedSizeListArray as Arrow2FixedSizeListArray, + ListArray as Arrow2ListArray, PrimitiveArray as Arrow2PrimitiveArray, + Utf8Array as Arrow2Utf8Array, }, bitmap::Bitmap as Arrow2Bitmap, Either, diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 4014d6b974b2..f79ab9c2bebd 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -484,15 +484,9 @@ async fn stream_catalog_async( )))?; let recording_uri_arrays: Vec> = chunk - .iter_component_arrays(&"id".into()) + .iter_string(&"id".into()) .map(|id| { - let rec_id = id - .as_any() - .downcast_ref::>() - .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: format!("id must be a utf8 array: {:?}", tc.schema), - }))? - .value(0); // each component batch is of length 1 i.e. single 'id' value + let rec_id = &id[0]; // each component batch is of length 1 i.e. single 'id' value let recording_uri = format!("rerun://{host}:{port}/recording/{rec_id}"); diff --git a/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs b/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs index 735cba3f50ff..327d24b60db5 100644 --- a/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs +++ b/crates/viewer/re_view_spatial/src/max_image_dimension_subscriber.rs @@ -1,4 +1,3 @@ -use arrow2::array::Array; use nohash_hasher::IntMap; use once_cell::sync::OnceCell; @@ -85,41 +84,38 @@ impl PerStoreChunkSubscriber for MaxImageDimensionsStoreSubscriber { } // Handle `ImageEncoded`, `AssetVideo`… - let blobs = event.diff.chunk.iter_component_arrays(&Blob::name()); - let media_types = event.diff.chunk.iter_component_arrays(&MediaType::name()); + let blobs = event.diff.chunk.iter_buffer(&Blob::name()); + let media_types = event.diff.chunk.iter_string(&MediaType::name()); for (blob, media_type) in itertools::izip!(blobs, media_types.map(Some).chain(std::iter::repeat(None))) { - if let Some([width, height]) = size_from_blob(blob.as_ref(), media_type.as_deref()) - { - let max_dim = self - .max_dimensions - .entry(event.diff.chunk.entity_path().clone()) - .or_default(); - max_dim.width = max_dim.width.max(width); - max_dim.height = max_dim.height.max(height); + if let Some(blob) = blob.first() { + if let Some([width, height]) = size_from_blob( + blob.as_slice(), + media_type.and_then(|v| v.first().map(|v| MediaType(v.clone().into()))), + ) { + let max_dim = self + .max_dimensions + .entry(event.diff.chunk.entity_path().clone()) + .or_default(); + max_dim.width = max_dim.width.max(width); + max_dim.height = max_dim.height.max(height); + } } } } } } -fn size_from_blob(blob: &dyn Array, media_type: Option<&dyn Array>) -> Option<[u32; 2]> { +fn size_from_blob(blob: &[u8], media_type: Option) -> Option<[u32; 2]> { re_tracing::profile_function!(); - let blob = Blob::from_arrow2_opt(blob).ok()?.first()?.clone()?; - - let media_type: Option = media_type - .and_then(|media_type| MediaType::from_arrow2_opt(media_type).ok()) - .and_then(|list| list.first().cloned()) - .flatten(); - - let media_type = MediaType::or_guess_from_data(media_type, &blob)?; + let media_type = MediaType::or_guess_from_data(media_type, blob)?; if media_type.is_image() { re_tracing::profile_scope!("image"); - let image_bytes = blob.0.as_slice(); + let image_bytes = blob; let mut reader = image::ImageReader::new(std::io::Cursor::new(image_bytes)); @@ -137,7 +133,7 @@ fn size_from_blob(blob: &dyn Array, media_type: Option<&dyn Array>) -> Option<[u reader.into_dimensions().ok().map(|size| size.into()) } else if media_type.is_video() { re_tracing::profile_scope!("video"); - re_video::VideoData::load_from_bytes(&blob, &media_type) + re_video::VideoData::load_from_bytes(blob, &media_type) .ok() .map(|video| video.dimensions()) } else { From 4c4319d7b6b9aed46eba23a623d170e4df8304b6 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 20 Dec 2024 09:56:13 +0100 Subject: [PATCH 3/3] lints --- crates/store/re_chunk/src/iter.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/store/re_chunk/src/iter.rs b/crates/store/re_chunk/src/iter.rs index 89e118f8d9a0..4bce90046356 100644 --- a/crates/store/re_chunk/src/iter.rs +++ b/crates/store/re_chunk/src/iter.rs @@ -212,7 +212,6 @@ impl Chunk { /// * [`Self::iter_primitive_array_list`] /// * [`Self::iter_string`] /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component_arrays`]. /// * [`Self::iter_component`]. #[inline] pub fn iter_primitive( @@ -255,7 +254,6 @@ impl Chunk { /// * [`Self::iter_primitive_array_list`] /// * [`Self::iter_string`] /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component_arrays`]. /// * [`Self::iter_component`]. #[inline] pub fn iter_bool( @@ -298,7 +296,6 @@ impl Chunk { /// * [`Self::iter_primitive`] /// * [`Self::iter_string`] /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component_arrays`]. /// * [`Self::iter_component`]. pub fn iter_primitive_array( &self, @@ -360,7 +357,6 @@ impl Chunk { /// * [`Self::iter_primitive_array`] /// * [`Self::iter_string`] /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component_arrays`]. /// * [`Self::iter_component`]. pub fn iter_primitive_array_list( &self, @@ -445,7 +441,6 @@ impl Chunk { /// * [`Self::iter_primitive_array`] /// * [`Self::iter_primitive_array_list`] /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component_arrays`]. /// * [`Self::iter_component`]. pub fn iter_string( &self, @@ -496,7 +491,6 @@ impl Chunk { /// * [`Self::iter_primitive_array`] /// * [`Self::iter_primitive_array_list`] /// * [`Self::iter_string`]. - /// * [`Self::iter_component_arrays`]. /// * [`Self::iter_component`]. pub fn iter_buffer( &self, @@ -719,7 +713,6 @@ impl Chunk { /// * [`Self::iter_primitive_array_list`] /// * [`Self::iter_string`] /// * [`Self::iter_buffer`]. - /// * [`Self::iter_component_arrays`]. #[inline] pub fn iter_component( &self,