diff --git a/crates/re_arrow_store/src/store_helpers.rs b/crates/re_arrow_store/src/store_helpers.rs index 0c40b83edce0..067a535a9712 100644 --- a/crates/re_arrow_store/src/store_helpers.rs +++ b/crates/re_arrow_store/src/store_helpers.rs @@ -6,8 +6,35 @@ use crate::{DataStore, LatestAtQuery}; // --- Read --- +/// A [`Component`] versioned with a specific [`RowId`]. +/// +/// This is not enough to globally, uniquely identify an instance of a component. +/// For that you will need to combine the `InstancePath` that was used to query +/// the versioned component with the returned [`RowId`], therefore creating a +/// `VersionedInstancePath`. +#[derive(Debug, Clone)] +pub struct VersionedComponent { + pub row_id: RowId, + pub value: C, +} + +impl From<(RowId, C)> for VersionedComponent { + #[inline] + fn from((row_id, value): (RowId, C)) -> Self { + Self { row_id, value } + } +} + +impl std::ops::Deref for VersionedComponent { + type Target = C; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + impl DataStore { - /// Get the latest value for a given [`re_types::Component`]. + /// Get the latest value for a given [`re_types::Component`] and the associated [`RowId`]. /// /// This assumes that the row we get from the store only contains a single instance for this /// component; it will log a warning otherwise. @@ -19,10 +46,10 @@ impl DataStore { &self, entity_path: &EntityPath, query: &LatestAtQuery, - ) -> Option { + ) -> Option> { re_tracing::profile_function!(); - let (_, cells) = self.latest_at(query, entity_path, C::name(), &[C::name()])?; + let (row_id, cells) = self.latest_at(query, entity_path, C::name(), &[C::name()])?; let cell = cells.get(0)?.as_ref()?; cell.try_to_native_mono::() @@ -61,27 +88,28 @@ impl DataStore { err }) .ok()? + .map(|c| (row_id, c).into()) } - /// Call `query_latest_component` at the given path, walking up the hierarchy until an instance is found. + /// Call [`Self::query_latest_component`] at the given path, walking up the hierarchy until an instance is found. pub fn query_latest_component_at_closest_ancestor( &self, entity_path: &EntityPath, query: &LatestAtQuery, - ) -> Option<(EntityPath, C)> { + ) -> Option<(EntityPath, VersionedComponent)> { re_tracing::profile_function!(); let mut cur_path = Some(entity_path.clone()); while let Some(path) = cur_path { - if let Some(component) = self.query_latest_component::(&path, query) { - return Some((path, component)); + if let Some(c) = self.query_latest_component::(&path, query) { + return Some((path, c)); } cur_path = path.parent(); } None } - /// Get the latest value for a given [`re_types::Component`], assuming it is timeless. + /// Get the latest value for a given [`re_types::Component`] and the associated [`RowId`], assuming it is timeless. /// /// This assumes that the row we get from the store only contains a single instance for this /// component; it will log a warning otherwise. @@ -89,7 +117,10 @@ impl DataStore { /// This should only be used for "mono-components" such as `Transform` and `Tensor`. /// /// This is a best-effort helper, it will merely log errors on failure. - pub fn query_timeless_component(&self, entity_path: &EntityPath) -> Option { + pub fn query_timeless_component( + &self, + entity_path: &EntityPath, + ) -> Option> { re_tracing::profile_function!(); let query = LatestAtQuery::latest(Timeline::default()); diff --git a/crates/re_components/src/lib.rs b/crates/re_components/src/lib.rs index 9859d7163596..54a701ec56a2 100644 --- a/crates/re_components/src/lib.rs +++ b/crates/re_components/src/lib.rs @@ -47,14 +47,13 @@ pub use self::{ bbox::Box3D, coordinates::ViewCoordinates, mat::LegacyMat3x3, - mesh3d::{EncodedMesh3D, Mesh3D, MeshFormat, MeshId, RawMesh3D}, + mesh3d::{EncodedMesh3D, Mesh3D, MeshFormat, RawMesh3D}, pinhole::Pinhole, quaternion::Quaternion, rect::Rect2D, scalar::{Scalar, ScalarPlotProps}, tensor::{ DecodedTensor, Tensor, TensorCastError, TensorData, TensorDataMeaning, TensorDimension, - TensorId, }, tensor_data::{TensorDataType, TensorDataTypeTrait, TensorElement}, text_box::TextBox, diff --git a/crates/re_components/src/load_file.rs b/crates/re_components/src/load_file.rs index d9af2d21d0ea..4a4ae89ea9e0 100644 --- a/crates/re_components/src/load_file.rs +++ b/crates/re_components/src/load_file.rs @@ -117,7 +117,6 @@ pub fn data_cell_from_mesh_file_contents( format: crate::MeshFormat, ) -> Result { let mesh = crate::EncodedMesh3D { - mesh_id: crate::MeshId::random(), format, bytes: bytes.into(), transform: [ diff --git a/crates/re_components/src/mesh3d.rs b/crates/re_components/src/mesh3d.rs index f17c84a9a0b0..84570ab87121 100644 --- a/crates/re_components/src/mesh3d.rs +++ b/crates/re_components/src/mesh3d.rs @@ -1,4 +1,3 @@ -use arrow2::array::{FixedSizeBinaryArray, MutableFixedSizeBinaryArray}; use arrow2::buffer::Buffer; use arrow2::datatypes::DataType; use arrow2_convert::arrow_enable_vec_for_type; @@ -10,71 +9,6 @@ use super::{FieldError, LegacyVec4D}; // ---------------------------------------------------------------------------- -/// A unique id per [`Mesh3D`]. -/// -/// TODO(emilk): this should be a hash of the mesh (CAS). -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct MeshId(pub uuid::Uuid); - -impl nohash_hasher::IsEnabled for MeshId {} - -// required for [`nohash_hasher`]. -#[allow(clippy::derived_hash_with_manual_eq)] -impl std::hash::Hash for MeshId { - #[inline] - fn hash(&self, state: &mut H) { - state.write_u64(self.0.as_u128() as u64); - } -} - -impl MeshId { - #[inline] - pub fn random() -> Self { - Self(uuid::Uuid::new_v4()) - } -} - -impl ArrowField for MeshId { - type Type = Self; - - #[inline] - fn data_type() -> arrow2::datatypes::DataType { - arrow2::datatypes::DataType::FixedSizeBinary(16) - } -} - -impl ArrowSerialize for MeshId { - type MutableArrayType = MutableFixedSizeBinaryArray; - - #[inline] - fn new_array() -> Self::MutableArrayType { - MutableFixedSizeBinaryArray::new(16) - } - - #[inline] - fn arrow_serialize( - v: &::Type, - array: &mut Self::MutableArrayType, - ) -> arrow2::error::Result<()> { - array.try_push(Some(v.0.as_bytes())) - } -} - -impl ArrowDeserialize for MeshId { - type ArrayType = FixedSizeBinaryArray; - - #[inline] - fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, - ) -> Option<::Type> { - v.and_then(|bytes| uuid::Uuid::from_slice(bytes).ok()) - .map(Self) - } -} - -// ---------------------------------------------------------------------------- - // TODO(cmc): Let's make both mesh Component types use friendlier types for their inner elements // (e.g. positions should be a vec of Vec3D, transform should be a Mat4, etc). // This will also make error checking for invalid user data much nicer. @@ -113,7 +47,6 @@ pub enum RawMeshError { /// assert_eq!( /// RawMesh3D::data_type(), /// DataType::Struct(vec![ -/// Field::new("mesh_id", DataType::FixedSizeBinary(16), false), /// Field::new("vertex_positions", DataType::List(Box::new( /// Field::new("item", DataType::Float32, false)), /// ), false), @@ -135,8 +68,6 @@ pub enum RawMeshError { /// ``` #[derive(ArrowField, ArrowSerialize, ArrowDeserialize, Clone, Debug, PartialEq)] pub struct RawMesh3D { - pub mesh_id: MeshId, - /// The flattened vertex positions array of this mesh. /// /// The length of this vector should always be divisible by three (since this is a 3D mesh). @@ -239,7 +170,6 @@ impl RawMesh3D { /// assert_eq!( /// EncodedMesh3D::data_type(), /// DataType::Struct(vec![ -/// Field::new("mesh_id", DataType::FixedSizeBinary(16), false), /// Field::new("format", DataType::Union(vec![ /// Field::new("Gltf", DataType::Boolean, false), /// Field::new("Glb", DataType::Boolean, false), @@ -255,8 +185,6 @@ impl RawMesh3D { /// ``` #[derive(Clone, Debug, PartialEq)] pub struct EncodedMesh3D { - pub mesh_id: MeshId, - pub format: MeshFormat, pub bytes: Buffer, @@ -268,8 +196,6 @@ pub struct EncodedMesh3D { /// Helper struct for converting `EncodedMesh3D` to arrow #[derive(ArrowField, ArrowSerialize, ArrowDeserialize)] pub struct EncodedMesh3DArrow { - pub mesh_id: MeshId, - pub format: MeshFormat, pub bytes: Buffer, @@ -281,13 +207,11 @@ pub struct EncodedMesh3DArrow { impl From<&EncodedMesh3D> for EncodedMesh3DArrow { fn from(v: &EncodedMesh3D) -> Self { let EncodedMesh3D { - mesh_id, format, bytes, transform, } = v; Self { - mesh_id: *mesh_id, format: *format, bytes: bytes.clone(), transform: transform.iter().flat_map(|c| c.iter().cloned()).collect(), @@ -300,14 +224,12 @@ impl TryFrom for EncodedMesh3D { fn try_from(v: EncodedMesh3DArrow) -> super::Result { let EncodedMesh3DArrow { - mesh_id, format, bytes, transform, } = v; Ok(Self { - mesh_id, format, bytes, transform: [ @@ -414,23 +336,12 @@ impl re_log_types::LegacyComponent for Mesh3D { } } -impl Mesh3D { - #[inline] - pub fn mesh_id(&self) -> MeshId { - match self { - Mesh3D::Encoded(mesh) => mesh.mesh_id, - Mesh3D::Raw(mesh) => mesh.mesh_id, - } - } -} - #[cfg(test)] mod tests { use super::*; fn example_raw_mesh() -> RawMesh3D { let mesh = RawMesh3D { - mesh_id: MeshId::random(), vertex_positions: vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 8.0, 9.0, 10.0].into(), vertex_colors: Some(vec![0xff0000ff, 0x00ff00ff, 0x0000ffff].into()), indices: Some(vec![0, 1, 2].into()), @@ -451,7 +362,6 @@ mod tests { // Encoded { let mesh_in = vec![Mesh3D::Encoded(EncodedMesh3D { - mesh_id: MeshId::random(), format: MeshFormat::Glb, bytes: vec![5, 9, 13, 95, 38, 42, 98, 17].into(), transform: [ diff --git a/crates/re_components/src/tensor.rs b/crates/re_components/src/tensor.rs index b1012a8d0c81..382f8916d4e0 100644 --- a/crates/re_components/src/tensor.rs +++ b/crates/re_components/src/tensor.rs @@ -1,79 +1,10 @@ -use arrow2::array::{FixedSizeBinaryArray, MutableFixedSizeBinaryArray}; use arrow2::buffer::Buffer; -use arrow2_convert::deserialize::ArrowDeserialize; -use arrow2_convert::field::ArrowField; -use arrow2_convert::{serialize::ArrowSerialize, ArrowDeserialize, ArrowField, ArrowSerialize}; +use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; use crate::{TensorDataType, TensorElement}; // ---------------------------------------------------------------------------- -/// A unique id per [`Tensor`]. -/// -/// TODO(emilk): this should be a hash of the tensor (CAS). -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct TensorId(pub uuid::Uuid); - -impl nohash_hasher::IsEnabled for TensorId {} - -// required for [`nohash_hasher`]. -#[allow(clippy::derived_hash_with_manual_eq)] -impl std::hash::Hash for TensorId { - #[inline] - fn hash(&self, state: &mut H) { - state.write_u64(self.0.as_u128() as u64); - } -} - -impl TensorId { - #[inline] - pub fn random() -> Self { - Self(uuid::Uuid::new_v4()) - } -} - -impl ArrowField for TensorId { - type Type = Self; - - #[inline] - fn data_type() -> arrow2::datatypes::DataType { - arrow2::datatypes::DataType::FixedSizeBinary(16) - } -} - -//TODO(https://github.com/DataEngineeringLabs/arrow2-convert/issues/79#issue-1415520918) -impl ArrowSerialize for TensorId { - type MutableArrayType = MutableFixedSizeBinaryArray; - - #[inline] - fn new_array() -> Self::MutableArrayType { - MutableFixedSizeBinaryArray::new(16) - } - - #[inline] - fn arrow_serialize( - v: &::Type, - array: &mut Self::MutableArrayType, - ) -> arrow2::error::Result<()> { - array.try_push(Some(v.0.as_bytes())) - } -} - -impl ArrowDeserialize for TensorId { - type ArrayType = FixedSizeBinaryArray; - - #[inline] - fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, - ) -> Option<::Type> { - v.and_then(|bytes| uuid::Uuid::from_slice(bytes).ok()) - .map(Self) - } -} - -// ---------------------------------------------------------------------------- - /// Flattened `Tensor` data payload /// /// ## Examples @@ -352,7 +283,6 @@ pub enum TensorDataMeaning { /// assert_eq!( /// Tensor::data_type(), /// DataType::Struct(vec![ -/// Field::new("tensor_id", DataType::FixedSizeBinary(16), false), /// Field::new( /// "shape", /// DataType::List(Box::new(Field::new( @@ -382,9 +312,6 @@ pub enum TensorDataMeaning { /// ``` #[derive(Clone, Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)] pub struct Tensor { - /// Unique identifier for the tensor - pub tensor_id: TensorId, - /// Dimensionality and length pub shape: Vec, @@ -400,11 +327,6 @@ pub struct Tensor { } impl Tensor { - #[inline] - pub fn id(&self) -> TensorId { - self.tensor_id - } - #[inline] pub fn shape(&self) -> &[TensorDimension] { self.shape.as_slice() @@ -613,14 +535,12 @@ macro_rules! tensor_type { match view.to_slice() { Some(slice) => Ok(Tensor { - tensor_id: TensorId::random(), shape, data: TensorData::$variant(Vec::from(slice).into()), meaning: TensorDataMeaning::Unknown, meter: None, }), None => Ok(Tensor { - tensor_id: TensorId::random(), shape, data: TensorData::$variant(view.iter().cloned().collect::>().into()), meaning: TensorDataMeaning::Unknown, @@ -645,7 +565,6 @@ macro_rules! tensor_type { value .is_standard_layout() .then(|| Tensor { - tensor_id: TensorId::random(), shape, data: TensorData::$variant(value.into_raw_vec().into()), meaning: TensorDataMeaning::Unknown, @@ -706,14 +625,12 @@ impl<'a, D: ::ndarray::Dimension> TryFrom<::ndarray::ArrayView<'a, half::f16, D> .collect(); match view.to_slice() { Some(slice) => Ok(Tensor { - tensor_id: TensorId::random(), shape, data: TensorData::F16(Vec::from(bytemuck::cast_slice(slice)).into()), meaning: TensorDataMeaning::Unknown, meter: None, }), None => Ok(Tensor { - tensor_id: TensorId::random(), shape, data: TensorData::F16( view.iter() @@ -743,7 +660,6 @@ impl TryFrom<::ndarray::Array> for Tensor value .is_standard_layout() .then(|| Tensor { - tensor_id: TensorId::random(), shape, data: TensorData::F16( bytemuck::cast_slice(value.into_raw_vec().as_slice()) @@ -814,14 +730,12 @@ pub enum TensorImageSaveError { impl Tensor { pub fn new( - tensor_id: TensorId, shape: Vec, data: TensorData, meaning: TensorDataMeaning, meter: Option, ) -> Self { Self { - tensor_id, shape, data, meaning, @@ -913,7 +827,6 @@ impl Tensor { let (w, h) = decoder.dimensions().unwrap(); // Can't fail after a successful decode_headers Ok(Self { - tensor_id: TensorId::random(), shape: vec![ TensorDimension::height(h as _), TensorDimension::width(w as _), @@ -1165,7 +1078,6 @@ impl DecodedTensor { } }; let tensor = Tensor { - tensor_id: TensorId::random(), shape: vec![ TensorDimension::height(h as _), TensorDimension::width(w as _), @@ -1252,7 +1164,6 @@ impl DecodedTensor { } let tensor = Tensor { - tensor_id: TensorId::random(), shape: vec![ TensorDimension::height(h), TensorDimension::width(w), @@ -1299,7 +1210,6 @@ re_log_types::component_legacy_shim!(Tensor); #[test] fn test_ndarray() { let t0 = Tensor { - tensor_id: TensorId::random(), shape: vec![ TensorDimension { size: 2, @@ -1328,7 +1238,6 @@ fn test_arrow() { let tensors_in = vec![ Tensor { - tensor_id: TensorId(std::default::Default::default()), shape: vec![TensorDimension { size: 4, name: None, @@ -1338,7 +1247,6 @@ fn test_arrow() { meter: Some(1000.0), }, Tensor { - tensor_id: TensorId(std::default::Default::default()), shape: vec![TensorDimension { size: 2, name: None, @@ -1365,7 +1273,6 @@ fn test_tensor_shape_utilities() { let data = (0..num_elements).map(|i| i as u32).collect::>(); Tensor { - tensor_id: TensorId(std::default::Default::default()), shape, data: TensorData::U32(data.into()), meaning: TensorDataMeaning::Unknown, diff --git a/crates/re_data_store/src/instance_path.rs b/crates/re_data_store/src/instance_path.rs index 6b6e3b02225b..4cec42900773 100644 --- a/crates/re_data_store/src/instance_path.rs +++ b/crates/re_data_store/src/instance_path.rs @@ -1,9 +1,9 @@ use std::hash::Hash; -use re_log_types::{EntityPath, EntityPathHash}; +use re_log_types::{EntityPath, EntityPathHash, RowId}; use re_types::components::InstanceKey; -use crate::store_db::EntityDb; +use crate::{store_db::EntityDb, VersionedInstancePath, VersionedInstancePathHash}; // ---------------------------------------------------------------------------- @@ -49,6 +49,15 @@ impl InstancePath { self.instance_key.is_splat() } + /// Versions this instance path by stamping it with the specified [`RowId`]. + #[inline] + pub fn versioned(&self, row_id: RowId) -> VersionedInstancePath { + VersionedInstancePath { + instance_path: self.clone(), + row_id, + } + } + #[inline] pub fn hash(&self) -> InstancePathHash { InstancePathHash { @@ -88,15 +97,25 @@ pub struct InstancePathHash { impl std::hash::Hash for InstancePathHash { #[inline] fn hash(&self, state: &mut H) { - state.write_u64(self.entity_path_hash.hash64()); - state.write_u64(self.instance_key.0); + let Self { + entity_path_hash, + instance_key, + } = self; + + state.write_u64(entity_path_hash.hash64()); + state.write_u64(instance_key.0); } } impl std::cmp::PartialEq for InstancePathHash { #[inline] fn eq(&self, other: &Self) -> bool { - self.entity_path_hash == other.entity_path_hash && self.instance_key == other.instance_key + let Self { + entity_path_hash, + instance_key, + } = self; + + entity_path_hash == &other.entity_path_hash && instance_key == &other.instance_key } } @@ -137,6 +156,15 @@ impl InstancePathHash { self.entity_path_hash.is_none() } + /// Versions this hashed instance path by stamping it with the specified [`RowId`]. + #[inline] + pub fn versioned(&self, row_id: RowId) -> VersionedInstancePathHash { + VersionedInstancePathHash { + instance_path_hash: *self, + row_id, + } + } + pub fn resolve(&self, entity_db: &EntityDb) -> Option { let entity_path = entity_db .entity_path_from_hash(&self.entity_path_hash) diff --git a/crates/re_data_store/src/lib.rs b/crates/re_data_store/src/lib.rs index bb1b4a312e46..ed65d04e40ad 100644 --- a/crates/re_data_store/src/lib.rs +++ b/crates/re_data_store/src/lib.rs @@ -11,12 +11,14 @@ pub mod entity_tree; mod instance_path; pub mod store_db; mod util; - -pub use entity_properties::*; -pub use entity_tree::*; -pub use instance_path::*; -pub use store_db::StoreDb; -pub use util::*; +mod versioned_instance_path; + +pub use self::entity_properties::*; +pub use self::entity_tree::*; +pub use self::instance_path::{InstancePath, InstancePathHash}; +pub use self::store_db::StoreDb; +pub use self::util::*; +pub use self::versioned_instance_path::{VersionedInstancePath, VersionedInstancePathHash}; #[cfg(feature = "serde")] pub use editable_auto_value::EditableAutoValue; diff --git a/crates/re_data_store/src/versioned_instance_path.rs b/crates/re_data_store/src/versioned_instance_path.rs new file mode 100644 index 000000000000..efd6754024eb --- /dev/null +++ b/crates/re_data_store/src/versioned_instance_path.rs @@ -0,0 +1,107 @@ +use std::hash::Hash; + +use re_log_types::RowId; + +use crate::{InstancePath, InstancePathHash}; + +// ---------------------------------------------------------------------------- + +/// A versioned path (i.e. pinned to a specific [`RowId`]) to either a specific instance of an entity, +/// or the whole entity (splat). +/// +/// The easiest way to construct this type is via [`crate::InstancePath::versioned`]. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct VersionedInstancePath { + pub instance_path: InstancePath, + pub row_id: RowId, +} + +impl VersionedInstancePath { + /// Do we refer to the whole entity (all instances of it)? + /// + /// For example: the whole point cloud, rather than a specific point. + #[inline] + pub fn is_splat(&self) -> bool { + self.instance_path.is_splat() + } + + #[inline] + pub fn hash(&self) -> VersionedInstancePathHash { + VersionedInstancePathHash { + instance_path_hash: self.instance_path.hash(), + row_id: self.row_id, + } + } +} + +impl std::fmt::Display for VersionedInstancePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + format!("{} @ {}", self.instance_path, self.row_id).fmt(f) + } +} + +// ---------------------------------------------------------------------------- + +/// Hashes of the components of a [`VersionedInstancePath`]. +/// +/// The easiest way to construct this type is to use either [`crate::InstancePathHash::versioned`] +/// or [`crate::VersionedInstancePath::hash`]. +#[derive(Clone, Copy, Debug, Eq)] +pub struct VersionedInstancePathHash { + pub instance_path_hash: InstancePathHash, + pub row_id: RowId, +} + +impl std::hash::Hash for VersionedInstancePathHash { + #[inline] + fn hash(&self, state: &mut H) { + let Self { + instance_path_hash, + row_id, + } = self; + let InstancePathHash { + entity_path_hash, + instance_key, + } = instance_path_hash; + + state.write_u64(entity_path_hash.hash64()); + state.write_u64(instance_key.0); + state.write_u128(row_id.as_u128()); + } +} + +impl std::cmp::PartialEq for VersionedInstancePathHash { + #[inline] + fn eq(&self, other: &Self) -> bool { + let Self { + instance_path_hash, + row_id, + } = self; + + instance_path_hash == &other.instance_path_hash && row_id == &other.row_id + } +} + +impl VersionedInstancePathHash { + pub const NONE: Self = Self { + instance_path_hash: InstancePathHash::NONE, + row_id: RowId::ZERO, + }; + + #[inline] + pub fn is_some(&self) -> bool { + self.instance_path_hash.is_some() + } + + #[inline] + pub fn is_none(&self) -> bool { + self.instance_path_hash.is_none() + } +} + +impl Default for VersionedInstancePathHash { + fn default() -> Self { + Self::NONE + } +} diff --git a/crates/re_data_ui/src/annotation_context.rs b/crates/re_data_ui/src/annotation_context.rs index 399b55ae204c..4cc109674593 100644 --- a/crates/re_data_ui/src/annotation_context.rs +++ b/crates/re_data_ui/src/annotation_context.rs @@ -91,7 +91,7 @@ fn annotation_info( .data_store .query_latest_component::(entity_path, query)?; let annotations = crate::annotations(ctx, query, entity_path); - let class = annotations.resolved_class_description(Some(class_id)); + let class = annotations.resolved_class_description(Some(*class_id)); class.keypoint_map?.get(&keypoint_id).cloned() } diff --git a/crates/re_data_ui/src/image.rs b/crates/re_data_ui/src/image.rs index 524b9343cec7..8757a47a6b5a 100644 --- a/crates/re_data_ui/src/image.rs +++ b/crates/re_data_ui/src/image.rs @@ -2,6 +2,8 @@ use egui::{Color32, Vec2}; use itertools::Itertools as _; use re_components::{DecodedTensor, Tensor, TensorDataMeaning, TensorElement}; +use re_data_store::{InstancePathHash, VersionedInstancePathHash}; +use re_log_types::RowId; use re_renderer::renderer::ColormappedTexture; use re_types::components::ClassId; use re_ui::ReUi; @@ -27,9 +29,18 @@ impl EntityDataUi for Tensor { ) { re_tracing::profile_function!(); + let row_id = ctx + .store_db + .entity_db + .data_store + .query_latest_component::(entity_path, query) + .map_or(RowId::ZERO, |tensor| tensor.row_id); + + // NOTE: Tensors don't support batches at the moment so always splat. + let tensor_path_hash = InstancePathHash::entity_splat(entity_path).versioned(row_id); let decoded = ctx .cache - .entry(|c: &mut TensorDecodeCache| c.entry(self.clone())); + .entry(|c: &mut TensorDecodeCache| c.entry(tensor_path_hash, self.clone())); match decoded { Ok(decoded) => { let annotations = crate::annotations(ctx, query, entity_path); @@ -39,6 +50,7 @@ impl EntityDataUi for Tensor { verbosity, entity_path, &annotations, + tensor_path_hash, self, &decoded, ); @@ -50,22 +62,27 @@ impl EntityDataUi for Tensor { } } +#[allow(clippy::too_many_arguments)] fn tensor_ui( ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui, verbosity: UiVerbosity, entity_path: &re_data_store::EntityPath, annotations: &Annotations, + tensor_path_hash: VersionedInstancePathHash, original_tensor: &Tensor, tensor: &DecodedTensor, ) { // See if we can convert the tensor to a GPU texture. // Even if not, we will show info about the tensor. - let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(tensor)); + let tensor_stats = ctx + .cache + .entry(|c: &mut TensorStatsCache| c.entry(tensor_path_hash, tensor)); let debug_name = entity_path.to_string(); let texture_result = gpu_bridge::tensor_to_gpu( ctx.render_ctx, &debug_name, + tensor_path_hash, tensor, &tensor_stats, annotations, @@ -141,6 +158,7 @@ fn tensor_ui( ctx.render_ctx, ui, response, + tensor_path_hash, tensor, &tensor_stats, annotations, @@ -217,7 +235,6 @@ pub fn tensor_summary_ui_grid_contents( tensor_stats: &TensorStats, ) { let Tensor { - tensor_id: _, shape, data: _, meaning, @@ -334,6 +351,7 @@ fn show_zoomed_image_region_tooltip( render_ctx: &mut re_renderer::RenderContext, parent_ui: &mut egui::Ui, response: egui::Response, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, annotations: &Annotations, @@ -365,6 +383,7 @@ fn show_zoomed_image_region_tooltip( show_zoomed_image_region( render_ctx, ui, + tensor_path_hash, tensor, tensor_stats, annotations, @@ -419,6 +438,7 @@ pub fn show_zoomed_image_region_area_outline( pub fn show_zoomed_image_region( render_ctx: &mut re_renderer::RenderContext, ui: &mut egui::Ui, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, annotations: &Annotations, @@ -429,6 +449,7 @@ pub fn show_zoomed_image_region( if let Err(err) = try_show_zoomed_image_region( render_ctx, ui, + tensor_path_hash, tensor, tensor_stats, annotations, @@ -445,6 +466,7 @@ pub fn show_zoomed_image_region( fn try_show_zoomed_image_region( render_ctx: &mut re_renderer::RenderContext, ui: &mut egui::Ui, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, annotations: &Annotations, @@ -456,8 +478,14 @@ fn try_show_zoomed_image_region( return Ok(()); }; - let texture = - gpu_bridge::tensor_to_gpu(render_ctx, debug_name, tensor, tensor_stats, annotations)?; + let texture = gpu_bridge::tensor_to_gpu( + render_ctx, + debug_name, + tensor_path_hash, + tensor, + tensor_stats, + annotations, + )?; const POINTS_PER_TEXEL: f32 = 5.0; let size = Vec2::splat((ZOOMED_IMAGE_TEXEL_RADIUS * 2 + 1) as f32 * POINTS_PER_TEXEL); diff --git a/crates/re_query/src/archetype_view.rs b/crates/re_query/src/archetype_view.rs index dcd272a57b58..2eb96193e6d4 100644 --- a/crates/re_query/src/archetype_view.rs +++ b/crates/re_query/src/archetype_view.rs @@ -221,7 +221,7 @@ where /// the required [`Component`]s using [`InstanceKey`] values. #[derive(Clone, Debug)] pub struct ArchetypeView { - pub(crate) row_id: RowId, + pub(crate) primary_row_id: RowId, pub(crate) components: BTreeMap, pub(crate) phantom: PhantomData, } @@ -248,9 +248,15 @@ impl ArchetypeView { self.required_comp().len() } + /// Returns the [`RowId`] associated with the _primary_ component that was used to drive this + /// entire query. + /// + /// Beware: when using this [`RowId`] for caching/versioning purposes, make sure the component + /// you are about to cache is in fact the primary component of the query! + /// See also . #[inline] - pub fn row_id(&self) -> RowId { - self.row_id + pub fn primary_row_id(&self) -> RowId { + self.primary_row_id } } @@ -348,7 +354,7 @@ impl ArchetypeView { components: impl IntoIterator, ) -> Self { Self { - row_id, + primary_row_id: row_id, components: components .into_iter() .map(|comp| (comp.name(), comp)) diff --git a/crates/re_query/src/entity_view.rs b/crates/re_query/src/entity_view.rs index c0fd1a2f7994..9c5e014d02d0 100644 --- a/crates/re_query/src/entity_view.rs +++ b/crates/re_query/src/entity_view.rs @@ -13,7 +13,7 @@ use crate::{archetype_view::ComponentJoinedIterator, ComponentWithInstances}; /// the primary component using instance keys. #[derive(Clone, Debug)] pub struct EntityView { - pub(crate) row_id: RowId, + pub(crate) primary_row_id: RowId, pub(crate) primary: ComponentWithInstances, pub(crate) components: BTreeMap, pub(crate) phantom: PhantomData, @@ -42,9 +42,15 @@ where self.primary.len() } + /// Returns the [`RowId`] associated with the _primary_ component that was used to drive this + /// entire query. + /// + /// Beware: when using this [`RowId`] for caching/versioning purposes, make sure the component + /// you are about to cache is in fact the primary component of the query! + /// See also . #[inline] - pub fn row_id(&self) -> RowId { - self.row_id + pub fn primary_row_id(&self) -> RowId { + self.primary_row_id } } @@ -115,7 +121,7 @@ where // Need to convert to new-style keys let primary = ComponentWithInstances::from_native(c0.0, c0.1); Self { - row_id: RowId::ZERO, + primary_row_id: RowId::ZERO, primary, components: Default::default(), phantom: PhantomData, @@ -138,7 +144,7 @@ where let components = [(component_c1.name(), component_c1)].into(); Self { - row_id: RowId::ZERO, + primary_row_id: RowId::ZERO, primary, components, phantom: PhantomData, diff --git a/crates/re_query/src/query.rs b/crates/re_query/src/query.rs index 38846c8a90bb..bb7185a8834f 100644 --- a/crates/re_query/src/query.rs +++ b/crates/re_query/src/query.rs @@ -148,7 +148,7 @@ pub fn query_entity_with_primary( .collect(); Ok(EntityView { - row_id, + primary_row_id: row_id, primary, components, phantom: std::marker::PhantomData, diff --git a/crates/re_query/src/range.rs b/crates/re_query/src/range.rs index eda09b559332..07393e092f44 100644 --- a/crates/re_query/src/range.rs +++ b/crates/re_query/src/range.rs @@ -134,7 +134,7 @@ pub fn range_entity_with_primary< let (row_id, cwi) = state[primary_col].clone().unwrap(); // shallow let ent_view = EntityView { - row_id, + primary_row_id: row_id, primary: cwi, components: components .iter() diff --git a/crates/re_renderer/src/resource_managers/texture_manager.rs b/crates/re_renderer/src/resource_managers/texture_manager.rs index c9464b10afd1..b0f99959a86e 100644 --- a/crates/re_renderer/src/resource_managers/texture_manager.rs +++ b/crates/re_renderer/src/resource_managers/texture_manager.rs @@ -151,14 +151,15 @@ pub struct TextureManager2D { #[derive(Default)] struct Inner { - // Cache the texture using a user-provided u64 id. This is expected - // to be derived from the `TensorId` which isn't available here for - // dependency reasons. - // TODO(jleibs): Introduce a proper key here. - // - // Any texture which wasn't accessed on the previous frame - // is ejected from the cache during [`begin_frame`]. + /// Caches textures using a `VersionedInstancePathHash`, i.e. a specific instance of a specific + /// entity path for a specific row in the store. + /// + /// For dependency reasons, the versioned path has to be provided pre-hashed as a u64 by the user. + /// + /// Any texture which wasn't accessed on the previous frame is ejected from the cache + /// during [`Self::begin_frame`]. texture_cache: HashMap, + accessed_textures: HashSet, } diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs index 6f359941c0f1..c75db327138d 100644 --- a/crates/re_sdk/src/lib.rs +++ b/crates/re_sdk/src/lib.rs @@ -70,9 +70,9 @@ pub mod time { /// and can be used in [`RecordingStream::log_component_lists`]. pub mod components { pub use re_components::{ - Box3D, EncodedMesh3D, Mesh3D, MeshFormat, MeshId, Pinhole, Quaternion, RawMesh3D, Rect2D, - Scalar, ScalarPlotProps, Tensor, TensorData, TensorDataMeaning, TensorDimension, TensorId, - TextEntry, ViewCoordinates, + Box3D, EncodedMesh3D, Mesh3D, MeshFormat, Pinhole, Quaternion, RawMesh3D, Rect2D, Scalar, + ScalarPlotProps, Tensor, TensorData, TensorDataMeaning, TensorDimension, TextEntry, + ViewCoordinates, }; pub use re_types::components::{ AnnotationContext, ClassId, Color, DisconnectedSpace, DrawOrder, InstanceKey, KeypointId, diff --git a/crates/re_space_view_bar_chart/src/view_part_system.rs b/crates/re_space_view_bar_chart/src/view_part_system.rs index 13618fa24232..a3f04bf508b7 100644 --- a/crates/re_space_view_bar_chart/src/view_part_system.rs +++ b/crates/re_space_view_bar_chart/src/view_part_system.rs @@ -67,7 +67,7 @@ impl ViewPartSystem for BarChartViewPartSystem { if let Some(tensor) = tensor { if tensor.is_vector() { - self.charts.insert(ent_path.clone(), tensor.clone()); // shallow clones + self.charts.insert(ent_path.clone(), tensor.value.clone()); // shallow clones } } } diff --git a/crates/re_space_view_spatial/src/contexts/depth_offsets.rs b/crates/re_space_view_spatial/src/contexts/depth_offsets.rs index 4f2af1d7ebbf..ab8f2a3ad1b4 100644 --- a/crates/re_space_view_spatial/src/contexts/depth_offsets.rs +++ b/crates/re_space_view_spatial/src/contexts/depth_offsets.rs @@ -54,7 +54,7 @@ impl ViewContextSystem for EntityDepthOffsets { &ctx.rec_cfg.time_ctrl.current_query(), ) { entities_per_draw_order - .entry(draw_order) + .entry(draw_order.value) .or_default() .insert(DrawOrderTarget::Entity(ent_path.hash())); } diff --git a/crates/re_space_view_spatial/src/contexts/transform_context.rs b/crates/re_space_view_spatial/src/contexts/transform_context.rs index 06ac4e3a875c..be36a415b667 100644 --- a/crates/re_space_view_spatial/src/contexts/transform_context.rs +++ b/crates/re_space_view_spatial/src/contexts/transform_context.rs @@ -252,7 +252,7 @@ impl TransformContext { t * store .query_latest_component::(ent_path, query) .map_or(glam::Affine3A::IDENTITY, |transform| { - transform.into_parent_from_child_transform() + transform.value.into_parent_from_child_transform() }) }) } else { @@ -299,7 +299,7 @@ fn transform_at( let transform3d = store .query_latest_component::(entity_path, query) - .map(|transform| transform.into_parent_from_child_transform()); + .map(|transform| transform.value.into_parent_from_child_transform()); let pinhole = pinhole.map(|pinhole| { // Everything under a pinhole camera is a 2D projection, thus doesn't actually have a proper 3D representation. @@ -367,5 +367,5 @@ pub fn pinhole_camera_view_coordinates( ) -> ViewCoordinates { store .query_latest_component(entity_path, query) - .unwrap_or(ViewCoordinates::RDF) + .map_or(ViewCoordinates::RDF, |c| c.value) } diff --git a/crates/re_space_view_spatial/src/mesh_cache.rs b/crates/re_space_view_spatial/src/mesh_cache.rs index 1e33b36349d8..a6fd28bba69a 100644 --- a/crates/re_space_view_spatial/src/mesh_cache.rs +++ b/crates/re_space_view_spatial/src/mesh_cache.rs @@ -1,6 +1,7 @@ use std::sync::Arc; -use re_components::{Mesh3D, MeshId}; +use re_components::Mesh3D; +use re_data_store::VersionedInstancePathHash; use re_renderer::RenderContext; use re_viewer_context::Cache; @@ -8,22 +9,23 @@ use crate::mesh_loader::LoadedMesh; // ---------------------------------------------------------------------------- +/// Caches meshes based on their [`VersionedInstancePathHash`], i.e. a specific instance of a specific +/// entity path for a specific row in the store. #[derive(Default)] -pub struct MeshCache(nohash_hasher::IntMap>>); +pub struct MeshCache(ahash::HashMap>>); impl MeshCache { pub fn entry( &mut self, name: &str, + key: VersionedInstancePathHash, mesh: &Mesh3D, render_ctx: &RenderContext, ) -> Option> { re_tracing::profile_function!(); - let mesh_id = mesh.mesh_id(); - self.0 - .entry(mesh_id) + .entry(key) .or_insert_with(|| { re_log::debug!("Loading CPU mesh {name:?}…"); diff --git a/crates/re_space_view_spatial/src/mesh_loader.rs b/crates/re_space_view_spatial/src/mesh_loader.rs index d7b0bef0958a..fc0035b20002 100644 --- a/crates/re_space_view_spatial/src/mesh_loader.rs +++ b/crates/re_space_view_spatial/src/mesh_loader.rs @@ -60,7 +60,6 @@ impl LoadedMesh { ) -> anyhow::Result { re_tracing::profile_function!(); let EncodedMesh3D { - mesh_id: _, format, bytes, transform, @@ -93,7 +92,6 @@ impl LoadedMesh { // Rust. Need to clean all of that up later. let RawMesh3D { - mesh_id: _, vertex_positions, vertex_colors, vertex_normals, diff --git a/crates/re_space_view_spatial/src/parts/cameras.rs b/crates/re_space_view_spatial/src/parts/cameras.rs index a9c56540919a..df2c6ff594c3 100644 --- a/crates/re_space_view_spatial/src/parts/cameras.rs +++ b/crates/re_space_view_spatial/src/parts/cameras.rs @@ -204,8 +204,10 @@ impl ViewPartSystem for CamerasPart { shared_render_builders, ent_path, &props, - pinhole, - store.query_latest_component::(ent_path, &time_query), + pinhole.value, + store + .query_latest_component::(ent_path, &time_query) + .map(|c| c.value), pinhole_view_coordinates, entity_highlight, ); diff --git a/crates/re_space_view_spatial/src/parts/images.rs b/crates/re_space_view_spatial/src/parts/images.rs index 025328f6f748..ca8b8f651dbc 100644 --- a/crates/re_space_view_spatial/src/parts/images.rs +++ b/crates/re_space_view_spatial/src/parts/images.rs @@ -6,7 +6,7 @@ use itertools::Itertools as _; use nohash_hasher::IntSet; use re_arrow_store::LatestAtQuery; use re_components::{DecodedTensor, Pinhole, Tensor, TensorDataMeaning}; -use re_data_store::{EntityPath, EntityProperties}; +use re_data_store::{EntityPath, EntityProperties, InstancePathHash, VersionedInstancePathHash}; use re_log_types::{ComponentName, EntityPathHash, TimeInt, Timeline}; use re_query::{EntityView, QueryError}; use re_renderer::{ @@ -52,6 +52,7 @@ fn to_textured_rect( ctx: &mut ViewerContext<'_>, ent_path: &EntityPath, ent_context: &SpatialSceneEntityContext<'_>, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, multiplicative_tint: egui::Rgba, ) -> Option { @@ -62,11 +63,14 @@ fn to_textured_rect( }; let debug_name = ent_path.to_string(); - let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(tensor)); + let tensor_stats = ctx + .cache + .entry(|c: &mut TensorStatsCache| c.entry(tensor_path_hash, tensor)); match gpu_bridge::tensor_to_gpu( ctx.render_ctx, &debug_name, + tensor_path_hash, tensor, &tensor_stats, &ent_context.annotations, @@ -219,7 +223,13 @@ impl ImagesPart { return Ok(()); } - let tensor = match ctx.cache.entry(|c: &mut TensorDecodeCache| c.entry(tensor)) { + // NOTE: Tensors don't support batches at the moment so always splat. + let tensor_path_hash = + InstancePathHash::entity_splat(ent_path).versioned(ent_view.primary_row_id()); + let tensor = match ctx + .cache + .entry(|c: &mut TensorDecodeCache| c.entry(tensor_path_hash, tensor)) + { Ok(tensor) => tensor, Err(err) => { re_log::warn_once!( @@ -239,6 +249,7 @@ impl ImagesPart { transforms, ent_context, ent_props, + tensor_path_hash, &tensor, ent_path, parent_pinhole_path, @@ -266,9 +277,14 @@ impl ImagesPart { DefaultColor::OpaqueWhite, ); - if let Some(textured_rect) = - to_textured_rect(ctx, ent_path, ent_context, &tensor, color.into()) - { + if let Some(textured_rect) = to_textured_rect( + ctx, + ent_path, + ent_context, + tensor_path_hash, + &tensor, + color.into(), + ) { { let top_left = textured_rect.top_left_corner_position; self.data.bounding_box.extend(top_left); @@ -296,11 +312,13 @@ impl ImagesPart { Ok(()) } + #[allow(clippy::too_many_arguments)] fn process_entity_view_as_depth_cloud( ctx: &mut ViewerContext<'_>, transforms: &TransformContext, ent_context: &SpatialSceneEntityContext<'_>, properties: &EntityProperties, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, ent_path: &EntityPath, parent_pinhole_path: &EntityPath, @@ -337,10 +355,13 @@ impl ImagesPart { let dimensions = glam::UVec2::new(width as _, height as _); let debug_name = ent_path.to_string(); - let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(tensor)); + let tensor_stats = ctx + .cache + .entry(|c: &mut TensorStatsCache| c.entry(tensor_path_hash, tensor)); let depth_texture = re_viewer_context::gpu_bridge::depth_tensor_to_gpu( ctx.render_ctx, &debug_name, + tensor_path_hash, tensor, &tensor_stats, )?; diff --git a/crates/re_space_view_spatial/src/parts/meshes.rs b/crates/re_space_view_spatial/src/parts/meshes.rs index 469286ad8379..bb14cb7fb39b 100644 --- a/crates/re_space_view_spatial/src/parts/meshes.rs +++ b/crates/re_space_view_spatial/src/parts/meshes.rs @@ -37,6 +37,7 @@ impl MeshPart { ) -> Result<(), QueryError> { let _default_color = DefaultColor::EntityPath(ent_path); + let primary_row_id = ent_view.primary_row_id(); let visitor = |instance_key: InstanceKey, mesh: re_components::Mesh3D, _color: Option| { @@ -45,9 +46,14 @@ impl MeshPart { let outline_mask_ids = ent_context.highlight.index_outline_mask(instance_key); - let mesh = ctx - .cache - .entry(|c: &mut MeshCache| c.entry(&ent_path.to_string(), &mesh, ctx.render_ctx)); + let mesh = ctx.cache.entry(|c: &mut MeshCache| { + c.entry( + &ent_path.to_string(), + picking_instance_hash.versioned(primary_row_id), + &mesh, + ctx.render_ctx, + ) + }); if let Some(mesh) = mesh { instances.extend(mesh.mesh_instances.iter().map(move |mesh_instance| { MeshInstance { diff --git a/crates/re_space_view_spatial/src/ui.rs b/crates/re_space_view_spatial/src/ui.rs index d140552635ba..4ec1e799a593 100644 --- a/crates/re_space_view_spatial/src/ui.rs +++ b/crates/re_space_view_spatial/src/ui.rs @@ -125,7 +125,8 @@ impl SpatialSpaceViewState { let view_coordinates = ctx .store_db .store() - .query_latest_component(space_origin, &ctx.current_query()); + .query_latest_component(space_origin, &ctx.current_query()) + .map(|c| c.value); ctx.re_ui.selection_grid(ui, "spatial_settings_ui") .show(ui, |ui| { @@ -554,11 +555,12 @@ pub fn picking( None } else { tensor.image_height_width_channels().map(|[_, w, _]| { + let tensor_path_hash = hit.instance_path_hash.versioned(tensor.row_id); let coordinates = hit .instance_path_hash .instance_key .to_2d_image_coordinate(w); - (tensor, coordinates) + (tensor_path_hash, tensor.value, coordinates) }) } }) @@ -581,7 +583,7 @@ pub fn picking( instance_path.clone(), )); - response = if let Some((tensor, coords)) = picked_image_with_coords { + response = if let Some((tensor_path_hash, tensor, coords)) = picked_image_with_coords { if let Some(meter) = tensor.meter { if let Some(raw_value) = tensor.get(&[ picking_context.pointer_in_space2d.y.round() as _, @@ -627,14 +629,15 @@ pub fn picking( let tensor_name = instance_path.to_string(); - let decoded_tensor = ctx.cache.entry(|c: &mut TensorDecodeCache| c.entry(tensor)); + let decoded_tensor = ctx.cache.entry(|c: &mut TensorDecodeCache| c.entry(tensor_path_hash, tensor)); match decoded_tensor { Ok(decoded_tensor) => { let annotations = annotations.0.find(&instance_path.entity_path); - let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(&decoded_tensor)); + let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(tensor_path_hash, &decoded_tensor)); show_zoomed_image_region( ctx.render_ctx, ui, + tensor_path_hash, &decoded_tensor, &tensor_stats, &annotations, diff --git a/crates/re_space_view_spatial/src/ui_2d.rs b/crates/re_space_view_spatial/src/ui_2d.rs index a58278db6650..cddd22f4b2e3 100644 --- a/crates/re_space_view_spatial/src/ui_2d.rs +++ b/crates/re_space_view_spatial/src/ui_2d.rs @@ -250,10 +250,12 @@ pub fn view_2d( // For that we need to check if this is defined by a pinhole camera. // Note that we can't rely on the camera being part of scene.space_cameras since that requires // the camera to be added to the scene! - let pinhole = store.query_latest_component::( - query.space_origin, - &ctx.rec_cfg.time_ctrl.current_query(), - ); + let pinhole = store + .query_latest_component::( + query.space_origin, + &ctx.rec_cfg.time_ctrl.current_query(), + ) + .map(|c| c.value); let canvas_rect = pinhole .and_then(|p| p.resolution()) .map_or(scene_rect_accum, |res| { diff --git a/crates/re_space_view_spatial/src/ui_3d.rs b/crates/re_space_view_spatial/src/ui_3d.rs index b2a4e6cadd4e..8cdd5236faae 100644 --- a/crates/re_space_view_spatial/src/ui_3d.rs +++ b/crates/re_space_view_spatial/src/ui_3d.rs @@ -325,7 +325,8 @@ pub fn view_3d( let view_coordinates = ctx .store_db .store() - .query_latest_component(query.space_origin, &ctx.current_query()); + .query_latest_component(query.space_origin, &ctx.current_query()) + .map(|c| c.value); let (rect, mut response) = ui.allocate_at_least(ui.available_size(), egui::Sense::click_and_drag()); diff --git a/crates/re_space_view_tensor/src/space_view_class.rs b/crates/re_space_view_tensor/src/space_view_class.rs index 3714b2d9d665..f2625f29b68f 100644 --- a/crates/re_space_view_tensor/src/space_view_class.rs +++ b/crates/re_space_view_tensor/src/space_view_class.rs @@ -5,7 +5,7 @@ use egui::{epaint::TextShape, NumExt as _, Vec2}; use ndarray::Axis; use re_components::{DecodedTensor, Tensor, TensorDimension}; -use re_data_store::InstancePath; +use re_data_store::{InstancePath, VersionedInstancePathHash}; use re_data_ui::tensor_summary_ui_grid_contents; use re_log_types::EntityPath; use re_renderer::Colormap; @@ -61,11 +61,14 @@ pub struct PerTensorState { /// Last viewed tensor, copied each frame. /// Used for the selection view. - tensor: Option, + tensor: Option<(VersionedInstancePathHash, DecodedTensor)>, } impl PerTensorState { - pub fn create(tensor: &DecodedTensor) -> PerTensorState { + pub fn create( + tensor_path_hash: VersionedInstancePathHash, + tensor: &DecodedTensor, + ) -> PerTensorState { Self { slice: SliceSelection { dim_mapping: DimensionMapping::create(tensor.shape()), @@ -73,7 +76,7 @@ impl PerTensorState { }, color_mapping: ColorMapping::default(), texture_settings: TextureSettings::default(), - tensor: Some(tensor.clone()), + tensor: Some((tensor_path_hash, tensor.clone())), } } @@ -86,12 +89,14 @@ impl PerTensorState { } pub fn ui(&mut self, ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui) { - let Some(tensor) = &self.tensor else { + let Some((tensor_path_hash, tensor)) = &self.tensor else { ui.label("No Tensor shown in this Space View."); return; }; - let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(tensor)); + let tensor_stats = ctx + .cache + .entry(|c: &mut TensorStatsCache| c.entry(*tensor_path_hash, tensor)); ctx.re_ui .selection_grid(ui, "tensor_selection_ui") .show(ui, |ui| { @@ -203,12 +208,13 @@ impl SpaceViewClass for TensorSpaceView { } if let Some(selected_tensor) = &state.selected_tensor { - if let Some(tensor) = tensors.get(selected_tensor) { + if let Some((row_id, tensor)) = tensors.get(selected_tensor) { + let tensor_path_hash = selected_tensor.hash().versioned(*row_id); let state_tensor = state .state_tensors .entry(selected_tensor.clone()) - .or_insert_with(|| PerTensorState::create(tensor)); - view_tensor(ctx, ui, state_tensor, tensor); + .or_insert_with(|| PerTensorState::create(tensor_path_hash, tensor)); + view_tensor(ctx, ui, state_tensor, tensor_path_hash, tensor); } } } @@ -221,11 +227,12 @@ fn view_tensor( ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui, state: &mut PerTensorState, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, ) { re_tracing::profile_function!(); - state.tensor = Some(tensor.clone()); + state.tensor = Some((tensor_path_hash, tensor.clone())); if !state.slice.dim_mapping.is_valid(tensor.num_dim()) { state.slice.dim_mapping = DimensionMapping::create(tensor.shape()); @@ -266,7 +273,9 @@ fn view_tensor( }; egui::ScrollArea::both().show(ui, |ui| { - if let Err(err) = tensor_slice_ui(ctx, ui, state, tensor, dimension_labels) { + if let Err(err) = + tensor_slice_ui(ctx, ui, state, tensor_path_hash, tensor, dimension_labels) + { ui.label(ctx.re_ui.error_text(err.to_string())); } }); @@ -276,10 +285,12 @@ fn tensor_slice_ui( ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui, state: &mut PerTensorState, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, dimension_labels: [(String, bool); 2], ) -> anyhow::Result<()> { - let (response, painter, image_rect) = paint_tensor_slice(ctx, ui, state, tensor)?; + let (response, painter, image_rect) = + paint_tensor_slice(ctx, ui, state, tensor_path_hash, tensor)?; if !response.hovered() { let font_id = egui::TextStyle::Body.resolve(ui.style()); @@ -293,13 +304,17 @@ fn paint_tensor_slice( ctx: &mut ViewerContext<'_>, ui: &mut egui::Ui, state: &mut PerTensorState, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, ) -> anyhow::Result<(egui::Response, egui::Painter, egui::Rect)> { re_tracing::profile_function!(); - let tensor_stats = ctx.cache.entry(|c: &mut TensorStatsCache| c.entry(tensor)); + let tensor_stats = ctx + .cache + .entry(|c: &mut TensorStatsCache| c.entry(tensor_path_hash, tensor)); let colormapped_texture = super::tensor_slice_to_gpu::colormapped_texture( ctx.render_ctx, + tensor_path_hash, tensor, &tensor_stats, state, diff --git a/crates/re_space_view_tensor/src/tensor_slice_to_gpu.rs b/crates/re_space_view_tensor/src/tensor_slice_to_gpu.rs index 2478912e7127..8b58018ace0f 100644 --- a/crates/re_space_view_tensor/src/tensor_slice_to_gpu.rs +++ b/crates/re_space_view_tensor/src/tensor_slice_to_gpu.rs @@ -1,4 +1,5 @@ use re_components::{DecodedTensor, TensorCastError, TensorDataType}; +use re_data_store::VersionedInstancePathHash; use re_renderer::{ renderer::ColormappedTexture, resource_managers::{GpuTexture2D, Texture2DCreationDesc, TextureManager2DError}, @@ -24,6 +25,7 @@ pub enum TensorUploadError { pub fn colormapped_texture( render_ctx: &mut re_renderer::RenderContext, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, state: &PerTensorState, @@ -32,7 +34,7 @@ pub fn colormapped_texture( let range = tensor_data_range_heuristic(tensor_stats, tensor.dtype()) .map_err(|err| TextureManager2DError::DataCreation(err.into()))?; - let texture = upload_texture_slice_to_gpu(render_ctx, tensor, state.slice())?; + let texture = upload_texture_slice_to_gpu(render_ctx, tensor_path_hash, tensor, state.slice())?; let color_mapping = state.color_mapping(); @@ -50,10 +52,11 @@ pub fn colormapped_texture( fn upload_texture_slice_to_gpu( render_ctx: &mut re_renderer::RenderContext, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, slice_selection: &SliceSelection, ) -> Result> { - let id = egui::util::hash((tensor.id(), slice_selection)); + let id = egui::util::hash((tensor_path_hash, slice_selection)); gpu_bridge::try_get_or_create_texture(render_ctx, id, || { texture_desc_from_tensor(tensor, slice_selection) diff --git a/crates/re_space_view_tensor/src/view_part_system.rs b/crates/re_space_view_tensor/src/view_part_system.rs index 28ed97b88303..fb17d8c45475 100644 --- a/crates/re_space_view_tensor/src/view_part_system.rs +++ b/crates/re_space_view_tensor/src/view_part_system.rs @@ -1,7 +1,7 @@ use re_arrow_store::LatestAtQuery; use re_components::{DecodedTensor, Tensor}; -use re_data_store::{EntityPath, EntityProperties, InstancePath}; -use re_log_types::{TimeInt, Timeline}; +use re_data_store::{EntityPath, EntityProperties, InstancePath, InstancePathHash}; +use re_log_types::{RowId, TimeInt, Timeline}; use re_types::{components::InstanceKey, ComponentName, Loggable as _}; use re_viewer_context::{ ArchetypeDefinition, NamedViewSystem, SpaceViewSystemExecutionError, TensorDecodeCache, @@ -10,7 +10,7 @@ use re_viewer_context::{ #[derive(Default)] pub struct TensorSystem { - pub tensors: std::collections::BTreeMap, + pub tensors: std::collections::BTreeMap, } impl NamedViewSystem for TensorSystem { @@ -59,7 +59,7 @@ impl ViewPartSystem for TensorSystem { if let Some(tensor) = store.query_latest_component::(ent_path, &timeline_query) { - self.load_tensor_entity(ctx, ent_path, &props, tensor); + self.load_tensor_entity(ctx, ent_path, tensor.row_id, &props, tensor.value); } } @@ -76,14 +76,20 @@ impl TensorSystem { &mut self, ctx: &mut ViewerContext<'_>, ent_path: &EntityPath, + row_id: RowId, _props: &EntityProperties, tensor: Tensor, ) { if !tensor.is_shaped_like_an_image() { - match ctx.cache.entry(|c: &mut TensorDecodeCache| c.entry(tensor)) { + // NOTE: Tensors don't support batches at the moment so always splat. + let tensor_path_hash = InstancePathHash::entity_splat(ent_path).versioned(row_id); + match ctx + .cache + .entry(|c: &mut TensorDecodeCache| c.entry(tensor_path_hash, tensor)) + { Ok(tensor) => { let instance_path = InstancePath::instance(ent_path.clone(), InstanceKey(0)); - self.tensors.insert(instance_path, tensor); + self.tensors.insert(instance_path, (row_id, tensor)); } Err(err) => { re_log::warn_once!( diff --git a/crates/re_space_view_text/src/view_part_system.rs b/crates/re_space_view_text/src/view_part_system.rs index b009a2fd8504..9b11dc5234b7 100644 --- a/crates/re_space_view_text/src/view_part_system.rs +++ b/crates/re_space_view_text/src/view_part_system.rs @@ -73,7 +73,7 @@ impl ViewPartSystem for TextSystem { let re_components::TextEntry { body, level } = text_entry; self.text_entries.push(TextEntry { - row_id: ent_view.row_id(), + row_id: ent_view.primary_row_id(), entity_path: ent_path.clone(), time: time.map(|time| time.as_i64()), color: color.map(|c| c.to_array()), diff --git a/crates/re_tensor_ops/tests/tensor_tests.rs b/crates/re_tensor_ops/tests/tensor_tests.rs index 7a6e4abf9be6..68dfac9795e4 100644 --- a/crates/re_tensor_ops/tests/tensor_tests.rs +++ b/crates/re_tensor_ops/tests/tensor_tests.rs @@ -1,13 +1,10 @@ #![allow(clippy::unwrap_used)] -use re_components::{ - Tensor, TensorCastError, TensorData, TensorDataMeaning, TensorDimension, TensorId, -}; +use re_components::{Tensor, TensorCastError, TensorData, TensorDataMeaning, TensorDimension}; #[test] fn convert_tensor_to_ndarray_u8() { let t = Tensor::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -26,7 +23,6 @@ fn convert_tensor_to_ndarray_u8() { #[test] fn convert_tensor_to_ndarray_u16() { let t = Tensor::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -45,7 +41,6 @@ fn convert_tensor_to_ndarray_u16() { #[test] fn convert_tensor_to_ndarray_f32() { let t = Tensor::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -84,7 +79,6 @@ fn convert_ndarray_slice_to_tensor() { #[test] fn check_slices() { let t = Tensor::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -126,7 +120,6 @@ fn check_slices() { #[test] fn check_tensor_shape_error() { let t = Tensor::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -150,7 +143,6 @@ fn check_tensor_shape_error() { #[test] fn check_tensor_type_error() { let t = Tensor::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), diff --git a/crates/re_types/.gitattributes b/crates/re_types/.gitattributes index 37d2d35d4936..4b0d719c670f 100644 --- a/crates/re_types/.gitattributes +++ b/crates/re_types/.gitattributes @@ -50,7 +50,6 @@ src/datatypes/scale3d.rs linguist-generated=true src/datatypes/tensor_buffer.rs linguist-generated=true src/datatypes/tensor_data.rs linguist-generated=true src/datatypes/tensor_dimension.rs linguist-generated=true -src/datatypes/tensor_id.rs linguist-generated=true src/datatypes/transform3d.rs linguist-generated=true src/datatypes/translation_and_mat3x3.rs linguist-generated=true src/datatypes/translation_rotation_scale3d.rs linguist-generated=true diff --git a/crates/re_types/definitions/rerun/datatypes.fbs b/crates/re_types/definitions/rerun/datatypes.fbs index 88c995b26b46..befe0103cbcd 100644 --- a/crates/re_types/definitions/rerun/datatypes.fbs +++ b/crates/re_types/definitions/rerun/datatypes.fbs @@ -17,7 +17,6 @@ include "./datatypes/scale3d.fbs"; include "./datatypes/tensor_buffer.fbs"; include "./datatypes/tensor_dimension.fbs"; include "./datatypes/tensor_data.fbs"; -include "./datatypes/tensor_id.fbs"; include "./datatypes/transform3d.fbs"; include "./datatypes/translation_and_mat3x3.fbs"; include "./datatypes/translation_rotation_scale3d.fbs"; diff --git a/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs b/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs index d0f88a8bf085..cd0a4d291aab 100644 --- a/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs +++ b/crates/re_types/definitions/rerun/datatypes/tensor_data.fbs @@ -1,7 +1,6 @@ include "arrow/attributes.fbs"; include "fbs/attributes.fbs"; -include "./tensor_id.fbs"; include "./tensor_dimension.fbs"; include "./tensor_data.fbs"; @@ -22,7 +21,6 @@ table TensorData ( "attr.python.array_aliases": "npt.ArrayLike", "attr.rust.derive": "PartialEq," ) { - id: rerun.datatypes.TensorId (order: 100); shape: [rerun.datatypes.TensorDimension] (order: 200); buffer: rerun.datatypes.TensorBuffer (order: 300); } diff --git a/crates/re_types/definitions/rerun/datatypes/tensor_id.fbs b/crates/re_types/definitions/rerun/datatypes/tensor_id.fbs deleted file mode 100644 index d90f34443bef..000000000000 --- a/crates/re_types/definitions/rerun/datatypes/tensor_id.fbs +++ /dev/null @@ -1,16 +0,0 @@ -include "arrow/attributes.fbs"; -include "fbs/attributes.fbs"; - -namespace rerun.datatypes; - -// --- - -// A unique id per [`Tensor`]. -struct TensorId ( - order: 100, - "attr.python.aliases": "uuid.UUID", - "attr.arrow.transparent", - "attr.rust.derive": "Copy, Default, Eq, PartialEq" -) { - uuid: [ubyte:16] (order: 100); -} diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 208aa24c2fe8..528cc6bdc998 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -f6259141991af01c7c3585b703b8149d0e9cd1e2449ae88ff5b0cb3a5d225058 +9f6a533804a841b00f687708378252c3aec7769cce554fd751aaa575262ad01a diff --git a/crates/re_types/src/archetypes/image_ext.rs b/crates/re_types/src/archetypes/image_ext.rs index 6660fd4e7502..543d9dcfb691 100644 --- a/crates/re_types/src/archetypes/image_ext.rs +++ b/crates/re_types/src/archetypes/image_ext.rs @@ -42,18 +42,6 @@ impl Image { draw_order: None, }) } - - pub fn with_id(self, id: crate::datatypes::TensorId) -> Self { - Self { - data: TensorData { - id, - shape: self.data.0.shape, - buffer: self.data.0.buffer, - } - .into(), - draw_order: None, - } - } } // Returns the indices of an appropriate set of non-empty dimensions diff --git a/crates/re_types/src/archetypes/tensor_ext.rs b/crates/re_types/src/archetypes/tensor_ext.rs index d1f6e9aba189..9579c66ee1d0 100644 --- a/crates/re_types/src/archetypes/tensor_ext.rs +++ b/crates/re_types/src/archetypes/tensor_ext.rs @@ -1,5 +1,5 @@ use crate::{ - datatypes::{TensorData, TensorDimension, TensorId}, + datatypes::{TensorData, TensorDimension}, ArrowString, }; @@ -19,18 +19,6 @@ impl Tensor { Ok(Self { data: data.into() }) } - /// Replace the `id` of the contained [`TensorData`] with a new [`TensorId`] - pub fn with_id(self, id: TensorId) -> Self { - Self { - data: TensorData { - id, - shape: self.data.0.shape, - buffer: self.data.0.buffer, - } - .into(), - } - } - /// Update the `names` of the contained [`TensorData`] dimensions. /// /// Any existing Dimension names will be be overwritten. @@ -48,7 +36,6 @@ impl Tensor { } Self { data: crate::datatypes::TensorData { - id: self.data.0.id, shape: self .data .0 diff --git a/crates/re_types/src/components/tensor_data.rs b/crates/re_types/src/components/tensor_data.rs index 8a67a361a6ec..3b4e42dbab4f 100644 --- a/crates/re_types/src/components/tensor_data.rs +++ b/crates/re_types/src/components/tensor_data.rs @@ -49,12 +49,6 @@ impl crate::Loggable for TensorData { fn arrow_datatype() -> arrow2::datatypes::DataType { use ::arrow2::datatypes::*; DataType::Struct(vec![ - Field { - name: "id".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: false, - metadata: [].into(), - }, Field { name: "shape".to_owned(), data_type: DataType::List(Box::new(Field { diff --git a/crates/re_types/src/datatypes/mod.rs b/crates/re_types/src/datatypes/mod.rs index ca2671a75a83..32a6b0e11fb3 100644 --- a/crates/re_types/src/datatypes/mod.rs +++ b/crates/re_types/src/datatypes/mod.rs @@ -36,8 +36,6 @@ mod tensor_data; mod tensor_data_ext; mod tensor_dimension; mod tensor_dimension_ext; -mod tensor_id; -mod tensor_id_ext; mod transform3d; mod transform3d_ext; mod translation_and_mat3x3; @@ -70,7 +68,6 @@ pub use self::scale3d::Scale3D; pub use self::tensor_buffer::TensorBuffer; pub use self::tensor_data::TensorData; pub use self::tensor_dimension::TensorDimension; -pub use self::tensor_id::TensorId; pub use self::transform3d::Transform3D; pub use self::translation_and_mat3x3::TranslationAndMat3x3; pub use self::translation_rotation_scale3d::TranslationRotationScale3D; diff --git a/crates/re_types/src/datatypes/tensor_data.rs b/crates/re_types/src/datatypes/tensor_data.rs index 275e9ec9e098..3bca123c18f6 100644 --- a/crates/re_types/src/datatypes/tensor_data.rs +++ b/crates/re_types/src/datatypes/tensor_data.rs @@ -22,7 +22,6 @@ /// which stores a contiguous array of typed values. #[derive(Clone, Debug, PartialEq)] pub struct TensorData { - pub id: crate::datatypes::TensorId, pub shape: Vec, pub buffer: crate::datatypes::TensorBuffer, } @@ -54,12 +53,6 @@ impl crate::Loggable for TensorData { fn arrow_datatype() -> arrow2::datatypes::DataType { use ::arrow2::datatypes::*; DataType::Struct(vec![ - Field { - name: "id".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: false, - metadata: [].into(), - }, Field { name: "shape".to_owned(), data_type: DataType::List(Box::new(Field { @@ -104,69 +97,6 @@ impl crate::Loggable for TensorData { StructArray::new( ::arrow_datatype(), vec![ - { - let (somes, id): (Vec<_>, Vec<_>) = data - .iter() - .map(|datum| { - let datum = datum.as_ref().map(|datum| { - let Self { id, .. } = &**datum; - id.clone() - }); - (datum.is_some(), datum) - }) - .unzip(); - let id_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let id_inner_data: Vec<_> = id - .iter() - .map(|datum| { - datum - .map(|datum| { - let crate::datatypes::TensorId { uuid } = datum; - uuid - }) - .unwrap_or_default() - }) - .flatten() - .map(Some) - .collect(); - let id_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = - id_bitmap.as_ref().map(|bitmap| { - bitmap - .iter() - .map(|i| std::iter::repeat(i).take(16usize)) - .flatten() - .collect::>() - .into() - }); - FixedSizeListArray::new( - DataType::FixedSizeList( - Box::new(Field { - name: "item".to_owned(), - data_type: DataType::UInt8, - is_nullable: false, - metadata: [].into(), - }), - 16usize, - ), - PrimitiveArray::new( - DataType::UInt8, - id_inner_data - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - id_inner_bitmap, - ) - .boxed(), - id_bitmap, - ) - .boxed() - } - }, { let (somes, shape): (Vec<_>, Vec<_>) = data .iter() @@ -262,12 +192,6 @@ impl crate::Loggable for TensorData { .ok_or_else(|| { crate::DeserializationError::datatype_mismatch( DataType::Struct(vec![ - Field { - name: "id".to_owned(), - data_type: ::arrow_datatype(), - is_nullable: false, - metadata: [].into(), - }, Field { name: "shape".to_owned(), data_type: DataType::List(Box::new(Field { @@ -301,90 +225,6 @@ impl crate::Loggable for TensorData { .map(|field| field.name.as_str()) .zip(arrow_data_arrays) .collect(); - let id = { - if !arrays_by_name.contains_key("id") { - return Err(crate::DeserializationError::missing_struct_field( - Self::arrow_datatype(), - "id", - )) - .with_context("rerun.datatypes.TensorData"); - } - let arrow_data = &**arrays_by_name["id"]; - { - let arrow_data = arrow_data - .as_any() - .downcast_ref::<::arrow2::array::FixedSizeListArray>() - .ok_or_else(|| { - crate::DeserializationError::datatype_mismatch( - DataType::FixedSizeList( - Box::new(Field { - name: "item".to_owned(), - data_type: DataType::UInt8, - is_nullable: false, - metadata: [].into(), - }), - 16usize, - ), - arrow_data.data_type().clone(), - ) - }) - .with_context("rerun.datatypes.TensorData#id")?; - if arrow_data.is_empty() { - Vec::new() - } else { - let offsets = (0..) - .step_by(16usize) - .zip((16usize..).step_by(16usize).take(arrow_data.len())); - let arrow_data_inner = { - let arrow_data_inner = &**arrow_data.values(); - arrow_data_inner - .as_any() - .downcast_ref::() - .ok_or_else(|| { - crate::DeserializationError::datatype_mismatch( - DataType::UInt8, - arrow_data_inner.data_type().clone(), - ) - }) - .with_context("rerun.datatypes.TensorData#id")? - .into_iter() - .map(|opt| opt.copied()) - .collect::>() - }; - arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets, - arrow_data.validity(), - ) - .map(|elem| { - elem.map(|(start, end)| { - debug_assert!(end - start == 16usize); - if end as usize > arrow_data_inner.len() { - return Err(crate::DeserializationError::offset_slice_oob( - (start, end), - arrow_data_inner.len(), - )); - } - - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - let data = unsafe { - arrow_data_inner.get_unchecked(start as usize..end as usize) - }; - let data = data.iter().cloned().map(Option::unwrap_or_default); - let arr = array_init::from_iter(data).unwrap(); - Ok(arr) - }) - .transpose() - }) - .map(|res_or_opt| { - res_or_opt.map(|res_or_opt| { - res_or_opt.map(|uuid| crate::datatypes::TensorId { uuid }) - }) - }) - .collect::>>>()? - } - .into_iter() - } - }; let shape = { if !arrays_by_name.contains_key("shape") { return Err(crate::DeserializationError::missing_struct_field( @@ -471,15 +311,12 @@ impl crate::Loggable for TensorData { .into_iter() }; arrow2::bitmap::utils::ZipValidity::new_with_validity( - ::itertools::izip!(id, shape, buffer), + ::itertools::izip!(shape, buffer), arrow_data.validity(), ) .map(|opt| { - opt.map(|(id, shape, buffer)| { + opt.map(|(shape, buffer)| { Ok(Self { - id: id - .ok_or_else(crate::DeserializationError::missing_data) - .with_context("rerun.datatypes.TensorData#id")?, shape: shape .ok_or_else(crate::DeserializationError::missing_data) .with_context("rerun.datatypes.TensorData#shape")?, diff --git a/crates/re_types/src/datatypes/tensor_data_ext.rs b/crates/re_types/src/datatypes/tensor_data_ext.rs index 614fedeceb7f..ae3d481baf11 100644 --- a/crates/re_types/src/datatypes/tensor_data_ext.rs +++ b/crates/re_types/src/datatypes/tensor_data_ext.rs @@ -3,7 +3,7 @@ use crate::tensor_data::{TensorCastError, TensorDataType, TensorElement}; #[cfg(feature = "image")] use crate::tensor_data::{DecodedTensor, TensorImageLoadError, TensorImageSaveError}; -use super::{TensorBuffer, TensorData, TensorDimension, TensorId}; +use super::{TensorBuffer, TensorData, TensorDimension}; // Much of the following duplicates code from: `crates/re_components/src/tensor.rs`, which // will eventually go away as the Tensor migration is completed. @@ -12,13 +12,8 @@ use super::{TensorBuffer, TensorData, TensorDimension, TensorId}; impl TensorData { #[inline] - pub fn new(id: TensorId, shape: Vec, buffer: TensorBuffer) -> Self { - Self { id, shape, buffer } - } - - #[inline] - pub fn id(&self) -> TensorId { - self.id + pub fn new(shape: Vec, buffer: TensorBuffer) -> Self { + Self { shape, buffer } } #[inline] @@ -210,12 +205,10 @@ macro_rules! tensor_type { match view.to_slice() { Some(slice) => Ok(TensorData { - id: TensorId::random(), shape, buffer: TensorBuffer::$variant(Vec::from(slice).into()), }), None => Ok(TensorData { - id: TensorId::random(), shape, buffer: TensorBuffer::$variant( view.iter().cloned().collect::>().into(), @@ -241,7 +234,6 @@ macro_rules! tensor_type { value .is_standard_layout() .then(|| TensorData { - id: TensorId::random(), shape, buffer: TensorBuffer::$variant(value.to_owned().into_raw_vec().into()), }) @@ -304,12 +296,10 @@ impl<'a, D: ::ndarray::Dimension> TryFrom<::ndarray::ArrayView<'a, half::f16, D> .collect(); match view.to_slice() { Some(slice) => Ok(TensorData { - uuid: TensorId::random(), shape, buffer: TensorBuffer::F16(Vec::from(bytemuck::cast_slice(slice)).into()), }), None => Ok(TensorData { - uuid: TensorId::random(), shape, buffer: TensorBuffer::F16( view.iter() @@ -337,7 +327,6 @@ impl TryFrom<::ndarray::Array> for Tensor value .is_standard_layout() .then(|| TensorData { - tensor_id: TensorId::random(), shape, data: TensorBuffer::F16( bytemuck::cast_slice(value.into_raw_vec().as_slice()) @@ -440,7 +429,6 @@ impl TensorData { let (w, h) = decoder.dimensions().unwrap(); // Can't fail after a successful decode_headers Ok(Self { - id: TensorId::random(), shape: vec![ TensorDimension::height(h as _), TensorDimension::width(w as _), diff --git a/crates/re_types/src/datatypes/tensor_id.rs b/crates/re_types/src/datatypes/tensor_id.rs deleted file mode 100644 index 4e9f283c4264..000000000000 --- a/crates/re_types/src/datatypes/tensor_id.rs +++ /dev/null @@ -1,196 +0,0 @@ -// DO NOT EDIT!: This file was auto-generated by crates/re_types_builder/src/codegen/rust/api.rs:165. - -#![allow(trivial_numeric_casts)] -#![allow(unused_parens)] -#![allow(clippy::clone_on_copy)] -#![allow(clippy::iter_on_single_items)] -#![allow(clippy::map_flatten)] -#![allow(clippy::match_wildcard_for_single_variants)] -#![allow(clippy::needless_question_mark)] -#![allow(clippy::redundant_closure)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::too_many_lines)] -#![allow(clippy::unnecessary_cast)] - -#[derive(Clone, Debug, Copy, Default, Eq, PartialEq)] -pub struct TensorId { - pub uuid: [u8; 16usize], -} - -impl<'a> From for ::std::borrow::Cow<'a, TensorId> { - #[inline] - fn from(value: TensorId) -> Self { - std::borrow::Cow::Owned(value) - } -} - -impl<'a> From<&'a TensorId> for ::std::borrow::Cow<'a, TensorId> { - #[inline] - fn from(value: &'a TensorId) -> Self { - std::borrow::Cow::Borrowed(value) - } -} - -impl crate::Loggable for TensorId { - type Name = crate::DatatypeName; - - #[inline] - fn name() -> Self::Name { - "rerun.datatypes.TensorId".into() - } - - #[allow(unused_imports, clippy::wildcard_imports)] - #[inline] - fn arrow_datatype() -> arrow2::datatypes::DataType { - use ::arrow2::datatypes::*; - DataType::FixedSizeList( - Box::new(Field { - name: "item".to_owned(), - data_type: DataType::UInt8, - is_nullable: false, - metadata: [].into(), - }), - 16usize, - ) - } - - #[allow(unused_imports, clippy::wildcard_imports)] - fn try_to_arrow_opt<'a>( - data: impl IntoIterator>>>, - ) -> crate::SerializationResult> - where - Self: Clone + 'a, - { - use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; - Ok({ - let (somes, uuid): (Vec<_>, Vec<_>) = data - .into_iter() - .map(|datum| { - let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); - let datum = datum.map(|datum| { - let Self { uuid } = datum.into_owned(); - uuid - }); - (datum.is_some(), datum) - }) - .unzip(); - let uuid_bitmap: Option<::arrow2::bitmap::Bitmap> = { - let any_nones = somes.iter().any(|some| !*some); - any_nones.then(|| somes.into()) - }; - { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let uuid_inner_data: Vec<_> = - uuid.iter().flatten().flatten().cloned().map(Some).collect(); - let uuid_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = - uuid_bitmap.as_ref().map(|bitmap| { - bitmap - .iter() - .map(|i| std::iter::repeat(i).take(16usize)) - .flatten() - .collect::>() - .into() - }); - FixedSizeListArray::new( - Self::arrow_datatype(), - PrimitiveArray::new( - DataType::UInt8, - uuid_inner_data - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - uuid_inner_bitmap, - ) - .boxed(), - uuid_bitmap, - ) - .boxed() - } - }) - } - - #[allow(unused_imports, clippy::wildcard_imports)] - fn try_from_arrow_opt( - arrow_data: &dyn ::arrow2::array::Array, - ) -> crate::DeserializationResult>> - where - Self: Sized, - { - use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, buffer::*, datatypes::*}; - Ok({ - let arrow_data = arrow_data - .as_any() - .downcast_ref::<::arrow2::array::FixedSizeListArray>() - .ok_or_else(|| { - crate::DeserializationError::datatype_mismatch( - DataType::FixedSizeList( - Box::new(Field { - name: "item".to_owned(), - data_type: DataType::UInt8, - is_nullable: false, - metadata: [].into(), - }), - 16usize, - ), - arrow_data.data_type().clone(), - ) - }) - .with_context("rerun.datatypes.TensorId#uuid")?; - if arrow_data.is_empty() { - Vec::new() - } else { - let offsets = (0..) - .step_by(16usize) - .zip((16usize..).step_by(16usize).take(arrow_data.len())); - let arrow_data_inner = { - let arrow_data_inner = &**arrow_data.values(); - arrow_data_inner - .as_any() - .downcast_ref::() - .ok_or_else(|| { - crate::DeserializationError::datatype_mismatch( - DataType::UInt8, - arrow_data_inner.data_type().clone(), - ) - }) - .with_context("rerun.datatypes.TensorId#uuid")? - .into_iter() - .map(|opt| opt.copied()) - .collect::>() - }; - arrow2::bitmap::utils::ZipValidity::new_with_validity( - offsets, - arrow_data.validity(), - ) - .map(|elem| { - elem.map(|(start, end)| { - debug_assert!(end - start == 16usize); - if end as usize > arrow_data_inner.len() { - return Err(crate::DeserializationError::offset_slice_oob( - (start, end), - arrow_data_inner.len(), - )); - } - - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - let data = - unsafe { arrow_data_inner.get_unchecked(start as usize..end as usize) }; - let data = data.iter().cloned().map(Option::unwrap_or_default); - let arr = array_init::from_iter(data).unwrap(); - Ok(arr) - }) - .transpose() - }) - .collect::>>>()? - } - .into_iter() - } - .map(|v| v.ok_or_else(crate::DeserializationError::missing_data)) - .map(|res| res.map(|uuid| Some(Self { uuid }))) - .collect::>>>() - .with_context("rerun.datatypes.TensorId#uuid") - .with_context("rerun.datatypes.TensorId")?) - } -} diff --git a/crates/re_types/src/datatypes/tensor_id_ext.rs b/crates/re_types/src/datatypes/tensor_id_ext.rs deleted file mode 100644 index b47fd2f3c66f..000000000000 --- a/crates/re_types/src/datatypes/tensor_id_ext.rs +++ /dev/null @@ -1,29 +0,0 @@ -use super::TensorId; - -impl nohash_hasher::IsEnabled for TensorId {} - -// required for [`nohash_hasher`]. -#[allow(clippy::derived_hash_with_manual_eq)] -impl std::hash::Hash for TensorId { - #[inline] - fn hash(&self, state: &mut H) { - state.write_u64(bytemuck::cast::<[u8; 16], [u64; 2]>(self.uuid)[0]); - } -} - -impl TensorId { - #[inline] - pub fn random() -> Self { - Self { - uuid: *uuid::Uuid::new_v4().as_bytes(), - } - } -} - -impl From for TensorId { - fn from(value: uuid::Uuid) -> Self { - Self { - uuid: *value.as_bytes(), - } - } -} diff --git a/crates/re_types/src/tensor_data.rs b/crates/re_types/src/tensor_data.rs index 3e583442f335..c091e74abd4f 100644 --- a/crates/re_types/src/tensor_data.rs +++ b/crates/re_types/src/tensor_data.rs @@ -3,7 +3,7 @@ use half::f16; use crate::datatypes::{TensorBuffer, TensorData}; #[cfg(feature = "image")] -use crate::datatypes::{TensorDimension, TensorId}; +use crate::datatypes::TensorDimension; // Much of the following duplicates code from: `crates/re_components/src/tensor.rs`, which // will eventually go away as the Tensor migration is completed. @@ -489,7 +489,6 @@ impl DecodedTensor { } }; let tensor = TensorData { - id: TensorId::random(), shape: vec![ TensorDimension::height(h as _), TensorDimension::width(w as _), @@ -575,7 +574,6 @@ impl DecodedTensor { } let tensor = TensorData { - id: TensorId::random(), shape: vec![ TensorDimension::height(h), TensorDimension::width(w), diff --git a/crates/re_types/tests/image.rs b/crates/re_types/tests/image.rs index 44615274373d..56560bd83ac1 100644 --- a/crates/re_types/tests/image.rs +++ b/crates/re_types/tests/image.rs @@ -2,22 +2,16 @@ use std::collections::HashMap; use re_types::{ archetypes::Image, - datatypes::{TensorBuffer, TensorData, TensorDimension, TensorId}, + datatypes::{TensorBuffer, TensorData, TensorDimension}, Archetype as _, }; mod util; -fn some_id(x: u8) -> TensorId { - TensorId { - uuid: [x, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - } -} #[test] fn image_roundtrip() { let all_expected = [Image { data: TensorData { - id: some_id(0), shape: vec![ TensorDimension { size: 2, @@ -36,7 +30,6 @@ fn image_roundtrip() { let all_arch_serialized = [Image::try_from(ndarray::array![[1u8, 2, 3], [4, 5, 6]]) .unwrap() - .with_id(some_id(0)) .to_arrow()]; let expected_extensions: HashMap<_, _> = [("data", vec!["rerun.components.TensorData"])].into(); @@ -70,7 +63,6 @@ fn dynamic_image_roundtrip() { let all_expected = [Image { data: TensorData { - id: some_id(0), shape: vec![ TensorDimension { size: 2, @@ -105,7 +97,7 @@ fn dynamic_image_roundtrip() { } } - let all_arch_serialized = [Image::try_from(img).unwrap().with_id(some_id(0)).to_arrow()]; + let all_arch_serialized = [Image::try_from(img).unwrap().to_arrow()]; let expected_extensions: HashMap<_, _> = [("data", vec!["rerun.components.TensorData"])].into(); diff --git a/crates/re_types/tests/tensor.rs b/crates/re_types/tests/tensor.rs index b3fe24db5868..11c355673f46 100644 --- a/crates/re_types/tests/tensor.rs +++ b/crates/re_types/tests/tensor.rs @@ -2,24 +2,17 @@ use std::collections::HashMap; use re_types::{ archetypes::Tensor, - datatypes::{TensorBuffer, TensorData, TensorDimension, TensorId}, + datatypes::{TensorBuffer, TensorData, TensorDimension}, tensor_data::TensorCastError, Archetype as _, }; mod util; -fn some_id(x: u8) -> TensorId { - TensorId { - uuid: [x, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - } -} - #[test] fn tensor_roundtrip() { let all_expected = [Tensor { data: TensorData { - id: some_id(0), shape: vec![ TensorDimension { size: 2, @@ -37,7 +30,6 @@ fn tensor_roundtrip() { let all_arch_serialized = [Tensor::try_from(ndarray::array![[1u8, 2, 3], [4, 5, 6]]) .unwrap() - .with_id(some_id(0)) .to_arrow()]; let expected_extensions: HashMap<_, _> = [("data", vec!["rerun.components.TensorData"])].into(); @@ -67,7 +59,6 @@ fn tensor_roundtrip() { #[test] fn convert_tensor_to_ndarray_u8() { let t = TensorData::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -84,7 +75,6 @@ fn convert_tensor_to_ndarray_u8() { #[test] fn convert_tensor_to_ndarray_u16() { let t = TensorData::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -101,7 +91,6 @@ fn convert_tensor_to_ndarray_u16() { #[test] fn convert_tensor_to_ndarray_f32() { let t = TensorData::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -138,7 +127,6 @@ fn convert_ndarray_slice_to_tensor() { #[test] fn check_slices() { let t = TensorData::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -178,7 +166,6 @@ fn check_slices() { #[test] fn check_tensor_shape_error() { let t = TensorData::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), @@ -200,7 +187,6 @@ fn check_tensor_shape_error() { #[test] fn check_tensor_type_error() { let t = TensorData::new( - TensorId::random(), vec![ TensorDimension::unnamed(3), TensorDimension::unnamed(4), diff --git a/crates/re_viewer_context/src/annotations.rs b/crates/re_viewer_context/src/annotations.rs index 4b19794eb77b..0e63d8c9015a 100644 --- a/crates/re_viewer_context/src/annotations.rs +++ b/crates/re_viewer_context/src/annotations.rs @@ -28,7 +28,7 @@ impl Annotations { .ok() .and_then(|mut iter| iter.next()) .map(|ctx| Self { - row_id: view.row_id(), + row_id: view.primary_row_id(), class_map: ctx .0 .into_iter() diff --git a/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs b/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs index 938f1fec6efd..b44ffa0d409a 100644 --- a/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs +++ b/crates/re_viewer_context/src/gpu_bridge/tensor_to_gpu.rs @@ -1,6 +1,7 @@ //! Upload [`Tensor`] to [`re_renderer`]. use anyhow::Context; +use re_data_store::VersionedInstancePathHash; use re_types::components::ClassId; use std::borrow::Cow; @@ -32,6 +33,7 @@ use super::{get_or_create_texture, try_get_or_create_texture}; pub fn tensor_to_gpu( render_ctx: &RenderContext, debug_name: &str, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, annotations: &Annotations, @@ -46,15 +48,28 @@ pub fn tensor_to_gpu( use re_components::TensorDataMeaning; match tensor.meaning { - TensorDataMeaning::Unknown => { - color_tensor_to_gpu(render_ctx, debug_name, tensor, tensor_stats) - } - TensorDataMeaning::ClassId => { - class_id_tensor_to_gpu(render_ctx, debug_name, tensor, tensor_stats, annotations) - } - TensorDataMeaning::Depth => { - depth_tensor_to_gpu(render_ctx, debug_name, tensor, tensor_stats) - } + TensorDataMeaning::Unknown => color_tensor_to_gpu( + render_ctx, + debug_name, + tensor_path_hash, + tensor, + tensor_stats, + ), + TensorDataMeaning::ClassId => class_id_tensor_to_gpu( + render_ctx, + debug_name, + tensor_path_hash, + tensor, + tensor_stats, + annotations, + ), + TensorDataMeaning::Depth => depth_tensor_to_gpu( + render_ctx, + debug_name, + tensor_path_hash, + tensor, + tensor_stats, + ), } } @@ -64,12 +79,13 @@ pub fn tensor_to_gpu( fn color_tensor_to_gpu( render_ctx: &RenderContext, debug_name: &str, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, ) -> anyhow::Result { let [height, width, depth] = height_width_depth(tensor)?; - let texture_handle = try_get_or_create_texture(render_ctx, hash(tensor.id()), || { + let texture_handle = try_get_or_create_texture(render_ctx, hash(tensor_path_hash), || { let (data, format) = match (depth, &tensor.data) { // Normalize sRGB(A) textures to 0-1 range, and let the GPU premultiply alpha. // Why? Because premul must happen _before_ sRGB decode, so we can't @@ -144,6 +160,7 @@ fn color_tensor_to_gpu( fn class_id_tensor_to_gpu( render_ctx: &RenderContext, debug_name: &str, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, annotations: &Annotations, @@ -194,7 +211,7 @@ fn class_id_tensor_to_gpu( }) .context("Failed to create class_id_colormap.")?; - let main_texture_handle = try_get_or_create_texture(render_ctx, hash(tensor.id()), || { + let main_texture_handle = try_get_or_create_texture(render_ctx, hash(tensor_path_hash), || { general_texture_creation_desc_from_tensor(debug_name, tensor) }) .map_err(|err| anyhow::anyhow!("Failed to create texture for class id tensor: {err}"))?; @@ -215,6 +232,7 @@ fn class_id_tensor_to_gpu( pub fn depth_tensor_to_gpu( render_ctx: &RenderContext, debug_name: &str, + tensor_path_hash: VersionedInstancePathHash, tensor: &DecodedTensor, tensor_stats: &TensorStats, ) -> anyhow::Result { @@ -226,7 +244,7 @@ pub fn depth_tensor_to_gpu( ); let (min, max) = depth_tensor_range(tensor, tensor_stats)?; - let texture = try_get_or_create_texture(render_ctx, hash(tensor.id()), || { + let texture = try_get_or_create_texture(render_ctx, hash(tensor_path_hash), || { general_texture_creation_desc_from_tensor(debug_name, tensor) }) .map_err(|err| anyhow::anyhow!("Failed to create depth tensor texture: {err}"))?; diff --git a/crates/re_viewer_context/src/tensor/tensor_decode_cache.rs b/crates/re_viewer_context/src/tensor/tensor_decode_cache.rs index 472645fc862b..fe5684ab1350 100644 --- a/crates/re_viewer_context/src/tensor/tensor_decode_cache.rs +++ b/crates/re_viewer_context/src/tensor/tensor_decode_cache.rs @@ -1,4 +1,5 @@ -use re_components::{DecodedTensor, Tensor, TensorId, TensorImageLoadError}; +use re_components::{DecodedTensor, Tensor, TensorImageLoadError}; +use re_data_store::VersionedInstancePathHash; use crate::Cache; @@ -13,10 +14,11 @@ struct DecodedTensorResult { last_use_generation: u64, } -/// A cache of decoded [`Tensor`] entities, indexed by `TensorId`. +/// Caches decoded tensors using a [`VersionedInstancePathHash`], i.e. a specific instance of +/// a specific entity path for a specific row in the store. #[derive(Default)] pub struct TensorDecodeCache { - cache: nohash_hasher::IntMap, + cache: ahash::HashMap, memory_used: u64, generation: u64, } @@ -30,6 +32,7 @@ impl TensorDecodeCache { /// Currently supports JPEG encoded tensors. pub fn entry( &mut self, + key: VersionedInstancePathHash, maybe_encoded_tensor: Tensor, ) -> Result { re_tracing::profile_function!(); @@ -38,7 +41,7 @@ impl TensorDecodeCache { Ok(decoded_tensor) => Ok(decoded_tensor), Err(encoded_tensor) => { - let lookup = self.cache.entry(encoded_tensor.id()).or_insert_with(|| { + let lookup = self.cache.entry(key).or_insert_with(|| { let tensor_result = DecodedTensor::try_decode(encoded_tensor); let memory_used = match &tensor_result { Ok(tensor) => tensor.size_in_bytes() as u64, diff --git a/crates/re_viewer_context/src/tensor/tensor_stats_cache.rs b/crates/re_viewer_context/src/tensor/tensor_stats_cache.rs index 885db479ffca..df1b1f73efcc 100644 --- a/crates/re_viewer_context/src/tensor/tensor_stats_cache.rs +++ b/crates/re_viewer_context/src/tensor/tensor_stats_cache.rs @@ -1,16 +1,19 @@ -use re_components::{Tensor, TensorId}; +use re_components::Tensor; +use re_data_store::VersionedInstancePathHash; use super::TensorStats; use crate::Cache; +/// Caches tensor stats using a [`VersionedInstancePathHash`], i.e. a specific instance of +/// a specific entity path for a specific row in the store. #[derive(Default)] -pub struct TensorStatsCache(nohash_hasher::IntMap); +pub struct TensorStatsCache(ahash::HashMap); impl TensorStatsCache { - pub fn entry(&mut self, tensor: &Tensor) -> TensorStats { + pub fn entry(&mut self, key: VersionedInstancePathHash, tensor: &Tensor) -> TensorStats { *self .0 - .entry(tensor.tensor_id) + .entry(key) .or_insert_with(|| TensorStats::new(tensor)) } } diff --git a/crates/re_viewport/src/space_info.rs b/crates/re_viewport/src/space_info.rs index 78fb64c03b9d..4f1a923812ec 100644 --- a/crates/re_viewport/src/space_info.rs +++ b/crates/re_viewport/src/space_info.rs @@ -126,8 +126,12 @@ impl SpaceInfoCollection { ) { // Determine how the paths are connected. let store = &entity_db.data_store; - let transform3d = store.query_latest_component::(&tree.path, query); - let pinhole = store.query_latest_component::(&tree.path, query); + let transform3d = store + .query_latest_component::(&tree.path, query) + .map(|c| c.value); + let pinhole = store + .query_latest_component::(&tree.path, query) + .map(|c| c.value); let connection = if transform3d.is_some() || pinhole.is_some() { Some(SpaceInfoConnection::Connected { diff --git a/crates/re_viewport/src/viewport_blueprint.rs b/crates/re_viewport/src/viewport_blueprint.rs index 57b97d1dd8ca..f9f4c15852ce 100644 --- a/crates/re_viewport/src/viewport_blueprint.rs +++ b/crates/re_viewport/src/viewport_blueprint.rs @@ -303,7 +303,7 @@ pub fn load_space_view_blueprint( let mut space_view = blueprint_db .store() .query_timeless_component::(path) - .map(|c| c.space_view); + .map(|c| c.value.space_view); // Blueprint data migrations can leave us unable to parse the expected id from the source-data // We always want the id to match the one derived from the EntityPath since this id is how @@ -344,23 +344,28 @@ pub fn load_viewport_blueprint(blueprint_db: &re_data_store::StoreDb) -> Viewpor let auto_space_views = blueprint_db .store() .query_timeless_component::(&VIEWPORT_PATH.into()) - .unwrap_or_else(|| { - // Only enable auto-space-views if this is the app-default blueprint - AutoSpaceViews( - blueprint_db - .store_info() - .map_or(false, |ri| ri.is_app_default_blueprint()), - ) - }); + .map_or_else( + || { + // Only enable auto-space-views if this is the app-default blueprint + AutoSpaceViews( + blueprint_db + .store_info() + .map_or(false, |ri| ri.is_app_default_blueprint()), + ) + }, + |auto| auto.value, + ); let space_view_maximized = blueprint_db .store() .query_timeless_component::(&VIEWPORT_PATH.into()) + .map(|space_view| space_view.value) .unwrap_or_default(); let viewport_layout: ViewportLayout = blueprint_db .store() .query_timeless_component::(&VIEWPORT_PATH.into()) + .map(|space_view| space_view.value) .unwrap_or_default(); let unknown_space_views: HashMap<_, _> = space_views diff --git a/docs/code-examples/mesh_simple.rs b/docs/code-examples/mesh_simple.rs index 7f46a6bf3a1d..bfb324d52c7b 100644 --- a/docs/code-examples/mesh_simple.rs +++ b/docs/code-examples/mesh_simple.rs @@ -1,7 +1,7 @@ //! Log a simple colored triangle. use rerun::{ - components::{Mesh3D, MeshId, RawMesh3D}, + components::{Mesh3D, RawMesh3D}, RecordingStreamBuilder, }; @@ -9,7 +9,6 @@ fn main() -> Result<(), Box> { let (rec, storage) = RecordingStreamBuilder::new("rerun_example_mesh").memory()?; let mesh = RawMesh3D { - mesh_id: MeshId::random(), vertex_positions: [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]] .into_iter() .flatten() diff --git a/examples/rust/raw_mesh/src/main.rs b/examples/rust/raw_mesh/src/main.rs index f6d5c04c467b..44835ee076a7 100644 --- a/examples/rust/raw_mesh/src/main.rs +++ b/examples/rust/raw_mesh/src/main.rs @@ -13,7 +13,7 @@ use std::path::PathBuf; use anyhow::anyhow; use bytes::Bytes; use rerun::{ - components::{Color, Mesh3D, MeshId, RawMesh3D, Transform3D, ViewCoordinates}, + components::{Color, Mesh3D, RawMesh3D, Transform3D, ViewCoordinates}, datatypes::Vec4D, external::{re_log, re_memory::AccountingAllocator}, transform::TranslationRotationScale3D, @@ -38,7 +38,6 @@ impl From for Mesh3D { } = primitive; let raw = RawMesh3D { - mesh_id: MeshId::random(), albedo_factor: albedo_factor.map(Vec4D).map(Into::into), indices: indices.map(|i| i.into()), vertex_positions: vertex_positions.into_iter().flatten().collect(), diff --git a/rerun_cpp/.gitattributes b/rerun_cpp/.gitattributes index 25d6aaf9458a..e1e0c0307eea 100644 --- a/rerun_cpp/.gitattributes +++ b/rerun_cpp/.gitattributes @@ -95,8 +95,6 @@ src/rerun/datatypes/tensor_data.cpp linguist-generated=true src/rerun/datatypes/tensor_data.hpp linguist-generated=true src/rerun/datatypes/tensor_dimension.cpp linguist-generated=true src/rerun/datatypes/tensor_dimension.hpp linguist-generated=true -src/rerun/datatypes/tensor_id.cpp linguist-generated=true -src/rerun/datatypes/tensor_id.hpp linguist-generated=true src/rerun/datatypes/transform3d.cpp linguist-generated=true src/rerun/datatypes/transform3d.hpp linguist-generated=true src/rerun/datatypes/translation_and_mat3x3.cpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/datatypes.hpp b/rerun_cpp/src/rerun/datatypes.hpp index af85735fede1..2a02ec2de651 100644 --- a/rerun_cpp/src/rerun/datatypes.hpp +++ b/rerun_cpp/src/rerun/datatypes.hpp @@ -21,7 +21,6 @@ #include "datatypes/tensor_buffer.hpp" #include "datatypes/tensor_data.hpp" #include "datatypes/tensor_dimension.hpp" -#include "datatypes/tensor_id.hpp" #include "datatypes/transform3d.hpp" #include "datatypes/translation_and_mat3x3.hpp" #include "datatypes/translation_rotation_scale3d.hpp" diff --git a/rerun_cpp/src/rerun/datatypes/tensor_data.cpp b/rerun_cpp/src/rerun/datatypes/tensor_data.cpp index c046960f158f..39282ab0b5b9 100644 --- a/rerun_cpp/src/rerun/datatypes/tensor_data.cpp +++ b/rerun_cpp/src/rerun/datatypes/tensor_data.cpp @@ -5,7 +5,6 @@ #include "tensor_buffer.hpp" #include "tensor_dimension.hpp" -#include "tensor_id.hpp" #include #include @@ -14,7 +13,6 @@ namespace rerun { namespace datatypes { const std::shared_ptr &TensorData::arrow_datatype() { static const auto datatype = arrow::struct_({ - arrow::field("id", rerun::datatypes::TensorId::arrow_datatype(), false), arrow::field( "shape", arrow::list(arrow::field( @@ -40,7 +38,6 @@ namespace rerun { arrow_datatype(), memory_pool, std::vector>({ - rerun::datatypes::TensorId::new_arrow_array_builder(memory_pool).value, std::make_shared( memory_pool, rerun::datatypes::TensorDimension::new_arrow_array_builder(memory_pool) @@ -65,19 +62,7 @@ namespace rerun { } { - auto field_builder = - static_cast(builder->field_builder(0)); - ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); - for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { - RR_RETURN_NOT_OK(rerun::datatypes::TensorId::fill_arrow_array_builder( - field_builder, - &elements[elem_idx].id, - 1 - )); - } - } - { - auto field_builder = static_cast(builder->field_builder(1)); + auto field_builder = static_cast(builder->field_builder(0)); auto value_builder = static_cast(field_builder->value_builder()); ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); @@ -99,7 +84,7 @@ namespace rerun { } { auto field_builder = - static_cast(builder->field_builder(2)); + static_cast(builder->field_builder(1)); ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { RR_RETURN_NOT_OK(rerun::datatypes::TensorBuffer::fill_arrow_array_builder( diff --git a/rerun_cpp/src/rerun/datatypes/tensor_data.hpp b/rerun_cpp/src/rerun/datatypes/tensor_data.hpp index 987fbfac0a3a..cb97d1bb6ed9 100644 --- a/rerun_cpp/src/rerun/datatypes/tensor_data.hpp +++ b/rerun_cpp/src/rerun/datatypes/tensor_data.hpp @@ -6,7 +6,6 @@ #include "../result.hpp" #include "tensor_buffer.hpp" #include "tensor_dimension.hpp" -#include "tensor_id.hpp" #include #include @@ -29,8 +28,6 @@ namespace rerun { /// These dimensions are combined with an index to look up values from the `buffer` field, /// which stores a contiguous array of typed values. struct TensorData { - rerun::datatypes::TensorId id; - std::vector shape; rerun::datatypes::TensorBuffer buffer; diff --git a/rerun_cpp/src/rerun/datatypes/tensor_id.cpp b/rerun_cpp/src/rerun/datatypes/tensor_id.cpp deleted file mode 100644 index e45e995fa61d..000000000000 --- a/rerun_cpp/src/rerun/datatypes/tensor_id.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// DO NOT EDIT!: This file was auto-generated by crates/re_types_builder/src/codegen/cpp/mod.rs:54. -// Based on "crates/re_types/definitions/rerun/datatypes/tensor_id.fbs". - -#include "tensor_id.hpp" - -#include -#include - -namespace rerun { - namespace datatypes { - const std::shared_ptr &TensorId::arrow_datatype() { - static const auto datatype = - arrow::fixed_size_list(arrow::field("item", arrow::uint8(), false), 16); - return datatype; - } - - Result> TensorId::new_arrow_array_builder( - arrow::MemoryPool *memory_pool - ) { - if (!memory_pool) { - return Error(ErrorCode::UnexpectedNullArgument, "Memory pool is null."); - } - - return Result(std::make_shared( - memory_pool, - std::make_shared(memory_pool), - 16 - )); - } - - Error TensorId::fill_arrow_array_builder( - arrow::FixedSizeListBuilder *builder, const TensorId *elements, size_t num_elements - ) { - if (!builder) { - return Error(ErrorCode::UnexpectedNullArgument, "Passed array builder is null."); - } - if (!elements) { - return Error( - ErrorCode::UnexpectedNullArgument, - "Cannot serialize null pointer to arrow array." - ); - } - - auto value_builder = static_cast(builder->value_builder()); - - ARROW_RETURN_NOT_OK(builder->AppendValues(static_cast(num_elements))); - static_assert(sizeof(elements[0].uuid) == sizeof(elements[0])); - ARROW_RETURN_NOT_OK(value_builder->AppendValues( - elements[0].uuid, - static_cast(num_elements * 16), - nullptr - )); - - return Error::ok(); - } - } // namespace datatypes -} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/tensor_id.hpp b/rerun_cpp/src/rerun/datatypes/tensor_id.hpp deleted file mode 100644 index 1481c56022b7..000000000000 --- a/rerun_cpp/src/rerun/datatypes/tensor_id.hpp +++ /dev/null @@ -1,58 +0,0 @@ -// DO NOT EDIT!: This file was auto-generated by crates/re_types_builder/src/codegen/cpp/mod.rs:54. -// Based on "crates/re_types/definitions/rerun/datatypes/tensor_id.fbs". - -#pragma once - -#include "../result.hpp" - -#include -#include - -namespace arrow { - class DataType; - class FixedSizeListBuilder; - class MemoryPool; -} // namespace arrow - -namespace rerun { - namespace datatypes { - struct TensorId { - uint8_t uuid[16]; - - public: - TensorId() = default; - - TensorId(const uint8_t (&_uuid)[16]) - : uuid{ - _uuid[0], - _uuid[1], - _uuid[2], - _uuid[3], - _uuid[4], - _uuid[5], - _uuid[6], - _uuid[7], - _uuid[8], - _uuid[9], - _uuid[10], - _uuid[11], - _uuid[12], - _uuid[13], - _uuid[14], - _uuid[15]} {} - - /// Returns the arrow data type this type corresponds to. - static const std::shared_ptr& arrow_datatype(); - - /// Creates a new array builder with an array of this type. - static Result> new_arrow_array_builder( - arrow::MemoryPool* memory_pool - ); - - /// Fills an arrow array builder with an array of this type. - static Error fill_arrow_array_builder( - arrow::FixedSizeListBuilder* builder, const TensorId* elements, size_t num_elements - ); - }; - } // namespace datatypes -} // namespace rerun diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/.gitattributes b/rerun_py/rerun_sdk/rerun/_rerun2/.gitattributes index 2f268f6f7c75..1f52de8cfa8d 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/_rerun2/.gitattributes @@ -55,7 +55,6 @@ datatypes/scale3d.py linguist-generated=true datatypes/tensor_buffer.py linguist-generated=true datatypes/tensor_data.py linguist-generated=true datatypes/tensor_dimension.py linguist-generated=true -datatypes/tensor_id.py linguist-generated=true datatypes/transform3d.py linguist-generated=true datatypes/translation_and_mat3x3.py linguist-generated=true datatypes/translation_rotation_scale3d.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/image.py b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/image.py index f8eef199b1c5..e30bfce29c31 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/image.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/archetypes/_overrides/image.py @@ -69,12 +69,10 @@ def image_data_converter(data: TensorDataArrayLike) -> TensorDataArray: TensorDataArray._EXTENSION_TYPE().wrap_array( pa.StructArray.from_arrays( [ - tensor_data.storage.field(0), new_shape, - tensor_data.storage.field(2), + tensor_data.storage.field(1), ], fields=[ - tensor_data_type.field("id"), tensor_data_type.field("shape"), tensor_data_type.field("buffer"), ], diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py index 5d911266ade7..e29d3e0ec3ee 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py @@ -100,7 +100,6 @@ TensorDimensionLike, TensorDimensionType, ) -from .tensor_id import TensorId, TensorIdArray, TensorIdArrayLike, TensorIdLike, TensorIdType from .transform3d import Transform3D, Transform3DArray, Transform3DArrayLike, Transform3DLike, Transform3DType from .translation_and_mat3x3 import ( TranslationAndMat3x3, @@ -261,11 +260,6 @@ "TensorDimensionArrayLike", "TensorDimensionLike", "TensorDimensionType", - "TensorId", - "TensorIdArray", - "TensorIdArrayLike", - "TensorIdLike", - "TensorIdType", "Transform3D", "Transform3DArray", "Transform3DArrayLike", diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py index b8f41eb750be..795a08b66580 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/__init__.py @@ -22,7 +22,6 @@ from .scale3d import scale3d_inner_converter from .tensor_buffer import tensorbuffer_inner_converter from .tensor_data import tensordata_init, tensordata_native_to_pa_array -from .tensor_id import tensorid_uuid_converter from .transform3d import transform3d_native_to_pa_array from .translation_and_mat3x3 import translationandmat3x3_init from .translation_rotation_scale3d import translationrotationscale3d_init diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_data.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_data.py index 1831c3679c6c..81ad9b8dd27a 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_data.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_data.py @@ -1,7 +1,6 @@ from __future__ import annotations import collections -import uuid from math import prod from typing import TYPE_CHECKING, Any, Final, Protocol, Sequence, Union @@ -12,7 +11,7 @@ from rerun.log.error_utils import _send_warning if TYPE_CHECKING: - from .. import TensorBufferLike, TensorData, TensorDataArrayLike, TensorDimension, TensorDimensionLike, TensorIdLike + from .. import TensorBufferLike, TensorData, TensorDataArrayLike, TensorDimension, TensorDimensionLike ################################################################################ @@ -55,7 +54,6 @@ def _to_numpy(tensor: Tensor) -> npt.NDArray[Any]: def tensordata_init( self: TensorData, *, - id: TensorIdLike | None = None, shape: Sequence[TensorDimensionLike] | None = None, buffer: TensorBufferLike | None = None, array: Tensor | None = None, @@ -64,7 +62,7 @@ def tensordata_init( """ Construct a `TensorData` object. - The `TensorData` object is internally represented by three fields: `id`, `shape`, and `buffer`. + The `TensorData` object is internally represented by three fields: `shape` and `buffer`. This constructor provides additional arguments 'array', and 'names'. When passing in a multi-dimensional array such as a `np.ndarray`, the `shape` and `buffer` fields will be @@ -74,8 +72,6 @@ def tensordata_init( ---------- self: TensorData The TensorData object to construct. - id: TensorIdLike | None - The id of the tensor. If None, a random id will be generated. shape: Sequence[TensorDimensionLike] | None The shape of the tensor. If None, and an array is proviced, the shape will be inferred from the shape of the array. @@ -99,13 +95,7 @@ def tensordata_init( raise ValueError("Can only provide one of 'shape' or 'names'") from .. import TensorBuffer, TensorDimension - from ..tensor_data import _tensordata_buffer_converter, _tensordata_id_converter - - # Assign an id if one wasn't provided - if id is not None: - self.id = _tensordata_id_converter(id) - else: - self.id = _tensordata_id_converter(uuid.uuid4()) + from ..tensor_data import _tensordata_buffer_converter if shape is not None: resolved_shape = list(shape) @@ -185,17 +175,15 @@ def tensordata_native_to_pa_array(data: TensorDataArrayLike, data_type: pa.DataT data = TensorData(array=array) # Now build the actual arrow fields - tensor_id = _build_tensorid(data.id) shape = _build_shape_array(data.shape).cast(data_type.field("shape").type) buffer = _build_buffer_array(data.buffer) return pa.StructArray.from_arrays( [ - tensor_id, shape, buffer, ], - fields=[data_type.field("id"), data_type.field("shape"), data_type.field("buffer")], + fields=[data_type.field("shape"), data_type.field("buffer")], ).cast(data_type) @@ -235,21 +223,6 @@ def _build_dense_union(data_type: pa.DenseUnionType, discriminant: str, child: p raise ValueError(e.args) -def _build_tensorid(id: TensorIdLike) -> pa.Array: - from .. import TensorId, TensorIdType - - if isinstance(id, uuid.UUID): - array = np.asarray(list(id.bytes), dtype=np.uint8) - elif isinstance(id, TensorId): - array = id.uuid - else: - raise ValueError("Unsupported TensorId input") - - data_type = TensorIdType().storage_type - - return pa.FixedSizeListArray.from_arrays(array, type=data_type) - - def _build_shape_array(dims: list[TensorDimension]) -> pa.Array: from .. import TensorDimensionType diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_id.py deleted file mode 100644 index 45b7afeba55e..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_id.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -import uuid - -import numpy as np -import numpy.typing as npt - - -def tensorid_uuid_converter(id: bytes | uuid.UUID | npt.ArrayLike) -> npt.NDArray[np.uint8]: - if isinstance(id, uuid.UUID): - id = id.bytes - - if isinstance(id, bytes): - id = list(id) - - id = np.asarray(id, dtype=np.uint8) - - if len(id) != 16: - raise ValueError("TensorId must be 16 bytes") - - return id diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_data.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_data.py index 0ea65a3553ed..29e144e9e5b4 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_data.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_data.py @@ -18,13 +18,6 @@ __all__ = ["TensorData", "TensorDataArray", "TensorDataArrayLike", "TensorDataLike", "TensorDataType"] -def _tensordata_id_converter(x: datatypes.TensorIdLike) -> datatypes.TensorId: - if isinstance(x, datatypes.TensorId): - return x - else: - return datatypes.TensorId(x) - - def _tensordata_buffer_converter(x: datatypes.TensorBufferLike) -> datatypes.TensorBuffer: if isinstance(x, datatypes.TensorBuffer): return x @@ -48,7 +41,6 @@ class TensorData: def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] tensordata_init(self, *args, **kwargs) - id: datatypes.TensorId = field(converter=_tensordata_id_converter) shape: list[datatypes.TensorDimension] = field() buffer: datatypes.TensorBuffer = field(converter=_tensordata_buffer_converter) @@ -66,12 +58,6 @@ def __init__(self) -> None: self, pa.struct( [ - pa.field( - "id", - pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={}), 16), - nullable=False, - metadata={}, - ), pa.field( "shape", pa.list_( diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_id.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_id.py deleted file mode 100644 index b6dee83693be..000000000000 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/tensor_id.py +++ /dev/null @@ -1,63 +0,0 @@ -# DO NOT EDIT!: This file was auto-generated by crates/re_types_builder/src/codegen/python.rs:277. - -from __future__ import annotations - -import uuid -from typing import TYPE_CHECKING, Any, Sequence, Union - -import numpy as np -import numpy.typing as npt -import pyarrow as pa -from attrs import define, field - -from .._baseclasses import ( - BaseExtensionArray, - BaseExtensionType, -) -from ._overrides import tensorid_uuid_converter # noqa: F401 - -__all__ = ["TensorId", "TensorIdArray", "TensorIdArrayLike", "TensorIdLike", "TensorIdType"] - - -@define -class TensorId: - uuid: npt.NDArray[np.uint8] = field(converter=tensorid_uuid_converter) - - def __array__(self, dtype: npt.DTypeLike = None) -> npt.NDArray[Any]: - return np.asarray(self.uuid, dtype=dtype) - - -if TYPE_CHECKING: - TensorIdLike = Union[TensorId, uuid.UUID] -else: - TensorIdLike = Any - -TensorIdArrayLike = Union[ - TensorId, - Sequence[TensorIdLike], -] - - -# --- Arrow support --- - - -class TensorIdType(BaseExtensionType): - def __init__(self) -> None: - pa.ExtensionType.__init__( - self, pa.list_(pa.field("item", pa.uint8(), nullable=False, metadata={}), 16), "rerun.datatypes.TensorId" - ) - - -class TensorIdArray(BaseExtensionArray[TensorIdArrayLike]): - _EXTENSION_NAME = "rerun.datatypes.TensorId" - _EXTENSION_TYPE = TensorIdType - - @staticmethod - def _native_to_pa_array(data: TensorIdArrayLike, data_type: pa.DataType) -> pa.Array: - raise NotImplementedError # You need to implement "tensorid_native_to_pa_array" in rerun_py/rerun_sdk/rerun/_rerun2/datatypes/_overrides/tensor_id.py - - -TensorIdType._ARRAY_TYPE = TensorIdArray - -# TODO(cmc): bring back registration to pyarrow once legacy types are gone -# pa.register_extension_type(TensorIdType()) diff --git a/rerun_py/rerun_sdk/rerun/components/tensor.py b/rerun_py/rerun_sdk/rerun/components/tensor.py index 3ccc8ee70c02..68cedc4a8d94 100644 --- a/rerun_py/rerun_sdk/rerun/components/tensor.py +++ b/rerun_py/rerun_sdk/rerun/components/tensor.py @@ -1,6 +1,5 @@ from __future__ import annotations -import uuid from typing import Final, Iterable, Union, cast import numpy as np @@ -54,8 +53,6 @@ def from_numpy( meter: float | None = None, ) -> TensorArray: """Build a `TensorArray` from an numpy array.""" - # Build a random tensor_id - tensor_id = pa.repeat(pa.scalar(uuid.uuid4().bytes, type=TensorType.storage_type["tensor_id"].type), 1) if not names: names = [None] * len(array.shape) @@ -96,7 +93,6 @@ def from_numpy( storage = pa.StructArray.from_arrays( [ - tensor_id, shape, data, meaning, diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 36568165c613..7f58badecd7e 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -29,10 +29,9 @@ use rerun::{ pub use rerun::{ components::{ AnnotationContext, Box3D, ClassId, Color, DisconnectedSpace, DrawOrder, EncodedMesh3D, - InstanceKey, KeypointId, Label, LineStrip2D, LineStrip3D, Mesh3D, MeshFormat, MeshId, - Origin3D, Pinhole, Point2D, Point3D, Quaternion, Radius, RawMesh3D, Rect2D, Scalar, - ScalarPlotProps, Tensor, TensorData, TensorDimension, TensorId, TextEntry, Transform3D, - Vector3D, ViewCoordinates, + InstanceKey, KeypointId, Label, LineStrip2D, LineStrip3D, Mesh3D, MeshFormat, Origin3D, + Pinhole, Point2D, Point3D, Quaternion, Radius, RawMesh3D, Rect2D, Scalar, ScalarPlotProps, + Tensor, TensorData, TensorDimension, TextEntry, Transform3D, Vector3D, ViewCoordinates, }, coordinates::{Axis3, Handedness, Sign, SignedAxis3}, datatypes::{AnnotationInfo, ClassDescription}, @@ -883,7 +882,6 @@ fn log_meshes( }; let raw = RawMesh3D { - mesh_id: MeshId::random(), vertex_positions: vertex_positions.as_array().to_vec().into(), vertex_colors, indices: indices.map(|indices| indices.as_array().to_vec().into()), @@ -981,7 +979,6 @@ fn log_mesh_file( }; let mesh3d = Mesh3D::Encoded(EncodedMesh3D { - mesh_id: MeshId::random(), format, bytes: mesh_bytes.into(), transform, diff --git a/rerun_py/tests/unit/test_image.py b/rerun_py/tests/unit/test_image.py index 4d58d34a8bc0..1e259e91df20 100644 --- a/rerun_py/tests/unit/test_image.py +++ b/rerun_py/tests/unit/test_image.py @@ -15,7 +15,6 @@ IMAGE_INPUTS: list[rrd.TensorDataArrayLike] = [ # Full explicit construction rrd.TensorData( - id=rrd.TensorId(uuid=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), shape=[ rrd.TensorDimension(10, "height"), rrd.TensorDimension(20, "width"), @@ -27,12 +26,11 @@ RANDOM_IMAGE_SOURCE, ] -# 0 = id -# 1 = shape -# 2 = buffer +# 0 = shape +# 1 = buffer CHECK_FIELDS: list[list[int]] = [ - [0, 1, 2], - [2], + [0, 1], + [1], ] diff --git a/rerun_py/tests/unit/test_tensor.py b/rerun_py/tests/unit/test_tensor.py index eec614424d24..551b27658185 100644 --- a/rerun_py/tests/unit/test_tensor.py +++ b/rerun_py/tests/unit/test_tensor.py @@ -1,6 +1,5 @@ from __future__ import annotations -import uuid from typing import Any import numpy as np @@ -16,7 +15,6 @@ TENSOR_DATA_INPUTS: list[rrd.TensorDataArrayLike] = [ # Full explicit construction rrd.TensorData( - id=rrd.TensorId(uuid=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), shape=[ rrd.TensorDimension(8, name="a"), rrd.TensorDimension(6, name="b"), @@ -32,18 +30,17 @@ # Explicit construction from array rrd.TensorData(array=RANDOM_TENSOR_SOURCE, names=["a", "b", "c", "d"]), # Explicit construction from array - rrd.TensorData(id=uuid.uuid4(), array=RANDOM_TENSOR_SOURCE, names=["a", "b", "c", "d"]), + rrd.TensorData(array=RANDOM_TENSOR_SOURCE, names=["a", "b", "c", "d"]), ] -# 0 = id -# 1 = shape -# 2 = buffer +# 0 = shape +# 1 = buffer CHECK_FIELDS: list[list[int]] = [ - [0, 1, 2], - [2], - [2], - [1, 2], - [1, 2], + [0, 1], + [1], + [1], + [0, 1], + [0, 1], ] diff --git a/tests/cpp/roundtrips/tensor/main.cpp b/tests/cpp/roundtrips/tensor/main.cpp index 49fbe2fee0f2..a66790908a01 100644 --- a/tests/cpp/roundtrips/tensor/main.cpp +++ b/tests/cpp/roundtrips/tensor/main.cpp @@ -9,8 +9,6 @@ int main(int argc, char** argv) { auto rec = rr::RecordingStream("rerun_example_roundtrip_tensor"); rec.save(argv[1]).throw_on_failure(); - uint8_t id[16] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}; - std::vector dimensions{ rr::datatypes::TensorDimension{3, std::nullopt}, rr::datatypes::TensorDimension{4, std::nullopt}, @@ -26,9 +24,8 @@ int main(int argc, char** argv) { // don't supported nested list-types. rec.log( "tensor", - rr::archetypes::Tensor(rr::datatypes::TensorData{ - rr::datatypes::TensorId(id), - dimensions, - rr::datatypes::TensorBuffer::i32(data)}) + rr::archetypes::Tensor( + rr::datatypes::TensorData{dimensions, rr::datatypes::TensorBuffer::i32(data)} + ) ); } diff --git a/tests/python/roundtrips/image/main.py b/tests/python/roundtrips/image/main.py index fdd4da50618f..b805ea63621e 100755 --- a/tests/python/roundtrips/image/main.py +++ b/tests/python/roundtrips/image/main.py @@ -26,7 +26,7 @@ def main() -> None: image[i, :, 1] = i image[:, :, 2] = 128 - image = rr2.dt.TensorData(array=image, id=np.arange(10, 26)) + image = rr2.dt.TensorData(array=image) rr2.log("image", rr2.Image(image)) diff --git a/tests/python/roundtrips/tensor/main.py b/tests/python/roundtrips/tensor/main.py index 16e17959d93e..7fabc7da57cb 100755 --- a/tests/python/roundtrips/tensor/main.py +++ b/tests/python/roundtrips/tensor/main.py @@ -19,7 +19,7 @@ def main() -> None: rr.script_setup(args, "rerun_example_roundtrip_tensor") tensor = np.array(np.arange(0, 360), dtype=np.int32).reshape((3, 4, 5, 6)) - tensor = rr2.dt.TensorData(array=tensor, id=np.arange(10, 26)) + tensor = rr2.dt.TensorData(array=tensor) rr2.log("tensor", rr2.Tensor(tensor)) diff --git a/tests/rust/roundtrips/image/src/main.rs b/tests/rust/roundtrips/image/src/main.rs index 30d31bf02228..67847e42e5a1 100644 --- a/tests/rust/roundtrips/image/src/main.rs +++ b/tests/rust/roundtrips/image/src/main.rs @@ -1,7 +1,7 @@ //! Logs an `Image` archetype for roundtrip checks. use image::{Rgb, RgbImage}; -use rerun::{archetypes::Image, datatypes::TensorId, external::re_log, RecordingStream}; +use rerun::{archetypes::Image, external::re_log, RecordingStream}; #[derive(Debug, clap::Parser)] #[clap(author, version, about)] @@ -11,11 +11,6 @@ struct Args { } fn run(rec: &RecordingStream, _args: &Args) -> anyhow::Result<()> { - // Need a deterministic id for round-trip tests. Used (10..26) - let id = TensorId { - uuid: core::array::from_fn(|i| (i + 10) as u8), - }; - let mut img = RgbImage::new(3, 2); // 2x3x3 image. Red channel = x. Green channel = y. Blue channel = 128. @@ -25,7 +20,7 @@ fn run(rec: &RecordingStream, _args: &Args) -> anyhow::Result<()> { } } - rec.log("image", &Image::try_from(img)?.with_id(id))?; + rec.log("image", &Image::try_from(img)?)?; Ok(()) } diff --git a/tests/rust/roundtrips/tensor/src/main.rs b/tests/rust/roundtrips/tensor/src/main.rs index 52951e7c3bbb..f7a2198512d5 100644 --- a/tests/rust/roundtrips/tensor/src/main.rs +++ b/tests/rust/roundtrips/tensor/src/main.rs @@ -1,6 +1,6 @@ //! Logs a `Tensor` archetype for roundtrip checks. -use rerun::{archetypes::Tensor, datatypes::TensorId, external::re_log, RecordingStream}; +use rerun::{archetypes::Tensor, external::re_log, RecordingStream}; #[derive(Debug, clap::Parser)] #[clap(author, version, about)] @@ -12,12 +12,7 @@ struct Args { fn run(rec: &RecordingStream, _args: &Args) -> anyhow::Result<()> { let tensor = ndarray::Array::from_shape_vec((3, 4, 5, 6), (0..360).collect::>())?; - // Need a deterministic id for round-trip tests. Used (10..26) - let id = TensorId { - uuid: core::array::from_fn(|i| (i + 10) as u8), - }; - - rec.log("tensor", &Tensor::try_from(tensor)?.with_id(id))?; + rec.log("tensor", &Tensor::try_from(tensor)?)?; Ok(()) }