From e59c1c69a03ac0b3a6fa927a4d38bf594db827a3 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 11:50:38 +0200 Subject: [PATCH 01/12] rename asset_video fbs file --- .../rerun/archetypes/{video.fbs => asset_video.fbs} | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) rename crates/store/re_types/definitions/rerun/archetypes/{video.fbs => asset_video.fbs} (71%) diff --git a/crates/store/re_types/definitions/rerun/archetypes/video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs similarity index 71% rename from crates/store/re_types/definitions/rerun/archetypes/video.fbs rename to crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 535ee5a6d09b..3b6480568a70 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -1,10 +1,13 @@ namespace rerun.archetypes; -/// A video file. +/// A video binary. /// /// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 and AV1 is currently supported, and not in all browsers. +/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. /// Follow for updates on the native support. +/// +/// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame. +// TODO(andreas): More docs and examples on how to use this. table AssetVideo ( "attr.rerun.experimental" ) { From c9bdfbd8fc732bbcf9eaef6e2c544e4e30aa2ea0 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 12:04:04 +0200 Subject: [PATCH 02/12] add video frame reference archetype --- .../re_types/definitions/rerun/archetypes.fbs | 3 ++- .../archetypes/video_frame_reference.fbs | 25 +++++++++++++++++++ .../re_types/definitions/rerun/components.fbs | 2 ++ .../rerun/components/entity_path.fbs | 12 +++++++++ .../rerun/components/video_reference.fbs | 11 ++++++++ .../re_types/definitions/rerun/datatypes.fbs | 1 + .../rerun/datatypes/video_timestamp.fbs | 24 ++++++++++++++++++ 7 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs create mode 100644 crates/store/re_types/definitions/rerun/components/entity_path.fbs create mode 100644 crates/store/re_types/definitions/rerun/components/video_reference.fbs create mode 100644 crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs diff --git a/crates/store/re_types/definitions/rerun/archetypes.fbs b/crates/store/re_types/definitions/rerun/archetypes.fbs index 1e679c77fc91..e8cc336b3afb 100644 --- a/crates/store/re_types/definitions/rerun/archetypes.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes.fbs @@ -4,6 +4,7 @@ include "./archetypes/annotation_context.fbs"; include "./archetypes/arrows2d.fbs"; include "./archetypes/arrows3d.fbs"; include "./archetypes/asset3d.fbs"; +include "./archetypes/asset_video.fbs"; include "./archetypes/bar_chart.fbs"; include "./archetypes/boxes2d.fbs"; include "./archetypes/boxes3d.fbs"; @@ -28,5 +29,5 @@ include "./archetypes/tensor.fbs"; include "./archetypes/text_document.fbs"; include "./archetypes/text_log.fbs"; include "./archetypes/transform3d.fbs"; -include "./archetypes/video.fbs"; +include "./archetypes/video_frame_reference.fbs"; include "./archetypes/view_coordinates.fbs"; diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs new file mode 100644 index 000000000000..1c1189530819 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -0,0 +1,25 @@ +namespace rerun.archetypes; + +/// References a single video frame. +/// +/// Used to display video frames from a [archetypes.AssetVideo]. +// TODO(andreas): More docs and examples on how to use this. +table VideoFrameReference ( + "attr.rerun.experimental" +){ + // --- Required --- + + /// References the closest video frame to this time. + /// + /// Note that this uses the closest video frame instead of the latest at this timestamp + /// in order to be more forgiving of rounding errors. + // TODO(andreas): Once this can also be a frame index, point out that this is an accurate measure. + timestamp: rerun.components.VideoTimestamp ("attr.rerun.component_required", required, order: 1000); + + // --- Optional --- + + /// Optional reference to an entity with a [archetypes.AssetVideo]. + /// + /// If none is specified, the video is assumed to be at the same entity. + video_reference: rerun.components.EntityPath ("attr.rerun.component_optional", nullable, order: 2000); +} diff --git a/crates/store/re_types/definitions/rerun/components.fbs b/crates/store/re_types/definitions/rerun/components.fbs index 764453c92d1e..d72129ca055e 100644 --- a/crates/store/re_types/definitions/rerun/components.fbs +++ b/crates/store/re_types/definitions/rerun/components.fbs @@ -12,6 +12,7 @@ include "./components/colormap.fbs"; include "./components/depth_meter.fbs"; include "./components/disconnected_space.fbs"; include "./components/draw_order.fbs"; +include "./components/entity_path.fbs"; include "./components/fill_mode.fbs"; include "./components/fill_ratio.fbs"; include "./components/gamma_correction.fbs"; @@ -52,4 +53,5 @@ include "./components/translation3d.fbs"; include "./components/triangle_indices.fbs"; include "./components/vector2d.fbs"; include "./components/vector3d.fbs"; +include "./components/video_reference.fbs"; include "./components/view_coordinates.fbs"; diff --git a/crates/store/re_types/definitions/rerun/components/entity_path.fbs b/crates/store/re_types/definitions/rerun/components/entity_path.fbs new file mode 100644 index 000000000000..3868835b0995 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/components/entity_path.fbs @@ -0,0 +1,12 @@ +namespace rerun.components; + +/// A path to an entity, usually to reference some data that is part of the target entity. +table EntityPath ( + "attr.arrow.transparent", + "attr.python.aliases": "str", + "attr.python.array_aliases": "str, Sequence[str]", + "attr.rust.derive": "Default, PartialEq, Eq, PartialOrd, Ord", + "attr.rust.repr": "transparent" +) { + value: rerun.datatypes.EntityPath (order: 100); +} diff --git a/crates/store/re_types/definitions/rerun/components/video_reference.fbs b/crates/store/re_types/definitions/rerun/components/video_reference.fbs new file mode 100644 index 000000000000..f3a0b0eb0ef7 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/components/video_reference.fbs @@ -0,0 +1,11 @@ + +namespace rerun.components; + +/// Timestamp inside a [archetypes.AssetVideo]. +struct VideoTimestamp ( + "attr.rust.derive": "Copy, PartialEq, Eq", + "attr.rust.repr": "transparent", + "attr.rerun.experimental" +) { + timestamp: rerun.datatypes.VideoTimestamp (order: 100); +} diff --git a/crates/store/re_types/definitions/rerun/datatypes.fbs b/crates/store/re_types/definitions/rerun/datatypes.fbs index 4b258e4a96e3..203c6105aa1a 100644 --- a/crates/store/re_types/definitions/rerun/datatypes.fbs +++ b/crates/store/re_types/definitions/rerun/datatypes.fbs @@ -39,5 +39,6 @@ include "./datatypes/uvec4d.fbs"; include "./datatypes/vec2d.fbs"; include "./datatypes/vec3d.fbs"; include "./datatypes/vec4d.fbs"; +include "./datatypes/video_timestamp.fbs"; include "./datatypes/view_coordinates.fbs"; include "./datatypes/visible_time_range.fbs"; diff --git a/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs b/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs new file mode 100644 index 000000000000..2c29b7d882d1 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs @@ -0,0 +1,24 @@ +namespace rerun.datatypes; + +/// Specifies how to interpret the `video_time` field of a [datatypes.VideoTimestamp]. +enum VideoTimeMode: ubyte{ + /// Invalid value. Won't show up in generated types. + Invalid = 0, + + /// Presentation timestamp in nanoseconds since the beginning of the video. + Nanoseconds = 1 (default), + + // Future values: FrameNr +} + +/// Timestamp inside a [archetypes.AssetVideo]. +struct VideoTimestamp ( + "attr.rust.derive": "Copy, PartialEq, Eq", + "attr.rerun.experimental" +) { + /// Timestamp value, type defined by `time_mode`. + video_time: long (order: 100); + + /// How to interpret `video_time`. + time_mode: VideoTimeMode (order: 200); +} From 219d2034751bd20d21ed1dabd48488fe8e30e239 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 12:04:35 +0200 Subject: [PATCH 03/12] codegen --- .../re_types/src/archetypes/.gitattributes | 1 + .../re_types/src/archetypes/asset_video.rs | 8 +- crates/store/re_types/src/archetypes/mod.rs | 2 + .../src/archetypes/video_frame_reference.rs | 199 +++++++++++++++ .../re_types/src/components/.gitattributes | 2 + .../re_types/src/components/entity_path.rs | 105 ++++++++ crates/store/re_types/src/components/mod.rs | 4 + .../src/components/video_timestamp.rs | 107 +++++++++ .../re_types/src/datatypes/.gitattributes | 2 + crates/store/re_types/src/datatypes/mod.rs | 4 + .../re_types/src/datatypes/video_time_mode.rs | 146 +++++++++++ .../re_types/src/datatypes/video_timestamp.rs | 226 ++++++++++++++++++ crates/viewer/re_viewer/src/reflection/mod.rs | 31 +++ docs/content/reference/types/archetypes.md | 3 +- .../reference/types/archetypes/.gitattributes | 1 + .../reference/types/archetypes/asset_video.md | 6 +- .../types/archetypes/video_frame_reference.md | 23 ++ docs/content/reference/types/components.md | 2 + .../reference/types/components/.gitattributes | 2 + .../reference/types/components/entity_path.md | 20 ++ .../types/components/video_timestamp.md | 23 ++ docs/content/reference/types/datatypes.md | 2 + .../reference/types/datatypes/.gitattributes | 2 + .../reference/types/datatypes/entity_path.md | 3 + .../types/datatypes/video_time_mode.md | 20 ++ .../types/datatypes/video_timestamp.md | 24 ++ rerun_cpp/src/rerun/archetypes.hpp | 1 + rerun_cpp/src/rerun/archetypes/.gitattributes | 2 + .../src/rerun/archetypes/asset_video.cpp | 2 +- .../src/rerun/archetypes/asset_video.hpp | 8 +- .../archetypes/video_frame_reference.cpp | 38 +++ .../archetypes/video_frame_reference.hpp | 77 ++++++ rerun_cpp/src/rerun/components.hpp | 2 + rerun_cpp/src/rerun/components/.gitattributes | 2 + .../src/rerun/components/entity_path.hpp | 66 +++++ .../src/rerun/components/video_timestamp.hpp | 59 +++++ rerun_cpp/src/rerun/datatypes.hpp | 2 + rerun_cpp/src/rerun/datatypes/.gitattributes | 4 + .../src/rerun/datatypes/video_time_mode.cpp | 56 +++++ .../src/rerun/datatypes/video_time_mode.hpp | 54 +++++ .../src/rerun/datatypes/video_timestamp.cpp | 84 +++++++ .../src/rerun/datatypes/video_timestamp.hpp | 57 +++++ .../rerun_sdk/rerun/archetypes/.gitattributes | 1 + .../rerun_sdk/rerun/archetypes/__init__.py | 2 + .../rerun_sdk/rerun/archetypes/asset_video.py | 8 +- .../rerun/archetypes/video_frame_reference.py | 94 ++++++++ .../rerun_sdk/rerun/components/.gitattributes | 2 + .../rerun_sdk/rerun/components/__init__.py | 8 + .../rerun_sdk/rerun/components/entity_path.py | 36 +++ .../rerun/components/video_timestamp.py | 40 ++++ .../rerun_sdk/rerun/datatypes/.gitattributes | 2 + .../rerun_sdk/rerun/datatypes/__init__.py | 24 ++ .../rerun/datatypes/video_time_mode.py | 71 ++++++ .../rerun/datatypes/video_timestamp.py | 110 +++++++++ 54 files changed, 1867 insertions(+), 13 deletions(-) create mode 100644 crates/store/re_types/src/archetypes/video_frame_reference.rs create mode 100644 crates/store/re_types/src/components/entity_path.rs create mode 100644 crates/store/re_types/src/components/video_timestamp.rs create mode 100644 crates/store/re_types/src/datatypes/video_time_mode.rs create mode 100644 crates/store/re_types/src/datatypes/video_timestamp.rs create mode 100644 docs/content/reference/types/archetypes/video_frame_reference.md create mode 100644 docs/content/reference/types/components/entity_path.md create mode 100644 docs/content/reference/types/components/video_timestamp.md create mode 100644 docs/content/reference/types/datatypes/video_time_mode.md create mode 100644 docs/content/reference/types/datatypes/video_timestamp.md create mode 100644 rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp create mode 100644 rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp create mode 100644 rerun_cpp/src/rerun/components/entity_path.hpp create mode 100644 rerun_cpp/src/rerun/components/video_timestamp.hpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_time_mode.cpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_time_mode.hpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_timestamp.cpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_timestamp.hpp create mode 100644 rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py create mode 100644 rerun_py/rerun_sdk/rerun/components/entity_path.py create mode 100644 rerun_py/rerun_sdk/rerun/components/video_timestamp.py create mode 100644 rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py create mode 100644 rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py diff --git a/crates/store/re_types/src/archetypes/.gitattributes b/crates/store/re_types/src/archetypes/.gitattributes index ee4909e04967..1dbfddfe109b 100644 --- a/crates/store/re_types/src/archetypes/.gitattributes +++ b/crates/store/re_types/src/archetypes/.gitattributes @@ -30,4 +30,5 @@ tensor.rs linguist-generated=true text_document.rs linguist-generated=true text_log.rs linguist-generated=true transform3d.rs linguist-generated=true +video_frame_reference.rs linguist-generated=true view_coordinates.rs linguist-generated=true diff --git a/crates/store/re_types/src/archetypes/asset_video.rs b/crates/store/re_types/src/archetypes/asset_video.rs index e54e3930eaeb..1133476159b8 100644 --- a/crates/store/re_types/src/archetypes/asset_video.rs +++ b/crates/store/re_types/src/archetypes/asset_video.rs @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs -// Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +// Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". #![allow(unused_imports)] #![allow(unused_parens)] @@ -18,12 +18,14 @@ use ::re_types_core::SerializationResult; use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; -/// **Archetype**: A video file. +/// **Archetype**: A video binary. /// /// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 and AV1 is currently supported, and not in all browsers. +/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. /// Follow for updates on the native support. /// +/// In order to display a video, you need to log a [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference] for each frame. +/// /// ⚠️ **This type is experimental and may be removed in future versions** #[derive(Clone, Debug)] pub struct AssetVideo { diff --git a/crates/store/re_types/src/archetypes/mod.rs b/crates/store/re_types/src/archetypes/mod.rs index 3c902398a8e7..46c6b161b698 100644 --- a/crates/store/re_types/src/archetypes/mod.rs +++ b/crates/store/re_types/src/archetypes/mod.rs @@ -45,6 +45,7 @@ mod text_document_ext; mod text_log; mod transform3d; mod transform3d_ext; +mod video_frame_reference; mod view_coordinates; mod view_coordinates_ext; @@ -76,4 +77,5 @@ pub use self::tensor::Tensor; pub use self::text_document::TextDocument; pub use self::text_log::TextLog; pub use self::transform3d::Transform3D; +pub use self::video_frame_reference::VideoFrameReference; pub use self::view_coordinates::ViewCoordinates; diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs new file mode 100644 index 000000000000..a9ebf9800269 --- /dev/null +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -0,0 +1,199 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Archetype**: References a single video frame. +/// +/// Used to display video frames from a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. +/// +/// ⚠️ **This type is experimental and may be removed in future versions** +#[derive(Clone, Debug)] +pub struct VideoFrameReference { + /// References the closest video frame to this time. + /// + /// Note that this uses the closest video frame instead of the latest at this timestamp + /// in order to be more forgiving of rounding errors. + pub timestamp: crate::components::VideoTimestamp, + + /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. + /// + /// If none is specified, the video is assumed to be at the same entity. + pub video_reference: Option, +} + +impl ::re_types_core::SizeBytes for VideoFrameReference { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.timestamp.heap_size_bytes() + self.video_reference.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + && >::is_pod() + } +} + +static REQUIRED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.VideoTimestamp".into()]); + +static RECOMMENDED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.VideoFrameReferenceIndicator".into()]); + +static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.EntityPath".into()]); + +static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 3usize]> = + once_cell::sync::Lazy::new(|| { + [ + "rerun.components.VideoTimestamp".into(), + "rerun.components.VideoFrameReferenceIndicator".into(), + "rerun.components.EntityPath".into(), + ] + }); + +impl VideoFrameReference { + /// The total number of components in the archetype: 1 required, 1 recommended, 1 optional + pub const NUM_COMPONENTS: usize = 3usize; +} + +/// Indicator component for the [`VideoFrameReference`] [`::re_types_core::Archetype`] +pub type VideoFrameReferenceIndicator = + ::re_types_core::GenericIndicatorComponent; + +impl ::re_types_core::Archetype for VideoFrameReference { + type Indicator = VideoFrameReferenceIndicator; + + #[inline] + fn name() -> ::re_types_core::ArchetypeName { + "rerun.archetypes.VideoFrameReference".into() + } + + #[inline] + fn display_name() -> &'static str { + "Video frame reference" + } + + #[inline] + fn indicator() -> MaybeOwnedComponentBatch<'static> { + static INDICATOR: VideoFrameReferenceIndicator = VideoFrameReferenceIndicator::DEFAULT; + MaybeOwnedComponentBatch::Ref(&INDICATOR) + } + + #[inline] + fn required_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + REQUIRED_COMPONENTS.as_slice().into() + } + + #[inline] + fn recommended_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + RECOMMENDED_COMPONENTS.as_slice().into() + } + + #[inline] + fn optional_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + OPTIONAL_COMPONENTS.as_slice().into() + } + + #[inline] + fn all_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + ALL_COMPONENTS.as_slice().into() + } + + #[inline] + fn from_arrow_components( + arrow_data: impl IntoIterator)>, + ) -> DeserializationResult { + re_tracing::profile_function!(); + use ::re_types_core::{Loggable as _, ResultExt as _}; + let arrays_by_name: ::std::collections::HashMap<_, _> = arrow_data + .into_iter() + .map(|(name, array)| (name.full_name(), array)) + .collect(); + let timestamp = { + let array = arrays_by_name + .get("rerun.components.VideoTimestamp") + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.VideoFrameReference#timestamp")?; + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.VideoFrameReference#timestamp")? + .into_iter() + .next() + .flatten() + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.VideoFrameReference#timestamp")? + }; + let video_reference = if let Some(array) = arrays_by_name.get("rerun.components.EntityPath") + { + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.VideoFrameReference#video_reference")? + .into_iter() + .next() + .flatten() + } else { + None + }; + Ok(Self { + timestamp, + video_reference, + }) + } +} + +impl ::re_types_core::AsComponents for VideoFrameReference { + fn as_component_batches(&self) -> Vec> { + re_tracing::profile_function!(); + use ::re_types_core::Archetype as _; + [ + Some(Self::indicator()), + Some((&self.timestamp as &dyn ComponentBatch).into()), + self.video_reference + .as_ref() + .map(|comp| (comp as &dyn ComponentBatch).into()), + ] + .into_iter() + .flatten() + .collect() + } +} + +impl ::re_types_core::ArchetypeReflectionMarker for VideoFrameReference {} + +impl VideoFrameReference { + /// Create a new `VideoFrameReference`. + #[inline] + pub fn new(timestamp: impl Into) -> Self { + Self { + timestamp: timestamp.into(), + video_reference: None, + } + } + + /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. + /// + /// If none is specified, the video is assumed to be at the same entity. + #[inline] + pub fn with_video_reference( + mut self, + video_reference: impl Into, + ) -> Self { + self.video_reference = Some(video_reference.into()); + self + } +} diff --git a/crates/store/re_types/src/components/.gitattributes b/crates/store/re_types/src/components/.gitattributes index 0473984ef016..ce1cd46d7000 100644 --- a/crates/store/re_types/src/components/.gitattributes +++ b/crates/store/re_types/src/components/.gitattributes @@ -12,6 +12,7 @@ colormap.rs linguist-generated=true depth_meter.rs linguist-generated=true disconnected_space.rs linguist-generated=true draw_order.rs linguist-generated=true +entity_path.rs linguist-generated=true fill_mode.rs linguist-generated=true fill_ratio.rs linguist-generated=true gamma_correction.rs linguist-generated=true @@ -60,4 +61,5 @@ translation3d.rs linguist-generated=true triangle_indices.rs linguist-generated=true vector2d.rs linguist-generated=true vector3d.rs linguist-generated=true +video_timestamp.rs linguist-generated=true view_coordinates.rs linguist-generated=true diff --git a/crates/store/re_types/src/components/entity_path.rs b/crates/store/re_types/src/components/entity_path.rs new file mode 100644 index 000000000000..2275f2cb5a79 --- /dev/null +++ b/crates/store/re_types/src/components/entity_path.rs @@ -0,0 +1,105 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/components/entity_path.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Component**: A path to an entity, usually to reference some data that is part of the target entity. +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct EntityPath(pub crate::datatypes::EntityPath); + +impl ::re_types_core::SizeBytes for EntityPath { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.0.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + } +} + +impl> From for EntityPath { + fn from(v: T) -> Self { + Self(v.into()) + } +} + +impl std::borrow::Borrow for EntityPath { + #[inline] + fn borrow(&self) -> &crate::datatypes::EntityPath { + &self.0 + } +} + +impl std::ops::Deref for EntityPath { + type Target = crate::datatypes::EntityPath; + + #[inline] + fn deref(&self) -> &crate::datatypes::EntityPath { + &self.0 + } +} + +impl std::ops::DerefMut for EntityPath { + #[inline] + fn deref_mut(&mut self) -> &mut crate::datatypes::EntityPath { + &mut self.0 + } +} + +::re_types_core::macros::impl_into_cow!(EntityPath); + +impl ::re_types_core::Loggable for EntityPath { + type Name = ::re_types_core::ComponentName; + + #[inline] + fn name() -> Self::Name { + "rerun.components.EntityPath".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + crate::datatypes::EntityPath::arrow_datatype() + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { + datum.map(|datum| match datum.into() { + ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), + ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), + }) + })) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + crate::datatypes::EntityPath::from_arrow_opt(arrow_data) + .map(|v| v.into_iter().map(|v| v.map(Self)).collect()) + } +} diff --git a/crates/store/re_types/src/components/mod.rs b/crates/store/re_types/src/components/mod.rs index 8ece75aaea25..0e499a64e2fa 100644 --- a/crates/store/re_types/src/components/mod.rs +++ b/crates/store/re_types/src/components/mod.rs @@ -18,6 +18,7 @@ mod disconnected_space; mod disconnected_space_ext; mod draw_order; mod draw_order_ext; +mod entity_path; mod fill_mode; mod fill_ratio; mod fill_ratio_ext; @@ -105,6 +106,7 @@ mod vector2d; mod vector2d_ext; mod vector3d; mod vector3d_ext; +mod video_timestamp; mod view_coordinates; mod view_coordinates_ext; @@ -119,6 +121,7 @@ pub use self::colormap::Colormap; pub use self::depth_meter::DepthMeter; pub use self::disconnected_space::DisconnectedSpace; pub use self::draw_order::DrawOrder; +pub use self::entity_path::EntityPath; pub use self::fill_mode::FillMode; pub use self::fill_ratio::FillRatio; pub use self::gamma_correction::GammaCorrection; @@ -166,4 +169,5 @@ pub use self::translation3d::Translation3D; pub use self::triangle_indices::TriangleIndices; pub use self::vector2d::Vector2D; pub use self::vector3d::Vector3D; +pub use self::video_timestamp::VideoTimestamp; pub use self::view_coordinates::ViewCoordinates; diff --git a/crates/store/re_types/src/components/video_timestamp.rs b/crates/store/re_types/src/components/video_timestamp.rs new file mode 100644 index 000000000000..10daf8833187 --- /dev/null +++ b/crates/store/re_types/src/components/video_timestamp.rs @@ -0,0 +1,107 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/components/video_reference.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Component**: Timestamp inside a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. +/// +/// ⚠️ **This type is experimental and may be removed in future versions** +#[derive(Clone, Debug, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct VideoTimestamp(pub crate::datatypes::VideoTimestamp); + +impl ::re_types_core::SizeBytes for VideoTimestamp { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.0.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + } +} + +impl> From for VideoTimestamp { + fn from(v: T) -> Self { + Self(v.into()) + } +} + +impl std::borrow::Borrow for VideoTimestamp { + #[inline] + fn borrow(&self) -> &crate::datatypes::VideoTimestamp { + &self.0 + } +} + +impl std::ops::Deref for VideoTimestamp { + type Target = crate::datatypes::VideoTimestamp; + + #[inline] + fn deref(&self) -> &crate::datatypes::VideoTimestamp { + &self.0 + } +} + +impl std::ops::DerefMut for VideoTimestamp { + #[inline] + fn deref_mut(&mut self) -> &mut crate::datatypes::VideoTimestamp { + &mut self.0 + } +} + +::re_types_core::macros::impl_into_cow!(VideoTimestamp); + +impl ::re_types_core::Loggable for VideoTimestamp { + type Name = ::re_types_core::ComponentName; + + #[inline] + fn name() -> Self::Name { + "rerun.components.VideoTimestamp".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + crate::datatypes::VideoTimestamp::arrow_datatype() + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + crate::datatypes::VideoTimestamp::to_arrow_opt(data.into_iter().map(|datum| { + datum.map(|datum| match datum.into() { + ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), + ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), + }) + })) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + crate::datatypes::VideoTimestamp::from_arrow_opt(arrow_data) + .map(|v| v.into_iter().map(|v| v.map(Self)).collect()) + } +} diff --git a/crates/store/re_types/src/datatypes/.gitattributes b/crates/store/re_types/src/datatypes/.gitattributes index af98d9b2ad4b..a79b23c34fb8 100644 --- a/crates/store/re_types/src/datatypes/.gitattributes +++ b/crates/store/re_types/src/datatypes/.gitattributes @@ -33,4 +33,6 @@ uvec4d.rs linguist-generated=true vec2d.rs linguist-generated=true vec3d.rs linguist-generated=true vec4d.rs linguist-generated=true +video_time_mode.rs linguist-generated=true +video_timestamp.rs linguist-generated=true view_coordinates.rs linguist-generated=true diff --git a/crates/store/re_types/src/datatypes/mod.rs b/crates/store/re_types/src/datatypes/mod.rs index e77eee7490d1..8f069af53f6c 100644 --- a/crates/store/re_types/src/datatypes/mod.rs +++ b/crates/store/re_types/src/datatypes/mod.rs @@ -61,6 +61,8 @@ mod vec3d; mod vec3d_ext; mod vec4d; mod vec4d_ext; +mod video_time_mode; +mod video_timestamp; mod view_coordinates; mod view_coordinates_ext; @@ -95,4 +97,6 @@ pub use self::uvec4d::UVec4D; pub use self::vec2d::Vec2D; pub use self::vec3d::Vec3D; pub use self::vec4d::Vec4D; +pub use self::video_time_mode::VideoTimeMode; +pub use self::video_timestamp::VideoTimestamp; pub use self::view_coordinates::ViewCoordinates; diff --git a/crates/store/re_types/src/datatypes/video_time_mode.rs b/crates/store/re_types/src/datatypes/video_time_mode.rs new file mode 100644 index 000000000000..d0e8e7ee6ded --- /dev/null +++ b/crates/store/re_types/src/datatypes/video_time_mode.rs @@ -0,0 +1,146 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Datatype**: Specifies how to interpret the `video_time` field of a [`datatypes::VideoTimestamp`][crate::datatypes::VideoTimestamp]. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Default)] +#[repr(u8)] +pub enum VideoTimeMode { + /// Presentation timestamp in nanoseconds since the beginning of the video. + #[default] + Nanoseconds = 1, +} + +impl ::re_types_core::reflection::Enum for VideoTimeMode { + #[inline] + fn variants() -> &'static [Self] { + &[Self::Nanoseconds] + } + + #[inline] + fn docstring_md(self) -> &'static str { + match self { + Self::Nanoseconds => { + "Presentation timestamp in nanoseconds since the beginning of the video." + } + } + } +} + +impl ::re_types_core::SizeBytes for VideoTimeMode { + #[inline] + fn heap_size_bytes(&self) -> u64 { + 0 + } + + #[inline] + fn is_pod() -> bool { + true + } +} + +impl std::fmt::Display for VideoTimeMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Nanoseconds => write!(f, "Nanoseconds"), + } + } +} + +::re_types_core::macros::impl_into_cow!(VideoTimeMode); + +impl ::re_types_core::Loggable for VideoTimeMode { + type Name = ::re_types_core::DatatypeName; + + #[inline] + fn name() -> Self::Name { + "rerun.datatypes.VideoTimeMode".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + #![allow(clippy::wildcard_imports)] + use arrow2::datatypes::*; + DataType::UInt8 + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| *datum as u8); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + Self::arrow_datatype(), + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, buffer::*, datatypes::*}; + Ok(arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + let expected = Self::arrow_datatype(); + let actual = arrow_data.data_type().clone(); + DeserializationError::datatype_mismatch(expected, actual) + }) + .with_context("rerun.datatypes.VideoTimeMode#enum")? + .into_iter() + .map(|opt| opt.copied()) + .map(|typ| match typ { + Some(1) => Ok(Some(Self::Nanoseconds)), + None => Ok(None), + Some(invalid) => Err(DeserializationError::missing_union_arm( + Self::arrow_datatype(), + "", + invalid as _, + )), + }) + .collect::>>>() + .with_context("rerun.datatypes.VideoTimeMode")?) + } +} diff --git a/crates/store/re_types/src/datatypes/video_timestamp.rs b/crates/store/re_types/src/datatypes/video_timestamp.rs new file mode 100644 index 000000000000..d395bfb35338 --- /dev/null +++ b/crates/store/re_types/src/datatypes/video_timestamp.rs @@ -0,0 +1,226 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Datatype**: Timestamp inside a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. +/// +/// ⚠️ **This type is experimental and may be removed in future versions** +#[derive(Clone, Debug, Copy, PartialEq, Eq)] +pub struct VideoTimestamp { + /// Timestamp value, type defined by `time_mode`. + pub video_time: i64, + + /// How to interpret `video_time`. + pub time_mode: crate::datatypes::VideoTimeMode, +} + +impl ::re_types_core::SizeBytes for VideoTimestamp { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.video_time.heap_size_bytes() + self.time_mode.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() && ::is_pod() + } +} + +::re_types_core::macros::impl_into_cow!(VideoTimestamp); + +impl ::re_types_core::Loggable for VideoTimestamp { + type Name = ::re_types_core::DatatypeName; + + #[inline] + fn name() -> Self::Name { + "rerun.datatypes.VideoTimestamp".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + #![allow(clippy::wildcard_imports)] + use arrow2::datatypes::*; + DataType::Struct(std::sync::Arc::new(vec![ + Field::new("video_time", DataType::Int64, false), + Field::new( + "time_mode", + ::arrow_datatype(), + false, + ), + ])) + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + Self::arrow_datatype(), + vec![ + { + let (somes, video_time): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| datum.video_time.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let video_time_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::Int64, + video_time + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + video_time_bitmap, + ) + .boxed() + }, + { + let (somes, time_mode): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| datum.time_mode.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let time_mode_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = time_mode_bitmap; + crate::datatypes::VideoTimeMode::to_arrow_opt(time_mode)? + } + }, + ], + bitmap, + ) + .boxed() + }) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, buffer::*, datatypes::*}; + Ok({ + let arrow_data = arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + let expected = Self::arrow_datatype(); + let actual = arrow_data.data_type().clone(); + DeserializationError::datatype_mismatch(expected, actual) + }) + .with_context("rerun.datatypes.VideoTimestamp")?; + if arrow_data.is_empty() { + Vec::new() + } else { + let (arrow_data_fields, arrow_data_arrays) = + (arrow_data.fields(), arrow_data.values()); + let arrays_by_name: ::std::collections::HashMap<_, _> = arrow_data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(arrow_data_arrays) + .collect(); + let video_time = { + if !arrays_by_name.contains_key("video_time") { + return Err(DeserializationError::missing_struct_field( + Self::arrow_datatype(), + "video_time", + )) + .with_context("rerun.datatypes.VideoTimestamp"); + } + let arrow_data = &**arrays_by_name["video_time"]; + arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + let expected = DataType::Int64; + let actual = arrow_data.data_type().clone(); + DeserializationError::datatype_mismatch(expected, actual) + }) + .with_context("rerun.datatypes.VideoTimestamp#video_time")? + .into_iter() + .map(|opt| opt.copied()) + }; + let time_mode = { + if !arrays_by_name.contains_key("time_mode") { + return Err(DeserializationError::missing_struct_field( + Self::arrow_datatype(), + "time_mode", + )) + .with_context("rerun.datatypes.VideoTimestamp"); + } + let arrow_data = &**arrays_by_name["time_mode"]; + crate::datatypes::VideoTimeMode::from_arrow_opt(arrow_data) + .with_context("rerun.datatypes.VideoTimestamp#time_mode")? + .into_iter() + }; + arrow2::bitmap::utils::ZipValidity::new_with_validity( + ::itertools::izip!(video_time, time_mode), + arrow_data.validity(), + ) + .map(|opt| { + opt.map(|(video_time, time_mode)| { + Ok(Self { + video_time: video_time + .ok_or_else(DeserializationError::missing_data) + .with_context( + "rerun.datatypes.VideoTimestamp#video_time", + )?, + time_mode: time_mode + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.datatypes.VideoTimestamp#time_mode")?, + }) + }) + .transpose() + }) + .collect::>>() + .with_context("rerun.datatypes.VideoTimestamp")? + } + }) + } +} diff --git a/crates/viewer/re_viewer/src/reflection/mod.rs b/crates/viewer/re_viewer/src/reflection/mod.rs index bb168edd9ddf..962bc073bb24 100644 --- a/crates/viewer/re_viewer/src/reflection/mod.rs +++ b/crates/viewer/re_viewer/src/reflection/mod.rs @@ -342,6 +342,13 @@ fn generate_component_reflection() -> Result::name(), + ComponentReflection { + docstring_md: "A path to an entity, usually to reference some data that is part of the target entity.", + placeholder: Some(EntityPath::default().to_arrow()?), + }, + ), ( ::name(), ComponentReflection { @@ -671,6 +678,13 @@ fn generate_component_reflection() -> Result::name(), + ComponentReflection { + docstring_md: "Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + placeholder: Some(VideoTimestamp::default().to_arrow()?), + }, + ), ( ::name(), ComponentReflection { @@ -1444,6 +1458,23 @@ fn generate_archetype_reflection() -> ArchetypeReflectionMap { ], }, ), + ( + ArchetypeName::new("rerun.archetypes.VideoFrameReference"), + ArchetypeReflection { + display_name: "Video frame reference", + fields: vec![ + ArchetypeFieldReflection { component_name : + "rerun.components.VideoTimestamp".into(), display_name : "Timestamp", + docstring_md : + "References the closest video frame to this time.\n\nNote that this uses the closest video frame instead of the latest at this timestamp\nin order to be more forgiving of rounding errors.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + is_required : true, }, ArchetypeFieldReflection { component_name : + "rerun.components.EntityPath".into(), display_name : + "Video reference", docstring_md : + "Optional reference to an entity with a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).\n\nIf none is specified, the video is assumed to be at the same entity.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + is_required : false, }, + ], + }, + ), ( ArchetypeName::new("rerun.archetypes.ViewCoordinates"), ArchetypeReflection { diff --git a/docs/content/reference/types/archetypes.md b/docs/content/reference/types/archetypes.md index 4e51b9910a27..be8c53d8ec70 100644 --- a/docs/content/reference/types/archetypes.md +++ b/docs/content/reference/types/archetypes.md @@ -56,7 +56,8 @@ This page lists all built-in archetypes. ## Other * [`AnnotationContext`](archetypes/annotation_context.md): The annotation context provides additional information on how to display entities. -* [`AssetVideo`](archetypes/asset_video.md): A video file. +* [`AssetVideo`](archetypes/asset_video.md): A video binary. * [`Clear`](archetypes/clear.md): Empties all the components of an entity. * [`DisconnectedSpace`](archetypes/disconnected_space.md): Spatially disconnect this entity from its parent. +* [`VideoFrameReference`](archetypes/video_frame_reference.md): References a single video frame. diff --git a/docs/content/reference/types/archetypes/.gitattributes b/docs/content/reference/types/archetypes/.gitattributes index e273ee9420db..c0ce3b7563c5 100644 --- a/docs/content/reference/types/archetypes/.gitattributes +++ b/docs/content/reference/types/archetypes/.gitattributes @@ -30,4 +30,5 @@ tensor.md linguist-generated=true text_document.md linguist-generated=true text_log.md linguist-generated=true transform3d.md linguist-generated=true +video_frame_reference.md linguist-generated=true view_coordinates.md linguist-generated=true diff --git a/docs/content/reference/types/archetypes/asset_video.md b/docs/content/reference/types/archetypes/asset_video.md index b24fdcebdf1f..110d10844a8b 100644 --- a/docs/content/reference/types/archetypes/asset_video.md +++ b/docs/content/reference/types/archetypes/asset_video.md @@ -6,12 +6,14 @@ title: "AssetVideo" ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** -A video file. +A video binary. NOTE: Videos can only be viewed in the Rerun web viewer. -Only MP4 and AV1 is currently supported, and not in all browsers. +Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. Follow for updates on the native support. +In order to display a video, you need to log a [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference) for each frame. + ## Components **Required**: [`Blob`](../components/blob.md) diff --git a/docs/content/reference/types/archetypes/video_frame_reference.md b/docs/content/reference/types/archetypes/video_frame_reference.md new file mode 100644 index 000000000000..641e9cbe7d6d --- /dev/null +++ b/docs/content/reference/types/archetypes/video_frame_reference.md @@ -0,0 +1,23 @@ +--- +title: "VideoFrameReference" +--- + + + +⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + +References a single video frame. + +Used to display video frames from a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). + +## Components + +**Required**: [`VideoTimestamp`](../components/video_timestamp.md) + +**Optional**: [`EntityPath`](../components/entity_path.md) + +## API reference links + * 🌊 [C++ API docs for `VideoFrameReference`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1archetypes_1_1VideoFrameReference.html) + * 🐍 [Python API docs for `VideoFrameReference`](https://ref.rerun.io/docs/python/stable/common/archetypes#rerun.archetypes.VideoFrameReference) + * 🦀 [Rust API docs for `VideoFrameReference`](https://docs.rs/rerun/latest/rerun/archetypes/struct.VideoFrameReference.html) + diff --git a/docs/content/reference/types/components.md b/docs/content/reference/types/components.md index 999b1c6e53b8..1139ef975b74 100644 --- a/docs/content/reference/types/components.md +++ b/docs/content/reference/types/components.md @@ -25,6 +25,7 @@ on [Entities and Components](../../concepts/entity-component.md). * [`DepthMeter`](components/depth_meter.md): The world->depth map scaling factor. * [`DisconnectedSpace`](components/disconnected_space.md): Spatially disconnect this entity from its parent. * [`DrawOrder`](components/draw_order.md): Draw order of 2D elements. Higher values are drawn on top of lower values. +* [`EntityPath`](components/entity_path.md): A path to an entity, usually to reference some data that is part of the target entity. * [`FillMode`](components/fill_mode.md): How a geometric shape is drawn and colored. * [`FillRatio`](components/fill_ratio.md): How much a primitive fills out the available space. * [`GammaCorrection`](components/gamma_correction.md): A gamma correction value to be used with a scalar value or color. @@ -72,5 +73,6 @@ on [Entities and Components](../../concepts/entity-component.md). * [`TriangleIndices`](components/triangle_indices.md): The three indices of a triangle in a triangle mesh. * [`Vector2D`](components/vector2d.md): A vector in 2D space. * [`Vector3D`](components/vector3d.md): A vector in 3D space. +* [`VideoTimestamp`](components/video_timestamp.md): Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). * [`ViewCoordinates`](components/view_coordinates.md): How we interpret the coordinate system of an entity/space. diff --git a/docs/content/reference/types/components/.gitattributes b/docs/content/reference/types/components/.gitattributes index a2b2fef2de52..25cfd40ed1e7 100644 --- a/docs/content/reference/types/components/.gitattributes +++ b/docs/content/reference/types/components/.gitattributes @@ -13,6 +13,7 @@ colormap.md linguist-generated=true depth_meter.md linguist-generated=true disconnected_space.md linguist-generated=true draw_order.md linguist-generated=true +entity_path.md linguist-generated=true fill_mode.md linguist-generated=true fill_ratio.md linguist-generated=true gamma_correction.md linguist-generated=true @@ -60,4 +61,5 @@ translation3d.md linguist-generated=true triangle_indices.md linguist-generated=true vector2d.md linguist-generated=true vector3d.md linguist-generated=true +video_timestamp.md linguist-generated=true view_coordinates.md linguist-generated=true diff --git a/docs/content/reference/types/components/entity_path.md b/docs/content/reference/types/components/entity_path.md new file mode 100644 index 000000000000..93cbe3bfecf4 --- /dev/null +++ b/docs/content/reference/types/components/entity_path.md @@ -0,0 +1,20 @@ +--- +title: "EntityPath" +--- + + +A path to an entity, usually to reference some data that is part of the target entity. + +## Fields + +* value: [`EntityPath`](../datatypes/entity_path.md) + +## API reference links + * 🌊 [C++ API docs for `EntityPath`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1EntityPath.html) + * 🐍 [Python API docs for `EntityPath`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.EntityPath) + * 🦀 [Rust API docs for `EntityPath`](https://docs.rs/rerun/latest/rerun/components/struct.EntityPath.html) + + +## Used by + +* [`VideoFrameReference`](../archetypes/video_frame_reference.md) diff --git a/docs/content/reference/types/components/video_timestamp.md b/docs/content/reference/types/components/video_timestamp.md new file mode 100644 index 000000000000..10ee10394b8f --- /dev/null +++ b/docs/content/reference/types/components/video_timestamp.md @@ -0,0 +1,23 @@ +--- +title: "VideoTimestamp" +--- + + + +⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + +Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). + +## Fields + +* timestamp: [`VideoTimestamp`](../datatypes/video_timestamp.md) + +## API reference links + * 🌊 [C++ API docs for `VideoTimestamp`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1VideoTimestamp.html) + * 🐍 [Python API docs for `VideoTimestamp`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.VideoTimestamp) + * 🦀 [Rust API docs for `VideoTimestamp`](https://docs.rs/rerun/latest/rerun/components/struct.VideoTimestamp.html) + + +## Used by + +* [`VideoFrameReference`](../archetypes/video_frame_reference.md) diff --git a/docs/content/reference/types/datatypes.md b/docs/content/reference/types/datatypes.md index b9cf705e2382..cd9a53251034 100644 --- a/docs/content/reference/types/datatypes.md +++ b/docs/content/reference/types/datatypes.md @@ -49,6 +49,8 @@ Data types are the lowest layer of the data model hierarchy. They are re-usable * [`Vec2D`](datatypes/vec2d.md): A vector in 2D space. * [`Vec3D`](datatypes/vec3d.md): A vector in 3D space. * [`Vec4D`](datatypes/vec4d.md): A vector in 4D space. +* [`VideoTimeMode`](datatypes/video_time_mode.md): Specifies how to interpret the `video_time` field of a [`datatypes.VideoTimestamp`](https://rerun.io/docs/reference/types/datatypes/video_timestamp). +* [`VideoTimestamp`](datatypes/video_timestamp.md): Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). * [`ViewCoordinates`](datatypes/view_coordinates.md): How we interpret the coordinate system of an entity/space. * [`VisibleTimeRange`](datatypes/visible_time_range.md): Visible time range bounds for a specific timeline. diff --git a/docs/content/reference/types/datatypes/.gitattributes b/docs/content/reference/types/datatypes/.gitattributes index 5c2e32f16a34..b051f19b2670 100644 --- a/docs/content/reference/types/datatypes/.gitattributes +++ b/docs/content/reference/types/datatypes/.gitattributes @@ -43,5 +43,7 @@ uvec4d.md linguist-generated=true vec2d.md linguist-generated=true vec3d.md linguist-generated=true vec4d.md linguist-generated=true +video_time_mode.md linguist-generated=true +video_timestamp.md linguist-generated=true view_coordinates.md linguist-generated=true visible_time_range.md linguist-generated=true diff --git a/docs/content/reference/types/datatypes/entity_path.md b/docs/content/reference/types/datatypes/entity_path.md index 54e00495e6cb..5e7a14c35223 100644 --- a/docs/content/reference/types/datatypes/entity_path.md +++ b/docs/content/reference/types/datatypes/entity_path.md @@ -15,3 +15,6 @@ A path to an entity in the `ChunkStore`. * 🦀 [Rust API docs for `EntityPath`](https://docs.rs/rerun/latest/rerun/datatypes/struct.EntityPath.html) +## Used by + +* [`EntityPath`](../components/entity_path.md) diff --git a/docs/content/reference/types/datatypes/video_time_mode.md b/docs/content/reference/types/datatypes/video_time_mode.md new file mode 100644 index 000000000000..247661b972d4 --- /dev/null +++ b/docs/content/reference/types/datatypes/video_time_mode.md @@ -0,0 +1,20 @@ +--- +title: "VideoTimeMode" +--- + + +Specifies how to interpret the `video_time` field of a [`datatypes.VideoTimestamp`](https://rerun.io/docs/reference/types/datatypes/video_timestamp). + +## Variants + +* Nanoseconds + +## API reference links + * 🌊 [C++ API docs for `VideoTimeMode`](https://ref.rerun.io/docs/cpp/stable/namespacererun_1_1datatypes.html) + * 🐍 [Python API docs for `VideoTimeMode`](https://ref.rerun.io/docs/python/stable/common/datatypes#rerun.datatypes.VideoTimeMode) + * 🦀 [Rust API docs for `VideoTimeMode`](https://docs.rs/rerun/latest/rerun/datatypes/enum.VideoTimeMode.html) + + +## Used by + +* [`VideoTimestamp`](../datatypes/video_timestamp.md) diff --git a/docs/content/reference/types/datatypes/video_timestamp.md b/docs/content/reference/types/datatypes/video_timestamp.md new file mode 100644 index 000000000000..ea04f86bb75c --- /dev/null +++ b/docs/content/reference/types/datatypes/video_timestamp.md @@ -0,0 +1,24 @@ +--- +title: "VideoTimestamp" +--- + + + +⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + +Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). + +## Fields + +* video_time: `i64` +* time_mode: [`VideoTimeMode`](../datatypes/video_time_mode.md) + +## API reference links + * 🌊 [C++ API docs for `VideoTimestamp`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1datatypes_1_1VideoTimestamp.html) + * 🐍 [Python API docs for `VideoTimestamp`](https://ref.rerun.io/docs/python/stable/common/datatypes#rerun.datatypes.VideoTimestamp) + * 🦀 [Rust API docs for `VideoTimestamp`](https://docs.rs/rerun/latest/rerun/datatypes/struct.VideoTimestamp.html) + + +## Used by + +* [`VideoTimestamp`](../components/video_timestamp.md) diff --git a/rerun_cpp/src/rerun/archetypes.hpp b/rerun_cpp/src/rerun/archetypes.hpp index bf8519498f20..4922c62f6956 100644 --- a/rerun_cpp/src/rerun/archetypes.hpp +++ b/rerun_cpp/src/rerun/archetypes.hpp @@ -31,4 +31,5 @@ #include "archetypes/text_document.hpp" #include "archetypes/text_log.hpp" #include "archetypes/transform3d.hpp" +#include "archetypes/video_frame_reference.hpp" #include "archetypes/view_coordinates.hpp" diff --git a/rerun_cpp/src/rerun/archetypes/.gitattributes b/rerun_cpp/src/rerun/archetypes/.gitattributes index cde22918ea6e..0151357a4f60 100644 --- a/rerun_cpp/src/rerun/archetypes/.gitattributes +++ b/rerun_cpp/src/rerun/archetypes/.gitattributes @@ -59,5 +59,7 @@ text_log.cpp linguist-generated=true text_log.hpp linguist-generated=true transform3d.cpp linguist-generated=true transform3d.hpp linguist-generated=true +video_frame_reference.cpp linguist-generated=true +video_frame_reference.hpp linguist-generated=true view_coordinates.cpp linguist-generated=true view_coordinates.hpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.cpp b/rerun_cpp/src/rerun/archetypes/asset_video.cpp index 3f85f212d452..d2322b27a28b 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.cpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.cpp @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs -// Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +// Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". #include "asset_video.hpp" diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.hpp b/rerun_cpp/src/rerun/archetypes/asset_video.hpp index 766c2608dc0c..d1510a5c8027 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.hpp @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs -// Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +// Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". #pragma once @@ -18,12 +18,14 @@ #include namespace rerun::archetypes { - /// **Archetype**: A video file. + /// **Archetype**: A video binary. /// /// NOTE: Videos can only be viewed in the Rerun web viewer. - /// Only MP4 and AV1 is currently supported, and not in all browsers. + /// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. /// Follow for updates on the native support. /// + /// In order to display a video, you need to log a `archetypes::VideoFrameReference` for each frame. + /// /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** struct AssetVideo { /// The asset's bytes. diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp new file mode 100644 index 000000000000..1c8535480c83 --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp @@ -0,0 +1,38 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +#include "video_frame_reference.hpp" + +#include "../collection_adapter_builtins.hpp" + +namespace rerun::archetypes {} + +namespace rerun { + + Result> AsComponents::serialize( + const archetypes::VideoFrameReference& archetype + ) { + using namespace archetypes; + std::vector cells; + cells.reserve(3); + + { + auto result = ComponentBatch::from_loggable(archetype.timestamp); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + if (archetype.video_reference.has_value()) { + auto result = ComponentBatch::from_loggable(archetype.video_reference.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + { + auto indicator = VideoFrameReference::IndicatorComponent(); + auto result = ComponentBatch::from_loggable(indicator); + RR_RETURN_NOT_OK(result.error); + cells.emplace_back(std::move(result.value)); + } + + return cells; + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp new file mode 100644 index 000000000000..80d6b7db752d --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -0,0 +1,77 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +#pragma once + +#include "../collection.hpp" +#include "../compiler_utils.hpp" +#include "../component_batch.hpp" +#include "../components/entity_path.hpp" +#include "../components/video_timestamp.hpp" +#include "../indicator_component.hpp" +#include "../result.hpp" + +#include +#include +#include +#include + +namespace rerun::archetypes { + /// **Archetype**: References a single video frame. + /// + /// Used to display video frames from a `archetypes::AssetVideo`. + /// + /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + struct VideoFrameReference { + /// References the closest video frame to this time. + /// + /// Note that this uses the closest video frame instead of the latest at this timestamp + /// in order to be more forgiving of rounding errors. + rerun::components::VideoTimestamp timestamp; + + /// Optional reference to an entity with a `archetypes::AssetVideo`. + /// + /// If none is specified, the video is assumed to be at the same entity. + std::optional video_reference; + + public: + static constexpr const char IndicatorComponentName[] = + "rerun.components.VideoFrameReferenceIndicator"; + + /// Indicator component, used to identify the archetype when converting to a list of components. + using IndicatorComponent = rerun::components::IndicatorComponent; + + public: + VideoFrameReference() = default; + VideoFrameReference(VideoFrameReference&& other) = default; + + explicit VideoFrameReference(rerun::components::VideoTimestamp _timestamp) + : timestamp(std::move(_timestamp)) {} + + /// Optional reference to an entity with a `archetypes::AssetVideo`. + /// + /// If none is specified, the video is assumed to be at the same entity. + VideoFrameReference with_video_reference(rerun::components::EntityPath _video_reference + ) && { + video_reference = std::move(_video_reference); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + }; + +} // namespace rerun::archetypes + +namespace rerun { + /// \private + template + struct AsComponents; + + /// \private + template <> + struct AsComponents { + /// Serialize all set component batches. + static Result> serialize( + const archetypes::VideoFrameReference& archetype + ); + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/components.hpp b/rerun_cpp/src/rerun/components.hpp index e3fa9d60b22a..3e05b8095a17 100644 --- a/rerun_cpp/src/rerun/components.hpp +++ b/rerun_cpp/src/rerun/components.hpp @@ -14,6 +14,7 @@ #include "components/depth_meter.hpp" #include "components/disconnected_space.hpp" #include "components/draw_order.hpp" +#include "components/entity_path.hpp" #include "components/fill_mode.hpp" #include "components/fill_ratio.hpp" #include "components/gamma_correction.hpp" @@ -61,4 +62,5 @@ #include "components/triangle_indices.hpp" #include "components/vector2d.hpp" #include "components/vector3d.hpp" +#include "components/video_timestamp.hpp" #include "components/view_coordinates.hpp" diff --git a/rerun_cpp/src/rerun/components/.gitattributes b/rerun_cpp/src/rerun/components/.gitattributes index 70e8b68eaff9..ef90d02b74d5 100644 --- a/rerun_cpp/src/rerun/components/.gitattributes +++ b/rerun_cpp/src/rerun/components/.gitattributes @@ -16,6 +16,7 @@ colormap.hpp linguist-generated=true depth_meter.hpp linguist-generated=true disconnected_space.hpp linguist-generated=true draw_order.hpp linguist-generated=true +entity_path.hpp linguist-generated=true fill_mode.cpp linguist-generated=true fill_mode.hpp linguist-generated=true fill_ratio.hpp linguist-generated=true @@ -69,4 +70,5 @@ translation3d.hpp linguist-generated=true triangle_indices.hpp linguist-generated=true vector2d.hpp linguist-generated=true vector3d.hpp linguist-generated=true +video_timestamp.hpp linguist-generated=true view_coordinates.hpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/components/entity_path.hpp b/rerun_cpp/src/rerun/components/entity_path.hpp new file mode 100644 index 000000000000..e62422c1ceb6 --- /dev/null +++ b/rerun_cpp/src/rerun/components/entity_path.hpp @@ -0,0 +1,66 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/components/entity_path.fbs". + +#pragma once + +#include "../datatypes/entity_path.hpp" +#include "../result.hpp" + +#include +#include +#include +#include + +namespace rerun::components { + /// **Component**: A path to an entity, usually to reference some data that is part of the target entity. + struct EntityPath { + rerun::datatypes::EntityPath value; + + public: + EntityPath() = default; + + EntityPath(rerun::datatypes::EntityPath value_) : value(std::move(value_)) {} + + EntityPath& operator=(rerun::datatypes::EntityPath value_) { + value = std::move(value_); + return *this; + } + + EntityPath(std::string path_) : value(std::move(path_)) {} + + EntityPath& operator=(std::string path_) { + value = std::move(path_); + return *this; + } + + /// Cast to the underlying EntityPath datatype + operator rerun::datatypes::EntityPath() const { + return value; + } + }; +} // namespace rerun::components + +namespace rerun { + static_assert(sizeof(rerun::datatypes::EntityPath) == sizeof(components::EntityPath)); + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.components.EntityPath"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype() { + return Loggable::arrow_datatype(); + } + + /// Serializes an array of `rerun::components::EntityPath` into an arrow array. + static Result> to_arrow( + const components::EntityPath* instances, size_t num_instances + ) { + return Loggable::to_arrow( + &instances->value, + num_instances + ); + } + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/components/video_timestamp.hpp b/rerun_cpp/src/rerun/components/video_timestamp.hpp new file mode 100644 index 000000000000..dd0e8a1e947f --- /dev/null +++ b/rerun_cpp/src/rerun/components/video_timestamp.hpp @@ -0,0 +1,59 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/components/video_reference.fbs". + +#pragma once + +#include "../datatypes/video_timestamp.hpp" +#include "../result.hpp" + +#include +#include + +namespace rerun::components { + /// **Component**: Timestamp inside a `archetypes::AssetVideo`. + /// + /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + struct VideoTimestamp { + rerun::datatypes::VideoTimestamp timestamp; + + public: + VideoTimestamp() = default; + + VideoTimestamp(rerun::datatypes::VideoTimestamp timestamp_) : timestamp(timestamp_) {} + + VideoTimestamp& operator=(rerun::datatypes::VideoTimestamp timestamp_) { + timestamp = timestamp_; + return *this; + } + + /// Cast to the underlying VideoTimestamp datatype + operator rerun::datatypes::VideoTimestamp() const { + return timestamp; + } + }; +} // namespace rerun::components + +namespace rerun { + static_assert(sizeof(rerun::datatypes::VideoTimestamp) == sizeof(components::VideoTimestamp)); + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.components.VideoTimestamp"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype() { + return Loggable::arrow_datatype(); + } + + /// Serializes an array of `rerun::components::VideoTimestamp` into an arrow array. + static Result> to_arrow( + const components::VideoTimestamp* instances, size_t num_instances + ) { + return Loggable::to_arrow( + &instances->timestamp, + num_instances + ); + } + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes.hpp b/rerun_cpp/src/rerun/datatypes.hpp index 18e56171ed93..46277320d00f 100644 --- a/rerun_cpp/src/rerun/datatypes.hpp +++ b/rerun_cpp/src/rerun/datatypes.hpp @@ -44,5 +44,7 @@ #include "datatypes/vec2d.hpp" #include "datatypes/vec3d.hpp" #include "datatypes/vec4d.hpp" +#include "datatypes/video_time_mode.hpp" +#include "datatypes/video_timestamp.hpp" #include "datatypes/view_coordinates.hpp" #include "datatypes/visible_time_range.hpp" diff --git a/rerun_cpp/src/rerun/datatypes/.gitattributes b/rerun_cpp/src/rerun/datatypes/.gitattributes index 936afb67fd18..5ccb732feac0 100644 --- a/rerun_cpp/src/rerun/datatypes/.gitattributes +++ b/rerun_cpp/src/rerun/datatypes/.gitattributes @@ -85,6 +85,10 @@ vec3d.cpp linguist-generated=true vec3d.hpp linguist-generated=true vec4d.cpp linguist-generated=true vec4d.hpp linguist-generated=true +video_time_mode.cpp linguist-generated=true +video_time_mode.hpp linguist-generated=true +video_timestamp.cpp linguist-generated=true +video_timestamp.hpp linguist-generated=true view_coordinates.cpp linguist-generated=true view_coordinates.hpp linguist-generated=true visible_time_range.cpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/datatypes/video_time_mode.cpp b/rerun_cpp/src/rerun/datatypes/video_time_mode.cpp new file mode 100644 index 000000000000..6f3859686b04 --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_time_mode.cpp @@ -0,0 +1,56 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#include "video_time_mode.hpp" + +#include +#include + +namespace rerun { + const std::shared_ptr& Loggable::arrow_datatype() { + static const auto datatype = arrow::uint8(); + return datatype; + } + + Result> Loggable::to_arrow( + const datatypes::VideoTimeMode* instances, size_t num_instances + ) { + // TODO(andreas): Allow configuring the memory pool. + arrow::MemoryPool* pool = arrow::default_memory_pool(); + auto datatype = arrow_datatype(); + + ARROW_ASSIGN_OR_RAISE(auto builder, arrow::MakeBuilder(datatype, pool)) + if (instances && num_instances > 0) { + RR_RETURN_NOT_OK(Loggable::fill_arrow_array_builder( + static_cast(builder.get()), + instances, + num_instances + )); + } + std::shared_ptr array; + ARROW_RETURN_NOT_OK(builder->Finish(&array)); + return array; + } + + rerun::Error Loggable::fill_arrow_array_builder( + arrow::UInt8Builder* builder, const datatypes::VideoTimeMode* elements, size_t num_elements + ) { + if (builder == nullptr) { + return rerun::Error(ErrorCode::UnexpectedNullArgument, "Passed array builder is null."); + } + if (elements == nullptr) { + return rerun::Error( + ErrorCode::UnexpectedNullArgument, + "Cannot serialize null pointer to arrow array." + ); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto variant = elements[elem_idx]; + ARROW_RETURN_NOT_OK(builder->Append(static_cast(variant))); + } + + return Error::ok(); + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/video_time_mode.hpp b/rerun_cpp/src/rerun/datatypes/video_time_mode.hpp new file mode 100644 index 000000000000..e0738745910e --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_time_mode.hpp @@ -0,0 +1,54 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#pragma once + +#include "../result.hpp" + +#include +#include + +namespace arrow { + /// \private + template + class NumericBuilder; + + class Array; + class DataType; + class UInt8Type; + using UInt8Builder = NumericBuilder; +} // namespace arrow + +namespace rerun::datatypes { + /// **Datatype**: Specifies how to interpret the `video_time` field of a `datatypes::VideoTimestamp`. + enum class VideoTimeMode : uint8_t { + + /// Presentation timestamp in nanoseconds since the beginning of the video. + Nanoseconds = 1, + }; +} // namespace rerun::datatypes + +namespace rerun { + template + struct Loggable; + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.datatypes.VideoTimeMode"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype(); + + /// Serializes an array of `rerun::datatypes::VideoTimeMode` into an arrow array. + static Result> to_arrow( + const datatypes::VideoTimeMode* instances, size_t num_instances + ); + + /// Fills an arrow array builder with an array of this type. + static rerun::Error fill_arrow_array_builder( + arrow::UInt8Builder* builder, const datatypes::VideoTimeMode* elements, + size_t num_elements + ); + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/video_timestamp.cpp b/rerun_cpp/src/rerun/datatypes/video_timestamp.cpp new file mode 100644 index 000000000000..0c7dcadc2037 --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_timestamp.cpp @@ -0,0 +1,84 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#include "video_timestamp.hpp" + +#include "video_time_mode.hpp" + +#include +#include + +namespace rerun::datatypes {} + +namespace rerun { + const std::shared_ptr& Loggable::arrow_datatype() { + static const auto datatype = arrow::struct_({ + arrow::field("video_time", arrow::int64(), false), + arrow::field( + "time_mode", + Loggable::arrow_datatype(), + false + ), + }); + return datatype; + } + + Result> Loggable::to_arrow( + const datatypes::VideoTimestamp* instances, size_t num_instances + ) { + // TODO(andreas): Allow configuring the memory pool. + arrow::MemoryPool* pool = arrow::default_memory_pool(); + auto datatype = arrow_datatype(); + + ARROW_ASSIGN_OR_RAISE(auto builder, arrow::MakeBuilder(datatype, pool)) + if (instances && num_instances > 0) { + RR_RETURN_NOT_OK(Loggable::fill_arrow_array_builder( + static_cast(builder.get()), + instances, + num_instances + )); + } + std::shared_ptr array; + ARROW_RETURN_NOT_OK(builder->Finish(&array)); + return array; + } + + rerun::Error Loggable::fill_arrow_array_builder( + arrow::StructBuilder* builder, const datatypes::VideoTimestamp* elements, + size_t num_elements + ) { + if (builder == nullptr) { + return rerun::Error(ErrorCode::UnexpectedNullArgument, "Passed array builder is null."); + } + if (elements == nullptr) { + return rerun::Error( + ErrorCode::UnexpectedNullArgument, + "Cannot serialize null pointer to arrow array." + ); + } + + { + auto field_builder = static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(field_builder->Append(elements[elem_idx].video_time)); + } + } + { + auto field_builder = static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + RR_RETURN_NOT_OK( + Loggable::fill_arrow_array_builder( + field_builder, + &elements[elem_idx].time_mode, + 1 + ) + ); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(static_cast(num_elements), nullptr)); + + return Error::ok(); + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/video_timestamp.hpp b/rerun_cpp/src/rerun/datatypes/video_timestamp.hpp new file mode 100644 index 000000000000..1d1110fa9d55 --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_timestamp.hpp @@ -0,0 +1,57 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#pragma once + +#include "../result.hpp" +#include "video_time_mode.hpp" + +#include +#include + +namespace arrow { + class Array; + class DataType; + class StructBuilder; +} // namespace arrow + +namespace rerun::datatypes { + /// **Datatype**: Timestamp inside a `archetypes::AssetVideo`. + /// + /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + struct VideoTimestamp { + /// Timestamp value, type defined by `time_mode`. + int64_t video_time; + + /// How to interpret `video_time`. + rerun::datatypes::VideoTimeMode time_mode; + + public: + VideoTimestamp() = default; + }; +} // namespace rerun::datatypes + +namespace rerun { + template + struct Loggable; + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.datatypes.VideoTimestamp"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype(); + + /// Serializes an array of `rerun::datatypes::VideoTimestamp` into an arrow array. + static Result> to_arrow( + const datatypes::VideoTimestamp* instances, size_t num_instances + ); + + /// Fills an arrow array builder with an array of this type. + static rerun::Error fill_arrow_array_builder( + arrow::StructBuilder* builder, const datatypes::VideoTimestamp* elements, + size_t num_elements + ); + }; +} // namespace rerun diff --git a/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes b/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes index 72603acdd483..b57110b5036e 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes @@ -31,4 +31,5 @@ tensor.py linguist-generated=true text_document.py linguist-generated=true text_log.py linguist-generated=true transform3d.py linguist-generated=true +video_frame_reference.py linguist-generated=true view_coordinates.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/archetypes/__init__.py b/rerun_py/rerun_sdk/rerun/archetypes/__init__.py index 088b8cb175b2..8ba5d3fcdee3 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/__init__.py @@ -31,6 +31,7 @@ from .text_document import TextDocument from .text_log import TextLog from .transform3d import Transform3D +from .video_frame_reference import VideoFrameReference from .view_coordinates import ViewCoordinates __all__ = [ @@ -63,5 +64,6 @@ "TextDocument", "TextLog", "Transform3D", + "VideoFrameReference", "ViewCoordinates", ] diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py index b12113bc71f9..ad3c40e8bd22 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py @@ -1,5 +1,5 @@ # DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs -# Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +# Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". # You can extend this class by creating a "AssetVideoExt" class in "asset_video_ext.py". @@ -19,12 +19,14 @@ @define(str=False, repr=False, init=False) class AssetVideo(AssetVideoExt, Archetype): """ - **Archetype**: A video file. + **Archetype**: A video binary. NOTE: Videos can only be viewed in the Rerun web viewer. - Only MP4 and AV1 is currently supported, and not in all browsers. + Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. Follow for updates on the native support. + In order to display a video, you need to log a [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference] for each frame. + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** """ diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py new file mode 100644 index 000000000000..986c53f75309 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -0,0 +1,94 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +# You can extend this class by creating a "VideoFrameReferenceExt" class in "video_frame_reference_ext.py". + +from __future__ import annotations + +from typing import Any + +from attrs import define, field + +from .. import components, datatypes +from .._baseclasses import ( + Archetype, +) +from ..error_utils import catch_and_log_exceptions + +__all__ = ["VideoFrameReference"] + + +@define(str=False, repr=False, init=False) +class VideoFrameReference(Archetype): + """ + **Archetype**: References a single video frame. + + Used to display video frames from a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + """ + + def __init__( + self: Any, timestamp: datatypes.VideoTimestampLike, *, video_reference: datatypes.EntityPathLike | None = None + ): + """ + Create a new instance of the VideoFrameReference archetype. + + Parameters + ---------- + timestamp: + References the closest video frame to this time. + + Note that this uses the closest video frame instead of the latest at this timestamp + in order to be more forgiving of rounding errors. + video_reference: + Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + If none is specified, the video is assumed to be at the same entity. + + """ + + # You can define your own __init__ function as a member of VideoFrameReferenceExt in video_frame_reference_ext.py + with catch_and_log_exceptions(context=self.__class__.__name__): + self.__attrs_init__(timestamp=timestamp, video_reference=video_reference) + return + self.__attrs_clear__() + + def __attrs_clear__(self) -> None: + """Convenience method for calling `__attrs_init__` with all `None`s.""" + self.__attrs_init__( + timestamp=None, # type: ignore[arg-type] + video_reference=None, # type: ignore[arg-type] + ) + + @classmethod + def _clear(cls) -> VideoFrameReference: + """Produce an empty VideoFrameReference, bypassing `__init__`.""" + inst = cls.__new__(cls) + inst.__attrs_clear__() + return inst + + timestamp: components.VideoTimestampBatch = field( + metadata={"component": "required"}, + converter=components.VideoTimestampBatch._required, # type: ignore[misc] + ) + # References the closest video frame to this time. + # + # Note that this uses the closest video frame instead of the latest at this timestamp + # in order to be more forgiving of rounding errors. + # + # (Docstring intentionally commented out to hide this field from the docs) + + video_reference: components.EntityPathBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.EntityPathBatch._optional, # type: ignore[misc] + ) + # Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + # + # If none is specified, the video is assumed to be at the same entity. + # + # (Docstring intentionally commented out to hide this field from the docs) + + __str__ = Archetype.__str__ + __repr__ = Archetype.__repr__ # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/components/.gitattributes b/rerun_py/rerun_sdk/rerun/components/.gitattributes index a238f07d07ce..bb7b80d1486c 100644 --- a/rerun_py/rerun_sdk/rerun/components/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/components/.gitattributes @@ -14,6 +14,7 @@ colormap.py linguist-generated=true depth_meter.py linguist-generated=true disconnected_space.py linguist-generated=true draw_order.py linguist-generated=true +entity_path.py linguist-generated=true fill_mode.py linguist-generated=true fill_ratio.py linguist-generated=true gamma_correction.py linguist-generated=true @@ -61,4 +62,5 @@ translation3d.py linguist-generated=true triangle_indices.py linguist-generated=true vector2d.py linguist-generated=true vector3d.py linguist-generated=true +video_timestamp.py linguist-generated=true view_coordinates.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/components/__init__.py b/rerun_py/rerun_sdk/rerun/components/__init__.py index 16625c133e9d..8855fcf59d3c 100644 --- a/rerun_py/rerun_sdk/rerun/components/__init__.py +++ b/rerun_py/rerun_sdk/rerun/components/__init__.py @@ -26,6 +26,7 @@ from .depth_meter import DepthMeter, DepthMeterBatch, DepthMeterType from .disconnected_space import DisconnectedSpace, DisconnectedSpaceBatch, DisconnectedSpaceType from .draw_order import DrawOrder, DrawOrderBatch, DrawOrderType +from .entity_path import EntityPath, EntityPathBatch, EntityPathType from .fill_mode import FillMode, FillModeArrayLike, FillModeBatch, FillModeLike, FillModeType from .fill_ratio import FillRatio, FillRatioBatch, FillRatioType from .gamma_correction import GammaCorrection, GammaCorrectionBatch, GammaCorrectionType @@ -89,6 +90,7 @@ from .triangle_indices import TriangleIndices, TriangleIndicesBatch, TriangleIndicesType from .vector2d import Vector2D, Vector2DBatch, Vector2DType from .vector3d import Vector3D, Vector3DBatch, Vector3DType +from .video_timestamp import VideoTimestamp, VideoTimestampBatch, VideoTimestampType from .view_coordinates import ViewCoordinates, ViewCoordinatesBatch, ViewCoordinatesType __all__ = [ @@ -134,6 +136,9 @@ "DrawOrder", "DrawOrderBatch", "DrawOrderType", + "EntityPath", + "EntityPathBatch", + "EntityPathType", "FillMode", "FillModeArrayLike", "FillModeBatch", @@ -287,6 +292,9 @@ "Vector3D", "Vector3DBatch", "Vector3DType", + "VideoTimestamp", + "VideoTimestampBatch", + "VideoTimestampType", "ViewCoordinates", "ViewCoordinatesBatch", "ViewCoordinatesType", diff --git a/rerun_py/rerun_sdk/rerun/components/entity_path.py b/rerun_py/rerun_sdk/rerun/components/entity_path.py new file mode 100644 index 000000000000..ad63a4793330 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/components/entity_path.py @@ -0,0 +1,36 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/components/entity_path.fbs". + +# You can extend this class by creating a "EntityPathExt" class in "entity_path_ext.py". + +from __future__ import annotations + +from .. import datatypes +from .._baseclasses import ( + ComponentBatchMixin, + ComponentMixin, +) + +__all__ = ["EntityPath", "EntityPathBatch", "EntityPathType"] + + +class EntityPath(datatypes.EntityPath, ComponentMixin): + """**Component**: A path to an entity, usually to reference some data that is part of the target entity.""" + + _BATCH_TYPE = None + # You can define your own __init__ function as a member of EntityPathExt in entity_path_ext.py + + # Note: there are no fields here because EntityPath delegates to datatypes.EntityPath + pass + + +class EntityPathType(datatypes.EntityPathType): + _TYPE_NAME: str = "rerun.components.EntityPath" + + +class EntityPathBatch(datatypes.EntityPathBatch, ComponentBatchMixin): + _ARROW_TYPE = EntityPathType() + + +# This is patched in late to avoid circular dependencies. +EntityPath._BATCH_TYPE = EntityPathBatch # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py new file mode 100644 index 000000000000..65ae824ff421 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py @@ -0,0 +1,40 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/components/video_reference.fbs". + +# You can extend this class by creating a "VideoTimestampExt" class in "video_timestamp_ext.py". + +from __future__ import annotations + +from .. import datatypes +from .._baseclasses import ( + ComponentBatchMixin, + ComponentMixin, +) + +__all__ = ["VideoTimestamp", "VideoTimestampBatch", "VideoTimestampType"] + + +class VideoTimestamp(datatypes.VideoTimestamp, ComponentMixin): + """ + **Component**: Timestamp inside a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + """ + + _BATCH_TYPE = None + # You can define your own __init__ function as a member of VideoTimestampExt in video_timestamp_ext.py + + # Note: there are no fields here because VideoTimestamp delegates to datatypes.VideoTimestamp + pass + + +class VideoTimestampType(datatypes.VideoTimestampType): + _TYPE_NAME: str = "rerun.components.VideoTimestamp" + + +class VideoTimestampBatch(datatypes.VideoTimestampBatch, ComponentBatchMixin): + _ARROW_TYPE = VideoTimestampType() + + +# This is patched in late to avoid circular dependencies. +VideoTimestamp._BATCH_TYPE = VideoTimestampBatch # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes b/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes index 549c47677d0e..42443440d38b 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes @@ -44,5 +44,7 @@ uvec4d.py linguist-generated=true vec2d.py linguist-generated=true vec3d.py linguist-generated=true vec4d.py linguist-generated=true +video_time_mode.py linguist-generated=true +video_timestamp.py linguist-generated=true view_coordinates.py linguist-generated=true visible_time_range.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/datatypes/__init__.py b/rerun_py/rerun_sdk/rerun/datatypes/__init__.py index 09616779d8c2..7c30b55ceb03 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/__init__.py @@ -98,6 +98,20 @@ from .vec2d import Vec2D, Vec2DArrayLike, Vec2DBatch, Vec2DLike, Vec2DType from .vec3d import Vec3D, Vec3DArrayLike, Vec3DBatch, Vec3DLike, Vec3DType from .vec4d import Vec4D, Vec4DArrayLike, Vec4DBatch, Vec4DLike, Vec4DType +from .video_time_mode import ( + VideoTimeMode, + VideoTimeModeArrayLike, + VideoTimeModeBatch, + VideoTimeModeLike, + VideoTimeModeType, +) +from .video_timestamp import ( + VideoTimestamp, + VideoTimestampArrayLike, + VideoTimestampBatch, + VideoTimestampLike, + VideoTimestampType, +) from .view_coordinates import ( ViewCoordinates, ViewCoordinatesArrayLike, @@ -324,6 +338,16 @@ "Vec4DBatch", "Vec4DLike", "Vec4DType", + "VideoTimeMode", + "VideoTimeModeArrayLike", + "VideoTimeModeBatch", + "VideoTimeModeLike", + "VideoTimeModeType", + "VideoTimestamp", + "VideoTimestampArrayLike", + "VideoTimestampBatch", + "VideoTimestampLike", + "VideoTimestampType", "ViewCoordinates", "ViewCoordinatesArrayLike", "ViewCoordinatesBatch", diff --git a/rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py b/rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py new file mode 100644 index 000000000000..b1b250eaa22d --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py @@ -0,0 +1,71 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +# You can extend this class by creating a "VideoTimeModeExt" class in "video_time_mode_ext.py". + +from __future__ import annotations + +from typing import Literal, Sequence, Union + +import pyarrow as pa + +from .._baseclasses import ( + BaseBatch, + BaseExtensionType, +) + +__all__ = ["VideoTimeMode", "VideoTimeModeArrayLike", "VideoTimeModeBatch", "VideoTimeModeLike", "VideoTimeModeType"] + + +from enum import Enum + + +class VideoTimeMode(Enum): + """**Datatype**: Specifies how to interpret the `video_time` field of a [`datatypes.VideoTimestamp`][rerun.datatypes.VideoTimestamp].""" + + Nanoseconds = 1 + """Presentation timestamp in nanoseconds since the beginning of the video.""" + + @classmethod + def auto(cls, val: str | int | VideoTimeMode) -> VideoTimeMode: + """Best-effort converter, including a case-insensitive string matcher.""" + if isinstance(val, VideoTimeMode): + return val + if isinstance(val, int): + return cls(val) + try: + return cls[val] + except KeyError: + val_lower = val.lower() + for variant in cls: + if variant.name.lower() == val_lower: + return variant + raise ValueError(f"Cannot convert {val} to {cls.__name__}") + + def __str__(self) -> str: + """Returns the variant name.""" + return self.name + + +VideoTimeModeLike = Union[VideoTimeMode, Literal["Nanoseconds", "nanoseconds"], int] +VideoTimeModeArrayLike = Union[VideoTimeModeLike, Sequence[VideoTimeModeLike]] + + +class VideoTimeModeType(BaseExtensionType): + _TYPE_NAME: str = "rerun.datatypes.VideoTimeMode" + + def __init__(self) -> None: + pa.ExtensionType.__init__(self, pa.uint8(), self._TYPE_NAME) + + +class VideoTimeModeBatch(BaseBatch[VideoTimeModeArrayLike]): + _ARROW_TYPE = VideoTimeModeType() + + @staticmethod + def _native_to_pa_array(data: VideoTimeModeArrayLike, data_type: pa.DataType) -> pa.Array: + if isinstance(data, (VideoTimeMode, int, str)): + data = [data] + + pa_data = [VideoTimeMode.auto(v).value if v is not None else None for v in data] # type: ignore[redundant-expr] + + return pa.array(pa_data, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py b/rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py new file mode 100644 index 000000000000..d0bf8deb7cac --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py @@ -0,0 +1,110 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +# You can extend this class by creating a "VideoTimestampExt" class in "video_timestamp_ext.py". + +from __future__ import annotations + +from typing import Any, Sequence, Union + +import numpy as np +import pyarrow as pa +from attrs import define, field + +from .. import datatypes +from .._baseclasses import ( + BaseBatch, + BaseExtensionType, +) + +__all__ = [ + "VideoTimestamp", + "VideoTimestampArrayLike", + "VideoTimestampBatch", + "VideoTimestampLike", + "VideoTimestampType", +] + + +def _video_timestamp__time_mode__special_field_converter_override( + x: datatypes.VideoTimeModeLike, +) -> datatypes.VideoTimeMode: + if isinstance(x, datatypes.VideoTimeMode): + return x + else: + return datatypes.VideoTimeMode(x) + + +@define(init=False) +class VideoTimestamp: + """ + **Datatype**: Timestamp inside a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + """ + + def __init__(self: Any, video_time: int, time_mode: datatypes.VideoTimeModeLike): + """ + Create a new instance of the VideoTimestamp datatype. + + Parameters + ---------- + video_time: + Timestamp value, type defined by `time_mode`. + time_mode: + How to interpret `video_time`. + + """ + + # You can define your own __init__ function as a member of VideoTimestampExt in video_timestamp_ext.py + self.__attrs_init__(video_time=video_time, time_mode=time_mode) + + video_time: int = field(converter=int) + # Timestamp value, type defined by `time_mode`. + # + # (Docstring intentionally commented out to hide this field from the docs) + + time_mode: datatypes.VideoTimeMode = field(converter=_video_timestamp__time_mode__special_field_converter_override) + # How to interpret `video_time`. + # + # (Docstring intentionally commented out to hide this field from the docs) + + +VideoTimestampLike = VideoTimestamp +VideoTimestampArrayLike = Union[ + VideoTimestamp, + Sequence[VideoTimestampLike], +] + + +class VideoTimestampType(BaseExtensionType): + _TYPE_NAME: str = "rerun.datatypes.VideoTimestamp" + + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([ + pa.field("video_time", pa.int64(), nullable=False, metadata={}), + pa.field("time_mode", pa.uint8(), nullable=False, metadata={}), + ]), + self._TYPE_NAME, + ) + + +class VideoTimestampBatch(BaseBatch[VideoTimestampArrayLike]): + _ARROW_TYPE = VideoTimestampType() + + @staticmethod + def _native_to_pa_array(data: VideoTimestampArrayLike, data_type: pa.DataType) -> pa.Array: + from rerun.datatypes import VideoTimeModeBatch + + if isinstance(data, VideoTimestamp): + data = [data] + + return pa.StructArray.from_arrays( + [ + pa.array(np.asarray([x.video_time for x in data], dtype=np.int64)), + VideoTimeModeBatch([x.time_mode for x in data]).as_arrow_array().storage, # type: ignore[misc, arg-type] + ], + fields=list(data_type), + ) From 4bcfc11e74b4cfa8123d82ae4ef4037b5c303549 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 15:03:03 +0200 Subject: [PATCH 04/12] Extensions for video timestamp --- .../re_types/definitions/rerun/components.fbs | 2 +- ...ideo_reference.fbs => video_timestamp.fbs} | 2 +- crates/store/re_types/src/components/mod.rs | 1 + .../src/components/video_timestamp.rs | 4 ++-- .../src/components/video_timestamp_ext.rs | 9 ++++++++ crates/store/re_types/src/datatypes/mod.rs | 1 + .../src/datatypes/video_timestamp_ext.rs | 21 +++++++++++++++++++ .../src/rerun/components/video_timestamp.hpp | 2 +- .../rerun/components/video_timestamp.py | 2 +- 9 files changed, 38 insertions(+), 6 deletions(-) rename crates/store/re_types/definitions/rerun/components/{video_reference.fbs => video_timestamp.fbs} (80%) create mode 100644 crates/store/re_types/src/components/video_timestamp_ext.rs create mode 100644 crates/store/re_types/src/datatypes/video_timestamp_ext.rs diff --git a/crates/store/re_types/definitions/rerun/components.fbs b/crates/store/re_types/definitions/rerun/components.fbs index d72129ca055e..b5916c757898 100644 --- a/crates/store/re_types/definitions/rerun/components.fbs +++ b/crates/store/re_types/definitions/rerun/components.fbs @@ -53,5 +53,5 @@ include "./components/translation3d.fbs"; include "./components/triangle_indices.fbs"; include "./components/vector2d.fbs"; include "./components/vector3d.fbs"; -include "./components/video_reference.fbs"; +include "./components/video_timestamp.fbs"; include "./components/view_coordinates.fbs"; diff --git a/crates/store/re_types/definitions/rerun/components/video_reference.fbs b/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs similarity index 80% rename from crates/store/re_types/definitions/rerun/components/video_reference.fbs rename to crates/store/re_types/definitions/rerun/components/video_timestamp.fbs index f3a0b0eb0ef7..b17d4411daf9 100644 --- a/crates/store/re_types/definitions/rerun/components/video_reference.fbs +++ b/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs @@ -3,7 +3,7 @@ namespace rerun.components; /// Timestamp inside a [archetypes.AssetVideo]. struct VideoTimestamp ( - "attr.rust.derive": "Copy, PartialEq, Eq", + "attr.rust.derive": "Copy, PartialEq, Eq, Default", "attr.rust.repr": "transparent", "attr.rerun.experimental" ) { diff --git a/crates/store/re_types/src/components/mod.rs b/crates/store/re_types/src/components/mod.rs index 0e499a64e2fa..e65ab7eed0c0 100644 --- a/crates/store/re_types/src/components/mod.rs +++ b/crates/store/re_types/src/components/mod.rs @@ -107,6 +107,7 @@ mod vector2d_ext; mod vector3d; mod vector3d_ext; mod video_timestamp; +mod video_timestamp_ext; mod view_coordinates; mod view_coordinates_ext; diff --git a/crates/store/re_types/src/components/video_timestamp.rs b/crates/store/re_types/src/components/video_timestamp.rs index 10daf8833187..64e30ae7553c 100644 --- a/crates/store/re_types/src/components/video_timestamp.rs +++ b/crates/store/re_types/src/components/video_timestamp.rs @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs -// Based on "crates/store/re_types/definitions/rerun/components/video_reference.fbs". +// Based on "crates/store/re_types/definitions/rerun/components/video_timestamp.fbs". #![allow(unused_imports)] #![allow(unused_parens)] @@ -21,7 +21,7 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Component**: Timestamp inside a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. /// /// ⚠️ **This type is experimental and may be removed in future versions** -#[derive(Clone, Debug, Copy, PartialEq, Eq)] +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] #[repr(transparent)] pub struct VideoTimestamp(pub crate::datatypes::VideoTimestamp); diff --git a/crates/store/re_types/src/components/video_timestamp_ext.rs b/crates/store/re_types/src/components/video_timestamp_ext.rs new file mode 100644 index 000000000000..df7a6f9947bc --- /dev/null +++ b/crates/store/re_types/src/components/video_timestamp_ext.rs @@ -0,0 +1,9 @@ +use super::VideoTimestamp; + +impl VideoTimestamp { + /// Create new timestamp from nanoseconds since video start. + #[inline] + pub fn new_nanoseconds(nanos: i64) -> Self { + crate::datatypes::VideoTimestamp::new_nanoseconds(nanos).into() + } +} diff --git a/crates/store/re_types/src/datatypes/mod.rs b/crates/store/re_types/src/datatypes/mod.rs index 8f069af53f6c..29fd05d8b9d5 100644 --- a/crates/store/re_types/src/datatypes/mod.rs +++ b/crates/store/re_types/src/datatypes/mod.rs @@ -63,6 +63,7 @@ mod vec4d; mod vec4d_ext; mod video_time_mode; mod video_timestamp; +mod video_timestamp_ext; mod view_coordinates; mod view_coordinates_ext; diff --git a/crates/store/re_types/src/datatypes/video_timestamp_ext.rs b/crates/store/re_types/src/datatypes/video_timestamp_ext.rs new file mode 100644 index 000000000000..342e0d1051db --- /dev/null +++ b/crates/store/re_types/src/datatypes/video_timestamp_ext.rs @@ -0,0 +1,21 @@ +use super::{VideoTimeMode, VideoTimestamp}; + +impl VideoTimestamp { + /// Create new timestamp from nanoseconds since video start. + #[inline] + pub fn new_nanoseconds(nanos: i64) -> Self { + Self { + video_time: nanos, + time_mode: VideoTimeMode::Nanoseconds, + } + } +} + +impl Default for VideoTimestamp { + fn default() -> Self { + Self { + video_time: 0, + time_mode: VideoTimeMode::Nanoseconds, + } + } +} diff --git a/rerun_cpp/src/rerun/components/video_timestamp.hpp b/rerun_cpp/src/rerun/components/video_timestamp.hpp index dd0e8a1e947f..7a09d14eba53 100644 --- a/rerun_cpp/src/rerun/components/video_timestamp.hpp +++ b/rerun_cpp/src/rerun/components/video_timestamp.hpp @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs -// Based on "crates/store/re_types/definitions/rerun/components/video_reference.fbs". +// Based on "crates/store/re_types/definitions/rerun/components/video_timestamp.fbs". #pragma once diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py index 65ae824ff421..433fe013f8cc 100644 --- a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py @@ -1,5 +1,5 @@ # DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs -# Based on "crates/store/re_types/definitions/rerun/components/video_reference.fbs". +# Based on "crates/store/re_types/definitions/rerun/components/video_timestamp.fbs". # You can extend this class by creating a "VideoTimestampExt" class in "video_timestamp_ext.py". From a5cd6c9b7871084b7e33482c087642cf532fd2ec Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 15:32:28 +0200 Subject: [PATCH 05/12] Replace `VideoTick` with new video timestamps --- crates/store/re_data_loader/Cargo.toml | 1 + .../re_data_loader/src/loader_archetype.rs | 184 ++++++++++-------- .../store/re_types_core/src/loggable_batch.rs | 14 +- crates/store/re_video/src/lib.rs | 9 + crates/top/re_sdk/src/recording_stream.rs | 23 +-- 5 files changed, 129 insertions(+), 102 deletions(-) diff --git a/crates/store/re_data_loader/Cargo.toml b/crates/store/re_data_loader/Cargo.toml index 11489e232e35..804ff9889f46 100644 --- a/crates/store/re_data_loader/Cargo.toml +++ b/crates/store/re_data_loader/Cargo.toml @@ -36,6 +36,7 @@ re_video.workspace = true ahash.workspace = true anyhow.workspace = true +arrow2.workspace = true image.workspace = true once_cell.workspace = true parking_lot.workspace = true diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs index cc13e83017ea..894605349ae4 100644 --- a/crates/store/re_data_loader/src/loader_archetype.rs +++ b/crates/store/re_data_loader/src/loader_archetype.rs @@ -1,7 +1,13 @@ use re_chunk::{Chunk, RowId}; -use re_log_types::NonMinI64; use re_log_types::{EntityPath, TimeInt, TimePoint}; -use re_types::components::MediaType; +use re_types::archetypes::VideoFrameReference; +use re_types::Archetype; +use re_types::{components::MediaType, ComponentBatch}; + +use arrow2::array::{ + ListArray as ArrowListArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray, +}; +use arrow2::Either; use crate::{DataLoader, DataLoaderError, LoadedData}; @@ -164,47 +170,6 @@ fn load_image( Ok(rows.into_iter()) } -/// TODO(#7272): fix this -/// Used to expand the timeline when logging a video, so that the video can be played back. -#[derive(Clone, Copy)] -struct VideoTick(re_types::datatypes::Float64); - -impl re_types::AsComponents for VideoTick { - fn as_component_batches(&self) -> Vec> { - vec![re_types::NamedIndicatorComponent("VideoTick".into()).to_batch()] - } -} - -impl re_types::Loggable for VideoTick { - type Name = re_types::ComponentName; - - fn name() -> Self::Name { - "rerun.components.VideoTick".into() - } - - fn arrow_datatype() -> re_chunk::external::arrow2::datatypes::DataType { - re_types::datatypes::Float64::arrow_datatype() - } - - fn to_arrow_opt<'a>( - data: impl IntoIterator>>>, - ) -> re_types::SerializationResult> - where - Self: 'a, - { - re_types::datatypes::Float64::to_arrow_opt( - data.into_iter() - .map(|datum| datum.map(|datum| datum.into().0)), - ) - } -} - -impl re_types::SizeBytes for VideoTick { - fn heap_size_bytes(&self) -> u64 { - 0 - } -} - #[derive(Clone, Copy)] struct ExperimentalFeature; @@ -252,51 +217,112 @@ fn load_video( ) -> Result, DataLoaderError> { re_tracing::profile_function!(); - timepoint.insert( - re_log_types::Timeline::new_temporal("video"), - re_log_types::TimeInt::new_temporal(0), - ); + let video_timeline = re_log_types::Timeline::new_temporal("video"); + timepoint.insert(video_timeline, re_log_types::TimeInt::new_temporal(0)); let media_type = MediaType::guess_from_path(filepath); - let duration_s = match media_type.as_ref().map(|v| v.as_str()) { - Some("video/mp4") => re_video::load_mp4(&contents) - .ok() - .map(|v| v.duration.as_f64() / 1_000.0), - _ => None, - } - .unwrap_or(100.0) - .ceil() as i64; - - let mut rows = vec![Chunk::builder(entity_path.clone()) + let video = if media_type.as_ref().map(|v| v.as_str()) == Some("video/mp4") { + match re_video::load_mp4(&contents) { + Ok(video) => Some(video), + Err(err) => { + re_log::warn!("Failed to load video asset {filepath:?}: {err}"); + None + } + } + } else { + re_log::warn!("Video asset {filepath:?} has an unsupported container format."); + None + }; + + // Log video time references on the `video` timeline. + let video_frame_reference_chunk = if let Some(video) = video { + let first_timestamp = video + .segments + .first() + .map_or(0, |segment| segment.timestamp.as_nanoseconds()); + + // Time column. + let is_sorted = Some(true); + let time_column_times = + ArrowPrimitiveArray::::from_values(video.segments.iter().flat_map(|segment| { + segment + .samples + .iter() + .map(|s| s.timestamp.as_nanoseconds() - first_timestamp) + })); + + let time_column = re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times); + + // VideoTimestamp component column. + let video_timestamps = video + .segments + .iter() + .flat_map(|segment| { + segment.samples.iter().map(|s| { + // TODO(andreas): Use sample indices instead of timestamps once possible. + re_types::components::VideoTimestamp::new_nanoseconds( + s.timestamp.as_nanoseconds(), + ) + }) + }) + .collect::>(); + let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch; + let video_timestmap_list_array = video_timestamp_batch + .to_arrow_list_array() + .map_err(re_chunk::ChunkError::from)?; + + // Indicator column. + let video_frame_reference_indicator_datatype = arrow2::datatypes::DataType::Null; + let video_frame_reference_indicator_list_array = ArrowListArray::::try_new( + ArrowListArray::::default_datatype( + video_frame_reference_indicator_datatype.clone(), + ), + video_timestmap_list_array.offsets().clone(), + Box::new(ArrowNullArray::new( + video_frame_reference_indicator_datatype, + video_timestamps.len(), + )), + None, + ) + .map_err(re_chunk::ChunkError::from)?; + + Some(Chunk::from_auto_row_ids( + re_chunk::ChunkId::new(), + entity_path.clone(), + std::iter::once((video_timeline, time_column)).collect(), + [ + ( + VideoFrameReference::indicator().name(), + video_frame_reference_indicator_list_array, + ), + (video_timestamp_batch.name(), video_timestmap_list_array), + ] + .into_iter() + .collect(), + )?) + } else { + None + }; + + // Put video asset into its own chunk since it can be fairly large. + let video_asset_chunk = Chunk::builder(entity_path.clone()) .with_archetype( RowId::new(), timepoint.clone(), - &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type), + &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type.clone()), ) .with_component_batch(RowId::new(), timepoint.clone(), &ExperimentalFeature) - .build()?]; - - for i in 0..duration_s { - // We need some breadcrumbs of timepoints because the video doesn't have a duration yet. - // TODO(#7272): fix this - timepoint.insert( - re_log_types::Timeline::new_temporal("video"), - re_log_types::TimeInt::from_seconds(NonMinI64::new(i).expect("i > i64::MIN")), - ); - - rows.push( - Chunk::builder(entity_path.clone()) - .with_component_batch( - RowId::new(), - timepoint.clone(), - &VideoTick(re_types::datatypes::Float64(i as f64)), - ) - .build()?, - ); + .build()?; + + if let Some(video_frame_reference_chunk) = video_frame_reference_chunk { + Ok(Either::Left( + [video_asset_chunk, video_frame_reference_chunk].into_iter(), + )) + } else { + // Still log the video asset, but don't include video frames. + Ok(Either::Right(std::iter::once(video_asset_chunk))) } - - Ok(rows.into_iter()) } fn load_mesh( diff --git a/crates/store/re_types_core/src/loggable_batch.rs b/crates/store/re_types_core/src/loggable_batch.rs index 802dbbf4e4a6..0ad382f06ad4 100644 --- a/crates/store/re_types_core/src/loggable_batch.rs +++ b/crates/store/re_types_core/src/loggable_batch.rs @@ -1,5 +1,7 @@ use crate::{Component, ComponentName, Loggable, SerializationResult}; +use arrow2::array::ListArray as ArrowListArray; + #[allow(unused_imports)] // used in docstrings use crate::Archetype; @@ -32,7 +34,17 @@ pub trait LoggableBatch { /// /// Any [`LoggableBatch`] with a [`Loggable::Name`] set to [`ComponentName`] automatically /// implements [`ComponentBatch`]. -pub trait ComponentBatch: LoggableBatch {} +pub trait ComponentBatch: LoggableBatch { + /// Serializes the batch into an Arrow list array with a single component per list. + fn to_arrow_list_array(&self) -> SerializationResult> { + let array = self.to_arrow()?; + let offsets = + arrow2::offset::Offsets::try_from_lengths(std::iter::repeat(1).take(array.len()))?; + let data_type = ArrowListArray::::default_datatype(array.data_type().clone()); + ArrowListArray::::try_new(data_type, offsets.into(), array.to_boxed(), None) + .map_err(|err| err.into()) + } +} /// Holds either an owned [`ComponentBatch`] that lives on heap, or a reference to one. /// diff --git a/crates/store/re_video/src/lib.rs b/crates/store/re_video/src/lib.rs index 204fa42fb1f4..f84a836aaaf5 100644 --- a/crates/store/re_video/src/lib.rs +++ b/crates/store/re_video/src/lib.rs @@ -72,18 +72,26 @@ pub struct TimeMs(OrderedFloat); impl TimeMs { pub const ZERO: Self = Self(OrderedFloat(0.0)); + #[inline] pub fn new(ms: f64) -> Self { Self(OrderedFloat(ms)) } + #[inline] pub fn as_f64(&self) -> f64 { self.0.into_inner() } + + #[inline] + pub fn as_nanoseconds(self) -> i64 { + (self.0 * 1_000_000.0).round() as i64 + } } impl std::ops::Add for TimeMs { type Output = Self; + #[inline] fn add(self, rhs: Self) -> Self::Output { Self(self.0 + rhs.0) } @@ -92,6 +100,7 @@ impl std::ops::Add for TimeMs { impl std::ops::Sub for TimeMs { type Output = Self; + #[inline] fn sub(self, rhs: Self) -> Self::Output { Self(self.0 - rhs.0) } diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index f2e794bbf64e..e12741e7016d 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -5,7 +5,6 @@ use std::sync::Weak; use std::sync::{atomic::AtomicI64, Arc}; use ahash::HashMap; -use arrow2::offset::Offsets; use crossbeam::channel::{Receiver, Sender}; use itertools::Either; use parking_lot::Mutex; @@ -921,27 +920,7 @@ impl RecordingStream { let components: Result, ChunkError> = components .into_iter() - .map(|batch| { - let array = batch.to_arrow()?; - - let offsets = Offsets::try_from_lengths(std::iter::repeat(1).take(array.len())) - .map_err(|err| ChunkError::Malformed { - reason: format!("Failed to create offsets: {err}"), - })?; - let data_type = ArrowListArray::::default_datatype(array.data_type().clone()); - - let array = ArrowListArray::::try_new( - data_type, - offsets.into(), - array.to_boxed(), - None, - ) - .map_err(|err| ChunkError::Malformed { - reason: format!("Failed to wrap in List array: {err}"), - })?; - - Ok((batch.name(), array)) - }) + .map(|batch| Ok((batch.name(), batch.to_arrow_list_array()?))) .collect(); let components: BTreeMap> = From 904303cb9606330fd5862da30f8f5a0349fa3a52 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 16:53:49 +0200 Subject: [PATCH 06/12] video visualizer now operates on video frame references --- .../src/visualizers/mod.rs | 4 +- .../src/visualizers/videos.rs | 302 +++++++++--------- 2 files changed, 158 insertions(+), 148 deletions(-) diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs index eccc0d5d5589..f60415b923e3 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs @@ -86,7 +86,7 @@ pub fn register_2d_spatial_visualizers( system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; - system_registry.register_visualizer::()?; + system_registry.register_visualizer::()?; Ok(()) } @@ -111,7 +111,7 @@ pub fn register_3d_spatial_visualizers( system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; - system_registry.register_visualizer::()?; + system_registry.register_visualizer::()?; Ok(()) } diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index 809a8e2c8d53..c3ed785197a2 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -1,50 +1,36 @@ -use glam::Vec3; -use re_chunk_store::RowId; -use re_chunk_store::TimeInt; -use re_log_types::hash::Hash64; -use re_log_types::TimeType; -use re_renderer::renderer::ColormappedTexture; -use re_renderer::renderer::RectangleOptions; -use re_renderer::renderer::TextureFilterMag; -use re_renderer::renderer::TextureFilterMin; -use re_renderer::renderer::TexturedRect; -use re_renderer::RenderContext; -use re_types::archetypes::AssetVideo; -use re_types::components::Blob; -use re_types::components::MediaType; -use re_types::ArrowBuffer; -use re_types::ArrowString; -use re_types::Loggable as _; -use re_viewer_context::SpaceViewClass as _; +use egui::mutex::Mutex; + +use re_log_types::EntityPath; +use re_renderer::renderer::{ + ColormappedTexture, RectangleOptions, TextureFilterMag, TextureFilterMin, TexturedRect, +}; +use re_types::{ + archetypes::{AssetVideo, VideoFrameReference}, + components::{Blob, EntityPath as EntityPathReferenceComponent, MediaType, VideoTimestamp}, + datatypes::VideoTimeMode, + Archetype, Loggable as _, +}; use re_viewer_context::{ - ApplicableEntities, IdentifiedViewSystem, QueryContext, SpaceViewSystemExecutionError, - ViewContext, ViewContextCollection, ViewQuery, VisualizableEntities, VisualizableFilterContext, - VisualizerQueryInfo, VisualizerSystem, + ApplicableEntities, IdentifiedViewSystem, SpaceViewClass as _, SpaceViewSystemExecutionError, + ViewContext, ViewContextCollection, ViewQuery, ViewerContext, VisualizableEntities, + VisualizableFilterContext, VisualizerQueryInfo, VisualizerSystem, }; -use crate::video_cache::VideoCache; -use crate::video_cache::VideoCacheKey; -use crate::visualizers::entity_iterator::iter_buffer; -use crate::SpatialSpaceView2D; use crate::{ - contexts::SpatialSceneEntityContext, view_kind::SpatialSpaceViewKind, - visualizers::filter_visualizable_2d_entities, + video_cache::{VideoCache, VideoCacheKey}, + view_kind::SpatialSpaceViewKind, + visualizers::{entity_iterator, filter_visualizable_2d_entities}, + SpatialSpaceView2D, }; use super::bounding_box_for_textured_rect; use super::{entity_iterator::process_archetype, SpatialViewVisualizerData}; -pub struct AssetVideoVisualizer { +pub struct VideoFrameReferenceVisualizer { pub data: SpatialViewVisualizerData, } -struct AssetVideoComponentData { - index: (TimeInt, RowId), - blob: ArrowBuffer, - media_type: Option, -} - -impl Default for AssetVideoVisualizer { +impl Default for VideoFrameReferenceVisualizer { fn default() -> Self { Self { data: SpatialViewVisualizerData::new(Some(SpatialSpaceViewKind::TwoD)), @@ -52,15 +38,15 @@ impl Default for AssetVideoVisualizer { } } -impl IdentifiedViewSystem for AssetVideoVisualizer { +impl IdentifiedViewSystem for VideoFrameReferenceVisualizer { fn identifier() -> re_viewer_context::ViewSystemIdentifier { - "Video".into() + "VideoFrameReference".into() } } -impl VisualizerSystem for AssetVideoVisualizer { +impl VisualizerSystem for VideoFrameReferenceVisualizer { fn visualizer_query_info(&self) -> VisualizerQueryInfo { - VisualizerQueryInfo::from_archetype::() + VisualizerQueryInfo::from_archetype::() } fn filter_visualizable_entities( @@ -84,47 +70,87 @@ impl VisualizerSystem for AssetVideoVisualizer { let mut rectangles = Vec::new(); - process_archetype::( + process_archetype::( ctx, view_query, context_systems, |ctx, spatial_ctx, results| { + // TODO(andreas): Should ignore range queries here and only do latest-at. use re_space_view::RangeResultsExt as _; - let Some(all_blob_chunks) = results.get_required_chunks(&Blob::name()) else { - return Ok(()); - }; let timeline = ctx.query.timeline(); - let all_blobs_indexed = iter_buffer::(&all_blob_chunks, timeline, Blob::name()); - let all_media_types = results.iter_as(timeline, MediaType::name()); - - let data = re_query::range_zip_1x1(all_blobs_indexed, all_media_types.string()) - .filter_map(|(index, blobs, media_types)| { - blobs.first().map(|blob| AssetVideoComponentData { - index, - blob: blob.clone(), - media_type: media_types - .and_then(|media_types| media_types.first().cloned()), - }) - }); - - let current_time_nanoseconds = match timeline.typ() { - TimeType::Time => view_query.latest_at.as_f64(), - // TODO(jan): scale by ticks per second - #[allow(clippy::match_same_arms)] - TimeType::Sequence => view_query.latest_at.as_f64(), + let entity_path = ctx.target_entity_path; + + let Some(all_video_timestamp_chunks) = + results.get_required_chunks(&VideoTimestamp::name()) + else { + return Ok(()); }; - let current_time_seconds = current_time_nanoseconds / 1e9; - - self.process_data( - ctx, - render_ctx, - &mut rectangles, - spatial_ctx, - data, - current_time_seconds, - results.query_result_hash(), - ); + let all_video_references = + results.iter_as(timeline, EntityPathReferenceComponent::name()); + + for (_index, video_timestamps, video_references) in re_query::range_zip_1x1( + entity_iterator::iter_component( + &all_video_timestamp_chunks, + timeline, + VideoTimestamp::name(), + ), + all_video_references.string(), + ) { + let Some(video_timestamp): Option<&VideoTimestamp> = video_timestamps.first() + else { + continue; + }; + + // Follow the reference to the video asset. + let video_reference = video_references + .and_then(|v| v.first().map(|e| e.as_str().into())) + .unwrap_or_else(|| entity_path.clone()); + let Some(video) = + latest_at_query_video_from_datastore(ctx.viewer_ctx, &video_reference) + else { + continue; + }; + + let timestamp_in_seconds = match video_timestamp.time_mode { + VideoTimeMode::Nanoseconds => video_timestamp.video_time as f64 / 1e9, + }; + + let (texture, video_width, video_height) = { + let mut video = video.lock(); // TODO(andreas): Interior mutability for re_renderer's video would be nice. + ( + video.frame_at(timestamp_in_seconds), + video.width(), + video.height(), + ) + }; + + let world_from_entity = + spatial_ctx.transform_info.single_entity_transform_required( + ctx.target_entity_path, + Self::identifier().as_str(), + ); + let textured_rect = textured_rect_for_video_frame( + world_from_entity, + video_width, + video_height, + texture, + ); + + if spatial_ctx.space_view_class_identifier == SpatialSpaceView2D::identifier() { + // Only update the bounding box if this is a 2D space view. + // This is avoids a cyclic relationship where the image plane grows + // the bounds which in turn influence the size of the image plane. + // See: https://github.com/rerun-io/rerun/issues/3728 + self.data.add_bounding_box( + entity_path.hash(), + bounding_box_for_textured_rect(&textured_rect), + world_from_entity, + ); + } + + rectangles.push(textured_rect); + } Ok(()) }, @@ -157,80 +183,64 @@ impl VisualizerSystem for AssetVideoVisualizer { } } -// NOTE: Do not put profile scopes in these methods. They are called for all entities and all -// timestamps within a time range -- it's _a lot_. -impl AssetVideoVisualizer { - #[allow(clippy::unused_self)] - #[allow(clippy::too_many_arguments)] - fn process_data( - &mut self, - ctx: &QueryContext<'_>, - render_ctx: &RenderContext, - rectangles: &mut Vec, - ent_context: &SpatialSceneEntityContext<'_>, - data: impl Iterator, - current_time_seconds: f64, - query_result_hash: Hash64, - ) { - let entity_path = ctx.target_entity_path; - - for data in data { - let timestamp_s = current_time_seconds - data.index.0.as_f64() / 1e9; - let video = AssetVideo { - blob: data.blob.clone().into(), - media_type: data.media_type.clone().map(Into::into), - }; - - let primary_row_id = data.index.1; - let picking_instance_hash = re_entity_db::InstancePathHash::entity_all(entity_path); - - let video = ctx.viewer_ctx.cache.entry(|c: &mut VideoCache| { - c.entry( - &entity_path.to_string(), - VideoCacheKey { - versioned_instance_path_hash: picking_instance_hash - .versioned(primary_row_id), - query_result_hash, - media_type: data.media_type.clone().map(Into::into), - }, - &video.blob, - video.media_type.as_ref().map(|v| v.as_str()), - render_ctx, - ) - }); - - if let Some(video) = video { - let mut video = video.lock(); - let texture = video.frame_at(timestamp_s); - - let world_from_entity = ent_context - .transform_info - .single_entity_transform_required(ctx.target_entity_path, "Video"); - let textured_rect = TexturedRect { - top_left_corner_position: world_from_entity.transform_point3(Vec3::ZERO), - extent_u: world_from_entity.transform_vector3(Vec3::X * video.width() as f32), - extent_v: world_from_entity.transform_vector3(Vec3::Y * video.height() as f32), - - colormapped_texture: ColormappedTexture::from_unorm_rgba(texture), - options: RectangleOptions { - texture_filter_magnification: TextureFilterMag::Nearest, - texture_filter_minification: TextureFilterMin::Linear, - ..Default::default() - }, - }; - - if ent_context.space_view_class_identifier == SpatialSpaceView2D::identifier() { - self.data.add_bounding_box( - entity_path.hash(), - bounding_box_for_textured_rect(&textured_rect), - world_from_entity, - ); - } - - rectangles.push(textured_rect); - }; - } +fn textured_rect_for_video_frame( + world_from_entity: glam::Affine3A, + video_width: u32, + video_height: u32, + texture: re_renderer::resource_managers::GpuTexture2D, +) -> TexturedRect { + TexturedRect { + top_left_corner_position: world_from_entity.transform_point3(glam::Vec3::ZERO), + // Make sure to use the video instead of texture size here, + // since it may be a placeholder which doesn't have the full size yet. + extent_u: world_from_entity.transform_vector3(glam::Vec3::X * video_width as f32), + extent_v: world_from_entity.transform_vector3(glam::Vec3::Y * video_height as f32), + + colormapped_texture: ColormappedTexture::from_unorm_rgba(texture), + options: RectangleOptions { + texture_filter_magnification: TextureFilterMag::Nearest, + texture_filter_minification: TextureFilterMin::Linear, + ..Default::default() + }, } } -re_viewer_context::impl_component_fallback_provider!(AssetVideoVisualizer => []); +/// Queries a video from the datatore and caches it in the video cache. +/// +/// Note that this does *NOT* check the blueprint store at all. +/// For this, we'd need a [`re_viewer_context::DataResult`] instead of merely a [`EntityPath`]. +fn latest_at_query_video_from_datastore( + ctx: &ViewerContext<'_>, + entity_path: &EntityPath, +) -> Option>> { + let query = ctx.current_query(); + + let results = ctx.recording().query_caches().latest_at( + ctx.recording_store(), + &query, + entity_path, + AssetVideo::all_components().iter().copied(), + ); + + let blob_row_id = results.component_row_id(&Blob::name())?; + let blob = results.component_instance::(0)?; + let media_type = results.component_instance::(0); + + ctx.cache.entry(|c: &mut VideoCache| { + c.entry( + &entity_path.to_string(), + VideoCacheKey { + versioned_instance_path_hash: re_entity_db::InstancePathHash::entity_all( + entity_path, + ) + .versioned(blob_row_id), + media_type: media_type.clone(), + }, + &blob, + media_type.as_ref().map(|v| v.as_str()), + ctx.render_ctx?, + ) + }) +} + +re_viewer_context::impl_component_fallback_provider!(VideoFrameReferenceVisualizer => []); From b040f4c5442482f92bcbbf5084f67c2fa164f04d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 16:57:32 +0200 Subject: [PATCH 07/12] use acquire release semantics on video cache --- .../re_space_view_spatial/src/video_cache.rs | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/viewer/re_space_view_spatial/src/video_cache.rs b/crates/viewer/re_space_view_spatial/src/video_cache.rs index 229dac9be86c..370a1464489b 100644 --- a/crates/viewer/re_space_view_spatial/src/video_cache.rs +++ b/crates/viewer/re_space_view_spatial/src/video_cache.rs @@ -1,20 +1,20 @@ -use egui::mutex::Mutex; use re_entity_db::VersionedInstancePathHash; -use re_log_types::hash::Hash64; -use re_renderer::renderer::Video; -use re_renderer::RenderContext; +use re_renderer::{renderer::Video, RenderContext}; use re_types::components::MediaType; use re_viewer_context::Cache; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; -use std::sync::Arc; + +use egui::mutex::Mutex; + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; // ---------------------------------------------------------------------------- #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub struct VideoCacheKey { pub versioned_instance_path_hash: VersionedInstancePathHash, - pub query_result_hash: Hash64, pub media_type: Option, } @@ -54,7 +54,11 @@ impl VideoCache { video, } }); - entry.used_this_frame.store(true, Ordering::Relaxed); + + // Using acquire/release here to be on the safe side and for semantical soundness: + // Whatever thread is acquiring the fact that this was used, should also see/acquire + // the side effect of having the entry contained in the cache. + entry.used_this_frame.store(true, Ordering::Release); entry.video.clone() } } @@ -62,13 +66,13 @@ impl VideoCache { impl Cache for VideoCache { fn begin_frame(&mut self) { for v in self.0.values() { - v.used_this_frame.store(false, Ordering::Relaxed); + v.used_this_frame.store(false, Ordering::Release); } } fn purge_memory(&mut self) { self.0 - .retain(|_, v| v.used_this_frame.load(Ordering::Relaxed)); + .retain(|_, v| v.used_this_frame.load(Ordering::Acquire)); } fn as_any_mut(&mut self) -> &mut dyn std::any::Any { From 5df57f332822f0df819db1358458f2560f9d9c23 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 16:58:24 +0200 Subject: [PATCH 08/12] improved commentry --- crates/store/re_data_loader/src/loader_archetype.rs | 4 +++- .../definitions/rerun/archetypes/video_frame_reference.fbs | 2 ++ crates/store/re_types/src/archetypes/video_frame_reference.rs | 4 ++++ crates/viewer/re_space_view_spatial/src/visualizers/videos.rs | 3 +++ crates/viewer/re_viewer/src/reflection/mod.rs | 2 +- rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp | 4 ++++ rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py | 4 ++++ 7 files changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs index 894605349ae4..e42a39a122d3 100644 --- a/crates/store/re_data_loader/src/loader_archetype.rs +++ b/crates/store/re_data_loader/src/loader_archetype.rs @@ -222,6 +222,8 @@ fn load_video( let media_type = MediaType::guess_from_path(filepath); + // TODO(andreas): Video frame reference generation should be available as a utility from the SDK. + let video = if media_type.as_ref().map(|v| v.as_str()) == Some("video/mp4") { match re_video::load_mp4(&contents) { Ok(video) => Some(video), @@ -235,7 +237,7 @@ fn load_video( None }; - // Log video time references on the `video` timeline. + // Log video frame references on the `video` timeline. let video_frame_reference_chunk = if let Some(video) = video { let first_timestamp = video .segments diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs index 1c1189530819..a4108fb89f6c 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -21,5 +21,7 @@ table VideoFrameReference ( /// Optional reference to an entity with a [archetypes.AssetVideo]. /// /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. video_reference: rerun.components.EntityPath ("attr.rerun.component_optional", nullable, order: 2000); } diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs index a9ebf9800269..963b15209cf5 100644 --- a/crates/store/re_types/src/archetypes/video_frame_reference.rs +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -34,6 +34,8 @@ pub struct VideoFrameReference { /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. /// /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. pub video_reference: Option, } @@ -188,6 +190,8 @@ impl VideoFrameReference { /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. /// /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. #[inline] pub fn with_video_reference( mut self, diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index c3ed785197a2..040687ef37c3 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -76,6 +76,9 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { context_systems, |ctx, spatial_ctx, results| { // TODO(andreas): Should ignore range queries here and only do latest-at. + // Not only would this simplify the code here quite a bit, it would also avoid lots of overhead. + // Same is true for the image visualizers in general - there seems to be no practial reason to do range queries + // for visualization here. use re_space_view::RangeResultsExt as _; let timeline = ctx.query.timeline(); diff --git a/crates/viewer/re_viewer/src/reflection/mod.rs b/crates/viewer/re_viewer/src/reflection/mod.rs index 962bc073bb24..b1cbae2aba0a 100644 --- a/crates/viewer/re_viewer/src/reflection/mod.rs +++ b/crates/viewer/re_viewer/src/reflection/mod.rs @@ -1470,7 +1470,7 @@ fn generate_archetype_reflection() -> ArchetypeReflectionMap { is_required : true, }, ArchetypeFieldReflection { component_name : "rerun.components.EntityPath".into(), display_name : "Video reference", docstring_md : - "Optional reference to an entity with a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).\n\nIf none is specified, the video is assumed to be at the same entity.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + "Optional reference to an entity with a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).\n\nIf none is specified, the video is assumed to be at the same entity.\nNote that blueprint overrides on the referenced video will be ignored regardless,\nas this is always interpreted as a reference to the data store.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", is_required : false, }, ], }, diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp index 80d6b7db752d..a3ff4476f3f8 100644 --- a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -32,6 +32,8 @@ namespace rerun::archetypes { /// Optional reference to an entity with a `archetypes::AssetVideo`. /// /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. std::optional video_reference; public: @@ -51,6 +53,8 @@ namespace rerun::archetypes { /// Optional reference to an entity with a `archetypes::AssetVideo`. /// /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. VideoFrameReference with_video_reference(rerun::components::EntityPath _video_reference ) && { video_reference = std::move(_video_reference); diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py index 986c53f75309..f2a8efd9b3fb 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -45,6 +45,8 @@ def __init__( Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. If none is specified, the video is assumed to be at the same entity. + Note that blueprint overrides on the referenced video will be ignored regardless, + as this is always interpreted as a reference to the data store. """ @@ -87,6 +89,8 @@ def _clear(cls) -> VideoFrameReference: # Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. # # If none is specified, the video is assumed to be at the same entity. + # Note that blueprint overrides on the referenced video will be ignored regardless, + # as this is always interpreted as a reference to the data store. # # (Docstring intentionally commented out to hide this field from the docs) From c0efad514e8475acb2b2c019fe9291505d17007f Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 17:19:39 +0200 Subject: [PATCH 09/12] typo fixes --- crates/store/re_data_loader/src/loader_archetype.rs | 6 +++--- .../viewer/re_space_view_spatial/src/visualizers/videos.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs index e42a39a122d3..6f29163ef7b6 100644 --- a/crates/store/re_data_loader/src/loader_archetype.rs +++ b/crates/store/re_data_loader/src/loader_archetype.rs @@ -270,7 +270,7 @@ fn load_video( }) .collect::>(); let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch; - let video_timestmap_list_array = video_timestamp_batch + let video_timestamp_list_array = video_timestamp_batch .to_arrow_list_array() .map_err(re_chunk::ChunkError::from)?; @@ -280,7 +280,7 @@ fn load_video( ArrowListArray::::default_datatype( video_frame_reference_indicator_datatype.clone(), ), - video_timestmap_list_array.offsets().clone(), + video_timestamp_list_array.offsets().clone(), Box::new(ArrowNullArray::new( video_frame_reference_indicator_datatype, video_timestamps.len(), @@ -298,7 +298,7 @@ fn load_video( VideoFrameReference::indicator().name(), video_frame_reference_indicator_list_array, ), - (video_timestamp_batch.name(), video_timestmap_list_array), + (video_timestamp_batch.name(), video_timestamp_list_array), ] .into_iter() .collect(), diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index 040687ef37c3..dcbe5bbb5755 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -77,7 +77,7 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { |ctx, spatial_ctx, results| { // TODO(andreas): Should ignore range queries here and only do latest-at. // Not only would this simplify the code here quite a bit, it would also avoid lots of overhead. - // Same is true for the image visualizers in general - there seems to be no practial reason to do range queries + // Same is true for the image visualizers in general - there seems to be no practical reason to do range queries // for visualization here. use re_space_view::RangeResultsExt as _; @@ -208,7 +208,7 @@ fn textured_rect_for_video_frame( } } -/// Queries a video from the datatore and caches it in the video cache. +/// Queries a video from the datstore and caches it in the video cache. /// /// Note that this does *NOT* check the blueprint store at all. /// For this, we'd need a [`re_viewer_context::DataResult`] instead of merely a [`EntityPath`]. From e7e1ab9196afc2e6ac74bf8972432f28a87192f3 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 10 Sep 2024 17:31:26 +0200 Subject: [PATCH 10/12] new video section in python docs --- rerun_py/docs/gen_common_index.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rerun_py/docs/gen_common_index.py b/rerun_py/docs/gen_common_index.py index f7b9cd39ccbc..147973087835 100755 --- a/rerun_py/docs/gen_common_index.py +++ b/rerun_py/docs/gen_common_index.py @@ -169,7 +169,14 @@ class Section: "archetypes.Image", "archetypes.EncodedImage", "archetypes.SegmentationImage", + ], + gen_page=False, + ), + Section( + title="Video", + class_list=[ "archetypes.AssetVideo", + "archetypes.VideoFrameReference", ], gen_page=False, ), From e1f408e825f5e6af26a729144ad27c17cfd73ef5 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 11 Sep 2024 11:07:58 +0200 Subject: [PATCH 11/12] todo note improvements --- Cargo.lock | 1 + .../re_types/definitions/rerun/archetypes/asset_video.fbs | 2 +- .../rerun/archetypes/video_frame_reference.fbs | 7 +++---- .../re_types/src/archetypes/video_frame_reference.rs | 4 ++-- crates/viewer/re_viewer/src/reflection/mod.rs | 2 +- rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp | 4 ++-- .../rerun_sdk/rerun/archetypes/video_frame_reference.py | 8 ++++---- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d30473d9371a..94b095c24953 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4916,6 +4916,7 @@ dependencies = [ "once_cell", "parking_lot", "rayon", + "re_arrow2", "re_build_info", "re_build_tools", "re_chunk", diff --git a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 3b6480568a70..ed4a1ad95245 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -7,7 +7,7 @@ namespace rerun.archetypes; /// Follow for updates on the native support. /// /// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame. -// TODO(andreas): More docs and examples on how to use this. +// TODO(#7368): More docs and examples on how to use this. table AssetVideo ( "attr.rerun.experimental" ) { diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs index a4108fb89f6c..94d7d29e7667 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -3,17 +3,16 @@ namespace rerun.archetypes; /// References a single video frame. /// /// Used to display video frames from a [archetypes.AssetVideo]. -// TODO(andreas): More docs and examples on how to use this. +// TODO(#7368): More docs and examples on how to use this. table VideoFrameReference ( "attr.rerun.experimental" ){ // --- Required --- - /// References the closest video frame to this time. + /// References the closest video frame to this timestamp. /// /// Note that this uses the closest video frame instead of the latest at this timestamp - /// in order to be more forgiving of rounding errors. - // TODO(andreas): Once this can also be a frame index, point out that this is an accurate measure. + /// in order to be more forgiving of rounding errors for inprecise timestamp types. timestamp: rerun.components.VideoTimestamp ("attr.rerun.component_required", required, order: 1000); // --- Optional --- diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs index 963b15209cf5..a70ca383cb02 100644 --- a/crates/store/re_types/src/archetypes/video_frame_reference.rs +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -25,10 +25,10 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// ⚠️ **This type is experimental and may be removed in future versions** #[derive(Clone, Debug)] pub struct VideoFrameReference { - /// References the closest video frame to this time. + /// References the closest video frame to this timestamp. /// /// Note that this uses the closest video frame instead of the latest at this timestamp - /// in order to be more forgiving of rounding errors. + /// in order to be more forgiving of rounding errors for inprecise timestamp types. pub timestamp: crate::components::VideoTimestamp, /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. diff --git a/crates/viewer/re_viewer/src/reflection/mod.rs b/crates/viewer/re_viewer/src/reflection/mod.rs index b1cbae2aba0a..38617ffe1f4b 100644 --- a/crates/viewer/re_viewer/src/reflection/mod.rs +++ b/crates/viewer/re_viewer/src/reflection/mod.rs @@ -1466,7 +1466,7 @@ fn generate_archetype_reflection() -> ArchetypeReflectionMap { ArchetypeFieldReflection { component_name : "rerun.components.VideoTimestamp".into(), display_name : "Timestamp", docstring_md : - "References the closest video frame to this time.\n\nNote that this uses the closest video frame instead of the latest at this timestamp\nin order to be more forgiving of rounding errors.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + "References the closest video frame to this timestamp.\n\nNote that this uses the closest video frame instead of the latest at this timestamp\nin order to be more forgiving of rounding errors for inprecise timestamp types.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", is_required : true, }, ArchetypeFieldReflection { component_name : "rerun.components.EntityPath".into(), display_name : "Video reference", docstring_md : diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp index a3ff4476f3f8..ef96331f1a1a 100644 --- a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -23,10 +23,10 @@ namespace rerun::archetypes { /// /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** struct VideoFrameReference { - /// References the closest video frame to this time. + /// References the closest video frame to this timestamp. /// /// Note that this uses the closest video frame instead of the latest at this timestamp - /// in order to be more forgiving of rounding errors. + /// in order to be more forgiving of rounding errors for inprecise timestamp types. rerun::components::VideoTimestamp timestamp; /// Optional reference to an entity with a `archetypes::AssetVideo`. diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py index f2a8efd9b3fb..d73ff771bdf3 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -37,10 +37,10 @@ def __init__( Parameters ---------- timestamp: - References the closest video frame to this time. + References the closest video frame to this timestamp. Note that this uses the closest video frame instead of the latest at this timestamp - in order to be more forgiving of rounding errors. + in order to be more forgiving of rounding errors for inprecise timestamp types. video_reference: Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. @@ -74,10 +74,10 @@ def _clear(cls) -> VideoFrameReference: metadata={"component": "required"}, converter=components.VideoTimestampBatch._required, # type: ignore[misc] ) - # References the closest video frame to this time. + # References the closest video frame to this timestamp. # # Note that this uses the closest video frame instead of the latest at this timestamp - # in order to be more forgiving of rounding errors. + # in order to be more forgiving of rounding errors for inprecise timestamp types. # # (Docstring intentionally commented out to hide this field from the docs) From 5cfd5497ff765be0a3720c1b0a3cfc98f10fddb7 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 11 Sep 2024 11:08:06 +0200 Subject: [PATCH 12/12] add missing components to check_all_components_ui.py --- tests/python/release_checklist/check_all_components_ui.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/python/release_checklist/check_all_components_ui.py b/tests/python/release_checklist/check_all_components_ui.py index 1560900ef91e..a7daab5b2fbc 100644 --- a/tests/python/release_checklist/check_all_components_ui.py +++ b/tests/python/release_checklist/check_all_components_ui.py @@ -111,6 +111,7 @@ def alternatives(self) -> list[Any] | None: "DepthMeterBatch": TestCase(1000.0), "DisconnectedSpaceBatch": TestCase(True), "DrawOrderBatch": TestCase(100.0), + "EntityPathBatch": TestCase("my/entity/path"), "FillModeBatch": TestCase( batch=[ rr.components.FillMode.MajorWireframe, @@ -217,6 +218,7 @@ def alternatives(self) -> list[Any] | None: "TriangleIndicesBatch": TestCase(batch=[(0, 1, 2), (3, 4, 5), (6, 7, 8)]), "Vector2DBatch": TestCase(batch=[(0, 1), (2, 3), (4, 5)]), "Vector3DBatch": TestCase(batch=[(0, 3, 4), (1, 4, 5), (2, 5, 6)]), + "VideoTimestampBatch": TestCase(rr.components.VideoTimestamp(0, rr.datatypes.VideoTimeMode.Nanoseconds)), "ViewCoordinatesBatch": TestCase(rr.components.ViewCoordinates.LBD), "VisualizerOverridesBatch": TestCase(disabled=True), # no Python-based serialization }