From e77459bbd4cf45176af61b1858f22012aa6f791d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 11 Sep 2024 11:57:20 +0200 Subject: [PATCH 1/3] Introduce `VideoFrameReference` archetype and base video visualization on it (#7396) ### What * Large chunk of #7368 --- * Introduce `VideoFrameReference` archetype * comes with a bunch of subtypes * Make the video visualizer use this * in the process I did some general iteration on how this visualizer ticks * note that I departed from the usual quite encrusted patterns of how to implement visualizers that naturally found its way into here as well. Imho the patterns don't hold up all that well in general (we had to grind over them many times in quick succession) and less so in particular for "special" archetypes like video or mesh (etc.) * video data loader emits `VideoFrameReference` instead of the adhoc `VideoTick Still missing for completion of the video frame reference task: * examples to use this from SDKs (by directly logging video frames) * this surely will show that we need more extensions * pretty/edit ui for video timestamps * @Wumpf are you saying that you can then scrub videos from a blueprint slider?! Yes. That's exactly what will end up happening ;D * pretty/edit ui for entity paths * exposed utilities to SDKs for generation of video frame references @ reviewers: Special attention please on the fbs definition, thank you! image ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7396?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7396?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7396) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- Cargo.lock | 1 + crates/store/re_data_loader/Cargo.toml | 1 + .../re_data_loader/src/loader_archetype.rs | 184 ++++++----- .../re_types/definitions/rerun/archetypes.fbs | 3 +- .../archetypes/{video.fbs => asset_video.fbs} | 7 +- .../archetypes/video_frame_reference.fbs | 26 ++ .../re_types/definitions/rerun/components.fbs | 2 + .../rerun/components/entity_path.fbs | 12 + .../rerun/components/video_timestamp.fbs | 11 + .../re_types/definitions/rerun/datatypes.fbs | 1 + .../rerun/datatypes/video_timestamp.fbs | 24 ++ .../re_types/src/archetypes/.gitattributes | 1 + .../re_types/src/archetypes/asset_video.rs | 8 +- crates/store/re_types/src/archetypes/mod.rs | 2 + .../src/archetypes/video_frame_reference.rs | 203 ++++++++++++ .../re_types/src/components/.gitattributes | 2 + .../re_types/src/components/entity_path.rs | 105 ++++++ crates/store/re_types/src/components/mod.rs | 5 + .../src/components/video_timestamp.rs | 107 ++++++ .../src/components/video_timestamp_ext.rs | 9 + .../re_types/src/datatypes/.gitattributes | 2 + crates/store/re_types/src/datatypes/mod.rs | 5 + .../re_types/src/datatypes/video_time_mode.rs | 146 +++++++++ .../re_types/src/datatypes/video_timestamp.rs | 226 +++++++++++++ .../src/datatypes/video_timestamp_ext.rs | 21 ++ .../store/re_types_core/src/loggable_batch.rs | 14 +- crates/store/re_video/src/lib.rs | 9 + crates/top/re_sdk/src/recording_stream.rs | 23 +- .../re_space_view_spatial/src/video_cache.rs | 26 +- .../src/visualizers/mod.rs | 4 +- .../src/visualizers/videos.rs | 305 +++++++++--------- crates/viewer/re_viewer/src/reflection/mod.rs | 31 ++ docs/content/reference/types/archetypes.md | 3 +- .../reference/types/archetypes/.gitattributes | 1 + .../reference/types/archetypes/asset_video.md | 6 +- .../types/archetypes/video_frame_reference.md | 23 ++ docs/content/reference/types/components.md | 2 + .../reference/types/components/.gitattributes | 2 + .../reference/types/components/entity_path.md | 20 ++ .../types/components/video_timestamp.md | 23 ++ docs/content/reference/types/datatypes.md | 2 + .../reference/types/datatypes/.gitattributes | 2 + .../reference/types/datatypes/entity_path.md | 3 + .../types/datatypes/video_time_mode.md | 20 ++ .../types/datatypes/video_timestamp.md | 24 ++ rerun_cpp/src/rerun/archetypes.hpp | 1 + rerun_cpp/src/rerun/archetypes/.gitattributes | 2 + .../src/rerun/archetypes/asset_video.cpp | 2 +- .../src/rerun/archetypes/asset_video.hpp | 8 +- .../archetypes/video_frame_reference.cpp | 38 +++ .../archetypes/video_frame_reference.hpp | 81 +++++ rerun_cpp/src/rerun/components.hpp | 2 + rerun_cpp/src/rerun/components/.gitattributes | 2 + .../src/rerun/components/entity_path.hpp | 66 ++++ .../src/rerun/components/video_timestamp.hpp | 59 ++++ rerun_cpp/src/rerun/datatypes.hpp | 2 + rerun_cpp/src/rerun/datatypes/.gitattributes | 4 + .../src/rerun/datatypes/video_time_mode.cpp | 56 ++++ .../src/rerun/datatypes/video_time_mode.hpp | 54 ++++ .../src/rerun/datatypes/video_timestamp.cpp | 84 +++++ .../src/rerun/datatypes/video_timestamp.hpp | 57 ++++ rerun_py/docs/gen_common_index.py | 7 + .../rerun_sdk/rerun/archetypes/.gitattributes | 1 + .../rerun_sdk/rerun/archetypes/__init__.py | 2 + .../rerun_sdk/rerun/archetypes/asset_video.py | 8 +- .../rerun/archetypes/video_frame_reference.py | 98 ++++++ .../rerun_sdk/rerun/components/.gitattributes | 2 + .../rerun_sdk/rerun/components/__init__.py | 8 + .../rerun_sdk/rerun/components/entity_path.py | 36 +++ .../rerun/components/video_timestamp.py | 40 +++ .../rerun_sdk/rerun/datatypes/.gitattributes | 2 + .../rerun_sdk/rerun/datatypes/__init__.py | 24 ++ .../rerun/datatypes/video_time_mode.py | 71 ++++ .../rerun/datatypes/video_timestamp.py | 110 +++++++ .../check_all_components_ui.py | 2 + 75 files changed, 2310 insertions(+), 276 deletions(-) rename crates/store/re_types/definitions/rerun/archetypes/{video.fbs => asset_video.fbs} (71%) create mode 100644 crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs create mode 100644 crates/store/re_types/definitions/rerun/components/entity_path.fbs create mode 100644 crates/store/re_types/definitions/rerun/components/video_timestamp.fbs create mode 100644 crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs create mode 100644 crates/store/re_types/src/archetypes/video_frame_reference.rs create mode 100644 crates/store/re_types/src/components/entity_path.rs create mode 100644 crates/store/re_types/src/components/video_timestamp.rs create mode 100644 crates/store/re_types/src/components/video_timestamp_ext.rs create mode 100644 crates/store/re_types/src/datatypes/video_time_mode.rs create mode 100644 crates/store/re_types/src/datatypes/video_timestamp.rs create mode 100644 crates/store/re_types/src/datatypes/video_timestamp_ext.rs create mode 100644 docs/content/reference/types/archetypes/video_frame_reference.md create mode 100644 docs/content/reference/types/components/entity_path.md create mode 100644 docs/content/reference/types/components/video_timestamp.md create mode 100644 docs/content/reference/types/datatypes/video_time_mode.md create mode 100644 docs/content/reference/types/datatypes/video_timestamp.md create mode 100644 rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp create mode 100644 rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp create mode 100644 rerun_cpp/src/rerun/components/entity_path.hpp create mode 100644 rerun_cpp/src/rerun/components/video_timestamp.hpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_time_mode.cpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_time_mode.hpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_timestamp.cpp create mode 100644 rerun_cpp/src/rerun/datatypes/video_timestamp.hpp create mode 100644 rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py create mode 100644 rerun_py/rerun_sdk/rerun/components/entity_path.py create mode 100644 rerun_py/rerun_sdk/rerun/components/video_timestamp.py create mode 100644 rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py create mode 100644 rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py diff --git a/Cargo.lock b/Cargo.lock index ccf83cfd8887..b01de3d39c17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4926,6 +4926,7 @@ dependencies = [ "once_cell", "parking_lot", "rayon", + "re_arrow2", "re_build_info", "re_build_tools", "re_chunk", diff --git a/crates/store/re_data_loader/Cargo.toml b/crates/store/re_data_loader/Cargo.toml index 11489e232e35..804ff9889f46 100644 --- a/crates/store/re_data_loader/Cargo.toml +++ b/crates/store/re_data_loader/Cargo.toml @@ -36,6 +36,7 @@ re_video.workspace = true ahash.workspace = true anyhow.workspace = true +arrow2.workspace = true image.workspace = true once_cell.workspace = true parking_lot.workspace = true diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs index cc13e83017ea..6f29163ef7b6 100644 --- a/crates/store/re_data_loader/src/loader_archetype.rs +++ b/crates/store/re_data_loader/src/loader_archetype.rs @@ -1,7 +1,13 @@ use re_chunk::{Chunk, RowId}; -use re_log_types::NonMinI64; use re_log_types::{EntityPath, TimeInt, TimePoint}; -use re_types::components::MediaType; +use re_types::archetypes::VideoFrameReference; +use re_types::Archetype; +use re_types::{components::MediaType, ComponentBatch}; + +use arrow2::array::{ + ListArray as ArrowListArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray, +}; +use arrow2::Either; use crate::{DataLoader, DataLoaderError, LoadedData}; @@ -164,47 +170,6 @@ fn load_image( Ok(rows.into_iter()) } -/// TODO(#7272): fix this -/// Used to expand the timeline when logging a video, so that the video can be played back. -#[derive(Clone, Copy)] -struct VideoTick(re_types::datatypes::Float64); - -impl re_types::AsComponents for VideoTick { - fn as_component_batches(&self) -> Vec> { - vec![re_types::NamedIndicatorComponent("VideoTick".into()).to_batch()] - } -} - -impl re_types::Loggable for VideoTick { - type Name = re_types::ComponentName; - - fn name() -> Self::Name { - "rerun.components.VideoTick".into() - } - - fn arrow_datatype() -> re_chunk::external::arrow2::datatypes::DataType { - re_types::datatypes::Float64::arrow_datatype() - } - - fn to_arrow_opt<'a>( - data: impl IntoIterator>>>, - ) -> re_types::SerializationResult> - where - Self: 'a, - { - re_types::datatypes::Float64::to_arrow_opt( - data.into_iter() - .map(|datum| datum.map(|datum| datum.into().0)), - ) - } -} - -impl re_types::SizeBytes for VideoTick { - fn heap_size_bytes(&self) -> u64 { - 0 - } -} - #[derive(Clone, Copy)] struct ExperimentalFeature; @@ -252,51 +217,114 @@ fn load_video( ) -> Result, DataLoaderError> { re_tracing::profile_function!(); - timepoint.insert( - re_log_types::Timeline::new_temporal("video"), - re_log_types::TimeInt::new_temporal(0), - ); + let video_timeline = re_log_types::Timeline::new_temporal("video"); + timepoint.insert(video_timeline, re_log_types::TimeInt::new_temporal(0)); let media_type = MediaType::guess_from_path(filepath); - let duration_s = match media_type.as_ref().map(|v| v.as_str()) { - Some("video/mp4") => re_video::load_mp4(&contents) - .ok() - .map(|v| v.duration.as_f64() / 1_000.0), - _ => None, - } - .unwrap_or(100.0) - .ceil() as i64; + // TODO(andreas): Video frame reference generation should be available as a utility from the SDK. - let mut rows = vec![Chunk::builder(entity_path.clone()) + let video = if media_type.as_ref().map(|v| v.as_str()) == Some("video/mp4") { + match re_video::load_mp4(&contents) { + Ok(video) => Some(video), + Err(err) => { + re_log::warn!("Failed to load video asset {filepath:?}: {err}"); + None + } + } + } else { + re_log::warn!("Video asset {filepath:?} has an unsupported container format."); + None + }; + + // Log video frame references on the `video` timeline. + let video_frame_reference_chunk = if let Some(video) = video { + let first_timestamp = video + .segments + .first() + .map_or(0, |segment| segment.timestamp.as_nanoseconds()); + + // Time column. + let is_sorted = Some(true); + let time_column_times = + ArrowPrimitiveArray::::from_values(video.segments.iter().flat_map(|segment| { + segment + .samples + .iter() + .map(|s| s.timestamp.as_nanoseconds() - first_timestamp) + })); + + let time_column = re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times); + + // VideoTimestamp component column. + let video_timestamps = video + .segments + .iter() + .flat_map(|segment| { + segment.samples.iter().map(|s| { + // TODO(andreas): Use sample indices instead of timestamps once possible. + re_types::components::VideoTimestamp::new_nanoseconds( + s.timestamp.as_nanoseconds(), + ) + }) + }) + .collect::>(); + let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch; + let video_timestamp_list_array = video_timestamp_batch + .to_arrow_list_array() + .map_err(re_chunk::ChunkError::from)?; + + // Indicator column. + let video_frame_reference_indicator_datatype = arrow2::datatypes::DataType::Null; + let video_frame_reference_indicator_list_array = ArrowListArray::::try_new( + ArrowListArray::::default_datatype( + video_frame_reference_indicator_datatype.clone(), + ), + video_timestamp_list_array.offsets().clone(), + Box::new(ArrowNullArray::new( + video_frame_reference_indicator_datatype, + video_timestamps.len(), + )), + None, + ) + .map_err(re_chunk::ChunkError::from)?; + + Some(Chunk::from_auto_row_ids( + re_chunk::ChunkId::new(), + entity_path.clone(), + std::iter::once((video_timeline, time_column)).collect(), + [ + ( + VideoFrameReference::indicator().name(), + video_frame_reference_indicator_list_array, + ), + (video_timestamp_batch.name(), video_timestamp_list_array), + ] + .into_iter() + .collect(), + )?) + } else { + None + }; + + // Put video asset into its own chunk since it can be fairly large. + let video_asset_chunk = Chunk::builder(entity_path.clone()) .with_archetype( RowId::new(), timepoint.clone(), - &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type), + &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type.clone()), ) .with_component_batch(RowId::new(), timepoint.clone(), &ExperimentalFeature) - .build()?]; - - for i in 0..duration_s { - // We need some breadcrumbs of timepoints because the video doesn't have a duration yet. - // TODO(#7272): fix this - timepoint.insert( - re_log_types::Timeline::new_temporal("video"), - re_log_types::TimeInt::from_seconds(NonMinI64::new(i).expect("i > i64::MIN")), - ); - - rows.push( - Chunk::builder(entity_path.clone()) - .with_component_batch( - RowId::new(), - timepoint.clone(), - &VideoTick(re_types::datatypes::Float64(i as f64)), - ) - .build()?, - ); + .build()?; + + if let Some(video_frame_reference_chunk) = video_frame_reference_chunk { + Ok(Either::Left( + [video_asset_chunk, video_frame_reference_chunk].into_iter(), + )) + } else { + // Still log the video asset, but don't include video frames. + Ok(Either::Right(std::iter::once(video_asset_chunk))) } - - Ok(rows.into_iter()) } fn load_mesh( diff --git a/crates/store/re_types/definitions/rerun/archetypes.fbs b/crates/store/re_types/definitions/rerun/archetypes.fbs index 1e679c77fc91..e8cc336b3afb 100644 --- a/crates/store/re_types/definitions/rerun/archetypes.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes.fbs @@ -4,6 +4,7 @@ include "./archetypes/annotation_context.fbs"; include "./archetypes/arrows2d.fbs"; include "./archetypes/arrows3d.fbs"; include "./archetypes/asset3d.fbs"; +include "./archetypes/asset_video.fbs"; include "./archetypes/bar_chart.fbs"; include "./archetypes/boxes2d.fbs"; include "./archetypes/boxes3d.fbs"; @@ -28,5 +29,5 @@ include "./archetypes/tensor.fbs"; include "./archetypes/text_document.fbs"; include "./archetypes/text_log.fbs"; include "./archetypes/transform3d.fbs"; -include "./archetypes/video.fbs"; +include "./archetypes/video_frame_reference.fbs"; include "./archetypes/view_coordinates.fbs"; diff --git a/crates/store/re_types/definitions/rerun/archetypes/video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs similarity index 71% rename from crates/store/re_types/definitions/rerun/archetypes/video.fbs rename to crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 535ee5a6d09b..ed4a1ad95245 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -1,10 +1,13 @@ namespace rerun.archetypes; -/// A video file. +/// A video binary. /// /// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 and AV1 is currently supported, and not in all browsers. +/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. /// Follow for updates on the native support. +/// +/// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame. +// TODO(#7368): More docs and examples on how to use this. table AssetVideo ( "attr.rerun.experimental" ) { diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs new file mode 100644 index 000000000000..94d7d29e7667 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -0,0 +1,26 @@ +namespace rerun.archetypes; + +/// References a single video frame. +/// +/// Used to display video frames from a [archetypes.AssetVideo]. +// TODO(#7368): More docs and examples on how to use this. +table VideoFrameReference ( + "attr.rerun.experimental" +){ + // --- Required --- + + /// References the closest video frame to this timestamp. + /// + /// Note that this uses the closest video frame instead of the latest at this timestamp + /// in order to be more forgiving of rounding errors for inprecise timestamp types. + timestamp: rerun.components.VideoTimestamp ("attr.rerun.component_required", required, order: 1000); + + // --- Optional --- + + /// Optional reference to an entity with a [archetypes.AssetVideo]. + /// + /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. + video_reference: rerun.components.EntityPath ("attr.rerun.component_optional", nullable, order: 2000); +} diff --git a/crates/store/re_types/definitions/rerun/components.fbs b/crates/store/re_types/definitions/rerun/components.fbs index 764453c92d1e..b5916c757898 100644 --- a/crates/store/re_types/definitions/rerun/components.fbs +++ b/crates/store/re_types/definitions/rerun/components.fbs @@ -12,6 +12,7 @@ include "./components/colormap.fbs"; include "./components/depth_meter.fbs"; include "./components/disconnected_space.fbs"; include "./components/draw_order.fbs"; +include "./components/entity_path.fbs"; include "./components/fill_mode.fbs"; include "./components/fill_ratio.fbs"; include "./components/gamma_correction.fbs"; @@ -52,4 +53,5 @@ include "./components/translation3d.fbs"; include "./components/triangle_indices.fbs"; include "./components/vector2d.fbs"; include "./components/vector3d.fbs"; +include "./components/video_timestamp.fbs"; include "./components/view_coordinates.fbs"; diff --git a/crates/store/re_types/definitions/rerun/components/entity_path.fbs b/crates/store/re_types/definitions/rerun/components/entity_path.fbs new file mode 100644 index 000000000000..3868835b0995 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/components/entity_path.fbs @@ -0,0 +1,12 @@ +namespace rerun.components; + +/// A path to an entity, usually to reference some data that is part of the target entity. +table EntityPath ( + "attr.arrow.transparent", + "attr.python.aliases": "str", + "attr.python.array_aliases": "str, Sequence[str]", + "attr.rust.derive": "Default, PartialEq, Eq, PartialOrd, Ord", + "attr.rust.repr": "transparent" +) { + value: rerun.datatypes.EntityPath (order: 100); +} diff --git a/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs b/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs new file mode 100644 index 000000000000..b17d4411daf9 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs @@ -0,0 +1,11 @@ + +namespace rerun.components; + +/// Timestamp inside a [archetypes.AssetVideo]. +struct VideoTimestamp ( + "attr.rust.derive": "Copy, PartialEq, Eq, Default", + "attr.rust.repr": "transparent", + "attr.rerun.experimental" +) { + timestamp: rerun.datatypes.VideoTimestamp (order: 100); +} diff --git a/crates/store/re_types/definitions/rerun/datatypes.fbs b/crates/store/re_types/definitions/rerun/datatypes.fbs index 4b258e4a96e3..203c6105aa1a 100644 --- a/crates/store/re_types/definitions/rerun/datatypes.fbs +++ b/crates/store/re_types/definitions/rerun/datatypes.fbs @@ -39,5 +39,6 @@ include "./datatypes/uvec4d.fbs"; include "./datatypes/vec2d.fbs"; include "./datatypes/vec3d.fbs"; include "./datatypes/vec4d.fbs"; +include "./datatypes/video_timestamp.fbs"; include "./datatypes/view_coordinates.fbs"; include "./datatypes/visible_time_range.fbs"; diff --git a/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs b/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs new file mode 100644 index 000000000000..2c29b7d882d1 --- /dev/null +++ b/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs @@ -0,0 +1,24 @@ +namespace rerun.datatypes; + +/// Specifies how to interpret the `video_time` field of a [datatypes.VideoTimestamp]. +enum VideoTimeMode: ubyte{ + /// Invalid value. Won't show up in generated types. + Invalid = 0, + + /// Presentation timestamp in nanoseconds since the beginning of the video. + Nanoseconds = 1 (default), + + // Future values: FrameNr +} + +/// Timestamp inside a [archetypes.AssetVideo]. +struct VideoTimestamp ( + "attr.rust.derive": "Copy, PartialEq, Eq", + "attr.rerun.experimental" +) { + /// Timestamp value, type defined by `time_mode`. + video_time: long (order: 100); + + /// How to interpret `video_time`. + time_mode: VideoTimeMode (order: 200); +} diff --git a/crates/store/re_types/src/archetypes/.gitattributes b/crates/store/re_types/src/archetypes/.gitattributes index ee4909e04967..1dbfddfe109b 100644 --- a/crates/store/re_types/src/archetypes/.gitattributes +++ b/crates/store/re_types/src/archetypes/.gitattributes @@ -30,4 +30,5 @@ tensor.rs linguist-generated=true text_document.rs linguist-generated=true text_log.rs linguist-generated=true transform3d.rs linguist-generated=true +video_frame_reference.rs linguist-generated=true view_coordinates.rs linguist-generated=true diff --git a/crates/store/re_types/src/archetypes/asset_video.rs b/crates/store/re_types/src/archetypes/asset_video.rs index e54e3930eaeb..1133476159b8 100644 --- a/crates/store/re_types/src/archetypes/asset_video.rs +++ b/crates/store/re_types/src/archetypes/asset_video.rs @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs -// Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +// Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". #![allow(unused_imports)] #![allow(unused_parens)] @@ -18,12 +18,14 @@ use ::re_types_core::SerializationResult; use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; -/// **Archetype**: A video file. +/// **Archetype**: A video binary. /// /// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 and AV1 is currently supported, and not in all browsers. +/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. /// Follow for updates on the native support. /// +/// In order to display a video, you need to log a [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference] for each frame. +/// /// ⚠️ **This type is experimental and may be removed in future versions** #[derive(Clone, Debug)] pub struct AssetVideo { diff --git a/crates/store/re_types/src/archetypes/mod.rs b/crates/store/re_types/src/archetypes/mod.rs index 3c902398a8e7..46c6b161b698 100644 --- a/crates/store/re_types/src/archetypes/mod.rs +++ b/crates/store/re_types/src/archetypes/mod.rs @@ -45,6 +45,7 @@ mod text_document_ext; mod text_log; mod transform3d; mod transform3d_ext; +mod video_frame_reference; mod view_coordinates; mod view_coordinates_ext; @@ -76,4 +77,5 @@ pub use self::tensor::Tensor; pub use self::text_document::TextDocument; pub use self::text_log::TextLog; pub use self::transform3d::Transform3D; +pub use self::video_frame_reference::VideoFrameReference; pub use self::view_coordinates::ViewCoordinates; diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs new file mode 100644 index 000000000000..a70ca383cb02 --- /dev/null +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -0,0 +1,203 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Archetype**: References a single video frame. +/// +/// Used to display video frames from a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. +/// +/// ⚠️ **This type is experimental and may be removed in future versions** +#[derive(Clone, Debug)] +pub struct VideoFrameReference { + /// References the closest video frame to this timestamp. + /// + /// Note that this uses the closest video frame instead of the latest at this timestamp + /// in order to be more forgiving of rounding errors for inprecise timestamp types. + pub timestamp: crate::components::VideoTimestamp, + + /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. + /// + /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. + pub video_reference: Option, +} + +impl ::re_types_core::SizeBytes for VideoFrameReference { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.timestamp.heap_size_bytes() + self.video_reference.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + && >::is_pod() + } +} + +static REQUIRED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.VideoTimestamp".into()]); + +static RECOMMENDED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.VideoFrameReferenceIndicator".into()]); + +static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.EntityPath".into()]); + +static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 3usize]> = + once_cell::sync::Lazy::new(|| { + [ + "rerun.components.VideoTimestamp".into(), + "rerun.components.VideoFrameReferenceIndicator".into(), + "rerun.components.EntityPath".into(), + ] + }); + +impl VideoFrameReference { + /// The total number of components in the archetype: 1 required, 1 recommended, 1 optional + pub const NUM_COMPONENTS: usize = 3usize; +} + +/// Indicator component for the [`VideoFrameReference`] [`::re_types_core::Archetype`] +pub type VideoFrameReferenceIndicator = + ::re_types_core::GenericIndicatorComponent; + +impl ::re_types_core::Archetype for VideoFrameReference { + type Indicator = VideoFrameReferenceIndicator; + + #[inline] + fn name() -> ::re_types_core::ArchetypeName { + "rerun.archetypes.VideoFrameReference".into() + } + + #[inline] + fn display_name() -> &'static str { + "Video frame reference" + } + + #[inline] + fn indicator() -> MaybeOwnedComponentBatch<'static> { + static INDICATOR: VideoFrameReferenceIndicator = VideoFrameReferenceIndicator::DEFAULT; + MaybeOwnedComponentBatch::Ref(&INDICATOR) + } + + #[inline] + fn required_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + REQUIRED_COMPONENTS.as_slice().into() + } + + #[inline] + fn recommended_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + RECOMMENDED_COMPONENTS.as_slice().into() + } + + #[inline] + fn optional_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + OPTIONAL_COMPONENTS.as_slice().into() + } + + #[inline] + fn all_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + ALL_COMPONENTS.as_slice().into() + } + + #[inline] + fn from_arrow_components( + arrow_data: impl IntoIterator)>, + ) -> DeserializationResult { + re_tracing::profile_function!(); + use ::re_types_core::{Loggable as _, ResultExt as _}; + let arrays_by_name: ::std::collections::HashMap<_, _> = arrow_data + .into_iter() + .map(|(name, array)| (name.full_name(), array)) + .collect(); + let timestamp = { + let array = arrays_by_name + .get("rerun.components.VideoTimestamp") + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.VideoFrameReference#timestamp")?; + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.VideoFrameReference#timestamp")? + .into_iter() + .next() + .flatten() + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.VideoFrameReference#timestamp")? + }; + let video_reference = if let Some(array) = arrays_by_name.get("rerun.components.EntityPath") + { + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.VideoFrameReference#video_reference")? + .into_iter() + .next() + .flatten() + } else { + None + }; + Ok(Self { + timestamp, + video_reference, + }) + } +} + +impl ::re_types_core::AsComponents for VideoFrameReference { + fn as_component_batches(&self) -> Vec> { + re_tracing::profile_function!(); + use ::re_types_core::Archetype as _; + [ + Some(Self::indicator()), + Some((&self.timestamp as &dyn ComponentBatch).into()), + self.video_reference + .as_ref() + .map(|comp| (comp as &dyn ComponentBatch).into()), + ] + .into_iter() + .flatten() + .collect() + } +} + +impl ::re_types_core::ArchetypeReflectionMarker for VideoFrameReference {} + +impl VideoFrameReference { + /// Create a new `VideoFrameReference`. + #[inline] + pub fn new(timestamp: impl Into) -> Self { + Self { + timestamp: timestamp.into(), + video_reference: None, + } + } + + /// Optional reference to an entity with a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. + /// + /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. + #[inline] + pub fn with_video_reference( + mut self, + video_reference: impl Into, + ) -> Self { + self.video_reference = Some(video_reference.into()); + self + } +} diff --git a/crates/store/re_types/src/components/.gitattributes b/crates/store/re_types/src/components/.gitattributes index 0473984ef016..ce1cd46d7000 100644 --- a/crates/store/re_types/src/components/.gitattributes +++ b/crates/store/re_types/src/components/.gitattributes @@ -12,6 +12,7 @@ colormap.rs linguist-generated=true depth_meter.rs linguist-generated=true disconnected_space.rs linguist-generated=true draw_order.rs linguist-generated=true +entity_path.rs linguist-generated=true fill_mode.rs linguist-generated=true fill_ratio.rs linguist-generated=true gamma_correction.rs linguist-generated=true @@ -60,4 +61,5 @@ translation3d.rs linguist-generated=true triangle_indices.rs linguist-generated=true vector2d.rs linguist-generated=true vector3d.rs linguist-generated=true +video_timestamp.rs linguist-generated=true view_coordinates.rs linguist-generated=true diff --git a/crates/store/re_types/src/components/entity_path.rs b/crates/store/re_types/src/components/entity_path.rs new file mode 100644 index 000000000000..2275f2cb5a79 --- /dev/null +++ b/crates/store/re_types/src/components/entity_path.rs @@ -0,0 +1,105 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/components/entity_path.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Component**: A path to an entity, usually to reference some data that is part of the target entity. +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct EntityPath(pub crate::datatypes::EntityPath); + +impl ::re_types_core::SizeBytes for EntityPath { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.0.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + } +} + +impl> From for EntityPath { + fn from(v: T) -> Self { + Self(v.into()) + } +} + +impl std::borrow::Borrow for EntityPath { + #[inline] + fn borrow(&self) -> &crate::datatypes::EntityPath { + &self.0 + } +} + +impl std::ops::Deref for EntityPath { + type Target = crate::datatypes::EntityPath; + + #[inline] + fn deref(&self) -> &crate::datatypes::EntityPath { + &self.0 + } +} + +impl std::ops::DerefMut for EntityPath { + #[inline] + fn deref_mut(&mut self) -> &mut crate::datatypes::EntityPath { + &mut self.0 + } +} + +::re_types_core::macros::impl_into_cow!(EntityPath); + +impl ::re_types_core::Loggable for EntityPath { + type Name = ::re_types_core::ComponentName; + + #[inline] + fn name() -> Self::Name { + "rerun.components.EntityPath".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + crate::datatypes::EntityPath::arrow_datatype() + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { + datum.map(|datum| match datum.into() { + ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), + ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), + }) + })) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + crate::datatypes::EntityPath::from_arrow_opt(arrow_data) + .map(|v| v.into_iter().map(|v| v.map(Self)).collect()) + } +} diff --git a/crates/store/re_types/src/components/mod.rs b/crates/store/re_types/src/components/mod.rs index 8ece75aaea25..e65ab7eed0c0 100644 --- a/crates/store/re_types/src/components/mod.rs +++ b/crates/store/re_types/src/components/mod.rs @@ -18,6 +18,7 @@ mod disconnected_space; mod disconnected_space_ext; mod draw_order; mod draw_order_ext; +mod entity_path; mod fill_mode; mod fill_ratio; mod fill_ratio_ext; @@ -105,6 +106,8 @@ mod vector2d; mod vector2d_ext; mod vector3d; mod vector3d_ext; +mod video_timestamp; +mod video_timestamp_ext; mod view_coordinates; mod view_coordinates_ext; @@ -119,6 +122,7 @@ pub use self::colormap::Colormap; pub use self::depth_meter::DepthMeter; pub use self::disconnected_space::DisconnectedSpace; pub use self::draw_order::DrawOrder; +pub use self::entity_path::EntityPath; pub use self::fill_mode::FillMode; pub use self::fill_ratio::FillRatio; pub use self::gamma_correction::GammaCorrection; @@ -166,4 +170,5 @@ pub use self::translation3d::Translation3D; pub use self::triangle_indices::TriangleIndices; pub use self::vector2d::Vector2D; pub use self::vector3d::Vector3D; +pub use self::video_timestamp::VideoTimestamp; pub use self::view_coordinates::ViewCoordinates; diff --git a/crates/store/re_types/src/components/video_timestamp.rs b/crates/store/re_types/src/components/video_timestamp.rs new file mode 100644 index 000000000000..64e30ae7553c --- /dev/null +++ b/crates/store/re_types/src/components/video_timestamp.rs @@ -0,0 +1,107 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/components/video_timestamp.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Component**: Timestamp inside a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. +/// +/// ⚠️ **This type is experimental and may be removed in future versions** +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] +#[repr(transparent)] +pub struct VideoTimestamp(pub crate::datatypes::VideoTimestamp); + +impl ::re_types_core::SizeBytes for VideoTimestamp { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.0.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + } +} + +impl> From for VideoTimestamp { + fn from(v: T) -> Self { + Self(v.into()) + } +} + +impl std::borrow::Borrow for VideoTimestamp { + #[inline] + fn borrow(&self) -> &crate::datatypes::VideoTimestamp { + &self.0 + } +} + +impl std::ops::Deref for VideoTimestamp { + type Target = crate::datatypes::VideoTimestamp; + + #[inline] + fn deref(&self) -> &crate::datatypes::VideoTimestamp { + &self.0 + } +} + +impl std::ops::DerefMut for VideoTimestamp { + #[inline] + fn deref_mut(&mut self) -> &mut crate::datatypes::VideoTimestamp { + &mut self.0 + } +} + +::re_types_core::macros::impl_into_cow!(VideoTimestamp); + +impl ::re_types_core::Loggable for VideoTimestamp { + type Name = ::re_types_core::ComponentName; + + #[inline] + fn name() -> Self::Name { + "rerun.components.VideoTimestamp".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + crate::datatypes::VideoTimestamp::arrow_datatype() + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + crate::datatypes::VideoTimestamp::to_arrow_opt(data.into_iter().map(|datum| { + datum.map(|datum| match datum.into() { + ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), + ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), + }) + })) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + crate::datatypes::VideoTimestamp::from_arrow_opt(arrow_data) + .map(|v| v.into_iter().map(|v| v.map(Self)).collect()) + } +} diff --git a/crates/store/re_types/src/components/video_timestamp_ext.rs b/crates/store/re_types/src/components/video_timestamp_ext.rs new file mode 100644 index 000000000000..df7a6f9947bc --- /dev/null +++ b/crates/store/re_types/src/components/video_timestamp_ext.rs @@ -0,0 +1,9 @@ +use super::VideoTimestamp; + +impl VideoTimestamp { + /// Create new timestamp from nanoseconds since video start. + #[inline] + pub fn new_nanoseconds(nanos: i64) -> Self { + crate::datatypes::VideoTimestamp::new_nanoseconds(nanos).into() + } +} diff --git a/crates/store/re_types/src/datatypes/.gitattributes b/crates/store/re_types/src/datatypes/.gitattributes index af98d9b2ad4b..a79b23c34fb8 100644 --- a/crates/store/re_types/src/datatypes/.gitattributes +++ b/crates/store/re_types/src/datatypes/.gitattributes @@ -33,4 +33,6 @@ uvec4d.rs linguist-generated=true vec2d.rs linguist-generated=true vec3d.rs linguist-generated=true vec4d.rs linguist-generated=true +video_time_mode.rs linguist-generated=true +video_timestamp.rs linguist-generated=true view_coordinates.rs linguist-generated=true diff --git a/crates/store/re_types/src/datatypes/mod.rs b/crates/store/re_types/src/datatypes/mod.rs index e77eee7490d1..29fd05d8b9d5 100644 --- a/crates/store/re_types/src/datatypes/mod.rs +++ b/crates/store/re_types/src/datatypes/mod.rs @@ -61,6 +61,9 @@ mod vec3d; mod vec3d_ext; mod vec4d; mod vec4d_ext; +mod video_time_mode; +mod video_timestamp; +mod video_timestamp_ext; mod view_coordinates; mod view_coordinates_ext; @@ -95,4 +98,6 @@ pub use self::uvec4d::UVec4D; pub use self::vec2d::Vec2D; pub use self::vec3d::Vec3D; pub use self::vec4d::Vec4D; +pub use self::video_time_mode::VideoTimeMode; +pub use self::video_timestamp::VideoTimestamp; pub use self::view_coordinates::ViewCoordinates; diff --git a/crates/store/re_types/src/datatypes/video_time_mode.rs b/crates/store/re_types/src/datatypes/video_time_mode.rs new file mode 100644 index 000000000000..d0e8e7ee6ded --- /dev/null +++ b/crates/store/re_types/src/datatypes/video_time_mode.rs @@ -0,0 +1,146 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Datatype**: Specifies how to interpret the `video_time` field of a [`datatypes::VideoTimestamp`][crate::datatypes::VideoTimestamp]. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Default)] +#[repr(u8)] +pub enum VideoTimeMode { + /// Presentation timestamp in nanoseconds since the beginning of the video. + #[default] + Nanoseconds = 1, +} + +impl ::re_types_core::reflection::Enum for VideoTimeMode { + #[inline] + fn variants() -> &'static [Self] { + &[Self::Nanoseconds] + } + + #[inline] + fn docstring_md(self) -> &'static str { + match self { + Self::Nanoseconds => { + "Presentation timestamp in nanoseconds since the beginning of the video." + } + } + } +} + +impl ::re_types_core::SizeBytes for VideoTimeMode { + #[inline] + fn heap_size_bytes(&self) -> u64 { + 0 + } + + #[inline] + fn is_pod() -> bool { + true + } +} + +impl std::fmt::Display for VideoTimeMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Nanoseconds => write!(f, "Nanoseconds"), + } + } +} + +::re_types_core::macros::impl_into_cow!(VideoTimeMode); + +impl ::re_types_core::Loggable for VideoTimeMode { + type Name = ::re_types_core::DatatypeName; + + #[inline] + fn name() -> Self::Name { + "rerun.datatypes.VideoTimeMode".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + #![allow(clippy::wildcard_imports)] + use arrow2::datatypes::*; + DataType::UInt8 + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data0): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + let datum = datum.map(|datum| *datum as u8); + (datum.is_some(), datum) + }) + .unzip(); + let data0_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + Self::arrow_datatype(), + data0.into_iter().map(|v| v.unwrap_or_default()).collect(), + data0_bitmap, + ) + .boxed() + }) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, buffer::*, datatypes::*}; + Ok(arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + let expected = Self::arrow_datatype(); + let actual = arrow_data.data_type().clone(); + DeserializationError::datatype_mismatch(expected, actual) + }) + .with_context("rerun.datatypes.VideoTimeMode#enum")? + .into_iter() + .map(|opt| opt.copied()) + .map(|typ| match typ { + Some(1) => Ok(Some(Self::Nanoseconds)), + None => Ok(None), + Some(invalid) => Err(DeserializationError::missing_union_arm( + Self::arrow_datatype(), + "", + invalid as _, + )), + }) + .collect::>>>() + .with_context("rerun.datatypes.VideoTimeMode")?) + } +} diff --git a/crates/store/re_types/src/datatypes/video_timestamp.rs b/crates/store/re_types/src/datatypes/video_timestamp.rs new file mode 100644 index 000000000000..d395bfb35338 --- /dev/null +++ b/crates/store/re_types/src/datatypes/video_timestamp.rs @@ -0,0 +1,226 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Datatype**: Timestamp inside a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. +/// +/// ⚠️ **This type is experimental and may be removed in future versions** +#[derive(Clone, Debug, Copy, PartialEq, Eq)] +pub struct VideoTimestamp { + /// Timestamp value, type defined by `time_mode`. + pub video_time: i64, + + /// How to interpret `video_time`. + pub time_mode: crate::datatypes::VideoTimeMode, +} + +impl ::re_types_core::SizeBytes for VideoTimestamp { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.video_time.heap_size_bytes() + self.time_mode.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() && ::is_pod() + } +} + +::re_types_core::macros::impl_into_cow!(VideoTimestamp); + +impl ::re_types_core::Loggable for VideoTimestamp { + type Name = ::re_types_core::DatatypeName; + + #[inline] + fn name() -> Self::Name { + "rerun.datatypes.VideoTimestamp".into() + } + + #[inline] + fn arrow_datatype() -> arrow2::datatypes::DataType { + #![allow(clippy::wildcard_imports)] + use arrow2::datatypes::*; + DataType::Struct(std::sync::Arc::new(vec![ + Field::new("video_time", DataType::Int64, false), + Field::new( + "time_mode", + ::arrow_datatype(), + false, + ), + ])) + } + + fn to_arrow_opt<'a>( + data: impl IntoIterator>>>, + ) -> SerializationResult> + where + Self: Clone + 'a, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + Self::arrow_datatype(), + vec![ + { + let (somes, video_time): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| datum.video_time.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let video_time_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + DataType::Int64, + video_time + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + video_time_bitmap, + ) + .boxed() + }, + { + let (somes, time_mode): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| datum.time_mode.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let time_mode_bitmap: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = time_mode_bitmap; + crate::datatypes::VideoTimeMode::to_arrow_opt(time_mode)? + } + }, + ], + bitmap, + ) + .boxed() + }) + } + + fn from_arrow_opt( + arrow_data: &dyn arrow2::array::Array, + ) -> DeserializationResult>> + where + Self: Sized, + { + #![allow(clippy::wildcard_imports)] + use ::re_types_core::{Loggable as _, ResultExt as _}; + use arrow2::{array::*, buffer::*, datatypes::*}; + Ok({ + let arrow_data = arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + let expected = Self::arrow_datatype(); + let actual = arrow_data.data_type().clone(); + DeserializationError::datatype_mismatch(expected, actual) + }) + .with_context("rerun.datatypes.VideoTimestamp")?; + if arrow_data.is_empty() { + Vec::new() + } else { + let (arrow_data_fields, arrow_data_arrays) = + (arrow_data.fields(), arrow_data.values()); + let arrays_by_name: ::std::collections::HashMap<_, _> = arrow_data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(arrow_data_arrays) + .collect(); + let video_time = { + if !arrays_by_name.contains_key("video_time") { + return Err(DeserializationError::missing_struct_field( + Self::arrow_datatype(), + "video_time", + )) + .with_context("rerun.datatypes.VideoTimestamp"); + } + let arrow_data = &**arrays_by_name["video_time"]; + arrow_data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + let expected = DataType::Int64; + let actual = arrow_data.data_type().clone(); + DeserializationError::datatype_mismatch(expected, actual) + }) + .with_context("rerun.datatypes.VideoTimestamp#video_time")? + .into_iter() + .map(|opt| opt.copied()) + }; + let time_mode = { + if !arrays_by_name.contains_key("time_mode") { + return Err(DeserializationError::missing_struct_field( + Self::arrow_datatype(), + "time_mode", + )) + .with_context("rerun.datatypes.VideoTimestamp"); + } + let arrow_data = &**arrays_by_name["time_mode"]; + crate::datatypes::VideoTimeMode::from_arrow_opt(arrow_data) + .with_context("rerun.datatypes.VideoTimestamp#time_mode")? + .into_iter() + }; + arrow2::bitmap::utils::ZipValidity::new_with_validity( + ::itertools::izip!(video_time, time_mode), + arrow_data.validity(), + ) + .map(|opt| { + opt.map(|(video_time, time_mode)| { + Ok(Self { + video_time: video_time + .ok_or_else(DeserializationError::missing_data) + .with_context( + "rerun.datatypes.VideoTimestamp#video_time", + )?, + time_mode: time_mode + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.datatypes.VideoTimestamp#time_mode")?, + }) + }) + .transpose() + }) + .collect::>>() + .with_context("rerun.datatypes.VideoTimestamp")? + } + }) + } +} diff --git a/crates/store/re_types/src/datatypes/video_timestamp_ext.rs b/crates/store/re_types/src/datatypes/video_timestamp_ext.rs new file mode 100644 index 000000000000..342e0d1051db --- /dev/null +++ b/crates/store/re_types/src/datatypes/video_timestamp_ext.rs @@ -0,0 +1,21 @@ +use super::{VideoTimeMode, VideoTimestamp}; + +impl VideoTimestamp { + /// Create new timestamp from nanoseconds since video start. + #[inline] + pub fn new_nanoseconds(nanos: i64) -> Self { + Self { + video_time: nanos, + time_mode: VideoTimeMode::Nanoseconds, + } + } +} + +impl Default for VideoTimestamp { + fn default() -> Self { + Self { + video_time: 0, + time_mode: VideoTimeMode::Nanoseconds, + } + } +} diff --git a/crates/store/re_types_core/src/loggable_batch.rs b/crates/store/re_types_core/src/loggable_batch.rs index 802dbbf4e4a6..0ad382f06ad4 100644 --- a/crates/store/re_types_core/src/loggable_batch.rs +++ b/crates/store/re_types_core/src/loggable_batch.rs @@ -1,5 +1,7 @@ use crate::{Component, ComponentName, Loggable, SerializationResult}; +use arrow2::array::ListArray as ArrowListArray; + #[allow(unused_imports)] // used in docstrings use crate::Archetype; @@ -32,7 +34,17 @@ pub trait LoggableBatch { /// /// Any [`LoggableBatch`] with a [`Loggable::Name`] set to [`ComponentName`] automatically /// implements [`ComponentBatch`]. -pub trait ComponentBatch: LoggableBatch {} +pub trait ComponentBatch: LoggableBatch { + /// Serializes the batch into an Arrow list array with a single component per list. + fn to_arrow_list_array(&self) -> SerializationResult> { + let array = self.to_arrow()?; + let offsets = + arrow2::offset::Offsets::try_from_lengths(std::iter::repeat(1).take(array.len()))?; + let data_type = ArrowListArray::::default_datatype(array.data_type().clone()); + ArrowListArray::::try_new(data_type, offsets.into(), array.to_boxed(), None) + .map_err(|err| err.into()) + } +} /// Holds either an owned [`ComponentBatch`] that lives on heap, or a reference to one. /// diff --git a/crates/store/re_video/src/lib.rs b/crates/store/re_video/src/lib.rs index 204fa42fb1f4..f84a836aaaf5 100644 --- a/crates/store/re_video/src/lib.rs +++ b/crates/store/re_video/src/lib.rs @@ -72,18 +72,26 @@ pub struct TimeMs(OrderedFloat); impl TimeMs { pub const ZERO: Self = Self(OrderedFloat(0.0)); + #[inline] pub fn new(ms: f64) -> Self { Self(OrderedFloat(ms)) } + #[inline] pub fn as_f64(&self) -> f64 { self.0.into_inner() } + + #[inline] + pub fn as_nanoseconds(self) -> i64 { + (self.0 * 1_000_000.0).round() as i64 + } } impl std::ops::Add for TimeMs { type Output = Self; + #[inline] fn add(self, rhs: Self) -> Self::Output { Self(self.0 + rhs.0) } @@ -92,6 +100,7 @@ impl std::ops::Add for TimeMs { impl std::ops::Sub for TimeMs { type Output = Self; + #[inline] fn sub(self, rhs: Self) -> Self::Output { Self(self.0 - rhs.0) } diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index f2e794bbf64e..e12741e7016d 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -5,7 +5,6 @@ use std::sync::Weak; use std::sync::{atomic::AtomicI64, Arc}; use ahash::HashMap; -use arrow2::offset::Offsets; use crossbeam::channel::{Receiver, Sender}; use itertools::Either; use parking_lot::Mutex; @@ -921,27 +920,7 @@ impl RecordingStream { let components: Result, ChunkError> = components .into_iter() - .map(|batch| { - let array = batch.to_arrow()?; - - let offsets = Offsets::try_from_lengths(std::iter::repeat(1).take(array.len())) - .map_err(|err| ChunkError::Malformed { - reason: format!("Failed to create offsets: {err}"), - })?; - let data_type = ArrowListArray::::default_datatype(array.data_type().clone()); - - let array = ArrowListArray::::try_new( - data_type, - offsets.into(), - array.to_boxed(), - None, - ) - .map_err(|err| ChunkError::Malformed { - reason: format!("Failed to wrap in List array: {err}"), - })?; - - Ok((batch.name(), array)) - }) + .map(|batch| Ok((batch.name(), batch.to_arrow_list_array()?))) .collect(); let components: BTreeMap> = diff --git a/crates/viewer/re_space_view_spatial/src/video_cache.rs b/crates/viewer/re_space_view_spatial/src/video_cache.rs index 229dac9be86c..370a1464489b 100644 --- a/crates/viewer/re_space_view_spatial/src/video_cache.rs +++ b/crates/viewer/re_space_view_spatial/src/video_cache.rs @@ -1,20 +1,20 @@ -use egui::mutex::Mutex; use re_entity_db::VersionedInstancePathHash; -use re_log_types::hash::Hash64; -use re_renderer::renderer::Video; -use re_renderer::RenderContext; +use re_renderer::{renderer::Video, RenderContext}; use re_types::components::MediaType; use re_viewer_context::Cache; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; -use std::sync::Arc; + +use egui::mutex::Mutex; + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; // ---------------------------------------------------------------------------- #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub struct VideoCacheKey { pub versioned_instance_path_hash: VersionedInstancePathHash, - pub query_result_hash: Hash64, pub media_type: Option, } @@ -54,7 +54,11 @@ impl VideoCache { video, } }); - entry.used_this_frame.store(true, Ordering::Relaxed); + + // Using acquire/release here to be on the safe side and for semantical soundness: + // Whatever thread is acquiring the fact that this was used, should also see/acquire + // the side effect of having the entry contained in the cache. + entry.used_this_frame.store(true, Ordering::Release); entry.video.clone() } } @@ -62,13 +66,13 @@ impl VideoCache { impl Cache for VideoCache { fn begin_frame(&mut self) { for v in self.0.values() { - v.used_this_frame.store(false, Ordering::Relaxed); + v.used_this_frame.store(false, Ordering::Release); } } fn purge_memory(&mut self) { self.0 - .retain(|_, v| v.used_this_frame.load(Ordering::Relaxed)); + .retain(|_, v| v.used_this_frame.load(Ordering::Acquire)); } fn as_any_mut(&mut self) -> &mut dyn std::any::Any { diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs index eccc0d5d5589..f60415b923e3 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs @@ -86,7 +86,7 @@ pub fn register_2d_spatial_visualizers( system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; - system_registry.register_visualizer::()?; + system_registry.register_visualizer::()?; Ok(()) } @@ -111,7 +111,7 @@ pub fn register_3d_spatial_visualizers( system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; - system_registry.register_visualizer::()?; + system_registry.register_visualizer::()?; Ok(()) } diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index 809a8e2c8d53..dcbe5bbb5755 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -1,50 +1,36 @@ -use glam::Vec3; -use re_chunk_store::RowId; -use re_chunk_store::TimeInt; -use re_log_types::hash::Hash64; -use re_log_types::TimeType; -use re_renderer::renderer::ColormappedTexture; -use re_renderer::renderer::RectangleOptions; -use re_renderer::renderer::TextureFilterMag; -use re_renderer::renderer::TextureFilterMin; -use re_renderer::renderer::TexturedRect; -use re_renderer::RenderContext; -use re_types::archetypes::AssetVideo; -use re_types::components::Blob; -use re_types::components::MediaType; -use re_types::ArrowBuffer; -use re_types::ArrowString; -use re_types::Loggable as _; -use re_viewer_context::SpaceViewClass as _; +use egui::mutex::Mutex; + +use re_log_types::EntityPath; +use re_renderer::renderer::{ + ColormappedTexture, RectangleOptions, TextureFilterMag, TextureFilterMin, TexturedRect, +}; +use re_types::{ + archetypes::{AssetVideo, VideoFrameReference}, + components::{Blob, EntityPath as EntityPathReferenceComponent, MediaType, VideoTimestamp}, + datatypes::VideoTimeMode, + Archetype, Loggable as _, +}; use re_viewer_context::{ - ApplicableEntities, IdentifiedViewSystem, QueryContext, SpaceViewSystemExecutionError, - ViewContext, ViewContextCollection, ViewQuery, VisualizableEntities, VisualizableFilterContext, - VisualizerQueryInfo, VisualizerSystem, + ApplicableEntities, IdentifiedViewSystem, SpaceViewClass as _, SpaceViewSystemExecutionError, + ViewContext, ViewContextCollection, ViewQuery, ViewerContext, VisualizableEntities, + VisualizableFilterContext, VisualizerQueryInfo, VisualizerSystem, }; -use crate::video_cache::VideoCache; -use crate::video_cache::VideoCacheKey; -use crate::visualizers::entity_iterator::iter_buffer; -use crate::SpatialSpaceView2D; use crate::{ - contexts::SpatialSceneEntityContext, view_kind::SpatialSpaceViewKind, - visualizers::filter_visualizable_2d_entities, + video_cache::{VideoCache, VideoCacheKey}, + view_kind::SpatialSpaceViewKind, + visualizers::{entity_iterator, filter_visualizable_2d_entities}, + SpatialSpaceView2D, }; use super::bounding_box_for_textured_rect; use super::{entity_iterator::process_archetype, SpatialViewVisualizerData}; -pub struct AssetVideoVisualizer { +pub struct VideoFrameReferenceVisualizer { pub data: SpatialViewVisualizerData, } -struct AssetVideoComponentData { - index: (TimeInt, RowId), - blob: ArrowBuffer, - media_type: Option, -} - -impl Default for AssetVideoVisualizer { +impl Default for VideoFrameReferenceVisualizer { fn default() -> Self { Self { data: SpatialViewVisualizerData::new(Some(SpatialSpaceViewKind::TwoD)), @@ -52,15 +38,15 @@ impl Default for AssetVideoVisualizer { } } -impl IdentifiedViewSystem for AssetVideoVisualizer { +impl IdentifiedViewSystem for VideoFrameReferenceVisualizer { fn identifier() -> re_viewer_context::ViewSystemIdentifier { - "Video".into() + "VideoFrameReference".into() } } -impl VisualizerSystem for AssetVideoVisualizer { +impl VisualizerSystem for VideoFrameReferenceVisualizer { fn visualizer_query_info(&self) -> VisualizerQueryInfo { - VisualizerQueryInfo::from_archetype::() + VisualizerQueryInfo::from_archetype::() } fn filter_visualizable_entities( @@ -84,47 +70,90 @@ impl VisualizerSystem for AssetVideoVisualizer { let mut rectangles = Vec::new(); - process_archetype::( + process_archetype::( ctx, view_query, context_systems, |ctx, spatial_ctx, results| { + // TODO(andreas): Should ignore range queries here and only do latest-at. + // Not only would this simplify the code here quite a bit, it would also avoid lots of overhead. + // Same is true for the image visualizers in general - there seems to be no practical reason to do range queries + // for visualization here. use re_space_view::RangeResultsExt as _; - let Some(all_blob_chunks) = results.get_required_chunks(&Blob::name()) else { - return Ok(()); - }; let timeline = ctx.query.timeline(); - let all_blobs_indexed = iter_buffer::(&all_blob_chunks, timeline, Blob::name()); - let all_media_types = results.iter_as(timeline, MediaType::name()); - - let data = re_query::range_zip_1x1(all_blobs_indexed, all_media_types.string()) - .filter_map(|(index, blobs, media_types)| { - blobs.first().map(|blob| AssetVideoComponentData { - index, - blob: blob.clone(), - media_type: media_types - .and_then(|media_types| media_types.first().cloned()), - }) - }); - - let current_time_nanoseconds = match timeline.typ() { - TimeType::Time => view_query.latest_at.as_f64(), - // TODO(jan): scale by ticks per second - #[allow(clippy::match_same_arms)] - TimeType::Sequence => view_query.latest_at.as_f64(), + let entity_path = ctx.target_entity_path; + + let Some(all_video_timestamp_chunks) = + results.get_required_chunks(&VideoTimestamp::name()) + else { + return Ok(()); }; - let current_time_seconds = current_time_nanoseconds / 1e9; - - self.process_data( - ctx, - render_ctx, - &mut rectangles, - spatial_ctx, - data, - current_time_seconds, - results.query_result_hash(), - ); + let all_video_references = + results.iter_as(timeline, EntityPathReferenceComponent::name()); + + for (_index, video_timestamps, video_references) in re_query::range_zip_1x1( + entity_iterator::iter_component( + &all_video_timestamp_chunks, + timeline, + VideoTimestamp::name(), + ), + all_video_references.string(), + ) { + let Some(video_timestamp): Option<&VideoTimestamp> = video_timestamps.first() + else { + continue; + }; + + // Follow the reference to the video asset. + let video_reference = video_references + .and_then(|v| v.first().map(|e| e.as_str().into())) + .unwrap_or_else(|| entity_path.clone()); + let Some(video) = + latest_at_query_video_from_datastore(ctx.viewer_ctx, &video_reference) + else { + continue; + }; + + let timestamp_in_seconds = match video_timestamp.time_mode { + VideoTimeMode::Nanoseconds => video_timestamp.video_time as f64 / 1e9, + }; + + let (texture, video_width, video_height) = { + let mut video = video.lock(); // TODO(andreas): Interior mutability for re_renderer's video would be nice. + ( + video.frame_at(timestamp_in_seconds), + video.width(), + video.height(), + ) + }; + + let world_from_entity = + spatial_ctx.transform_info.single_entity_transform_required( + ctx.target_entity_path, + Self::identifier().as_str(), + ); + let textured_rect = textured_rect_for_video_frame( + world_from_entity, + video_width, + video_height, + texture, + ); + + if spatial_ctx.space_view_class_identifier == SpatialSpaceView2D::identifier() { + // Only update the bounding box if this is a 2D space view. + // This is avoids a cyclic relationship where the image plane grows + // the bounds which in turn influence the size of the image plane. + // See: https://github.com/rerun-io/rerun/issues/3728 + self.data.add_bounding_box( + entity_path.hash(), + bounding_box_for_textured_rect(&textured_rect), + world_from_entity, + ); + } + + rectangles.push(textured_rect); + } Ok(()) }, @@ -157,80 +186,64 @@ impl VisualizerSystem for AssetVideoVisualizer { } } -// NOTE: Do not put profile scopes in these methods. They are called for all entities and all -// timestamps within a time range -- it's _a lot_. -impl AssetVideoVisualizer { - #[allow(clippy::unused_self)] - #[allow(clippy::too_many_arguments)] - fn process_data( - &mut self, - ctx: &QueryContext<'_>, - render_ctx: &RenderContext, - rectangles: &mut Vec, - ent_context: &SpatialSceneEntityContext<'_>, - data: impl Iterator, - current_time_seconds: f64, - query_result_hash: Hash64, - ) { - let entity_path = ctx.target_entity_path; - - for data in data { - let timestamp_s = current_time_seconds - data.index.0.as_f64() / 1e9; - let video = AssetVideo { - blob: data.blob.clone().into(), - media_type: data.media_type.clone().map(Into::into), - }; - - let primary_row_id = data.index.1; - let picking_instance_hash = re_entity_db::InstancePathHash::entity_all(entity_path); - - let video = ctx.viewer_ctx.cache.entry(|c: &mut VideoCache| { - c.entry( - &entity_path.to_string(), - VideoCacheKey { - versioned_instance_path_hash: picking_instance_hash - .versioned(primary_row_id), - query_result_hash, - media_type: data.media_type.clone().map(Into::into), - }, - &video.blob, - video.media_type.as_ref().map(|v| v.as_str()), - render_ctx, - ) - }); - - if let Some(video) = video { - let mut video = video.lock(); - let texture = video.frame_at(timestamp_s); - - let world_from_entity = ent_context - .transform_info - .single_entity_transform_required(ctx.target_entity_path, "Video"); - let textured_rect = TexturedRect { - top_left_corner_position: world_from_entity.transform_point3(Vec3::ZERO), - extent_u: world_from_entity.transform_vector3(Vec3::X * video.width() as f32), - extent_v: world_from_entity.transform_vector3(Vec3::Y * video.height() as f32), - - colormapped_texture: ColormappedTexture::from_unorm_rgba(texture), - options: RectangleOptions { - texture_filter_magnification: TextureFilterMag::Nearest, - texture_filter_minification: TextureFilterMin::Linear, - ..Default::default() - }, - }; - - if ent_context.space_view_class_identifier == SpatialSpaceView2D::identifier() { - self.data.add_bounding_box( - entity_path.hash(), - bounding_box_for_textured_rect(&textured_rect), - world_from_entity, - ); - } - - rectangles.push(textured_rect); - }; - } +fn textured_rect_for_video_frame( + world_from_entity: glam::Affine3A, + video_width: u32, + video_height: u32, + texture: re_renderer::resource_managers::GpuTexture2D, +) -> TexturedRect { + TexturedRect { + top_left_corner_position: world_from_entity.transform_point3(glam::Vec3::ZERO), + // Make sure to use the video instead of texture size here, + // since it may be a placeholder which doesn't have the full size yet. + extent_u: world_from_entity.transform_vector3(glam::Vec3::X * video_width as f32), + extent_v: world_from_entity.transform_vector3(glam::Vec3::Y * video_height as f32), + + colormapped_texture: ColormappedTexture::from_unorm_rgba(texture), + options: RectangleOptions { + texture_filter_magnification: TextureFilterMag::Nearest, + texture_filter_minification: TextureFilterMin::Linear, + ..Default::default() + }, } } -re_viewer_context::impl_component_fallback_provider!(AssetVideoVisualizer => []); +/// Queries a video from the datstore and caches it in the video cache. +/// +/// Note that this does *NOT* check the blueprint store at all. +/// For this, we'd need a [`re_viewer_context::DataResult`] instead of merely a [`EntityPath`]. +fn latest_at_query_video_from_datastore( + ctx: &ViewerContext<'_>, + entity_path: &EntityPath, +) -> Option>> { + let query = ctx.current_query(); + + let results = ctx.recording().query_caches().latest_at( + ctx.recording_store(), + &query, + entity_path, + AssetVideo::all_components().iter().copied(), + ); + + let blob_row_id = results.component_row_id(&Blob::name())?; + let blob = results.component_instance::(0)?; + let media_type = results.component_instance::(0); + + ctx.cache.entry(|c: &mut VideoCache| { + c.entry( + &entity_path.to_string(), + VideoCacheKey { + versioned_instance_path_hash: re_entity_db::InstancePathHash::entity_all( + entity_path, + ) + .versioned(blob_row_id), + media_type: media_type.clone(), + }, + &blob, + media_type.as_ref().map(|v| v.as_str()), + ctx.render_ctx?, + ) + }) +} + +re_viewer_context::impl_component_fallback_provider!(VideoFrameReferenceVisualizer => []); diff --git a/crates/viewer/re_viewer/src/reflection/mod.rs b/crates/viewer/re_viewer/src/reflection/mod.rs index bb168edd9ddf..38617ffe1f4b 100644 --- a/crates/viewer/re_viewer/src/reflection/mod.rs +++ b/crates/viewer/re_viewer/src/reflection/mod.rs @@ -342,6 +342,13 @@ fn generate_component_reflection() -> Result::name(), + ComponentReflection { + docstring_md: "A path to an entity, usually to reference some data that is part of the target entity.", + placeholder: Some(EntityPath::default().to_arrow()?), + }, + ), ( ::name(), ComponentReflection { @@ -671,6 +678,13 @@ fn generate_component_reflection() -> Result::name(), + ComponentReflection { + docstring_md: "Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + placeholder: Some(VideoTimestamp::default().to_arrow()?), + }, + ), ( ::name(), ComponentReflection { @@ -1444,6 +1458,23 @@ fn generate_archetype_reflection() -> ArchetypeReflectionMap { ], }, ), + ( + ArchetypeName::new("rerun.archetypes.VideoFrameReference"), + ArchetypeReflection { + display_name: "Video frame reference", + fields: vec![ + ArchetypeFieldReflection { component_name : + "rerun.components.VideoTimestamp".into(), display_name : "Timestamp", + docstring_md : + "References the closest video frame to this timestamp.\n\nNote that this uses the closest video frame instead of the latest at this timestamp\nin order to be more forgiving of rounding errors for inprecise timestamp types.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + is_required : true, }, ArchetypeFieldReflection { component_name : + "rerun.components.EntityPath".into(), display_name : + "Video reference", docstring_md : + "Optional reference to an entity with a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).\n\nIf none is specified, the video is assumed to be at the same entity.\nNote that blueprint overrides on the referenced video will be ignored regardless,\nas this is always interpreted as a reference to the data store.\n\n⚠\u{fe0f} **This type is experimental and may be removed in future versions**", + is_required : false, }, + ], + }, + ), ( ArchetypeName::new("rerun.archetypes.ViewCoordinates"), ArchetypeReflection { diff --git a/docs/content/reference/types/archetypes.md b/docs/content/reference/types/archetypes.md index 4e51b9910a27..be8c53d8ec70 100644 --- a/docs/content/reference/types/archetypes.md +++ b/docs/content/reference/types/archetypes.md @@ -56,7 +56,8 @@ This page lists all built-in archetypes. ## Other * [`AnnotationContext`](archetypes/annotation_context.md): The annotation context provides additional information on how to display entities. -* [`AssetVideo`](archetypes/asset_video.md): A video file. +* [`AssetVideo`](archetypes/asset_video.md): A video binary. * [`Clear`](archetypes/clear.md): Empties all the components of an entity. * [`DisconnectedSpace`](archetypes/disconnected_space.md): Spatially disconnect this entity from its parent. +* [`VideoFrameReference`](archetypes/video_frame_reference.md): References a single video frame. diff --git a/docs/content/reference/types/archetypes/.gitattributes b/docs/content/reference/types/archetypes/.gitattributes index e273ee9420db..c0ce3b7563c5 100644 --- a/docs/content/reference/types/archetypes/.gitattributes +++ b/docs/content/reference/types/archetypes/.gitattributes @@ -30,4 +30,5 @@ tensor.md linguist-generated=true text_document.md linguist-generated=true text_log.md linguist-generated=true transform3d.md linguist-generated=true +video_frame_reference.md linguist-generated=true view_coordinates.md linguist-generated=true diff --git a/docs/content/reference/types/archetypes/asset_video.md b/docs/content/reference/types/archetypes/asset_video.md index b24fdcebdf1f..110d10844a8b 100644 --- a/docs/content/reference/types/archetypes/asset_video.md +++ b/docs/content/reference/types/archetypes/asset_video.md @@ -6,12 +6,14 @@ title: "AssetVideo" ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** -A video file. +A video binary. NOTE: Videos can only be viewed in the Rerun web viewer. -Only MP4 and AV1 is currently supported, and not in all browsers. +Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. Follow for updates on the native support. +In order to display a video, you need to log a [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference) for each frame. + ## Components **Required**: [`Blob`](../components/blob.md) diff --git a/docs/content/reference/types/archetypes/video_frame_reference.md b/docs/content/reference/types/archetypes/video_frame_reference.md new file mode 100644 index 000000000000..641e9cbe7d6d --- /dev/null +++ b/docs/content/reference/types/archetypes/video_frame_reference.md @@ -0,0 +1,23 @@ +--- +title: "VideoFrameReference" +--- + + + +⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + +References a single video frame. + +Used to display video frames from a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). + +## Components + +**Required**: [`VideoTimestamp`](../components/video_timestamp.md) + +**Optional**: [`EntityPath`](../components/entity_path.md) + +## API reference links + * 🌊 [C++ API docs for `VideoFrameReference`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1archetypes_1_1VideoFrameReference.html) + * 🐍 [Python API docs for `VideoFrameReference`](https://ref.rerun.io/docs/python/stable/common/archetypes#rerun.archetypes.VideoFrameReference) + * 🦀 [Rust API docs for `VideoFrameReference`](https://docs.rs/rerun/latest/rerun/archetypes/struct.VideoFrameReference.html) + diff --git a/docs/content/reference/types/components.md b/docs/content/reference/types/components.md index 999b1c6e53b8..1139ef975b74 100644 --- a/docs/content/reference/types/components.md +++ b/docs/content/reference/types/components.md @@ -25,6 +25,7 @@ on [Entities and Components](../../concepts/entity-component.md). * [`DepthMeter`](components/depth_meter.md): The world->depth map scaling factor. * [`DisconnectedSpace`](components/disconnected_space.md): Spatially disconnect this entity from its parent. * [`DrawOrder`](components/draw_order.md): Draw order of 2D elements. Higher values are drawn on top of lower values. +* [`EntityPath`](components/entity_path.md): A path to an entity, usually to reference some data that is part of the target entity. * [`FillMode`](components/fill_mode.md): How a geometric shape is drawn and colored. * [`FillRatio`](components/fill_ratio.md): How much a primitive fills out the available space. * [`GammaCorrection`](components/gamma_correction.md): A gamma correction value to be used with a scalar value or color. @@ -72,5 +73,6 @@ on [Entities and Components](../../concepts/entity-component.md). * [`TriangleIndices`](components/triangle_indices.md): The three indices of a triangle in a triangle mesh. * [`Vector2D`](components/vector2d.md): A vector in 2D space. * [`Vector3D`](components/vector3d.md): A vector in 3D space. +* [`VideoTimestamp`](components/video_timestamp.md): Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). * [`ViewCoordinates`](components/view_coordinates.md): How we interpret the coordinate system of an entity/space. diff --git a/docs/content/reference/types/components/.gitattributes b/docs/content/reference/types/components/.gitattributes index a2b2fef2de52..25cfd40ed1e7 100644 --- a/docs/content/reference/types/components/.gitattributes +++ b/docs/content/reference/types/components/.gitattributes @@ -13,6 +13,7 @@ colormap.md linguist-generated=true depth_meter.md linguist-generated=true disconnected_space.md linguist-generated=true draw_order.md linguist-generated=true +entity_path.md linguist-generated=true fill_mode.md linguist-generated=true fill_ratio.md linguist-generated=true gamma_correction.md linguist-generated=true @@ -60,4 +61,5 @@ translation3d.md linguist-generated=true triangle_indices.md linguist-generated=true vector2d.md linguist-generated=true vector3d.md linguist-generated=true +video_timestamp.md linguist-generated=true view_coordinates.md linguist-generated=true diff --git a/docs/content/reference/types/components/entity_path.md b/docs/content/reference/types/components/entity_path.md new file mode 100644 index 000000000000..93cbe3bfecf4 --- /dev/null +++ b/docs/content/reference/types/components/entity_path.md @@ -0,0 +1,20 @@ +--- +title: "EntityPath" +--- + + +A path to an entity, usually to reference some data that is part of the target entity. + +## Fields + +* value: [`EntityPath`](../datatypes/entity_path.md) + +## API reference links + * 🌊 [C++ API docs for `EntityPath`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1EntityPath.html) + * 🐍 [Python API docs for `EntityPath`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.EntityPath) + * 🦀 [Rust API docs for `EntityPath`](https://docs.rs/rerun/latest/rerun/components/struct.EntityPath.html) + + +## Used by + +* [`VideoFrameReference`](../archetypes/video_frame_reference.md) diff --git a/docs/content/reference/types/components/video_timestamp.md b/docs/content/reference/types/components/video_timestamp.md new file mode 100644 index 000000000000..10ee10394b8f --- /dev/null +++ b/docs/content/reference/types/components/video_timestamp.md @@ -0,0 +1,23 @@ +--- +title: "VideoTimestamp" +--- + + + +⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + +Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). + +## Fields + +* timestamp: [`VideoTimestamp`](../datatypes/video_timestamp.md) + +## API reference links + * 🌊 [C++ API docs for `VideoTimestamp`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1VideoTimestamp.html) + * 🐍 [Python API docs for `VideoTimestamp`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.VideoTimestamp) + * 🦀 [Rust API docs for `VideoTimestamp`](https://docs.rs/rerun/latest/rerun/components/struct.VideoTimestamp.html) + + +## Used by + +* [`VideoFrameReference`](../archetypes/video_frame_reference.md) diff --git a/docs/content/reference/types/datatypes.md b/docs/content/reference/types/datatypes.md index b9cf705e2382..cd9a53251034 100644 --- a/docs/content/reference/types/datatypes.md +++ b/docs/content/reference/types/datatypes.md @@ -49,6 +49,8 @@ Data types are the lowest layer of the data model hierarchy. They are re-usable * [`Vec2D`](datatypes/vec2d.md): A vector in 2D space. * [`Vec3D`](datatypes/vec3d.md): A vector in 3D space. * [`Vec4D`](datatypes/vec4d.md): A vector in 4D space. +* [`VideoTimeMode`](datatypes/video_time_mode.md): Specifies how to interpret the `video_time` field of a [`datatypes.VideoTimestamp`](https://rerun.io/docs/reference/types/datatypes/video_timestamp). +* [`VideoTimestamp`](datatypes/video_timestamp.md): Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). * [`ViewCoordinates`](datatypes/view_coordinates.md): How we interpret the coordinate system of an entity/space. * [`VisibleTimeRange`](datatypes/visible_time_range.md): Visible time range bounds for a specific timeline. diff --git a/docs/content/reference/types/datatypes/.gitattributes b/docs/content/reference/types/datatypes/.gitattributes index 5c2e32f16a34..b051f19b2670 100644 --- a/docs/content/reference/types/datatypes/.gitattributes +++ b/docs/content/reference/types/datatypes/.gitattributes @@ -43,5 +43,7 @@ uvec4d.md linguist-generated=true vec2d.md linguist-generated=true vec3d.md linguist-generated=true vec4d.md linguist-generated=true +video_time_mode.md linguist-generated=true +video_timestamp.md linguist-generated=true view_coordinates.md linguist-generated=true visible_time_range.md linguist-generated=true diff --git a/docs/content/reference/types/datatypes/entity_path.md b/docs/content/reference/types/datatypes/entity_path.md index 54e00495e6cb..5e7a14c35223 100644 --- a/docs/content/reference/types/datatypes/entity_path.md +++ b/docs/content/reference/types/datatypes/entity_path.md @@ -15,3 +15,6 @@ A path to an entity in the `ChunkStore`. * 🦀 [Rust API docs for `EntityPath`](https://docs.rs/rerun/latest/rerun/datatypes/struct.EntityPath.html) +## Used by + +* [`EntityPath`](../components/entity_path.md) diff --git a/docs/content/reference/types/datatypes/video_time_mode.md b/docs/content/reference/types/datatypes/video_time_mode.md new file mode 100644 index 000000000000..247661b972d4 --- /dev/null +++ b/docs/content/reference/types/datatypes/video_time_mode.md @@ -0,0 +1,20 @@ +--- +title: "VideoTimeMode" +--- + + +Specifies how to interpret the `video_time` field of a [`datatypes.VideoTimestamp`](https://rerun.io/docs/reference/types/datatypes/video_timestamp). + +## Variants + +* Nanoseconds + +## API reference links + * 🌊 [C++ API docs for `VideoTimeMode`](https://ref.rerun.io/docs/cpp/stable/namespacererun_1_1datatypes.html) + * 🐍 [Python API docs for `VideoTimeMode`](https://ref.rerun.io/docs/python/stable/common/datatypes#rerun.datatypes.VideoTimeMode) + * 🦀 [Rust API docs for `VideoTimeMode`](https://docs.rs/rerun/latest/rerun/datatypes/enum.VideoTimeMode.html) + + +## Used by + +* [`VideoTimestamp`](../datatypes/video_timestamp.md) diff --git a/docs/content/reference/types/datatypes/video_timestamp.md b/docs/content/reference/types/datatypes/video_timestamp.md new file mode 100644 index 000000000000..ea04f86bb75c --- /dev/null +++ b/docs/content/reference/types/datatypes/video_timestamp.md @@ -0,0 +1,24 @@ +--- +title: "VideoTimestamp" +--- + + + +⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + +Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video). + +## Fields + +* video_time: `i64` +* time_mode: [`VideoTimeMode`](../datatypes/video_time_mode.md) + +## API reference links + * 🌊 [C++ API docs for `VideoTimestamp`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1datatypes_1_1VideoTimestamp.html) + * 🐍 [Python API docs for `VideoTimestamp`](https://ref.rerun.io/docs/python/stable/common/datatypes#rerun.datatypes.VideoTimestamp) + * 🦀 [Rust API docs for `VideoTimestamp`](https://docs.rs/rerun/latest/rerun/datatypes/struct.VideoTimestamp.html) + + +## Used by + +* [`VideoTimestamp`](../components/video_timestamp.md) diff --git a/rerun_cpp/src/rerun/archetypes.hpp b/rerun_cpp/src/rerun/archetypes.hpp index bf8519498f20..4922c62f6956 100644 --- a/rerun_cpp/src/rerun/archetypes.hpp +++ b/rerun_cpp/src/rerun/archetypes.hpp @@ -31,4 +31,5 @@ #include "archetypes/text_document.hpp" #include "archetypes/text_log.hpp" #include "archetypes/transform3d.hpp" +#include "archetypes/video_frame_reference.hpp" #include "archetypes/view_coordinates.hpp" diff --git a/rerun_cpp/src/rerun/archetypes/.gitattributes b/rerun_cpp/src/rerun/archetypes/.gitattributes index cde22918ea6e..0151357a4f60 100644 --- a/rerun_cpp/src/rerun/archetypes/.gitattributes +++ b/rerun_cpp/src/rerun/archetypes/.gitattributes @@ -59,5 +59,7 @@ text_log.cpp linguist-generated=true text_log.hpp linguist-generated=true transform3d.cpp linguist-generated=true transform3d.hpp linguist-generated=true +video_frame_reference.cpp linguist-generated=true +video_frame_reference.hpp linguist-generated=true view_coordinates.cpp linguist-generated=true view_coordinates.hpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.cpp b/rerun_cpp/src/rerun/archetypes/asset_video.cpp index 3f85f212d452..d2322b27a28b 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.cpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.cpp @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs -// Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +// Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". #include "asset_video.hpp" diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.hpp b/rerun_cpp/src/rerun/archetypes/asset_video.hpp index 766c2608dc0c..d1510a5c8027 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.hpp @@ -1,5 +1,5 @@ // DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs -// Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +// Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". #pragma once @@ -18,12 +18,14 @@ #include namespace rerun::archetypes { - /// **Archetype**: A video file. + /// **Archetype**: A video binary. /// /// NOTE: Videos can only be viewed in the Rerun web viewer. - /// Only MP4 and AV1 is currently supported, and not in all browsers. + /// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. /// Follow for updates on the native support. /// + /// In order to display a video, you need to log a `archetypes::VideoFrameReference` for each frame. + /// /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** struct AssetVideo { /// The asset's bytes. diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp new file mode 100644 index 000000000000..1c8535480c83 --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.cpp @@ -0,0 +1,38 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +#include "video_frame_reference.hpp" + +#include "../collection_adapter_builtins.hpp" + +namespace rerun::archetypes {} + +namespace rerun { + + Result> AsComponents::serialize( + const archetypes::VideoFrameReference& archetype + ) { + using namespace archetypes; + std::vector cells; + cells.reserve(3); + + { + auto result = ComponentBatch::from_loggable(archetype.timestamp); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + if (archetype.video_reference.has_value()) { + auto result = ComponentBatch::from_loggable(archetype.video_reference.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + { + auto indicator = VideoFrameReference::IndicatorComponent(); + auto result = ComponentBatch::from_loggable(indicator); + RR_RETURN_NOT_OK(result.error); + cells.emplace_back(std::move(result.value)); + } + + return cells; + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp new file mode 100644 index 000000000000..ef96331f1a1a --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -0,0 +1,81 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +#pragma once + +#include "../collection.hpp" +#include "../compiler_utils.hpp" +#include "../component_batch.hpp" +#include "../components/entity_path.hpp" +#include "../components/video_timestamp.hpp" +#include "../indicator_component.hpp" +#include "../result.hpp" + +#include +#include +#include +#include + +namespace rerun::archetypes { + /// **Archetype**: References a single video frame. + /// + /// Used to display video frames from a `archetypes::AssetVideo`. + /// + /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + struct VideoFrameReference { + /// References the closest video frame to this timestamp. + /// + /// Note that this uses the closest video frame instead of the latest at this timestamp + /// in order to be more forgiving of rounding errors for inprecise timestamp types. + rerun::components::VideoTimestamp timestamp; + + /// Optional reference to an entity with a `archetypes::AssetVideo`. + /// + /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. + std::optional video_reference; + + public: + static constexpr const char IndicatorComponentName[] = + "rerun.components.VideoFrameReferenceIndicator"; + + /// Indicator component, used to identify the archetype when converting to a list of components. + using IndicatorComponent = rerun::components::IndicatorComponent; + + public: + VideoFrameReference() = default; + VideoFrameReference(VideoFrameReference&& other) = default; + + explicit VideoFrameReference(rerun::components::VideoTimestamp _timestamp) + : timestamp(std::move(_timestamp)) {} + + /// Optional reference to an entity with a `archetypes::AssetVideo`. + /// + /// If none is specified, the video is assumed to be at the same entity. + /// Note that blueprint overrides on the referenced video will be ignored regardless, + /// as this is always interpreted as a reference to the data store. + VideoFrameReference with_video_reference(rerun::components::EntityPath _video_reference + ) && { + video_reference = std::move(_video_reference); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + }; + +} // namespace rerun::archetypes + +namespace rerun { + /// \private + template + struct AsComponents; + + /// \private + template <> + struct AsComponents { + /// Serialize all set component batches. + static Result> serialize( + const archetypes::VideoFrameReference& archetype + ); + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/components.hpp b/rerun_cpp/src/rerun/components.hpp index e3fa9d60b22a..3e05b8095a17 100644 --- a/rerun_cpp/src/rerun/components.hpp +++ b/rerun_cpp/src/rerun/components.hpp @@ -14,6 +14,7 @@ #include "components/depth_meter.hpp" #include "components/disconnected_space.hpp" #include "components/draw_order.hpp" +#include "components/entity_path.hpp" #include "components/fill_mode.hpp" #include "components/fill_ratio.hpp" #include "components/gamma_correction.hpp" @@ -61,4 +62,5 @@ #include "components/triangle_indices.hpp" #include "components/vector2d.hpp" #include "components/vector3d.hpp" +#include "components/video_timestamp.hpp" #include "components/view_coordinates.hpp" diff --git a/rerun_cpp/src/rerun/components/.gitattributes b/rerun_cpp/src/rerun/components/.gitattributes index 70e8b68eaff9..ef90d02b74d5 100644 --- a/rerun_cpp/src/rerun/components/.gitattributes +++ b/rerun_cpp/src/rerun/components/.gitattributes @@ -16,6 +16,7 @@ colormap.hpp linguist-generated=true depth_meter.hpp linguist-generated=true disconnected_space.hpp linguist-generated=true draw_order.hpp linguist-generated=true +entity_path.hpp linguist-generated=true fill_mode.cpp linguist-generated=true fill_mode.hpp linguist-generated=true fill_ratio.hpp linguist-generated=true @@ -69,4 +70,5 @@ translation3d.hpp linguist-generated=true triangle_indices.hpp linguist-generated=true vector2d.hpp linguist-generated=true vector3d.hpp linguist-generated=true +video_timestamp.hpp linguist-generated=true view_coordinates.hpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/components/entity_path.hpp b/rerun_cpp/src/rerun/components/entity_path.hpp new file mode 100644 index 000000000000..e62422c1ceb6 --- /dev/null +++ b/rerun_cpp/src/rerun/components/entity_path.hpp @@ -0,0 +1,66 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/components/entity_path.fbs". + +#pragma once + +#include "../datatypes/entity_path.hpp" +#include "../result.hpp" + +#include +#include +#include +#include + +namespace rerun::components { + /// **Component**: A path to an entity, usually to reference some data that is part of the target entity. + struct EntityPath { + rerun::datatypes::EntityPath value; + + public: + EntityPath() = default; + + EntityPath(rerun::datatypes::EntityPath value_) : value(std::move(value_)) {} + + EntityPath& operator=(rerun::datatypes::EntityPath value_) { + value = std::move(value_); + return *this; + } + + EntityPath(std::string path_) : value(std::move(path_)) {} + + EntityPath& operator=(std::string path_) { + value = std::move(path_); + return *this; + } + + /// Cast to the underlying EntityPath datatype + operator rerun::datatypes::EntityPath() const { + return value; + } + }; +} // namespace rerun::components + +namespace rerun { + static_assert(sizeof(rerun::datatypes::EntityPath) == sizeof(components::EntityPath)); + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.components.EntityPath"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype() { + return Loggable::arrow_datatype(); + } + + /// Serializes an array of `rerun::components::EntityPath` into an arrow array. + static Result> to_arrow( + const components::EntityPath* instances, size_t num_instances + ) { + return Loggable::to_arrow( + &instances->value, + num_instances + ); + } + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/components/video_timestamp.hpp b/rerun_cpp/src/rerun/components/video_timestamp.hpp new file mode 100644 index 000000000000..7a09d14eba53 --- /dev/null +++ b/rerun_cpp/src/rerun/components/video_timestamp.hpp @@ -0,0 +1,59 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/components/video_timestamp.fbs". + +#pragma once + +#include "../datatypes/video_timestamp.hpp" +#include "../result.hpp" + +#include +#include + +namespace rerun::components { + /// **Component**: Timestamp inside a `archetypes::AssetVideo`. + /// + /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + struct VideoTimestamp { + rerun::datatypes::VideoTimestamp timestamp; + + public: + VideoTimestamp() = default; + + VideoTimestamp(rerun::datatypes::VideoTimestamp timestamp_) : timestamp(timestamp_) {} + + VideoTimestamp& operator=(rerun::datatypes::VideoTimestamp timestamp_) { + timestamp = timestamp_; + return *this; + } + + /// Cast to the underlying VideoTimestamp datatype + operator rerun::datatypes::VideoTimestamp() const { + return timestamp; + } + }; +} // namespace rerun::components + +namespace rerun { + static_assert(sizeof(rerun::datatypes::VideoTimestamp) == sizeof(components::VideoTimestamp)); + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.components.VideoTimestamp"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype() { + return Loggable::arrow_datatype(); + } + + /// Serializes an array of `rerun::components::VideoTimestamp` into an arrow array. + static Result> to_arrow( + const components::VideoTimestamp* instances, size_t num_instances + ) { + return Loggable::to_arrow( + &instances->timestamp, + num_instances + ); + } + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes.hpp b/rerun_cpp/src/rerun/datatypes.hpp index 18e56171ed93..46277320d00f 100644 --- a/rerun_cpp/src/rerun/datatypes.hpp +++ b/rerun_cpp/src/rerun/datatypes.hpp @@ -44,5 +44,7 @@ #include "datatypes/vec2d.hpp" #include "datatypes/vec3d.hpp" #include "datatypes/vec4d.hpp" +#include "datatypes/video_time_mode.hpp" +#include "datatypes/video_timestamp.hpp" #include "datatypes/view_coordinates.hpp" #include "datatypes/visible_time_range.hpp" diff --git a/rerun_cpp/src/rerun/datatypes/.gitattributes b/rerun_cpp/src/rerun/datatypes/.gitattributes index 936afb67fd18..5ccb732feac0 100644 --- a/rerun_cpp/src/rerun/datatypes/.gitattributes +++ b/rerun_cpp/src/rerun/datatypes/.gitattributes @@ -85,6 +85,10 @@ vec3d.cpp linguist-generated=true vec3d.hpp linguist-generated=true vec4d.cpp linguist-generated=true vec4d.hpp linguist-generated=true +video_time_mode.cpp linguist-generated=true +video_time_mode.hpp linguist-generated=true +video_timestamp.cpp linguist-generated=true +video_timestamp.hpp linguist-generated=true view_coordinates.cpp linguist-generated=true view_coordinates.hpp linguist-generated=true visible_time_range.cpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/datatypes/video_time_mode.cpp b/rerun_cpp/src/rerun/datatypes/video_time_mode.cpp new file mode 100644 index 000000000000..6f3859686b04 --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_time_mode.cpp @@ -0,0 +1,56 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#include "video_time_mode.hpp" + +#include +#include + +namespace rerun { + const std::shared_ptr& Loggable::arrow_datatype() { + static const auto datatype = arrow::uint8(); + return datatype; + } + + Result> Loggable::to_arrow( + const datatypes::VideoTimeMode* instances, size_t num_instances + ) { + // TODO(andreas): Allow configuring the memory pool. + arrow::MemoryPool* pool = arrow::default_memory_pool(); + auto datatype = arrow_datatype(); + + ARROW_ASSIGN_OR_RAISE(auto builder, arrow::MakeBuilder(datatype, pool)) + if (instances && num_instances > 0) { + RR_RETURN_NOT_OK(Loggable::fill_arrow_array_builder( + static_cast(builder.get()), + instances, + num_instances + )); + } + std::shared_ptr array; + ARROW_RETURN_NOT_OK(builder->Finish(&array)); + return array; + } + + rerun::Error Loggable::fill_arrow_array_builder( + arrow::UInt8Builder* builder, const datatypes::VideoTimeMode* elements, size_t num_elements + ) { + if (builder == nullptr) { + return rerun::Error(ErrorCode::UnexpectedNullArgument, "Passed array builder is null."); + } + if (elements == nullptr) { + return rerun::Error( + ErrorCode::UnexpectedNullArgument, + "Cannot serialize null pointer to arrow array." + ); + } + + ARROW_RETURN_NOT_OK(builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + const auto variant = elements[elem_idx]; + ARROW_RETURN_NOT_OK(builder->Append(static_cast(variant))); + } + + return Error::ok(); + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/video_time_mode.hpp b/rerun_cpp/src/rerun/datatypes/video_time_mode.hpp new file mode 100644 index 000000000000..e0738745910e --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_time_mode.hpp @@ -0,0 +1,54 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#pragma once + +#include "../result.hpp" + +#include +#include + +namespace arrow { + /// \private + template + class NumericBuilder; + + class Array; + class DataType; + class UInt8Type; + using UInt8Builder = NumericBuilder; +} // namespace arrow + +namespace rerun::datatypes { + /// **Datatype**: Specifies how to interpret the `video_time` field of a `datatypes::VideoTimestamp`. + enum class VideoTimeMode : uint8_t { + + /// Presentation timestamp in nanoseconds since the beginning of the video. + Nanoseconds = 1, + }; +} // namespace rerun::datatypes + +namespace rerun { + template + struct Loggable; + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.datatypes.VideoTimeMode"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype(); + + /// Serializes an array of `rerun::datatypes::VideoTimeMode` into an arrow array. + static Result> to_arrow( + const datatypes::VideoTimeMode* instances, size_t num_instances + ); + + /// Fills an arrow array builder with an array of this type. + static rerun::Error fill_arrow_array_builder( + arrow::UInt8Builder* builder, const datatypes::VideoTimeMode* elements, + size_t num_elements + ); + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/video_timestamp.cpp b/rerun_cpp/src/rerun/datatypes/video_timestamp.cpp new file mode 100644 index 000000000000..0c7dcadc2037 --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_timestamp.cpp @@ -0,0 +1,84 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#include "video_timestamp.hpp" + +#include "video_time_mode.hpp" + +#include +#include + +namespace rerun::datatypes {} + +namespace rerun { + const std::shared_ptr& Loggable::arrow_datatype() { + static const auto datatype = arrow::struct_({ + arrow::field("video_time", arrow::int64(), false), + arrow::field( + "time_mode", + Loggable::arrow_datatype(), + false + ), + }); + return datatype; + } + + Result> Loggable::to_arrow( + const datatypes::VideoTimestamp* instances, size_t num_instances + ) { + // TODO(andreas): Allow configuring the memory pool. + arrow::MemoryPool* pool = arrow::default_memory_pool(); + auto datatype = arrow_datatype(); + + ARROW_ASSIGN_OR_RAISE(auto builder, arrow::MakeBuilder(datatype, pool)) + if (instances && num_instances > 0) { + RR_RETURN_NOT_OK(Loggable::fill_arrow_array_builder( + static_cast(builder.get()), + instances, + num_instances + )); + } + std::shared_ptr array; + ARROW_RETURN_NOT_OK(builder->Finish(&array)); + return array; + } + + rerun::Error Loggable::fill_arrow_array_builder( + arrow::StructBuilder* builder, const datatypes::VideoTimestamp* elements, + size_t num_elements + ) { + if (builder == nullptr) { + return rerun::Error(ErrorCode::UnexpectedNullArgument, "Passed array builder is null."); + } + if (elements == nullptr) { + return rerun::Error( + ErrorCode::UnexpectedNullArgument, + "Cannot serialize null pointer to arrow array." + ); + } + + { + auto field_builder = static_cast(builder->field_builder(0)); + ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + ARROW_RETURN_NOT_OK(field_builder->Append(elements[elem_idx].video_time)); + } + } + { + auto field_builder = static_cast(builder->field_builder(1)); + ARROW_RETURN_NOT_OK(field_builder->Reserve(static_cast(num_elements))); + for (size_t elem_idx = 0; elem_idx < num_elements; elem_idx += 1) { + RR_RETURN_NOT_OK( + Loggable::fill_arrow_array_builder( + field_builder, + &elements[elem_idx].time_mode, + 1 + ) + ); + } + } + ARROW_RETURN_NOT_OK(builder->AppendValues(static_cast(num_elements), nullptr)); + + return Error::ok(); + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/datatypes/video_timestamp.hpp b/rerun_cpp/src/rerun/datatypes/video_timestamp.hpp new file mode 100644 index 000000000000..1d1110fa9d55 --- /dev/null +++ b/rerun_cpp/src/rerun/datatypes/video_timestamp.hpp @@ -0,0 +1,57 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +#pragma once + +#include "../result.hpp" +#include "video_time_mode.hpp" + +#include +#include + +namespace arrow { + class Array; + class DataType; + class StructBuilder; +} // namespace arrow + +namespace rerun::datatypes { + /// **Datatype**: Timestamp inside a `archetypes::AssetVideo`. + /// + /// ⚠ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + struct VideoTimestamp { + /// Timestamp value, type defined by `time_mode`. + int64_t video_time; + + /// How to interpret `video_time`. + rerun::datatypes::VideoTimeMode time_mode; + + public: + VideoTimestamp() = default; + }; +} // namespace rerun::datatypes + +namespace rerun { + template + struct Loggable; + + /// \private + template <> + struct Loggable { + static constexpr const char Name[] = "rerun.datatypes.VideoTimestamp"; + + /// Returns the arrow data type this type corresponds to. + static const std::shared_ptr& arrow_datatype(); + + /// Serializes an array of `rerun::datatypes::VideoTimestamp` into an arrow array. + static Result> to_arrow( + const datatypes::VideoTimestamp* instances, size_t num_instances + ); + + /// Fills an arrow array builder with an array of this type. + static rerun::Error fill_arrow_array_builder( + arrow::StructBuilder* builder, const datatypes::VideoTimestamp* elements, + size_t num_elements + ); + }; +} // namespace rerun diff --git a/rerun_py/docs/gen_common_index.py b/rerun_py/docs/gen_common_index.py index f7b9cd39ccbc..147973087835 100755 --- a/rerun_py/docs/gen_common_index.py +++ b/rerun_py/docs/gen_common_index.py @@ -169,7 +169,14 @@ class Section: "archetypes.Image", "archetypes.EncodedImage", "archetypes.SegmentationImage", + ], + gen_page=False, + ), + Section( + title="Video", + class_list=[ "archetypes.AssetVideo", + "archetypes.VideoFrameReference", ], gen_page=False, ), diff --git a/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes b/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes index 72603acdd483..b57110b5036e 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes @@ -31,4 +31,5 @@ tensor.py linguist-generated=true text_document.py linguist-generated=true text_log.py linguist-generated=true transform3d.py linguist-generated=true +video_frame_reference.py linguist-generated=true view_coordinates.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/archetypes/__init__.py b/rerun_py/rerun_sdk/rerun/archetypes/__init__.py index 088b8cb175b2..8ba5d3fcdee3 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/__init__.py @@ -31,6 +31,7 @@ from .text_document import TextDocument from .text_log import TextLog from .transform3d import Transform3D +from .video_frame_reference import VideoFrameReference from .view_coordinates import ViewCoordinates __all__ = [ @@ -63,5 +64,6 @@ "TextDocument", "TextLog", "Transform3D", + "VideoFrameReference", "ViewCoordinates", ] diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py index b12113bc71f9..ad3c40e8bd22 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py @@ -1,5 +1,5 @@ # DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs -# Based on "crates/store/re_types/definitions/rerun/archetypes/video.fbs". +# Based on "crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs". # You can extend this class by creating a "AssetVideoExt" class in "asset_video_ext.py". @@ -19,12 +19,14 @@ @define(str=False, repr=False, init=False) class AssetVideo(AssetVideoExt, Archetype): """ - **Archetype**: A video file. + **Archetype**: A video binary. NOTE: Videos can only be viewed in the Rerun web viewer. - Only MP4 and AV1 is currently supported, and not in all browsers. + Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. Follow for updates on the native support. + In order to display a video, you need to log a [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference] for each frame. + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** """ diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py new file mode 100644 index 000000000000..d73ff771bdf3 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -0,0 +1,98 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs". + +# You can extend this class by creating a "VideoFrameReferenceExt" class in "video_frame_reference_ext.py". + +from __future__ import annotations + +from typing import Any + +from attrs import define, field + +from .. import components, datatypes +from .._baseclasses import ( + Archetype, +) +from ..error_utils import catch_and_log_exceptions + +__all__ = ["VideoFrameReference"] + + +@define(str=False, repr=False, init=False) +class VideoFrameReference(Archetype): + """ + **Archetype**: References a single video frame. + + Used to display video frames from a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + """ + + def __init__( + self: Any, timestamp: datatypes.VideoTimestampLike, *, video_reference: datatypes.EntityPathLike | None = None + ): + """ + Create a new instance of the VideoFrameReference archetype. + + Parameters + ---------- + timestamp: + References the closest video frame to this timestamp. + + Note that this uses the closest video frame instead of the latest at this timestamp + in order to be more forgiving of rounding errors for inprecise timestamp types. + video_reference: + Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + If none is specified, the video is assumed to be at the same entity. + Note that blueprint overrides on the referenced video will be ignored regardless, + as this is always interpreted as a reference to the data store. + + """ + + # You can define your own __init__ function as a member of VideoFrameReferenceExt in video_frame_reference_ext.py + with catch_and_log_exceptions(context=self.__class__.__name__): + self.__attrs_init__(timestamp=timestamp, video_reference=video_reference) + return + self.__attrs_clear__() + + def __attrs_clear__(self) -> None: + """Convenience method for calling `__attrs_init__` with all `None`s.""" + self.__attrs_init__( + timestamp=None, # type: ignore[arg-type] + video_reference=None, # type: ignore[arg-type] + ) + + @classmethod + def _clear(cls) -> VideoFrameReference: + """Produce an empty VideoFrameReference, bypassing `__init__`.""" + inst = cls.__new__(cls) + inst.__attrs_clear__() + return inst + + timestamp: components.VideoTimestampBatch = field( + metadata={"component": "required"}, + converter=components.VideoTimestampBatch._required, # type: ignore[misc] + ) + # References the closest video frame to this timestamp. + # + # Note that this uses the closest video frame instead of the latest at this timestamp + # in order to be more forgiving of rounding errors for inprecise timestamp types. + # + # (Docstring intentionally commented out to hide this field from the docs) + + video_reference: components.EntityPathBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.EntityPathBatch._optional, # type: ignore[misc] + ) + # Optional reference to an entity with a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + # + # If none is specified, the video is assumed to be at the same entity. + # Note that blueprint overrides on the referenced video will be ignored regardless, + # as this is always interpreted as a reference to the data store. + # + # (Docstring intentionally commented out to hide this field from the docs) + + __str__ = Archetype.__str__ + __repr__ = Archetype.__repr__ # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/components/.gitattributes b/rerun_py/rerun_sdk/rerun/components/.gitattributes index a238f07d07ce..bb7b80d1486c 100644 --- a/rerun_py/rerun_sdk/rerun/components/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/components/.gitattributes @@ -14,6 +14,7 @@ colormap.py linguist-generated=true depth_meter.py linguist-generated=true disconnected_space.py linguist-generated=true draw_order.py linguist-generated=true +entity_path.py linguist-generated=true fill_mode.py linguist-generated=true fill_ratio.py linguist-generated=true gamma_correction.py linguist-generated=true @@ -61,4 +62,5 @@ translation3d.py linguist-generated=true triangle_indices.py linguist-generated=true vector2d.py linguist-generated=true vector3d.py linguist-generated=true +video_timestamp.py linguist-generated=true view_coordinates.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/components/__init__.py b/rerun_py/rerun_sdk/rerun/components/__init__.py index 16625c133e9d..8855fcf59d3c 100644 --- a/rerun_py/rerun_sdk/rerun/components/__init__.py +++ b/rerun_py/rerun_sdk/rerun/components/__init__.py @@ -26,6 +26,7 @@ from .depth_meter import DepthMeter, DepthMeterBatch, DepthMeterType from .disconnected_space import DisconnectedSpace, DisconnectedSpaceBatch, DisconnectedSpaceType from .draw_order import DrawOrder, DrawOrderBatch, DrawOrderType +from .entity_path import EntityPath, EntityPathBatch, EntityPathType from .fill_mode import FillMode, FillModeArrayLike, FillModeBatch, FillModeLike, FillModeType from .fill_ratio import FillRatio, FillRatioBatch, FillRatioType from .gamma_correction import GammaCorrection, GammaCorrectionBatch, GammaCorrectionType @@ -89,6 +90,7 @@ from .triangle_indices import TriangleIndices, TriangleIndicesBatch, TriangleIndicesType from .vector2d import Vector2D, Vector2DBatch, Vector2DType from .vector3d import Vector3D, Vector3DBatch, Vector3DType +from .video_timestamp import VideoTimestamp, VideoTimestampBatch, VideoTimestampType from .view_coordinates import ViewCoordinates, ViewCoordinatesBatch, ViewCoordinatesType __all__ = [ @@ -134,6 +136,9 @@ "DrawOrder", "DrawOrderBatch", "DrawOrderType", + "EntityPath", + "EntityPathBatch", + "EntityPathType", "FillMode", "FillModeArrayLike", "FillModeBatch", @@ -287,6 +292,9 @@ "Vector3D", "Vector3DBatch", "Vector3DType", + "VideoTimestamp", + "VideoTimestampBatch", + "VideoTimestampType", "ViewCoordinates", "ViewCoordinatesBatch", "ViewCoordinatesType", diff --git a/rerun_py/rerun_sdk/rerun/components/entity_path.py b/rerun_py/rerun_sdk/rerun/components/entity_path.py new file mode 100644 index 000000000000..ad63a4793330 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/components/entity_path.py @@ -0,0 +1,36 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/components/entity_path.fbs". + +# You can extend this class by creating a "EntityPathExt" class in "entity_path_ext.py". + +from __future__ import annotations + +from .. import datatypes +from .._baseclasses import ( + ComponentBatchMixin, + ComponentMixin, +) + +__all__ = ["EntityPath", "EntityPathBatch", "EntityPathType"] + + +class EntityPath(datatypes.EntityPath, ComponentMixin): + """**Component**: A path to an entity, usually to reference some data that is part of the target entity.""" + + _BATCH_TYPE = None + # You can define your own __init__ function as a member of EntityPathExt in entity_path_ext.py + + # Note: there are no fields here because EntityPath delegates to datatypes.EntityPath + pass + + +class EntityPathType(datatypes.EntityPathType): + _TYPE_NAME: str = "rerun.components.EntityPath" + + +class EntityPathBatch(datatypes.EntityPathBatch, ComponentBatchMixin): + _ARROW_TYPE = EntityPathType() + + +# This is patched in late to avoid circular dependencies. +EntityPath._BATCH_TYPE = EntityPathBatch # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py new file mode 100644 index 000000000000..433fe013f8cc --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py @@ -0,0 +1,40 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/components/video_timestamp.fbs". + +# You can extend this class by creating a "VideoTimestampExt" class in "video_timestamp_ext.py". + +from __future__ import annotations + +from .. import datatypes +from .._baseclasses import ( + ComponentBatchMixin, + ComponentMixin, +) + +__all__ = ["VideoTimestamp", "VideoTimestampBatch", "VideoTimestampType"] + + +class VideoTimestamp(datatypes.VideoTimestamp, ComponentMixin): + """ + **Component**: Timestamp inside a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + """ + + _BATCH_TYPE = None + # You can define your own __init__ function as a member of VideoTimestampExt in video_timestamp_ext.py + + # Note: there are no fields here because VideoTimestamp delegates to datatypes.VideoTimestamp + pass + + +class VideoTimestampType(datatypes.VideoTimestampType): + _TYPE_NAME: str = "rerun.components.VideoTimestamp" + + +class VideoTimestampBatch(datatypes.VideoTimestampBatch, ComponentBatchMixin): + _ARROW_TYPE = VideoTimestampType() + + +# This is patched in late to avoid circular dependencies. +VideoTimestamp._BATCH_TYPE = VideoTimestampBatch # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes b/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes index 549c47677d0e..42443440d38b 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/datatypes/.gitattributes @@ -44,5 +44,7 @@ uvec4d.py linguist-generated=true vec2d.py linguist-generated=true vec3d.py linguist-generated=true vec4d.py linguist-generated=true +video_time_mode.py linguist-generated=true +video_timestamp.py linguist-generated=true view_coordinates.py linguist-generated=true visible_time_range.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/datatypes/__init__.py b/rerun_py/rerun_sdk/rerun/datatypes/__init__.py index 09616779d8c2..7c30b55ceb03 100644 --- a/rerun_py/rerun_sdk/rerun/datatypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/datatypes/__init__.py @@ -98,6 +98,20 @@ from .vec2d import Vec2D, Vec2DArrayLike, Vec2DBatch, Vec2DLike, Vec2DType from .vec3d import Vec3D, Vec3DArrayLike, Vec3DBatch, Vec3DLike, Vec3DType from .vec4d import Vec4D, Vec4DArrayLike, Vec4DBatch, Vec4DLike, Vec4DType +from .video_time_mode import ( + VideoTimeMode, + VideoTimeModeArrayLike, + VideoTimeModeBatch, + VideoTimeModeLike, + VideoTimeModeType, +) +from .video_timestamp import ( + VideoTimestamp, + VideoTimestampArrayLike, + VideoTimestampBatch, + VideoTimestampLike, + VideoTimestampType, +) from .view_coordinates import ( ViewCoordinates, ViewCoordinatesArrayLike, @@ -324,6 +338,16 @@ "Vec4DBatch", "Vec4DLike", "Vec4DType", + "VideoTimeMode", + "VideoTimeModeArrayLike", + "VideoTimeModeBatch", + "VideoTimeModeLike", + "VideoTimeModeType", + "VideoTimestamp", + "VideoTimestampArrayLike", + "VideoTimestampBatch", + "VideoTimestampLike", + "VideoTimestampType", "ViewCoordinates", "ViewCoordinatesArrayLike", "ViewCoordinatesBatch", diff --git a/rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py b/rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py new file mode 100644 index 000000000000..b1b250eaa22d --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/datatypes/video_time_mode.py @@ -0,0 +1,71 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +# You can extend this class by creating a "VideoTimeModeExt" class in "video_time_mode_ext.py". + +from __future__ import annotations + +from typing import Literal, Sequence, Union + +import pyarrow as pa + +from .._baseclasses import ( + BaseBatch, + BaseExtensionType, +) + +__all__ = ["VideoTimeMode", "VideoTimeModeArrayLike", "VideoTimeModeBatch", "VideoTimeModeLike", "VideoTimeModeType"] + + +from enum import Enum + + +class VideoTimeMode(Enum): + """**Datatype**: Specifies how to interpret the `video_time` field of a [`datatypes.VideoTimestamp`][rerun.datatypes.VideoTimestamp].""" + + Nanoseconds = 1 + """Presentation timestamp in nanoseconds since the beginning of the video.""" + + @classmethod + def auto(cls, val: str | int | VideoTimeMode) -> VideoTimeMode: + """Best-effort converter, including a case-insensitive string matcher.""" + if isinstance(val, VideoTimeMode): + return val + if isinstance(val, int): + return cls(val) + try: + return cls[val] + except KeyError: + val_lower = val.lower() + for variant in cls: + if variant.name.lower() == val_lower: + return variant + raise ValueError(f"Cannot convert {val} to {cls.__name__}") + + def __str__(self) -> str: + """Returns the variant name.""" + return self.name + + +VideoTimeModeLike = Union[VideoTimeMode, Literal["Nanoseconds", "nanoseconds"], int] +VideoTimeModeArrayLike = Union[VideoTimeModeLike, Sequence[VideoTimeModeLike]] + + +class VideoTimeModeType(BaseExtensionType): + _TYPE_NAME: str = "rerun.datatypes.VideoTimeMode" + + def __init__(self) -> None: + pa.ExtensionType.__init__(self, pa.uint8(), self._TYPE_NAME) + + +class VideoTimeModeBatch(BaseBatch[VideoTimeModeArrayLike]): + _ARROW_TYPE = VideoTimeModeType() + + @staticmethod + def _native_to_pa_array(data: VideoTimeModeArrayLike, data_type: pa.DataType) -> pa.Array: + if isinstance(data, (VideoTimeMode, int, str)): + data = [data] + + pa_data = [VideoTimeMode.auto(v).value if v is not None else None for v in data] # type: ignore[redundant-expr] + + return pa.array(pa_data, type=data_type) diff --git a/rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py b/rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py new file mode 100644 index 000000000000..d0bf8deb7cac --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/datatypes/video_timestamp.py @@ -0,0 +1,110 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs". + +# You can extend this class by creating a "VideoTimestampExt" class in "video_timestamp_ext.py". + +from __future__ import annotations + +from typing import Any, Sequence, Union + +import numpy as np +import pyarrow as pa +from attrs import define, field + +from .. import datatypes +from .._baseclasses import ( + BaseBatch, + BaseExtensionType, +) + +__all__ = [ + "VideoTimestamp", + "VideoTimestampArrayLike", + "VideoTimestampBatch", + "VideoTimestampLike", + "VideoTimestampType", +] + + +def _video_timestamp__time_mode__special_field_converter_override( + x: datatypes.VideoTimeModeLike, +) -> datatypes.VideoTimeMode: + if isinstance(x, datatypes.VideoTimeMode): + return x + else: + return datatypes.VideoTimeMode(x) + + +@define(init=False) +class VideoTimestamp: + """ + **Datatype**: Timestamp inside a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. + + ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** + """ + + def __init__(self: Any, video_time: int, time_mode: datatypes.VideoTimeModeLike): + """ + Create a new instance of the VideoTimestamp datatype. + + Parameters + ---------- + video_time: + Timestamp value, type defined by `time_mode`. + time_mode: + How to interpret `video_time`. + + """ + + # You can define your own __init__ function as a member of VideoTimestampExt in video_timestamp_ext.py + self.__attrs_init__(video_time=video_time, time_mode=time_mode) + + video_time: int = field(converter=int) + # Timestamp value, type defined by `time_mode`. + # + # (Docstring intentionally commented out to hide this field from the docs) + + time_mode: datatypes.VideoTimeMode = field(converter=_video_timestamp__time_mode__special_field_converter_override) + # How to interpret `video_time`. + # + # (Docstring intentionally commented out to hide this field from the docs) + + +VideoTimestampLike = VideoTimestamp +VideoTimestampArrayLike = Union[ + VideoTimestamp, + Sequence[VideoTimestampLike], +] + + +class VideoTimestampType(BaseExtensionType): + _TYPE_NAME: str = "rerun.datatypes.VideoTimestamp" + + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, + pa.struct([ + pa.field("video_time", pa.int64(), nullable=False, metadata={}), + pa.field("time_mode", pa.uint8(), nullable=False, metadata={}), + ]), + self._TYPE_NAME, + ) + + +class VideoTimestampBatch(BaseBatch[VideoTimestampArrayLike]): + _ARROW_TYPE = VideoTimestampType() + + @staticmethod + def _native_to_pa_array(data: VideoTimestampArrayLike, data_type: pa.DataType) -> pa.Array: + from rerun.datatypes import VideoTimeModeBatch + + if isinstance(data, VideoTimestamp): + data = [data] + + return pa.StructArray.from_arrays( + [ + pa.array(np.asarray([x.video_time for x in data], dtype=np.int64)), + VideoTimeModeBatch([x.time_mode for x in data]).as_arrow_array().storage, # type: ignore[misc, arg-type] + ], + fields=list(data_type), + ) diff --git a/tests/python/release_checklist/check_all_components_ui.py b/tests/python/release_checklist/check_all_components_ui.py index 1560900ef91e..a7daab5b2fbc 100644 --- a/tests/python/release_checklist/check_all_components_ui.py +++ b/tests/python/release_checklist/check_all_components_ui.py @@ -111,6 +111,7 @@ def alternatives(self) -> list[Any] | None: "DepthMeterBatch": TestCase(1000.0), "DisconnectedSpaceBatch": TestCase(True), "DrawOrderBatch": TestCase(100.0), + "EntityPathBatch": TestCase("my/entity/path"), "FillModeBatch": TestCase( batch=[ rr.components.FillMode.MajorWireframe, @@ -217,6 +218,7 @@ def alternatives(self) -> list[Any] | None: "TriangleIndicesBatch": TestCase(batch=[(0, 1, 2), (3, 4, 5), (6, 7, 8)]), "Vector2DBatch": TestCase(batch=[(0, 1), (2, 3), (4, 5)]), "Vector3DBatch": TestCase(batch=[(0, 3, 4), (1, 4, 5), (2, 5, 6)]), + "VideoTimestampBatch": TestCase(rr.components.VideoTimestamp(0, rr.datatypes.VideoTimeMode.Nanoseconds)), "ViewCoordinatesBatch": TestCase(rr.components.ViewCoordinates.LBD), "VisualizerOverridesBatch": TestCase(disabled=True), # no Python-based serialization } From 9ddf77e4563c16be955b3dcc4d37def325db219e Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 11 Sep 2024 13:15:45 +0200 Subject: [PATCH 2/3] Keep redrawing until video decoder is done decoding requested frame (#7398) ### What * bits and pieces for https://github.com/rerun-io/rerun/issues/7373 * but far from having that solved Video decoder forwards now more rich status information upon decode. This is useful for error handling and - the immediate motivation - to keep redrawing while we're waiting on pending frames. Before, when clicking somewhere on the timeline, we wouldn't redraw until the mouse moves, which for back-seeking (and far forward-seeking) usually means that we don't get the new frame until the decoder catches up. I also took the liberty to move the video re_renderer module around a bit - it was falsely sorted under `renderer` which are all about renderers that process draw data and get submitted to a view builder (which is not the case here!). https://github.com/user-attachments/assets/623d712b-384b-4ffa-bcbf-3a80885ccef6 ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7398?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7398?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7398) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- crates/viewer/re_renderer/src/lib.rs | 1 + crates/viewer/re_renderer/src/renderer/mod.rs | 3 - .../src/{renderer => }/video/decoder/mod.rs | 0 .../{renderer => }/video/decoder/native.rs | 18 ++- .../src/{renderer => }/video/decoder/web.rs | 134 ++++++++---------- .../src/{renderer => }/video/mod.rs | 66 ++++++--- .../re_space_view_spatial/src/video_cache.rs | 2 +- .../src/visualizers/videos.rs | 26 +++- 8 files changed, 145 insertions(+), 105 deletions(-) rename crates/viewer/re_renderer/src/{renderer => }/video/decoder/mod.rs (100%) rename crates/viewer/re_renderer/src/{renderer => }/video/decoder/native.rs (73%) rename crates/viewer/re_renderer/src/{renderer => }/video/decoder/web.rs (81%) rename crates/viewer/re_renderer/src/{renderer => }/video/mod.rs (55%) diff --git a/crates/viewer/re_renderer/src/lib.rs b/crates/viewer/re_renderer/src/lib.rs index 673e44e56a40..a0634e825a52 100644 --- a/crates/viewer/re_renderer/src/lib.rs +++ b/crates/viewer/re_renderer/src/lib.rs @@ -16,6 +16,7 @@ pub mod mesh; pub mod renderer; pub mod resource_managers; pub mod texture_info; +pub mod video; pub mod view_builder; mod allocator; diff --git a/crates/viewer/re_renderer/src/renderer/mod.rs b/crates/viewer/re_renderer/src/renderer/mod.rs index 339a86b265c5..ccca81217c8a 100644 --- a/crates/viewer/re_renderer/src/renderer/mod.rs +++ b/crates/viewer/re_renderer/src/renderer/mod.rs @@ -31,9 +31,6 @@ pub(crate) use compositor::CompositorDrawData; mod debug_overlay; pub use debug_overlay::{DebugOverlayDrawData, DebugOverlayError, DebugOverlayRenderer}; -mod video; -pub use video::Video; - pub mod gpu_data { pub use super::lines::gpu_data::{LineStripInfo, LineVertex}; pub use super::point_cloud::gpu_data::PositionRadius; diff --git a/crates/viewer/re_renderer/src/renderer/video/decoder/mod.rs b/crates/viewer/re_renderer/src/video/decoder/mod.rs similarity index 100% rename from crates/viewer/re_renderer/src/renderer/video/decoder/mod.rs rename to crates/viewer/re_renderer/src/video/decoder/mod.rs diff --git a/crates/viewer/re_renderer/src/renderer/video/decoder/native.rs b/crates/viewer/re_renderer/src/video/decoder/native.rs similarity index 73% rename from crates/viewer/re_renderer/src/renderer/video/decoder/native.rs rename to crates/viewer/re_renderer/src/video/decoder/native.rs index a587c9b786b6..524ff8d13deb 100644 --- a/crates/viewer/re_renderer/src/renderer/video/decoder/native.rs +++ b/crates/viewer/re_renderer/src/video/decoder/native.rs @@ -1,7 +1,10 @@ #![allow(dead_code, unused_variables, clippy::unnecessary_wraps)] -use crate::resource_managers::GpuTexture2D; -use crate::RenderContext; +use crate::{ + resource_managers::GpuTexture2D, + video::{DecodingError, FrameDecodingResult}, + RenderContext, +}; // TODO(#7298): remove `allow` once we have native video decoding #[allow(unused_imports)] @@ -17,7 +20,10 @@ pub struct VideoDecoder { } impl VideoDecoder { - pub fn new(render_context: &RenderContext, data: re_video::VideoData) -> Option { + pub fn new( + render_context: &RenderContext, + data: re_video::VideoData, + ) -> Result { re_log::warn_once!("Video playback not yet available in the native viewer, try the web viewer instead. See https://github.com/rerun-io/rerun/issues/7298 for more information."); let device = render_context.device.clone(); @@ -27,7 +33,7 @@ impl VideoDecoder { data.config.coded_width as u32, data.config.coded_height as u32, ); - Some(Self { + Ok(Self { data, zeroed_texture, }) @@ -45,7 +51,7 @@ impl VideoDecoder { self.data.config.coded_height as u32 } - pub fn frame_at(&mut self, timestamp: TimeMs) -> GpuTexture2D { - self.zeroed_texture.clone() + pub fn frame_at(&mut self, timestamp: TimeMs) -> FrameDecodingResult { + FrameDecodingResult::Ready(self.zeroed_texture.clone()) } } diff --git a/crates/viewer/re_renderer/src/renderer/video/decoder/web.rs b/crates/viewer/re_renderer/src/video/decoder/web.rs similarity index 81% rename from crates/viewer/re_renderer/src/renderer/video/decoder/web.rs rename to crates/viewer/re_renderer/src/video/decoder/web.rs index 6eeacc9de8b5..c72438854d11 100644 --- a/crates/viewer/re_renderer/src/renderer/video/decoder/web.rs +++ b/crates/viewer/re_renderer/src/video/decoder/web.rs @@ -1,22 +1,21 @@ -// TODO(emilk): proper error handling: pass errors to caller instead of logging them` +use std::sync::Arc; -use super::latest_at_idx; -use crate::resource_managers::GpuTexture2D; -use crate::RenderContext; -use js_sys::Function; -use js_sys::Uint8Array; +use js_sys::{Function, Uint8Array}; use parking_lot::Mutex; -use re_video::TimeMs; -use re_video::VideoData; -use std::ops::Deref; -use std::sync::Arc; -use wasm_bindgen::closure::Closure; -use wasm_bindgen::JsCast as _; -use web_sys::EncodedVideoChunk; -use web_sys::EncodedVideoChunkInit; -use web_sys::EncodedVideoChunkType; -use web_sys::VideoDecoderConfig; -use web_sys::VideoDecoderInit; +use wasm_bindgen::{closure::Closure, JsCast as _}; +use web_sys::{ + EncodedVideoChunk, EncodedVideoChunkInit, EncodedVideoChunkType, VideoDecoderConfig, + VideoDecoderInit, +}; + +use re_video::{TimeMs, VideoData}; + +use super::latest_at_idx; +use crate::{ + resource_managers::GpuTexture2D, + video::{DecodingError, FrameDecodingResult}, + RenderContext, +}; #[derive(Clone)] #[repr(transparent)] @@ -28,7 +27,7 @@ impl Drop for VideoFrame { } } -impl Deref for VideoFrame { +impl std::ops::Deref for VideoFrame { type Target = web_sys::VideoFrame; #[inline] @@ -41,7 +40,6 @@ pub struct VideoDecoder { data: re_video::VideoData, queue: Arc, texture: GpuTexture2D, - zeroed_texture: GpuTexture2D, decoder: web_sys::VideoDecoder, @@ -83,7 +81,7 @@ impl Drop for VideoDecoder { } impl VideoDecoder { - pub fn new(render_context: &RenderContext, data: VideoData) -> Option { + pub fn new(render_context: &RenderContext, data: VideoData) -> Result { let frames = Arc::new(Mutex::new(Vec::with_capacity(16))); let decoder = init_video_decoder({ @@ -105,18 +103,11 @@ impl VideoDecoder { data.config.coded_width as u32, data.config.coded_height as u32, ); - let zeroed_texture = super::alloc_video_frame_texture( - &render_context.device, - &render_context.gpu_resources.textures, - data.config.coded_width as u32, - data.config.coded_height as u32, - ); let mut this = Self { data, queue, texture, - zeroed_texture, decoder, @@ -127,10 +118,10 @@ impl VideoDecoder { }; // immediately enqueue some frames, assuming playback at start - this.reset(); + this.reset()?; let _ = this.frame_at(TimeMs::new(0.0)); - Some(this) + Ok(this) } pub fn duration_ms(&self) -> f64 { @@ -145,16 +136,15 @@ impl VideoDecoder { self.data.config.coded_height as u32 } - pub fn frame_at(&mut self, timestamp: TimeMs) -> GpuTexture2D { + pub fn frame_at(&mut self, timestamp: TimeMs) -> FrameDecodingResult { if timestamp < TimeMs::ZERO { - return self.zeroed_texture.clone(); + return FrameDecodingResult::Error(DecodingError::NegativeTimestamp); } let Some(requested_segment_idx) = latest_at_idx(&self.data.segments, |segment| segment.timestamp, ×tamp) else { - // This should only happen if the video is completely empty. - return self.zeroed_texture.clone(); + return FrameDecodingResult::Error(DecodingError::EmptyVideo); }; let Some(requested_sample_idx) = latest_at_idx( @@ -163,7 +153,7 @@ impl VideoDecoder { ×tamp, ) else { // This should never happen, because segments are never empty. - return self.zeroed_texture.clone(); + return FrameDecodingResult::Error(DecodingError::EmptySegment); }; // Enqueue segments as needed. We maintain a buffer of 2 segments, so we can @@ -179,12 +169,14 @@ impl VideoDecoder { requested_segment_idx as isize - self.current_segment_idx as isize; if segment_distance == 1 { // forward seek to next segment - queue up the one _after_ requested - self.enqueue_all(requested_segment_idx + 1); + self.enqueue_segment(requested_segment_idx + 1); } else { // forward seek by N>1 OR backward seek across segments - reset - self.reset(); - self.enqueue_all(requested_segment_idx); - self.enqueue_all(requested_segment_idx + 1); + if let Err(err) = self.reset() { + return FrameDecodingResult::Error(err); + } + self.enqueue_segment(requested_segment_idx); + self.enqueue_segment(requested_segment_idx + 1); } } else if requested_sample_idx != self.current_sample_idx { // special case: handle seeking backwards within a single segment @@ -192,9 +184,11 @@ impl VideoDecoder { // while maintaining a buffer of 2 segments let sample_distance = requested_sample_idx as isize - self.current_sample_idx as isize; if sample_distance < 0 { - self.reset(); - self.enqueue_all(requested_segment_idx); - self.enqueue_all(requested_segment_idx + 1); + if let Err(err) = self.reset() { + return FrameDecodingResult::Error(err); + } + self.enqueue_segment(requested_segment_idx); + self.enqueue_segment(requested_segment_idx + 1); } } @@ -205,10 +199,10 @@ impl VideoDecoder { let Some(frame_idx) = latest_at_idx(&frames, |(t, _)| *t, ×tamp) else { // no buffered frames - texture will be blank - // not return a zeroed texture, because we may just be behind on decoding + // Don't return a zeroed texture, because we may just be behind on decoding // and showing an old frame is better than showing a blank frame, // because it causes "black flashes" to appear - return self.texture.clone(); + return FrameDecodingResult::Pending(self.texture.clone()); }; // drain up-to (but not including) the frame idx, clearing out any frames @@ -226,9 +220,10 @@ impl VideoDecoder { let frame_duration_ms = frame.duration().map(TimeMs::new).unwrap_or_default(); // This handles the case when we have a buffered frame that's older than the requested timestamp. - // We don't want to show this frame to the user, because it's not actually the one they requested. + // We don't want to show this frame to the user, because it's not actually the one they requested, + // so instead return the last decoded frame. if timestamp - frame_timestamp_ms > frame_duration_ms { - return self.texture.clone(); + return FrameDecodingResult::Pending(self.texture.clone()); } if self.last_used_frame_timestamp != frame_timestamp_ms { @@ -236,25 +231,25 @@ impl VideoDecoder { self.last_used_frame_timestamp = frame_timestamp_ms; } - self.texture.clone() + FrameDecodingResult::Ready(self.texture.clone()) } /// Enqueue all samples in the given segment. /// /// Does nothing if the index is out of bounds. - fn enqueue_all(&self, segment_idx: usize) { + fn enqueue_segment(&self, segment_idx: usize) { let Some(segment) = self.data.segments.get(segment_idx) else { return; }; - self.enqueue(&segment.samples[0], true); + self.enqueue_sample(&segment.samples[0], true); for sample in &segment.samples[1..] { - self.enqueue(sample, false); + self.enqueue_sample(sample, false); } } /// Enqueue the given sample. - fn enqueue(&self, sample: &re_video::Sample, is_key: bool) { + fn enqueue_sample(&self, sample: &re_video::Sample, is_key: bool) { let data = Uint8Array::from( &self.data.data[sample.byte_offset as usize ..sample.byte_offset as usize + sample.byte_length as usize], @@ -268,6 +263,7 @@ impl VideoDecoder { chunk.set_duration(sample.duration.as_f64()); let Some(chunk) = EncodedVideoChunk::new(&chunk) .inspect_err(|err| { + // TODO(#7373): return this error once the decoder tries to return a frame for this sample. how exactly? re_log::error!("failed to create video chunk: {}", js_error_to_string(err)); }) .ok() @@ -276,31 +272,24 @@ impl VideoDecoder { }; if let Err(err) = self.decoder.decode(&chunk) { + // TODO(#7373): return this error once the decoder tries to return a frame for this sample. how exactly? re_log::error!("Failed to decode video chunk: {}", js_error_to_string(&err)); } } /// Reset the video decoder and discard all frames. - fn reset(&mut self) { - if let Err(err) = self.decoder.reset() { - re_log::error!( - "Failed to reset video decoder: {}", - js_error_to_string(&err) - ); - } - - if let Err(err) = self - .decoder + fn reset(&mut self) -> Result<(), DecodingError> { + self.decoder + .reset() + .map_err(|err| DecodingError::ResetFailure(js_error_to_string(&err)))?; + self.decoder .configure(&js_video_decoder_config(&self.data.config)) - { - re_log::error!( - "Failed to configure video decoder: {}", - js_error_to_string(&err) - ); - } + .map_err(|err| DecodingError::ConfigureFailure(js_error_to_string(&err)))?; let mut frames = self.frames.lock(); drop(frames.drain(..)); + + Ok(()) } } @@ -359,11 +348,11 @@ fn copy_video_frame_to_texture( fn init_video_decoder( on_output: impl Fn(web_sys::VideoFrame) + 'static, -) -> Option { +) -> Result { let on_output = Closure::wrap(Box::new(on_output) as Box); let on_error = Closure::wrap(Box::new(|err: js_sys::Error| { + // TODO(#7373): store this error and report during decode let err = std::string::ToString::to_string(&err.to_string()); - re_log::error!("failed to decode video: {err}"); }) as Box); @@ -373,13 +362,8 @@ fn init_video_decoder( let Ok(on_error) = on_error.into_js_value().dyn_into::() else { unreachable!() }; - let decoder = web_sys::VideoDecoder::new(&VideoDecoderInit::new(&on_error, &on_output)) - .inspect_err(|err| { - re_log::error!("failed to create VideoDecoder: {}", js_error_to_string(err)); - }) - .ok()?; - - Some(decoder) + web_sys::VideoDecoder::new(&VideoDecoderInit::new(&on_error, &on_output)) + .map_err(|err| DecodingError::DecoderSetupFailure(js_error_to_string(&err))) } fn js_video_decoder_config(config: &re_video::Config) -> VideoDecoderConfig { diff --git a/crates/viewer/re_renderer/src/renderer/video/mod.rs b/crates/viewer/re_renderer/src/video/mod.rs similarity index 55% rename from crates/viewer/re_renderer/src/renderer/video/mod.rs rename to crates/viewer/re_renderer/src/video/mod.rs index 62a08a9608d9..7b981b48fe28 100644 --- a/crates/viewer/re_renderer/src/renderer/video/mod.rs +++ b/crates/viewer/re_renderer/src/video/mod.rs @@ -1,9 +1,53 @@ mod decoder; -use crate::resource_managers::GpuTexture2D; -use crate::RenderContext; -use re_video::TimeMs; -use re_video::VideoLoadError; +use re_video::{TimeMs, VideoLoadError}; + +use crate::{resource_managers::GpuTexture2D, RenderContext}; + +#[derive(thiserror::Error, Debug)] +pub enum VideoError { + #[error(transparent)] + Load(#[from] VideoLoadError), + + #[error(transparent)] + Init(#[from] DecodingError), +} + +/// Error that can occur during frame decoding. +// TODO(jan, andreas): These errors are for the most part specific to the web decoder right now. +#[derive(thiserror::Error, Debug)] +pub enum DecodingError { + #[error("Failed to create VideoDecoder: {0}")] + DecoderSetupFailure(String), + + #[error("Video seems to be empty, no segments have beem found.")] + EmptyVideo, + + #[error("The current segment is empty.")] + EmptySegment, + + #[error("Failed to reset the decoder: {0}")] + ResetFailure(String), + + #[error("Failed to configure the video decoder: {0}")] + ConfigureFailure(String), + + #[error("The timestamp passed was negative.")] + NegativeTimestamp, +} + +/// Information about the status of a frame decoding. +pub enum FrameDecodingResult { + /// The requested frame got decoded and is ready to be used. + Ready(GpuTexture2D), + + /// The returned texture is from a previous frame or a placeholder, the decoder is still decoding the requested frame. + Pending(GpuTexture2D), + + /// The decoder encountered an error and was not able to produce a texture for the requested timestamp. + /// The returned texture is either a placeholder or the last successfully decoded texture. + Error(DecodingError), +} /// A video file. /// @@ -31,8 +75,7 @@ impl Video { } None => return Err(VideoError::Load(VideoLoadError::UnknownMediaType)), }; - let decoder = - decoder::VideoDecoder::new(render_context, data).ok_or_else(|| VideoError::Init)?; + let decoder = decoder::VideoDecoder::new(render_context, data)?; Ok(Self { decoder }) } @@ -62,17 +105,8 @@ impl Video { /// /// This takes `&mut self` because the decoder maintains a buffer of decoded frames, /// which requires mutation. It is also not thread-safe by default. - pub fn frame_at(&mut self, timestamp_s: f64) -> GpuTexture2D { + pub fn frame_at(&mut self, timestamp_s: f64) -> FrameDecodingResult { re_tracing::profile_function!(); self.decoder.frame_at(TimeMs::new(timestamp_s * 1e3)) } } - -#[derive(thiserror::Error, Debug)] -pub enum VideoError { - #[error("{0}")] - Load(#[from] VideoLoadError), - - #[error("failed to initialize video decoder")] - Init, -} diff --git a/crates/viewer/re_space_view_spatial/src/video_cache.rs b/crates/viewer/re_space_view_spatial/src/video_cache.rs index 370a1464489b..8c6768dee6f4 100644 --- a/crates/viewer/re_space_view_spatial/src/video_cache.rs +++ b/crates/viewer/re_space_view_spatial/src/video_cache.rs @@ -1,5 +1,5 @@ use re_entity_db::VersionedInstancePathHash; -use re_renderer::{renderer::Video, RenderContext}; +use re_renderer::{video::Video, RenderContext}; use re_types::components::MediaType; use re_viewer_context::Cache; diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index dcbe5bbb5755..b89fcebeb5d5 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -1,8 +1,11 @@ use egui::mutex::Mutex; use re_log_types::EntityPath; -use re_renderer::renderer::{ - ColormappedTexture, RectangleOptions, TextureFilterMag, TextureFilterMin, TexturedRect, +use re_renderer::{ + renderer::{ + ColormappedTexture, RectangleOptions, TextureFilterMag, TextureFilterMin, TexturedRect, + }, + video::{FrameDecodingResult, Video}, }; use re_types::{ archetypes::{AssetVideo, VideoFrameReference}, @@ -119,7 +122,7 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { VideoTimeMode::Nanoseconds => video_timestamp.video_time as f64 / 1e9, }; - let (texture, video_width, video_height) = { + let (texture_result, video_width, video_height) = { let mut video = video.lock(); // TODO(andreas): Interior mutability for re_renderer's video would be nice. ( video.frame_at(timestamp_in_seconds), @@ -128,6 +131,21 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { ) }; + let texture = match texture_result { + FrameDecodingResult::Ready(texture) => texture, + FrameDecodingResult::Pending(texture) => { + ctx.viewer_ctx.egui_ctx.request_repaint(); + texture + } + FrameDecodingResult::Error(err) => { + // TODO(#7373): show this error in the ui + re_log::error_once!( + "Failed to decode video frame for {entity_path}: {err}" + ); + continue; + } + }; + let world_from_entity = spatial_ctx.transform_info.single_entity_transform_required( ctx.target_entity_path, @@ -215,7 +233,7 @@ fn textured_rect_for_video_frame( fn latest_at_query_video_from_datastore( ctx: &ViewerContext<'_>, entity_path: &EntityPath, -) -> Option>> { +) -> Option>> { let query = ctx.current_query(); let results = ctx.recording().query_caches().latest_at( From 8ddd84fca78204092d183a66438f88a353758a53 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 11 Sep 2024 13:39:58 +0200 Subject: [PATCH 3/3] Split out (ultra-)slow cargo doc jobs and run them only on nightly (#7399) ### What * Fixes https://github.com/rerun-io/rerun/issues/7387 * well, sort of ;) ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7399?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7399?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7399) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- .github/workflows/contrib_checks.yml | 2 +- .github/workflows/reusable_checks_rust.yml | 4 +-- .../src/visualizers/meshes.rs | 3 +-- scripts/ci/rust_checks.py | 25 +++++++++++++------ 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/workflows/contrib_checks.yml b/.github/workflows/contrib_checks.yml index a3e5f847917b..29f38dcf9d45 100644 --- a/.github/workflows/contrib_checks.yml +++ b/.github/workflows/contrib_checks.yml @@ -104,7 +104,7 @@ jobs: pixi-version: v0.25.0 - name: Rust checks & tests - run: pixi run rs-check --skip individual_crates + run: pixi run rs-check --skip individual_crates docs_slow misc-rerun-lints: name: Rerun lints diff --git a/.github/workflows/reusable_checks_rust.yml b/.github/workflows/reusable_checks_rust.yml index b89ef812556c..57dd6e587a1f 100644 --- a/.github/workflows/reusable_checks_rust.yml +++ b/.github/workflows/reusable_checks_rust.yml @@ -67,11 +67,11 @@ jobs: - name: Rust checks & tests if: ${{ inputs.CHANNEL == 'pr' }} - run: pixi run rs-check --skip individual_crates tests + run: pixi run rs-check --skip individual_crates tests docs_slow - name: Rust checks & tests if: ${{ inputs.CHANNEL == 'main' }} - run: pixi run rs-check --skip individual_crates + run: pixi run rs-check --skip individual_crates docs_slow - name: Rust all checks & tests if: ${{ inputs.CHANNEL == 'nightly' }} diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/meshes.rs b/crates/viewer/re_space_view_spatial/src/visualizers/meshes.rs index 91ac57ab236e..23fbe03c1d0d 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/meshes.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/meshes.rs @@ -1,7 +1,6 @@ use re_chunk_store::RowId; use re_log_types::{hash::Hash64, Instance, TimeInt}; -use re_renderer::renderer::MeshInstance; -use re_renderer::RenderContext; +use re_renderer::{renderer::MeshInstance, RenderContext}; use re_types::{ archetypes::Mesh3D, components::{ diff --git a/scripts/ci/rust_checks.py b/scripts/ci/rust_checks.py index bca4d2bf44a3..49e4b169937e 100755 --- a/scripts/ci/rust_checks.py +++ b/scripts/ci/rust_checks.py @@ -13,7 +13,7 @@ pixi run rs-check --only wasm To run all tests except a few specific ones you can use the `--skip` argument: - pixi run rs-check --skip wasm docs + pixi run rs-check --skip wasm docs docs_slow To see a list of all available tests you can use the `--help` argument: pixi run rs-check --help @@ -91,6 +91,7 @@ def main() -> None: ("individual_examples", individual_examples), ("individual_crates", individual_crates), ("docs", docs), + ("docs_slow", docs_slow), ("tests", tests), ] check_names = [check[0] for check in checks] @@ -199,12 +200,22 @@ def individual_crates(timings: list[Timing]) -> None: def docs(timings: list[Timing]) -> None: - # Full doc build takes prohibitively long (over 17min as of writing), so we skip it: - # timings.append(run_cargo("doc", "--all-features")) - - # These take around 3m40s each on CI, but very useful for catching broken doclinks: - timings.append(run_cargo("doc", "--no-deps --all-features --workspace")) - timings.append(run_cargo("doc", "--document-private-items --no-deps --all-features --workspace")) + # ⚠️ This version skips the `rerun` crate itself + # Presumably due to https://github.com/rust-lang/rust/issues/114891, checking the `rerun` crate + # takes about 20minutes on CI (per command). + # Since this crate mostly combines & exposes other crates, it's not as important for iterating on the code. + # + # For details see https://github.com/rerun-io/rerun/issues/7387 + + # These take a few minutes each on CI, but very useful for catching broken doclinks. + timings.append(run_cargo("doc", "--no-deps --all-features --workspace --exclude rerun")) + timings.append(run_cargo("doc", "--document-private-items --no-deps --all-features --workspace --exclude rerun")) + + +def docs_slow(timings: list[Timing]) -> None: + # See `docs` above, this may take 20min each due to issues in cargo doc. + timings.append(run_cargo("doc", "--no-deps --all-features -p rerun")) + timings.append(run_cargo("doc", "--document-private-items --no-deps --all-features -p rerun")) def tests(timings: list[Timing]) -> None: