Introduce VideoFrameReference archetype and base video visualizatio…

…n on it (#7396)

### What

* Large chunk of  #7368
---

* Introduce `VideoFrameReference` archetype
   * comes with a bunch of subtypes
* Make the video visualizer use this
* in the process I did some general iteration on how this visualizer
ticks
* note that I departed from the usual quite encrusted patterns of how to
implement visualizers that naturally found its way into here as well.
Imho the patterns don't hold up all that well in general (we had to
grind over them many times in quick succession) and less so in
particular for "special" archetypes like video or mesh (etc.)
* video data loader emits `VideoFrameReference` instead of the adhoc
`VideoTick


Still missing for completion of the video frame reference task:
* examples to use this from SDKs (by directly logging video frames)
   * this surely will show that we need more extensions 
* pretty/edit ui for video timestamps
* @Wumpf are you saying that you can then scrub videos from a blueprint
slider?! Yes. That's exactly what will end up happening ;D
* pretty/edit ui for entity paths 
* exposed utilities to SDKs for generation of video frame references

@ reviewers: Special attention please on the fbs definition, thank you!

<img width="1813" alt="image"
src="https://github.com/user-attachments/assets/d221ff1e-7ec0-4067-8916-7a8dd2ed8002">


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7396?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7396?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7396)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

Cargo.lock

@@ -4926,6 +4926,7 @@ dependencies = [
  "once_cell",
  "parking_lot",
  "rayon",
+ "re_arrow2",
  "re_build_info",
  "re_build_tools",
  "re_chunk",

crates/store/re_data_loader/Cargo.toml

@@ -36,6 +36,7 @@ re_video.workspace = true
 
 ahash.workspace = true
 anyhow.workspace = true
+arrow2.workspace = true
 image.workspace = true
 once_cell.workspace = true
 parking_lot.workspace = true

crates/store/re_data_loader/src/loader_archetype.rs

@@ -1,7 +1,13 @@
 use re_chunk::{Chunk, RowId};
-use re_log_types::NonMinI64;
 use re_log_types::{EntityPath, TimeInt, TimePoint};
-use re_types::components::MediaType;
+use re_types::archetypes::VideoFrameReference;
+use re_types::Archetype;
+use re_types::{components::MediaType, ComponentBatch};
+
+use arrow2::array::{
+    ListArray as ArrowListArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray,
+};
+use arrow2::Either;
 
 use crate::{DataLoader, DataLoaderError, LoadedData};
 
@@ -164,47 +170,6 @@ fn load_image(
     Ok(rows.into_iter())
 }
 
-/// TODO(#7272): fix this
-/// Used to expand the timeline when logging a video, so that the video can be played back.
-#[derive(Clone, Copy)]
-struct VideoTick(re_types::datatypes::Float64);
-
-impl re_types::AsComponents for VideoTick {
-    fn as_component_batches(&self) -> Vec<re_types::MaybeOwnedComponentBatch<'_>> {
-        vec![re_types::NamedIndicatorComponent("VideoTick".into()).to_batch()]
-    }
-}
-
-impl re_types::Loggable for VideoTick {
-    type Name = re_types::ComponentName;
-
-    fn name() -> Self::Name {
-        "rerun.components.VideoTick".into()
-    }
-
-    fn arrow_datatype() -> re_chunk::external::arrow2::datatypes::DataType {
-        re_types::datatypes::Float64::arrow_datatype()
-    }
-
-    fn to_arrow_opt<'a>(
-        data: impl IntoIterator<Item = Option<impl Into<std::borrow::Cow<'a, Self>>>>,
-    ) -> re_types::SerializationResult<Box<dyn re_chunk::external::arrow2::array::Array>>
-    where
-        Self: 'a,
-    {
-        re_types::datatypes::Float64::to_arrow_opt(
-            data.into_iter()
-                .map(|datum| datum.map(|datum| datum.into().0)),
-        )
-    }
-}
-
-impl re_types::SizeBytes for VideoTick {
-    fn heap_size_bytes(&self) -> u64 {
-        0
-    }
-}
-
 #[derive(Clone, Copy)]
 struct ExperimentalFeature;
 
@@ -252,51 +217,114 @@ fn load_video(
 ) -> Result<impl ExactSizeIterator<Item = Chunk>, DataLoaderError> {
     re_tracing::profile_function!();
 
-    timepoint.insert(
-        re_log_types::Timeline::new_temporal("video"),
-        re_log_types::TimeInt::new_temporal(0),
-    );
+    let video_timeline = re_log_types::Timeline::new_temporal("video");
+    timepoint.insert(video_timeline, re_log_types::TimeInt::new_temporal(0));
 
     let media_type = MediaType::guess_from_path(filepath);
 
-    let duration_s = match media_type.as_ref().map(|v| v.as_str()) {
-        Some("video/mp4") => re_video::load_mp4(&contents)
-            .ok()
-            .map(|v| v.duration.as_f64() / 1_000.0),
-        _ => None,
-    }
-    .unwrap_or(100.0)
-    .ceil() as i64;
+    // TODO(andreas): Video frame reference generation should be available as a utility from the SDK.
 
-    let mut rows = vec![Chunk::builder(entity_path.clone())
+    let video = if media_type.as_ref().map(|v| v.as_str()) == Some("video/mp4") {
+        match re_video::load_mp4(&contents) {
+            Ok(video) => Some(video),
+            Err(err) => {
+                re_log::warn!("Failed to load video asset {filepath:?}: {err}");
+                None
+            }
+        }
+    } else {
+        re_log::warn!("Video asset {filepath:?} has an unsupported container format.");
+        None
+    };
+
+    // Log video frame references on the `video` timeline.
+    let video_frame_reference_chunk = if let Some(video) = video {
+        let first_timestamp = video
+            .segments
+            .first()
+            .map_or(0, |segment| segment.timestamp.as_nanoseconds());
+
+        // Time column.
+        let is_sorted = Some(true);
+        let time_column_times =
+            ArrowPrimitiveArray::<i64>::from_values(video.segments.iter().flat_map(|segment| {
+                segment
+                    .samples
+                    .iter()
+                    .map(|s| s.timestamp.as_nanoseconds() - first_timestamp)
+            }));
+
+        let time_column = re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times);
+
+        // VideoTimestamp component column.
+        let video_timestamps = video
+            .segments
+            .iter()
+            .flat_map(|segment| {
+                segment.samples.iter().map(|s| {
+                    // TODO(andreas): Use sample indices instead of timestamps once possible.
+                    re_types::components::VideoTimestamp::new_nanoseconds(
+                        s.timestamp.as_nanoseconds(),
+                    )
+                })
+            })
+            .collect::<Vec<_>>();
+        let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch;
+        let video_timestamp_list_array = video_timestamp_batch
+            .to_arrow_list_array()
+            .map_err(re_chunk::ChunkError::from)?;
+
+        // Indicator column.
+        let video_frame_reference_indicator_datatype = arrow2::datatypes::DataType::Null;
+        let video_frame_reference_indicator_list_array = ArrowListArray::<i32>::try_new(
+            ArrowListArray::<i32>::default_datatype(
+                video_frame_reference_indicator_datatype.clone(),
+            ),
+            video_timestamp_list_array.offsets().clone(),
+            Box::new(ArrowNullArray::new(
+                video_frame_reference_indicator_datatype,
+                video_timestamps.len(),
+            )),
+            None,
+        )
+        .map_err(re_chunk::ChunkError::from)?;
+
+        Some(Chunk::from_auto_row_ids(
+            re_chunk::ChunkId::new(),
+            entity_path.clone(),
+            std::iter::once((video_timeline, time_column)).collect(),
+            [
+                (
+                    VideoFrameReference::indicator().name(),
+                    video_frame_reference_indicator_list_array,
+                ),
+                (video_timestamp_batch.name(), video_timestamp_list_array),
+            ]
+            .into_iter()
+            .collect(),
+        )?)
+    } else {
+        None
+    };
+
+    // Put video asset into its own chunk since it can be fairly large.
+    let video_asset_chunk = Chunk::builder(entity_path.clone())
         .with_archetype(
             RowId::new(),
             timepoint.clone(),
-            &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type),
+            &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type.clone()),
         )
         .with_component_batch(RowId::new(), timepoint.clone(), &ExperimentalFeature)
-        .build()?];
-
-    for i in 0..duration_s {
-        // We need some breadcrumbs of timepoints because the video doesn't have a duration yet.
-        // TODO(#7272): fix this
-        timepoint.insert(
-            re_log_types::Timeline::new_temporal("video"),
-            re_log_types::TimeInt::from_seconds(NonMinI64::new(i).expect("i > i64::MIN")),
-        );
-
-        rows.push(
-            Chunk::builder(entity_path.clone())
-                .with_component_batch(
-                    RowId::new(),
-                    timepoint.clone(),
-                    &VideoTick(re_types::datatypes::Float64(i as f64)),
-                )
-                .build()?,
-        );
+        .build()?;
+
+    if let Some(video_frame_reference_chunk) = video_frame_reference_chunk {
+        Ok(Either::Left(
+            [video_asset_chunk, video_frame_reference_chunk].into_iter(),
+        ))
+    } else {
+        // Still log the video asset, but don't include video frames.
+        Ok(Either::Right(std::iter::once(video_asset_chunk)))
     }
-
-    Ok(rows.into_iter())
 }
 
 fn load_mesh(

crates/store/re_types/definitions/rerun/archetypes/video.fbs → crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs

@@ -1,10 +1,13 @@
 namespace rerun.archetypes;
 
-/// A video file.
+/// A video binary.
 ///
 /// NOTE: Videos can only be viewed in the Rerun web viewer.
-/// Only MP4 and AV1 is currently supported, and not in all browsers.
+/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers.
 /// Follow <https://github.com/rerun-io/rerun/issues/7298> for updates on the native support.
+///
+/// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame.
+// TODO(#7368): More docs and examples on how to use this.
 table AssetVideo (
   "attr.rerun.experimental"
 ) {

crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs

@@ -0,0 +1,26 @@
+namespace rerun.archetypes;
+
+/// References a single video frame.
+///
+/// Used to display video frames from a [archetypes.AssetVideo].
+// TODO(#7368): More docs and examples on how to use this.
+table VideoFrameReference (
+    "attr.rerun.experimental"
+){
+    // --- Required ---
+
+    /// References the closest video frame to this timestamp.
+    ///
+    /// Note that this uses the closest video frame instead of the latest at this timestamp
+    /// in order to be more forgiving of rounding errors for inprecise timestamp types.
+    timestamp: rerun.components.VideoTimestamp ("attr.rerun.component_required", required, order: 1000);
+
+    // --- Optional ---
+
+    /// Optional reference to an entity with a [archetypes.AssetVideo].
+    ///
+    /// If none is specified, the video is assumed to be at the same entity.
+    /// Note that blueprint overrides on the referenced video will be ignored regardless,
+    /// as this is always interpreted as a reference to the data store.
+    video_reference: rerun.components.EntityPath ("attr.rerun.component_optional", nullable, order: 2000);
+}

crates/store/re_types/definitions/rerun/components/entity_path.fbs

@@ -0,0 +1,12 @@
+namespace rerun.components;
+
+/// A path to an entity, usually to reference some data that is part of the target entity.
+table EntityPath (
+  "attr.arrow.transparent",
+  "attr.python.aliases": "str",
+  "attr.python.array_aliases": "str, Sequence[str]",
+  "attr.rust.derive": "Default, PartialEq, Eq, PartialOrd, Ord",
+  "attr.rust.repr": "transparent"
+) {
+  value: rerun.datatypes.EntityPath (order: 100);
+}

crates/store/re_types/definitions/rerun/components/video_timestamp.fbs

@@ -0,0 +1,11 @@
+
+namespace rerun.components;
+
+/// Timestamp inside a [archetypes.AssetVideo].
+struct VideoTimestamp (
+  "attr.rust.derive": "Copy, PartialEq, Eq, Default",
+  "attr.rust.repr": "transparent",
+  "attr.rerun.experimental"
+) {
+  timestamp: rerun.datatypes.VideoTimestamp (order: 100);
+}

crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs

@@ -0,0 +1,24 @@
+namespace rerun.datatypes;
+
+/// Specifies how to interpret the `video_time` field of a [datatypes.VideoTimestamp].
+enum VideoTimeMode: ubyte{
+    /// Invalid value. Won't show up in generated types.
+    Invalid = 0,
+
+    /// Presentation timestamp in nanoseconds since the beginning of the video.
+    Nanoseconds = 1 (default),
+
+    // Future values: FrameNr
+}
+
+/// Timestamp inside a [archetypes.AssetVideo].
+struct VideoTimestamp (
+  "attr.rust.derive": "Copy, PartialEq, Eq",
+  "attr.rerun.experimental"
+) {
+    /// Timestamp value, type defined by `time_mode`.
+    video_time: long (order: 100);
+
+    /// How to interpret `video_time`.
+    time_mode: VideoTimeMode (order: 200);
+}