From 684851a361f8092286894f24078de2040c7cb3ad Mon Sep 17 00:00:00 2001
From: Andreas Reich <andreas@rerun.io>
Date: Thu, 17 Oct 2024 11:59:52 +0200
Subject: [PATCH] Link to video reference from image & image encoded (#7797)

### What

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)

- [PR Build Summary](https://build.rerun.io/pr/7797)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.
---
 .../definitions/rerun/archetypes/encoded_image.fbs         | 1 +
 .../store/re_types/definitions/rerun/archetypes/image.fbs  | 7 ++++---
 crates/store/re_types/src/archetypes/encoded_image.rs      | 1 +
 crates/store/re_types/src/archetypes/image.rs              | 7 ++++---
 docs/content/reference/types/archetypes/encoded_image.md   | 1 +
 docs/content/reference/types/archetypes/image.md           | 7 ++++---
 rerun_cpp/src/rerun/archetypes/encoded_image.hpp           | 1 +
 rerun_cpp/src/rerun/archetypes/image.hpp                   | 7 ++++---
 rerun_py/rerun_sdk/rerun/archetypes/encoded_image.py       | 1 +
 rerun_py/rerun_sdk/rerun/archetypes/image.py               | 7 ++++---
 10 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/crates/store/re_types/definitions/rerun/archetypes/encoded_image.fbs b/crates/store/re_types/definitions/rerun/archetypes/encoded_image.fbs
index 2b319f178ce3..654b846e77ff 100644
--- a/crates/store/re_types/definitions/rerun/archetypes/encoded_image.fbs
+++ b/crates/store/re_types/definitions/rerun/archetypes/encoded_image.fbs
@@ -4,6 +4,7 @@ namespace rerun.archetypes;
 /// An image encoded as e.g. a JPEG or PNG.
 ///
 /// Rerun also supports uncompressed images with the [archetypes.Image].
+/// For images that refer to video frames see [archetypes.VideoFrameReference].
 ///
 /// \py To compress an image, use [`rerun.Image.compress`][].
 ///
diff --git a/crates/store/re_types/definitions/rerun/archetypes/image.fbs b/crates/store/re_types/definitions/rerun/archetypes/image.fbs
index a588790063da..39684dd9dc5c 100644
--- a/crates/store/re_types/definitions/rerun/archetypes/image.fbs
+++ b/crates/store/re_types/definitions/rerun/archetypes/image.fbs
@@ -5,6 +5,10 @@ namespace rerun.archetypes;
 ///
 /// See also [archetypes.DepthImage] and [archetypes.SegmentationImage].
 ///
+/// Rerun also supports compressed images (JPEG, PNG, …), using [archetypes.EncodedImage].
+/// For images that refer to video frames see [archetypes.VideoFrameReference].
+/// Compressing images or using video data instead can save a lot of bandwidth and memory.
+///
 /// The raw image data is stored as a single buffer of bytes in a [components.Blob].
 /// The meaning of these bytes is determined by the [components.ImageFormat] which specifies the resolution
 /// and the pixel format (e.g. RGB, RGBA, …).
@@ -12,9 +16,6 @@ namespace rerun.archetypes;
 /// The order of dimensions in the underlying [components.Blob] follows the typical
 /// row-major, interleaved-pixel image format.
 ///
-/// Rerun also supports compressed images (JPEG, PNG, …), using [archetypes.EncodedImage].
-/// Compressing images can save a lot of bandwidth and memory.
-///
 /// \cpp Since the underlying [rerun::components::Blob] uses `rerun::Collection` internally,
 /// \cpp data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays.
 /// \cpp If needed, this "borrow-behavior" can be extended by defining your own `rerun::CollectionAdapter`.
diff --git a/crates/store/re_types/src/archetypes/encoded_image.rs b/crates/store/re_types/src/archetypes/encoded_image.rs
index 511aee051b62..2962feba7c20 100644
--- a/crates/store/re_types/src/archetypes/encoded_image.rs
+++ b/crates/store/re_types/src/archetypes/encoded_image.rs
@@ -21,6 +21,7 @@ use ::re_types_core::{DeserializationError, DeserializationResult};
 /// **Archetype**: An image encoded as e.g. a JPEG or PNG.
 ///
 /// Rerun also supports uncompressed images with the [`archetypes::Image`][crate::archetypes::Image].
+/// For images that refer to video frames see [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference].
 ///
 /// ## Example
 ///
diff --git a/crates/store/re_types/src/archetypes/image.rs b/crates/store/re_types/src/archetypes/image.rs
index 1e87272285c7..bcfd540dcbb0 100644
--- a/crates/store/re_types/src/archetypes/image.rs
+++ b/crates/store/re_types/src/archetypes/image.rs
@@ -22,6 +22,10 @@ use ::re_types_core::{DeserializationError, DeserializationResult};
 ///
 /// See also [`archetypes::DepthImage`][crate::archetypes::DepthImage] and [`archetypes::SegmentationImage`][crate::archetypes::SegmentationImage].
 ///
+/// Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes::EncodedImage`][crate::archetypes::EncodedImage].
+/// For images that refer to video frames see [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference].
+/// Compressing images or using video data instead can save a lot of bandwidth and memory.
+///
 /// The raw image data is stored as a single buffer of bytes in a [`components::Blob`][crate::components::Blob].
 /// The meaning of these bytes is determined by the [`components::ImageFormat`][crate::components::ImageFormat] which specifies the resolution
 /// and the pixel format (e.g. RGB, RGBA, …).
@@ -29,9 +33,6 @@ use ::re_types_core::{DeserializationError, DeserializationResult};
 /// The order of dimensions in the underlying [`components::Blob`][crate::components::Blob] follows the typical
 /// row-major, interleaved-pixel image format.
 ///
-/// Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes::EncodedImage`][crate::archetypes::EncodedImage].
-/// Compressing images can save a lot of bandwidth and memory.
-///
 /// ## Examples
 ///
 /// ### `image_simple`:
diff --git a/docs/content/reference/types/archetypes/encoded_image.md b/docs/content/reference/types/archetypes/encoded_image.md
index e824f2f3ff8d..6b0931405ee4 100644
--- a/docs/content/reference/types/archetypes/encoded_image.md
+++ b/docs/content/reference/types/archetypes/encoded_image.md
@@ -6,6 +6,7 @@ title: "EncodedImage"
 An image encoded as e.g. a JPEG or PNG.
 
 Rerun also supports uncompressed images with the [`archetypes.Image`](https://rerun.io/docs/reference/types/archetypes/image).
+For images that refer to video frames see [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference?speculative-link).
 
 ## Components
 
diff --git a/docs/content/reference/types/archetypes/image.md b/docs/content/reference/types/archetypes/image.md
index a2f2f64bf9df..a6d23fbe46f8 100644
--- a/docs/content/reference/types/archetypes/image.md
+++ b/docs/content/reference/types/archetypes/image.md
@@ -7,6 +7,10 @@ A monochrome or color image.
 
 See also [`archetypes.DepthImage`](https://rerun.io/docs/reference/types/archetypes/depth_image) and [`archetypes.SegmentationImage`](https://rerun.io/docs/reference/types/archetypes/segmentation_image).
 
+Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes.EncodedImage`](https://rerun.io/docs/reference/types/archetypes/encoded_image).
+For images that refer to video frames see [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference?speculative-link).
+Compressing images or using video data instead can save a lot of bandwidth and memory.
+
 The raw image data is stored as a single buffer of bytes in a [`components.Blob`](https://rerun.io/docs/reference/types/components/blob).
 The meaning of these bytes is determined by the [`components.ImageFormat`](https://rerun.io/docs/reference/types/components/image_format) which specifies the resolution
 and the pixel format (e.g. RGB, RGBA, …).
@@ -14,9 +18,6 @@ and the pixel format (e.g. RGB, RGBA, …).
 The order of dimensions in the underlying [`components.Blob`](https://rerun.io/docs/reference/types/components/blob) follows the typical
 row-major, interleaved-pixel image format.
 
-Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes.EncodedImage`](https://rerun.io/docs/reference/types/archetypes/encoded_image).
-Compressing images can save a lot of bandwidth and memory.
-
 ## Components
 
 **Required**: [`ImageBuffer`](../components/image_buffer.md), [`ImageFormat`](../components/image_format.md)
diff --git a/rerun_cpp/src/rerun/archetypes/encoded_image.hpp b/rerun_cpp/src/rerun/archetypes/encoded_image.hpp
index 2e4fb48d1b0b..3b13684c4ddb 100644
--- a/rerun_cpp/src/rerun/archetypes/encoded_image.hpp
+++ b/rerun_cpp/src/rerun/archetypes/encoded_image.hpp
@@ -23,6 +23,7 @@ namespace rerun::archetypes {
     /// **Archetype**: An image encoded as e.g. a JPEG or PNG.
     ///
     /// Rerun also supports uncompressed images with the `archetypes::Image`.
+    /// For images that refer to video frames see `archetypes::VideoFrameReference`.
     ///
     /// ## Example
     ///
diff --git a/rerun_cpp/src/rerun/archetypes/image.hpp b/rerun_cpp/src/rerun/archetypes/image.hpp
index aac2f05a9cc1..a96850434d3e 100644
--- a/rerun_cpp/src/rerun/archetypes/image.hpp
+++ b/rerun_cpp/src/rerun/archetypes/image.hpp
@@ -24,6 +24,10 @@ namespace rerun::archetypes {
     ///
     /// See also `archetypes::DepthImage` and `archetypes::SegmentationImage`.
     ///
+    /// Rerun also supports compressed images (JPEG, PNG, …), using `archetypes::EncodedImage`.
+    /// For images that refer to video frames see `archetypes::VideoFrameReference`.
+    /// Compressing images or using video data instead can save a lot of bandwidth and memory.
+    ///
     /// The raw image data is stored as a single buffer of bytes in a `components::Blob`.
     /// The meaning of these bytes is determined by the `components::ImageFormat` which specifies the resolution
     /// and the pixel format (e.g. RGB, RGBA, …).
@@ -31,9 +35,6 @@ namespace rerun::archetypes {
     /// The order of dimensions in the underlying `components::Blob` follows the typical
     /// row-major, interleaved-pixel image format.
     ///
-    /// Rerun also supports compressed images (JPEG, PNG, …), using `archetypes::EncodedImage`.
-    /// Compressing images can save a lot of bandwidth and memory.
-    ///
     /// Since the underlying [rerun::components::Blob] uses `rerun::Collection` internally,
     /// data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays.
     /// If needed, this "borrow-behavior" can be extended by defining your own `rerun::CollectionAdapter`.
diff --git a/rerun_py/rerun_sdk/rerun/archetypes/encoded_image.py b/rerun_py/rerun_sdk/rerun/archetypes/encoded_image.py
index e11db623d3da..8cc606648010 100644
--- a/rerun_py/rerun_sdk/rerun/archetypes/encoded_image.py
+++ b/rerun_py/rerun_sdk/rerun/archetypes/encoded_image.py
@@ -22,6 +22,7 @@ class EncodedImage(EncodedImageExt, Archetype):
     **Archetype**: An image encoded as e.g. a JPEG or PNG.
 
     Rerun also supports uncompressed images with the [`archetypes.Image`][rerun.archetypes.Image].
+    For images that refer to video frames see [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference].
 
     To compress an image, use [`rerun.Image.compress`][].
 
diff --git a/rerun_py/rerun_sdk/rerun/archetypes/image.py b/rerun_py/rerun_sdk/rerun/archetypes/image.py
index 92d18465b192..6180b2b98e7c 100644
--- a/rerun_py/rerun_sdk/rerun/archetypes/image.py
+++ b/rerun_py/rerun_sdk/rerun/archetypes/image.py
@@ -23,6 +23,10 @@ class Image(ImageExt, Archetype):
 
     See also [`archetypes.DepthImage`][rerun.archetypes.DepthImage] and [`archetypes.SegmentationImage`][rerun.archetypes.SegmentationImage].
 
+    Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes.EncodedImage`][rerun.archetypes.EncodedImage].
+    For images that refer to video frames see [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference].
+    Compressing images or using video data instead can save a lot of bandwidth and memory.
+
     The raw image data is stored as a single buffer of bytes in a [`components.Blob`][rerun.components.Blob].
     The meaning of these bytes is determined by the [`components.ImageFormat`][rerun.components.ImageFormat] which specifies the resolution
     and the pixel format (e.g. RGB, RGBA, …).
@@ -30,9 +34,6 @@ class Image(ImageExt, Archetype):
     The order of dimensions in the underlying [`components.Blob`][rerun.components.Blob] follows the typical
     row-major, interleaved-pixel image format.
 
-    Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes.EncodedImage`][rerun.archetypes.EncodedImage].
-    Compressing images can save a lot of bandwidth and memory.
-
     Examples
     --------
     ### `image_simple`: