diff --git a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 0190d6f0eb33..2a4c27ac48a3 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -2,10 +2,10 @@ namespace rerun.archetypes; /// A video binary. /// -/// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +/// Only MP4 containers with AV1 are generally supported, +/// though the web viewer supports more video codecs, depending on browser. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a [archetypes.VideoFrameReference] for each frame. /// diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs index 481d31c92a0f..b0f749184cc5 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -5,7 +5,7 @@ namespace rerun.archetypes; /// Used to display individual video frames from a [archetypes.AssetVideo]. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" /// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/9f41c00f84a98cc3f26875fba7c1d2fa2bad7151/1200w.png" diff --git a/crates/store/re_types/src/archetypes/asset_video.rs b/crates/store/re_types/src/archetypes/asset_video.rs index d0c64526b632..ea790784319c 100644 --- a/crates/store/re_types/src/archetypes/asset_video.rs +++ b/crates/store/re_types/src/archetypes/asset_video.rs @@ -20,10 +20,10 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Archetype**: A video binary. /// -/// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +/// Only MP4 containers with AV1 are generally supported, +/// though the web viewer supports more video codecs, depending on browser. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference] for each frame. /// diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs index 42143f2935f6..08380d586d21 100644 --- a/crates/store/re_types/src/archetypes/video_frame_reference.rs +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -23,7 +23,7 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// Used to display individual video frames from a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// ## Examples /// diff --git a/docs/content/reference/types/archetypes/asset_video.md b/docs/content/reference/types/archetypes/asset_video.md index 34ffa59c7bf3..4e9f2c488d27 100644 --- a/docs/content/reference/types/archetypes/asset_video.md +++ b/docs/content/reference/types/archetypes/asset_video.md @@ -5,10 +5,10 @@ title: "AssetVideo" A video binary. -NOTE: Videos can only be viewed in the Rerun web viewer. -Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +Only MP4 containers with AV1 are generally supported, +though the web viewer supports more video codecs, depending on browser. -See for details of what is and isn't supported. +See for details of what is and isn't supported. In order to display a video, you also need to log a [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference?speculative-link) for each frame. diff --git a/docs/content/reference/types/archetypes/video_frame_reference.md b/docs/content/reference/types/archetypes/video_frame_reference.md index 311d6e6f7f2d..5f33f501aa74 100644 --- a/docs/content/reference/types/archetypes/video_frame_reference.md +++ b/docs/content/reference/types/archetypes/video_frame_reference.md @@ -8,7 +8,7 @@ References a single video frame. Used to display individual video frames from a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video?speculative-link). To show an entire video, a video frame reference for each frame of the video should be logged. -See for details of what is and isn't supported. +See for details of what is and isn't supported. ## Components diff --git a/docs/content/reference/video.md b/docs/content/reference/video.md index f40693e42cb8..e7948d51a202 100644 --- a/docs/content/reference/video.md +++ b/docs/content/reference/video.md @@ -23,26 +23,32 @@ This gives the best compression ratio, reducing file sizes and bandwidth require snippet: archetypes/video_auto_frames -## `AssetVideo` limitations -Video support is new in Rerun, and has several limitations: +## Video playback limitations +Video support is new in Rerun, and has a few limitations: -* [#7298](https://github.com/rerun-io/rerun/issues/7298): Video playback only works in the web viewer * [#7354](https://github.com/rerun-io/rerun/issues/7354): Only the MP4 container format is supported +* [#7298](https://github.com/rerun-io/rerun/issues/7298): Only the AV1 codec is supported in the native viewer * [#5181](https://github.com/rerun-io/rerun/issues/5181): There is no audio support +* [#7594](https://github.com/rerun-io/rerun/issues/7594): HDR video is not supported * There is no video encoder in the Rerun SDK, so you need to create the video file yourself -* Only a limited sets of codecs are supported (see below) +* A limited sets of codecs are supported on web (see below) -## Web viewer support -As of writing, playback of `AssetVideo` is only supported on the web viewer. -Native video playback is coming, and can be tracked [in this GitHub issue](https://github.com/rerun-io/rerun/issues/7298). - -Video playback is done using the browser's own video decoder, so the supported codecs depend on your browser. - -Overall, we recommend using Chrome or another Chromium-based browser, as it seems to have the best video support as of writing. +## Streaming video +Rerun does not yet support streaming video support. For scenarios where you don't need live video, you can work around this limitation by logging many small `AssetVideo`s to the same Entity Path. See [#7484](https://github.com/rerun-io/rerun/issues/7484) for more. +## Codec support When choosing a codec, we recommend [AV1](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs#av1), as it seems to have the best overall playback support while also having very high compression quality. -Since AV1 is patent-free, it is also likely the first codec we will support in the native viewer. +AV1 is also patent-free, and is the only codec we currently support in the native viewer (see [#7298](https://github.com/rerun-io/rerun/issues/7298)). H.264/avc is another popular choice, and native support for that is coming soon. + +### Native viewer +In the native viewer, AV1 is the only supported codec. H.264 is coming soon ([#7298](https://github.com/rerun-io/rerun/issues/7298)). + + +### Web viewer +Video playback in the Rerun Web Viewer is done using the browser's own video decoder, so the supported codecs depend on your browser. + +Overall, we recommend using Chrome or another Chromium-based browser, as it seems to have the best video support as of writing. For decoding video in the Web Viewer, we use the [WebCodecs API](https://developer.mozilla.org/en-US/docs/Web/API/WebCodecs_API). This API enables us to take advantage of the browser's hardware accelerated video decoding capabilities. @@ -58,7 +64,7 @@ With that in mind, here are the browsers which we have tested and verified to ge [^1]: Firefox on Linux has been observed to [stutter when playing back H.264 video](https://github.com/rerun-io/rerun/issues/7532). [^2]: Any Chromium-based browser should work, but we don't test all of them. -[^3]: Chrome on Windows has been observed to stutter on playback. It can be mitigated by [using software decoding](https://rerun.io/docs/getting-started/troubleshooting#video-stuttering), but this may lead to high memory usage. See also https://github.com/rerun-io/rerun/issues/7595. +[^3]: Chrome on Windows has been observed to stutter on playback. It can be mitigated by [using software decoding](https://rerun.io/docs/getting-started/troubleshooting#video-stuttering), but this may lead to high memory usage. See [#7595](https://github.com/rerun-io/rerun/issues/7595). When it comes to codecs, we aim to support any codec which the browser supports, but we currently cannot guarantee that all of them will work. For more information about @@ -77,7 +83,6 @@ At the moment, we test the following codecs: [^6]: Safari/WebKit has been observed suttering when playing `hvc1` but working fine with `hevc1`. Despite support being advertised Safari 16.5 has been observed not support H.265 decoding. [^7]: Only supported if hardware encoding is available. Therefore always affected by Windows stuttering issues, see [^3]. -(TODO(#7594))[https://github.com/rerun-io/rerun/issues/7594]: HDR video is generally not supported at this point. ## Links * [Web video codec guide, by Mozilla](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs) diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.hpp b/rerun_cpp/src/rerun/archetypes/asset_video.hpp index 3d9dd80f1054..6748876bccbf 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.hpp @@ -21,10 +21,10 @@ namespace rerun::archetypes { /// **Archetype**: A video binary. /// - /// NOTE: Videos can only be viewed in the Rerun web viewer. - /// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. + /// Only MP4 containers with AV1 are generally supported, + /// though the web viewer supports more video codecs, depending on browser. /// - /// See for details of what is and isn't supported. + /// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a `archetypes::VideoFrameReference` for each frame. /// diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp index 254279dde5ce..421841e33496 100644 --- a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -22,7 +22,7 @@ namespace rerun::archetypes { /// Used to display individual video frames from a `archetypes::AssetVideo`. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// - /// See for details of what is and isn't supported. + /// See for details of what is and isn't supported. /// /// ## Examples /// diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py index 4709296b2a58..65c41a918d95 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py @@ -21,10 +21,10 @@ class AssetVideo(AssetVideoExt, Archetype): """ **Archetype**: A video binary. - NOTE: Videos can only be viewed in the Rerun web viewer. - Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. + Only MP4 containers with AV1 are generally supported, + though the web viewer supports more video codecs, depending on browser. - See for details of what is and isn't supported. + See for details of what is and isn't supported. In order to display a video, you also need to log a [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference] for each frame. diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py index 5ae51042c52e..71ffdaaae9f1 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -26,7 +26,7 @@ class VideoFrameReference(Archetype): Used to display individual video frames from a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. To show an entire video, a video frame reference for each frame of the video should be logged. - See for details of what is and isn't supported. + See for details of what is and isn't supported. Examples --------