From ad631cd76e90099205ccf7ac021a2a41aaef3096 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Fri, 11 Oct 2024 16:30:39 +0200 Subject: [PATCH 1/3] Enable (n)asm rav1d on Mac & Windows via `nasm` feature (#7694) ### What Related to * #7671 * #7605 * #7588 Biting the bullet and split the feature on our rav1d fork: https://github.com/rerun-io/rav1d/compare/emilk/dav1d-interface...rerun-io:rav1d:andreas/disable-asm-linux Testing done: * wsl linux release build works now (didn't before) * windows build still works * shows no-asm warning if compiled with out nasm * works otherwise * added a `panic` under `cfg(asm)` in one of the interface functions (now removed again) .. then: * linux still worked * windows hit that panic when compiled with nasm feature ### Checklist * [x] main ci passes * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7694?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7694?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7694) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --------- Co-authored-by: Emil Ernerfeldt --- Cargo.lock | 3 ++- crates/store/re_video/Cargo.toml | 10 ++++++---- crates/store/re_video/src/decode/av1.rs | 4 ++-- crates/top/rerun-cli/Cargo.toml | 2 +- crates/top/rerun/Cargo.toml | 2 +- rerun_py/Cargo.toml | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab71bbc4dffc..390eb91b4723 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4758,7 +4758,7 @@ dependencies = [ [[package]] name = "rav1d" version = "1.0.0" -source = "git+https://github.com/rerun-io/rav1d?branch=emilk/dav1d-interface#d3acfb077eac7759f94279ec9a81edd7d0c409ed" +source = "git+https://github.com/rerun-io/rav1d?rev=a9059f6a02e68cf2aba3795cc370c72f695c4730#a9059f6a02e68cf2aba3795cc370c72f695c4730" dependencies = [ "assert_matches", "atomig", @@ -4766,6 +4766,7 @@ dependencies = [ "bitflags 2.6.0", "cc", "cfg-if", + "cfg_aliases 0.2.1", "libc", "nasm-rs", "parking_lot", diff --git a/crates/store/re_video/Cargo.toml b/crates/store/re_video/Cargo.toml index afc07c39dcd3..ab63551b1016 100644 --- a/crates/store/re_video/Cargo.toml +++ b/crates/store/re_video/Cargo.toml @@ -31,9 +31,11 @@ av1 = ["dep:dav1d"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing -# nasm = ["dav1d?/default"] # The default feature set of dav1d has asm enabled -nasm = [] ## TODO(#7671): fix Linux build +# TODO(#7671): this feature flag currently does nothing on Linux. +nasm = [ + # The default feature set of our dav1d fork has asm enabled (except on Linux, see above) + "dav1d?/default", +] [dependencies] re_log.workspace = true @@ -52,7 +54,7 @@ thiserror.workspace = true # If this package fails to build, install `nasm` locally, or build through `pixi`. # See https://github.com/rerun-io/rav1d/pull/1 # TODO(#7588): publish this fork of rav1d -dav1d = { git = "https://github.com/rerun-io/rav1d", branch = "emilk/dav1d-interface", package = "rav1d", optional = true, default-features = false, features = [ +dav1d = { git = "https://github.com/rerun-io/rav1d", rev = "a9059f6a02e68cf2aba3795cc370c72f695c4730", package = "rav1d", optional = true, default-features = false, features = [ # We opt-out of wasm features so we don't need `nasm` installed. It's still pretty fast. "bitdepth_8", ] } diff --git a/crates/store/re_video/src/decode/av1.rs b/crates/store/re_video/src/decode/av1.rs index dea52895e2e7..983b918a7fb6 100644 --- a/crates/store/re_video/src/decode/av1.rs +++ b/crates/store/re_video/src/decode/av1.rs @@ -16,9 +16,9 @@ impl SyncDav1dDecoder { pub fn new() -> Result { re_tracing::profile_function!(); - // TODO(#7671): enable this warning again when the `nasm` feature actually does something + // TODO(#7671): enable this warning again on Linux when the `nasm` feature actually does something #[allow(clippy::overly_complex_bool_expr)] - if false && !cfg!(feature = "nasm") { + if !cfg!(target_os = "linux") && !cfg!(feature = "nasm") { re_log::warn_once!( "NOTE: native AV1 video decoder is running extra slowly. \ Speed it up by compiling Rerun with the `nasm` feature enabled. \ diff --git a/crates/top/rerun-cli/Cargo.toml b/crates/top/rerun-cli/Cargo.toml index 1f0f0ef3c5b0..2cf4ac433a67 100644 --- a/crates/top/rerun-cli/Cargo.toml +++ b/crates/top/rerun-cli/Cargo.toml @@ -43,7 +43,7 @@ default = ["native_viewer", "web_viewer"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing +# TODO(#7671): this feature flag currently does nothing on linux. nasm = ["rerun/nasm"] ## Support spawning a native viewer. diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index dfc1e8bccdde..a6fe1b87e346 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -73,7 +73,7 @@ log = ["dep:env_logger", "dep:log"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing +# TODO(#7671): this feature flag currently does nothing on linux. nasm = ["re_video/nasm"] ## Support spawning a native viewer. diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 9c19c36591ba..992d5b5f5977 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -30,7 +30,7 @@ extension-module = ["pyo3/extension-module"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing +# TODO(#7671): this feature flag currently does nothing on linux. nasm = ["re_video/nasm"] ## Support serving a web viewer over HTTP with `serve()`. From 9f34cb72ad88a9289f913c10c0b4c1d6f748a061 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Fri, 11 Oct 2024 16:56:18 +0200 Subject: [PATCH 2/3] Improve docs about video support (#7687) We do support AV1 on native, and now the docs say so too. * Closes https://github.com/rerun-io/rerun/issues/7662 ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7687?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7687?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide * [x] I've read the docs preview at https://landing-e6yens1hp-rerun.vercel.app/docs/reference/video - [PR Build Summary](https://build.rerun.io/pr/7687) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --------- Co-authored-by: Andreas Reich --- .../rerun/archetypes/asset_video.fbs | 6 ++-- .../archetypes/video_frame_reference.fbs | 2 +- .../re_types/src/archetypes/asset_video.rs | 6 ++-- .../src/archetypes/video_frame_reference.rs | 2 +- .../reference/types/archetypes/asset_video.md | 6 ++-- .../types/archetypes/video_frame_reference.md | 2 +- docs/content/reference/video.md | 33 +++++++++++-------- .../src/rerun/archetypes/asset_video.hpp | 6 ++-- .../archetypes/video_frame_reference.hpp | 2 +- .../rerun_sdk/rerun/archetypes/asset_video.py | 6 ++-- .../rerun/archetypes/video_frame_reference.py | 2 +- 11 files changed, 39 insertions(+), 34 deletions(-) diff --git a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 0190d6f0eb33..2a4c27ac48a3 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -2,10 +2,10 @@ namespace rerun.archetypes; /// A video binary. /// -/// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +/// Only MP4 containers with AV1 are generally supported, +/// though the web viewer supports more video codecs, depending on browser. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a [archetypes.VideoFrameReference] for each frame. /// diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs index 481d31c92a0f..b0f749184cc5 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -5,7 +5,7 @@ namespace rerun.archetypes; /// Used to display individual video frames from a [archetypes.AssetVideo]. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" /// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/9f41c00f84a98cc3f26875fba7c1d2fa2bad7151/1200w.png" diff --git a/crates/store/re_types/src/archetypes/asset_video.rs b/crates/store/re_types/src/archetypes/asset_video.rs index d0c64526b632..ea790784319c 100644 --- a/crates/store/re_types/src/archetypes/asset_video.rs +++ b/crates/store/re_types/src/archetypes/asset_video.rs @@ -20,10 +20,10 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Archetype**: A video binary. /// -/// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +/// Only MP4 containers with AV1 are generally supported, +/// though the web viewer supports more video codecs, depending on browser. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference] for each frame. /// diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs index 42143f2935f6..08380d586d21 100644 --- a/crates/store/re_types/src/archetypes/video_frame_reference.rs +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -23,7 +23,7 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// Used to display individual video frames from a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// ## Examples /// diff --git a/docs/content/reference/types/archetypes/asset_video.md b/docs/content/reference/types/archetypes/asset_video.md index 34ffa59c7bf3..4e9f2c488d27 100644 --- a/docs/content/reference/types/archetypes/asset_video.md +++ b/docs/content/reference/types/archetypes/asset_video.md @@ -5,10 +5,10 @@ title: "AssetVideo" A video binary. -NOTE: Videos can only be viewed in the Rerun web viewer. -Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +Only MP4 containers with AV1 are generally supported, +though the web viewer supports more video codecs, depending on browser. -See for details of what is and isn't supported. +See for details of what is and isn't supported. In order to display a video, you also need to log a [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference?speculative-link) for each frame. diff --git a/docs/content/reference/types/archetypes/video_frame_reference.md b/docs/content/reference/types/archetypes/video_frame_reference.md index 311d6e6f7f2d..5f33f501aa74 100644 --- a/docs/content/reference/types/archetypes/video_frame_reference.md +++ b/docs/content/reference/types/archetypes/video_frame_reference.md @@ -8,7 +8,7 @@ References a single video frame. Used to display individual video frames from a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video?speculative-link). To show an entire video, a video frame reference for each frame of the video should be logged. -See for details of what is and isn't supported. +See for details of what is and isn't supported. ## Components diff --git a/docs/content/reference/video.md b/docs/content/reference/video.md index f40693e42cb8..e7948d51a202 100644 --- a/docs/content/reference/video.md +++ b/docs/content/reference/video.md @@ -23,26 +23,32 @@ This gives the best compression ratio, reducing file sizes and bandwidth require snippet: archetypes/video_auto_frames -## `AssetVideo` limitations -Video support is new in Rerun, and has several limitations: +## Video playback limitations +Video support is new in Rerun, and has a few limitations: -* [#7298](https://github.com/rerun-io/rerun/issues/7298): Video playback only works in the web viewer * [#7354](https://github.com/rerun-io/rerun/issues/7354): Only the MP4 container format is supported +* [#7298](https://github.com/rerun-io/rerun/issues/7298): Only the AV1 codec is supported in the native viewer * [#5181](https://github.com/rerun-io/rerun/issues/5181): There is no audio support +* [#7594](https://github.com/rerun-io/rerun/issues/7594): HDR video is not supported * There is no video encoder in the Rerun SDK, so you need to create the video file yourself -* Only a limited sets of codecs are supported (see below) +* A limited sets of codecs are supported on web (see below) -## Web viewer support -As of writing, playback of `AssetVideo` is only supported on the web viewer. -Native video playback is coming, and can be tracked [in this GitHub issue](https://github.com/rerun-io/rerun/issues/7298). - -Video playback is done using the browser's own video decoder, so the supported codecs depend on your browser. - -Overall, we recommend using Chrome or another Chromium-based browser, as it seems to have the best video support as of writing. +## Streaming video +Rerun does not yet support streaming video support. For scenarios where you don't need live video, you can work around this limitation by logging many small `AssetVideo`s to the same Entity Path. See [#7484](https://github.com/rerun-io/rerun/issues/7484) for more. +## Codec support When choosing a codec, we recommend [AV1](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs#av1), as it seems to have the best overall playback support while also having very high compression quality. -Since AV1 is patent-free, it is also likely the first codec we will support in the native viewer. +AV1 is also patent-free, and is the only codec we currently support in the native viewer (see [#7298](https://github.com/rerun-io/rerun/issues/7298)). H.264/avc is another popular choice, and native support for that is coming soon. + +### Native viewer +In the native viewer, AV1 is the only supported codec. H.264 is coming soon ([#7298](https://github.com/rerun-io/rerun/issues/7298)). + + +### Web viewer +Video playback in the Rerun Web Viewer is done using the browser's own video decoder, so the supported codecs depend on your browser. + +Overall, we recommend using Chrome or another Chromium-based browser, as it seems to have the best video support as of writing. For decoding video in the Web Viewer, we use the [WebCodecs API](https://developer.mozilla.org/en-US/docs/Web/API/WebCodecs_API). This API enables us to take advantage of the browser's hardware accelerated video decoding capabilities. @@ -58,7 +64,7 @@ With that in mind, here are the browsers which we have tested and verified to ge [^1]: Firefox on Linux has been observed to [stutter when playing back H.264 video](https://github.com/rerun-io/rerun/issues/7532). [^2]: Any Chromium-based browser should work, but we don't test all of them. -[^3]: Chrome on Windows has been observed to stutter on playback. It can be mitigated by [using software decoding](https://rerun.io/docs/getting-started/troubleshooting#video-stuttering), but this may lead to high memory usage. See also https://github.com/rerun-io/rerun/issues/7595. +[^3]: Chrome on Windows has been observed to stutter on playback. It can be mitigated by [using software decoding](https://rerun.io/docs/getting-started/troubleshooting#video-stuttering), but this may lead to high memory usage. See [#7595](https://github.com/rerun-io/rerun/issues/7595). When it comes to codecs, we aim to support any codec which the browser supports, but we currently cannot guarantee that all of them will work. For more information about @@ -77,7 +83,6 @@ At the moment, we test the following codecs: [^6]: Safari/WebKit has been observed suttering when playing `hvc1` but working fine with `hevc1`. Despite support being advertised Safari 16.5 has been observed not support H.265 decoding. [^7]: Only supported if hardware encoding is available. Therefore always affected by Windows stuttering issues, see [^3]. -(TODO(#7594))[https://github.com/rerun-io/rerun/issues/7594]: HDR video is generally not supported at this point. ## Links * [Web video codec guide, by Mozilla](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs) diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.hpp b/rerun_cpp/src/rerun/archetypes/asset_video.hpp index 3d9dd80f1054..6748876bccbf 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.hpp @@ -21,10 +21,10 @@ namespace rerun::archetypes { /// **Archetype**: A video binary. /// - /// NOTE: Videos can only be viewed in the Rerun web viewer. - /// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. + /// Only MP4 containers with AV1 are generally supported, + /// though the web viewer supports more video codecs, depending on browser. /// - /// See for details of what is and isn't supported. + /// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a `archetypes::VideoFrameReference` for each frame. /// diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp index 254279dde5ce..421841e33496 100644 --- a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -22,7 +22,7 @@ namespace rerun::archetypes { /// Used to display individual video frames from a `archetypes::AssetVideo`. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// - /// See for details of what is and isn't supported. + /// See for details of what is and isn't supported. /// /// ## Examples /// diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py index 4709296b2a58..65c41a918d95 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py @@ -21,10 +21,10 @@ class AssetVideo(AssetVideoExt, Archetype): """ **Archetype**: A video binary. - NOTE: Videos can only be viewed in the Rerun web viewer. - Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. + Only MP4 containers with AV1 are generally supported, + though the web viewer supports more video codecs, depending on browser. - See for details of what is and isn't supported. + See for details of what is and isn't supported. In order to display a video, you also need to log a [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference] for each frame. diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py index 5ae51042c52e..71ffdaaae9f1 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -26,7 +26,7 @@ class VideoFrameReference(Archetype): Used to display individual video frames from a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. To show an entire video, a video frame reference for each frame of the video should be logged. - See for details of what is and isn't supported. + See for details of what is and isn't supported. Examples -------- From fd46ce3142bbe823de0417b105bef0c794d0be3f Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 11 Oct 2024 17:04:03 +0200 Subject: [PATCH 3/3] Fix main (unordered map that shouldnt be here) (#7697) --- crates/store/re_chunk_store/src/store.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/store/re_chunk_store/src/store.rs b/crates/store/re_chunk_store/src/store.rs index 28d825aa9af3..4456a2a0e231 100644 --- a/crates/store/re_chunk_store/src/store.rs +++ b/crates/store/re_chunk_store/src/store.rs @@ -333,7 +333,8 @@ pub struct ChunkStore { // different datatype for a given component. pub(crate) type_registry: IntMap, - pub(crate) per_column_metadata: IntMap>, + pub(crate) per_column_metadata: + BTreeMap>, pub(crate) chunks_per_chunk_id: BTreeMap>,