diff --git a/Cargo.lock b/Cargo.lock index ab71bbc4dffc..390eb91b4723 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4758,7 +4758,7 @@ dependencies = [ [[package]] name = "rav1d" version = "1.0.0" -source = "git+https://github.com/rerun-io/rav1d?branch=emilk/dav1d-interface#d3acfb077eac7759f94279ec9a81edd7d0c409ed" +source = "git+https://github.com/rerun-io/rav1d?rev=a9059f6a02e68cf2aba3795cc370c72f695c4730#a9059f6a02e68cf2aba3795cc370c72f695c4730" dependencies = [ "assert_matches", "atomig", @@ -4766,6 +4766,7 @@ dependencies = [ "bitflags 2.6.0", "cc", "cfg-if", + "cfg_aliases 0.2.1", "libc", "nasm-rs", "parking_lot", diff --git a/crates/store/re_chunk_store/src/store.rs b/crates/store/re_chunk_store/src/store.rs index 28d825aa9af3..4456a2a0e231 100644 --- a/crates/store/re_chunk_store/src/store.rs +++ b/crates/store/re_chunk_store/src/store.rs @@ -333,7 +333,8 @@ pub struct ChunkStore { // different datatype for a given component. pub(crate) type_registry: IntMap, - pub(crate) per_column_metadata: IntMap>, + pub(crate) per_column_metadata: + BTreeMap>, pub(crate) chunks_per_chunk_id: BTreeMap>, diff --git a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 0190d6f0eb33..2a4c27ac48a3 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -2,10 +2,10 @@ namespace rerun.archetypes; /// A video binary. /// -/// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +/// Only MP4 containers with AV1 are generally supported, +/// though the web viewer supports more video codecs, depending on browser. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a [archetypes.VideoFrameReference] for each frame. /// diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs index 481d31c92a0f..b0f749184cc5 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -5,7 +5,7 @@ namespace rerun.archetypes; /// Used to display individual video frames from a [archetypes.AssetVideo]. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" /// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/9f41c00f84a98cc3f26875fba7c1d2fa2bad7151/1200w.png" diff --git a/crates/store/re_types/src/archetypes/asset_video.rs b/crates/store/re_types/src/archetypes/asset_video.rs index d0c64526b632..ea790784319c 100644 --- a/crates/store/re_types/src/archetypes/asset_video.rs +++ b/crates/store/re_types/src/archetypes/asset_video.rs @@ -20,10 +20,10 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Archetype**: A video binary. /// -/// NOTE: Videos can only be viewed in the Rerun web viewer. -/// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +/// Only MP4 containers with AV1 are generally supported, +/// though the web viewer supports more video codecs, depending on browser. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a [`archetypes::VideoFrameReference`][crate::archetypes::VideoFrameReference] for each frame. /// diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs index 42143f2935f6..08380d586d21 100644 --- a/crates/store/re_types/src/archetypes/video_frame_reference.rs +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -23,7 +23,7 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// Used to display individual video frames from a [`archetypes::AssetVideo`][crate::archetypes::AssetVideo]. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// -/// See for details of what is and isn't supported. +/// See for details of what is and isn't supported. /// /// ## Examples /// diff --git a/crates/store/re_video/Cargo.toml b/crates/store/re_video/Cargo.toml index afc07c39dcd3..ab63551b1016 100644 --- a/crates/store/re_video/Cargo.toml +++ b/crates/store/re_video/Cargo.toml @@ -31,9 +31,11 @@ av1 = ["dep:dav1d"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing -# nasm = ["dav1d?/default"] # The default feature set of dav1d has asm enabled -nasm = [] ## TODO(#7671): fix Linux build +# TODO(#7671): this feature flag currently does nothing on Linux. +nasm = [ + # The default feature set of our dav1d fork has asm enabled (except on Linux, see above) + "dav1d?/default", +] [dependencies] re_log.workspace = true @@ -52,7 +54,7 @@ thiserror.workspace = true # If this package fails to build, install `nasm` locally, or build through `pixi`. # See https://github.com/rerun-io/rav1d/pull/1 # TODO(#7588): publish this fork of rav1d -dav1d = { git = "https://github.com/rerun-io/rav1d", branch = "emilk/dav1d-interface", package = "rav1d", optional = true, default-features = false, features = [ +dav1d = { git = "https://github.com/rerun-io/rav1d", rev = "a9059f6a02e68cf2aba3795cc370c72f695c4730", package = "rav1d", optional = true, default-features = false, features = [ # We opt-out of wasm features so we don't need `nasm` installed. It's still pretty fast. "bitdepth_8", ] } diff --git a/crates/store/re_video/src/decode/av1.rs b/crates/store/re_video/src/decode/av1.rs index dea52895e2e7..983b918a7fb6 100644 --- a/crates/store/re_video/src/decode/av1.rs +++ b/crates/store/re_video/src/decode/av1.rs @@ -16,9 +16,9 @@ impl SyncDav1dDecoder { pub fn new() -> Result { re_tracing::profile_function!(); - // TODO(#7671): enable this warning again when the `nasm` feature actually does something + // TODO(#7671): enable this warning again on Linux when the `nasm` feature actually does something #[allow(clippy::overly_complex_bool_expr)] - if false && !cfg!(feature = "nasm") { + if !cfg!(target_os = "linux") && !cfg!(feature = "nasm") { re_log::warn_once!( "NOTE: native AV1 video decoder is running extra slowly. \ Speed it up by compiling Rerun with the `nasm` feature enabled. \ diff --git a/crates/top/rerun-cli/Cargo.toml b/crates/top/rerun-cli/Cargo.toml index 1f0f0ef3c5b0..2cf4ac433a67 100644 --- a/crates/top/rerun-cli/Cargo.toml +++ b/crates/top/rerun-cli/Cargo.toml @@ -43,7 +43,7 @@ default = ["native_viewer", "web_viewer"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing +# TODO(#7671): this feature flag currently does nothing on linux. nasm = ["rerun/nasm"] ## Support spawning a native viewer. diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index dfc1e8bccdde..a6fe1b87e346 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -73,7 +73,7 @@ log = ["dep:env_logger", "dep:log"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing +# TODO(#7671): this feature flag currently does nothing on linux. nasm = ["re_video/nasm"] ## Support spawning a native viewer. diff --git a/docs/content/reference/types/archetypes/asset_video.md b/docs/content/reference/types/archetypes/asset_video.md index 34ffa59c7bf3..4e9f2c488d27 100644 --- a/docs/content/reference/types/archetypes/asset_video.md +++ b/docs/content/reference/types/archetypes/asset_video.md @@ -5,10 +5,10 @@ title: "AssetVideo" A video binary. -NOTE: Videos can only be viewed in the Rerun web viewer. -Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. +Only MP4 containers with AV1 are generally supported, +though the web viewer supports more video codecs, depending on browser. -See for details of what is and isn't supported. +See for details of what is and isn't supported. In order to display a video, you also need to log a [`archetypes.VideoFrameReference`](https://rerun.io/docs/reference/types/archetypes/video_frame_reference?speculative-link) for each frame. diff --git a/docs/content/reference/types/archetypes/video_frame_reference.md b/docs/content/reference/types/archetypes/video_frame_reference.md index 311d6e6f7f2d..5f33f501aa74 100644 --- a/docs/content/reference/types/archetypes/video_frame_reference.md +++ b/docs/content/reference/types/archetypes/video_frame_reference.md @@ -8,7 +8,7 @@ References a single video frame. Used to display individual video frames from a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video?speculative-link). To show an entire video, a video frame reference for each frame of the video should be logged. -See for details of what is and isn't supported. +See for details of what is and isn't supported. ## Components diff --git a/docs/content/reference/video.md b/docs/content/reference/video.md index f40693e42cb8..e7948d51a202 100644 --- a/docs/content/reference/video.md +++ b/docs/content/reference/video.md @@ -23,26 +23,32 @@ This gives the best compression ratio, reducing file sizes and bandwidth require snippet: archetypes/video_auto_frames -## `AssetVideo` limitations -Video support is new in Rerun, and has several limitations: +## Video playback limitations +Video support is new in Rerun, and has a few limitations: -* [#7298](https://github.com/rerun-io/rerun/issues/7298): Video playback only works in the web viewer * [#7354](https://github.com/rerun-io/rerun/issues/7354): Only the MP4 container format is supported +* [#7298](https://github.com/rerun-io/rerun/issues/7298): Only the AV1 codec is supported in the native viewer * [#5181](https://github.com/rerun-io/rerun/issues/5181): There is no audio support +* [#7594](https://github.com/rerun-io/rerun/issues/7594): HDR video is not supported * There is no video encoder in the Rerun SDK, so you need to create the video file yourself -* Only a limited sets of codecs are supported (see below) +* A limited sets of codecs are supported on web (see below) -## Web viewer support -As of writing, playback of `AssetVideo` is only supported on the web viewer. -Native video playback is coming, and can be tracked [in this GitHub issue](https://github.com/rerun-io/rerun/issues/7298). - -Video playback is done using the browser's own video decoder, so the supported codecs depend on your browser. - -Overall, we recommend using Chrome or another Chromium-based browser, as it seems to have the best video support as of writing. +## Streaming video +Rerun does not yet support streaming video support. For scenarios where you don't need live video, you can work around this limitation by logging many small `AssetVideo`s to the same Entity Path. See [#7484](https://github.com/rerun-io/rerun/issues/7484) for more. +## Codec support When choosing a codec, we recommend [AV1](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs#av1), as it seems to have the best overall playback support while also having very high compression quality. -Since AV1 is patent-free, it is also likely the first codec we will support in the native viewer. +AV1 is also patent-free, and is the only codec we currently support in the native viewer (see [#7298](https://github.com/rerun-io/rerun/issues/7298)). H.264/avc is another popular choice, and native support for that is coming soon. + +### Native viewer +In the native viewer, AV1 is the only supported codec. H.264 is coming soon ([#7298](https://github.com/rerun-io/rerun/issues/7298)). + + +### Web viewer +Video playback in the Rerun Web Viewer is done using the browser's own video decoder, so the supported codecs depend on your browser. + +Overall, we recommend using Chrome or another Chromium-based browser, as it seems to have the best video support as of writing. For decoding video in the Web Viewer, we use the [WebCodecs API](https://developer.mozilla.org/en-US/docs/Web/API/WebCodecs_API). This API enables us to take advantage of the browser's hardware accelerated video decoding capabilities. @@ -58,7 +64,7 @@ With that in mind, here are the browsers which we have tested and verified to ge [^1]: Firefox on Linux has been observed to [stutter when playing back H.264 video](https://github.com/rerun-io/rerun/issues/7532). [^2]: Any Chromium-based browser should work, but we don't test all of them. -[^3]: Chrome on Windows has been observed to stutter on playback. It can be mitigated by [using software decoding](https://rerun.io/docs/getting-started/troubleshooting#video-stuttering), but this may lead to high memory usage. See also https://github.com/rerun-io/rerun/issues/7595. +[^3]: Chrome on Windows has been observed to stutter on playback. It can be mitigated by [using software decoding](https://rerun.io/docs/getting-started/troubleshooting#video-stuttering), but this may lead to high memory usage. See [#7595](https://github.com/rerun-io/rerun/issues/7595). When it comes to codecs, we aim to support any codec which the browser supports, but we currently cannot guarantee that all of them will work. For more information about @@ -77,7 +83,6 @@ At the moment, we test the following codecs: [^6]: Safari/WebKit has been observed suttering when playing `hvc1` but working fine with `hevc1`. Despite support being advertised Safari 16.5 has been observed not support H.265 decoding. [^7]: Only supported if hardware encoding is available. Therefore always affected by Windows stuttering issues, see [^3]. -(TODO(#7594))[https://github.com/rerun-io/rerun/issues/7594]: HDR video is generally not supported at this point. ## Links * [Web video codec guide, by Mozilla](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs) diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.hpp b/rerun_cpp/src/rerun/archetypes/asset_video.hpp index 3d9dd80f1054..6748876bccbf 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.hpp @@ -21,10 +21,10 @@ namespace rerun::archetypes { /// **Archetype**: A video binary. /// - /// NOTE: Videos can only be viewed in the Rerun web viewer. - /// Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. + /// Only MP4 containers with AV1 are generally supported, + /// though the web viewer supports more video codecs, depending on browser. /// - /// See for details of what is and isn't supported. + /// See for details of what is and isn't supported. /// /// In order to display a video, you also need to log a `archetypes::VideoFrameReference` for each frame. /// diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp index 254279dde5ce..421841e33496 100644 --- a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -22,7 +22,7 @@ namespace rerun::archetypes { /// Used to display individual video frames from a `archetypes::AssetVideo`. /// To show an entire video, a video frame reference for each frame of the video should be logged. /// - /// See for details of what is and isn't supported. + /// See for details of what is and isn't supported. /// /// ## Examples /// diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 9c19c36591ba..992d5b5f5977 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -30,7 +30,7 @@ extension-module = ["pyo3/extension-module"] ## Enable faster native video decoding with assembly. ## You need to install [nasm](https://nasm.us/) to compile with this feature. -# TODO(#7671): this feature flag currently does nothing +# TODO(#7671): this feature flag currently does nothing on linux. nasm = ["re_video/nasm"] ## Support serving a web viewer over HTTP with `serve()`. diff --git a/rerun_py/rerun_bindings/rerun_bindings.pyi b/rerun_py/rerun_bindings/rerun_bindings.pyi index 72dcdb44801a..9ebf12f9bdb4 100644 --- a/rerun_py/rerun_bindings/rerun_bindings.pyi +++ b/rerun_py/rerun_bindings/rerun_bindings.pyi @@ -61,12 +61,42 @@ class RecordingView: """ Filter the view to only include data at the given index values. + The index values returned will be the intersection between the provided values and the + original index values. + This requires index values to be a precise match. Index values in Rerun are represented as i64 sequence counts or nanoseconds. This API does not expose an interface in floating point seconds, as the numerical conversion would risk false mismatches. """ ... + def filter_is_not_null(self, column: AnyColumn) -> RecordingView: + """ + Filter the view to only include rows where the given column is not null. + + This corresponds to rows for index values where this component was provided to Rerun explicitly + via `.log()` or `.send_columns()`. + """ + ... + + def using_index_values(self, values: IndexValuesLike) -> RecordingView: + """ + Replace the index in the view with the provided values. + + The output view will always have the same number of rows as the provided values, even if + those rows are empty. Use with `.fill_latest_at()` to populate these rows with the most + recent data. + + This requires index values to be a precise match. Index values in Rerun are + represented as i64 sequence counts or nanoseconds. This API does not expose an interface + in floating point seconds, as the numerical conversion would risk false mismatches. + """ + ... + + def fill_latest_at(self) -> RecordingView: + """Populate any null values in a row with the latest valid data on the timeline.""" + ... + def select(self, *args: AnyColumn, columns: Optional[Sequence[AnyColumn]] = None) -> pa.RecordBatchReader: ... class Recording: diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py index 4709296b2a58..65c41a918d95 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py @@ -21,10 +21,10 @@ class AssetVideo(AssetVideoExt, Archetype): """ **Archetype**: A video binary. - NOTE: Videos can only be viewed in the Rerun web viewer. - Only MP4 containers with a limited number of codecs are currently supported, and not in all browsers. + Only MP4 containers with AV1 are generally supported, + though the web viewer supports more video codecs, depending on browser. - See for details of what is and isn't supported. + See for details of what is and isn't supported. In order to display a video, you also need to log a [`archetypes.VideoFrameReference`][rerun.archetypes.VideoFrameReference] for each frame. diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py index 5ae51042c52e..71ffdaaae9f1 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -26,7 +26,7 @@ class VideoFrameReference(Archetype): Used to display individual video frames from a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. To show an entire video, a video frame reference for each frame of the video should be logged. - See for details of what is and isn't supported. + See for details of what is and isn't supported. Examples -------- diff --git a/rerun_py/src/dataframe.rs b/rerun_py/src/dataframe.rs index 6651fd2020b6..00c573c9d964 100644 --- a/rerun_py/src/dataframe.rs +++ b/rerun_py/src/dataframe.rs @@ -553,6 +553,40 @@ impl PyRecordingView { query_expression, }) } + + fn filter_is_not_null(&self, column: AnyComponentColumn) -> Self { + let column = column.into_selector(); + + let mut query_expression = self.query_expression.clone(); + query_expression.filtered_point_of_view = Some(column); + + Self { + recording: self.recording.clone(), + query_expression, + } + } + + fn using_index_values(&self, values: IndexValuesLike<'_>) -> PyResult { + let values = values.to_index_values()?; + + let mut query_expression = self.query_expression.clone(); + query_expression.using_index_values = Some(values); + + Ok(Self { + recording: self.recording.clone(), + query_expression, + }) + } + + fn fill_latest_at(&self) -> Self { + let mut query_expression = self.query_expression.clone(); + query_expression.sparse_fill_strategy = SparseFillStrategy::LatestAtGlobal; + + Self { + recording: self.recording.clone(), + query_expression, + } + } } impl PyRecording { diff --git a/rerun_py/tests/unit/test_dataframe.py b/rerun_py/tests/unit/test_dataframe.py index 06b967bdb937..5532ca283fbc 100644 --- a/rerun_py/tests/unit/test_dataframe.py +++ b/rerun_py/tests/unit/test_dataframe.py @@ -86,6 +86,11 @@ def setup_method(self) -> None: type=rr.components.Position3D.arrow_type(), ) + self.empty_pos = pa.array( + [], + type=rr.components.Position3D.arrow_type(), + ) + def test_recording_info(self) -> None: assert self.recording.application_id() == APP_ID assert self.recording.recording_id() == str(RECORDING_ID) @@ -188,6 +193,55 @@ def test_index_values(self) -> None: with pytest.raises(TypeError): view.filter_index_values(pa.array([1.0, 2.0], type=pa.float64())) + def test_using_index_values(self) -> None: + view = self.recording.view(index="my_index", contents="points") + view = view.using_index_values([0, 5, 9]) + + table = view.select().read_all().combine_chunks() + + # my_index, log_time, log_tick, points, colors + assert table.num_columns == 5 + assert table.num_rows == 3 + + expected_index = pa.chunked_array([ + pa.array( + [0, 5, 9], + type=pa.int64(), + ) + ]) + + assert table.column("my_index").equals(expected_index) + assert not table.column("/points:Position3D")[0].is_valid + assert not table.column("/points:Position3D")[1].is_valid + assert not table.column("/points:Position3D")[2].is_valid + + table = view.fill_latest_at().select().read_all().combine_chunks() + + assert table.num_columns == 5 + assert table.num_rows == 3 + + assert table.column("my_index").equals(expected_index) + assert not table.column("/points:Position3D")[0].is_valid + assert table.column("/points:Position3D")[1].values.equals(self.expected_pos0) + assert table.column("/points:Position3D")[2].values.equals(self.expected_pos1) + + def test_filter_is_not_null(self) -> None: + view = self.recording.view(index="my_index", contents="points") + + color = rr.dataframe.ComponentColumnSelector("points", rr.components.Color) + + view = view.filter_is_not_null(color) + + table = view.select().read_all() + + # my_index, log_time, log_tick, points, colors + assert table.num_columns == 5 + assert table.num_rows == 1 + + assert table.column("my_index")[0].equals(self.expected_index1[0]) + + assert table.column("/points:Position3D")[0].values.equals(self.expected_pos1) + def test_view_syntax(self) -> None: good_content_expressions = [ {"points": rr.components.Position3D},