From e6f93d85970a68f9b6be1e09881057cec2dc8f12 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 3 Oct 2024 11:32:26 +0200 Subject: [PATCH] Make `VideoDecoder` a `trait` (#7577) ### What * Part of #7298 ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7577?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7577?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7577) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- .../re_renderer/src/video/decoder/mod.rs | 51 ++++++++++++++++--- .../re_renderer/src/video/decoder/native.rs | 16 +++--- .../re_renderer/src/video/decoder/web.rs | 21 +++++--- crates/viewer/re_renderer/src/video/mod.rs | 4 +- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/crates/viewer/re_renderer/src/video/decoder/mod.rs b/crates/viewer/re_renderer/src/video/decoder/mod.rs index cf0b76c9ddd9..f0151ecc1e48 100644 --- a/crates/viewer/re_renderer/src/video/decoder/mod.rs +++ b/crates/viewer/re_renderer/src/video/decoder/mod.rs @@ -1,17 +1,52 @@ #[cfg(target_arch = "wasm32")] mod web; -#[cfg(target_arch = "wasm32")] -pub use web::VideoDecoder; -// TODO(#7298): decode on native #[cfg(not(target_arch = "wasm32"))] mod native; -#[cfg(not(target_arch = "wasm32"))] -pub use native::VideoDecoder; +use crate::{ + resource_managers::GpuTexture2D, + wgpu_resources::{GpuTexturePool, TextureDesc}, + RenderContext, +}; + +use std::sync::Arc; + +use super::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult}; + +/// Decode video to a texture. +/// +/// If you want to sample multiple points in a video simultaneously, use multiple decoders. +pub trait VideoDecoder: 'static + Send { + /// Get the video frame at the given time stamp. + /// + /// This will seek in the video if needed. + /// If you want to sample multiple points in a video simultaneously, use multiple decoders. + fn frame_at( + &mut self, + render_ctx: &RenderContext, + presentation_timestamp_s: f64, + ) -> FrameDecodingResult; +} -use crate::resource_managers::GpuTexture2D; -use crate::wgpu_resources::GpuTexturePool; -use crate::wgpu_resources::TextureDesc; +#[cfg(target_arch = "wasm32")] +pub fn new_video_decoder( + render_context: &RenderContext, + data: Arc, + hw_acceleration: DecodeHardwareAcceleration, +) -> Result, DecodingError> { + let decoder = web::WebVideoDecoder::new(render_context, data, hw_acceleration)?; + Ok(Box::new(decoder)) +} + +#[cfg(not(target_arch = "wasm32"))] +pub fn new_video_decoder( + render_context: &RenderContext, + data: Arc, + _hw_acceleration: DecodeHardwareAcceleration, +) -> Result, DecodingError> { + let decoder = native::NoNativeVideoDecoder::new(render_context, data)?; + Ok(Box::new(decoder)) +} fn alloc_video_frame_texture( device: &wgpu::Device, diff --git a/crates/viewer/re_renderer/src/video/decoder/native.rs b/crates/viewer/re_renderer/src/video/decoder/native.rs index 87abc6c87917..b4ce6e3a789c 100644 --- a/crates/viewer/re_renderer/src/video/decoder/native.rs +++ b/crates/viewer/re_renderer/src/video/decoder/native.rs @@ -1,10 +1,12 @@ +// TODO(#7298): decode on native + #![allow(dead_code, unused_variables, clippy::unnecessary_wraps)] use std::sync::Arc; use crate::{ resource_managers::GpuTexture2D, - video::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult}, + video::{DecodingError, FrameDecodingResult}, RenderContext, }; @@ -12,18 +14,18 @@ use crate::{ #[allow(unused_imports)] use super::latest_at_idx; -use super::alloc_video_frame_texture; +use super::{alloc_video_frame_texture, VideoDecoder}; -pub struct VideoDecoder { +/// A [`VideoDecoder`] that always fails. +pub struct NoNativeVideoDecoder { data: Arc, zeroed_texture: GpuTexture2D, } -impl VideoDecoder { +impl NoNativeVideoDecoder { pub fn new( render_context: &RenderContext, data: Arc, - _hw_acceleration: DecodeHardwareAcceleration, ) -> Result { let device = render_context.device.clone(); let zeroed_texture = alloc_video_frame_texture( @@ -37,9 +39,11 @@ impl VideoDecoder { zeroed_texture, }) } +} +impl VideoDecoder for NoNativeVideoDecoder { #[allow(clippy::unused_self)] - pub fn frame_at( + fn frame_at( &mut self, _render_ctx: &RenderContext, _presentation_timestamp_s: f64, diff --git a/crates/viewer/re_renderer/src/video/decoder/web.rs b/crates/viewer/re_renderer/src/video/decoder/web.rs index 9bfa280385e1..7caf5dbbc411 100644 --- a/crates/viewer/re_renderer/src/video/decoder/web.rs +++ b/crates/viewer/re_renderer/src/video/decoder/web.rs @@ -10,13 +10,14 @@ use web_sys::{ }; use web_time::Instant; -use super::latest_at_idx; use crate::{ resource_managers::GpuTexture2D, video::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult, VideoFrameTexture}, DebugLabel, RenderContext, }; +use super::{latest_at_idx, VideoDecoder}; + #[derive(Clone)] #[repr(transparent)] struct VideoFrame(web_sys::VideoFrame); @@ -61,7 +62,7 @@ struct DecoderOutput { /// transient errors without flickering. const DECODING_ERROR_REPORTING_DELAY: Duration = Duration::from_millis(400); -pub struct VideoDecoder { +pub struct WebVideoDecoder { data: Arc, queue: Arc, texture: GpuTexture2D, @@ -83,11 +84,11 @@ pub struct VideoDecoder { #[allow(unsafe_code)] // Clippy did not recognize a safety comment on these impls no matter what I tried: #[allow(clippy::undocumented_unsafe_blocks)] -unsafe impl Send for VideoDecoder {} +unsafe impl Send for WebVideoDecoder {} #[allow(unsafe_code)] #[allow(clippy::undocumented_unsafe_blocks)] -unsafe impl Sync for VideoDecoder {} +unsafe impl Sync for WebVideoDecoder {} #[allow(unsafe_code)] #[allow(clippy::undocumented_unsafe_blocks)] @@ -97,9 +98,9 @@ unsafe impl Send for VideoFrame {} #[allow(clippy::undocumented_unsafe_blocks)] unsafe impl Sync for VideoFrame {} -impl Drop for VideoDecoder { +impl Drop for WebVideoDecoder { fn drop(&mut self) { - re_log::debug!("Dropping VideoDecoder"); + re_log::debug!("Dropping WebVideoDecoder"); if let Err(err) = self.decoder.close() { if let Some(dom_exception) = err.dyn_ref::() { if dom_exception.code() == web_sys::DomException::INVALID_STATE_ERR @@ -118,7 +119,7 @@ impl Drop for VideoDecoder { } } -impl VideoDecoder { +impl WebVideoDecoder { pub fn new( render_context: &RenderContext, data: Arc, @@ -157,8 +158,10 @@ impl VideoDecoder { error_on_last_frame_at: false, }) } +} - pub fn frame_at( +impl VideoDecoder for WebVideoDecoder { + fn frame_at( &mut self, render_ctx: &RenderContext, presentation_timestamp_s: f64, @@ -185,7 +188,9 @@ impl VideoDecoder { } result } +} +impl WebVideoDecoder { fn frame_at_internal(&mut self, presentation_timestamp_s: f64) -> FrameDecodingResult { if presentation_timestamp_s < 0.0 { return Err(DecodingError::NegativeTimestamp); diff --git a/crates/viewer/re_renderer/src/video/mod.rs b/crates/viewer/re_renderer/src/video/mod.rs index 41a885af6db8..b728ffd2cbb8 100644 --- a/crates/viewer/re_renderer/src/video/mod.rs +++ b/crates/viewer/re_renderer/src/video/mod.rs @@ -67,7 +67,7 @@ pub enum VideoFrameTexture { pub struct VideoDecodingStreamId(pub u64); struct DecoderEntry { - decoder: decoder::VideoDecoder, + decoder: Box, frame_index: u64, } @@ -192,7 +192,7 @@ impl Video { let decoder_entry = match decoders.entry(decoder_stream_id) { Entry::Occupied(occupied_entry) => occupied_entry.into_mut(), Entry::Vacant(vacant_entry) => { - let new_decoder = decoder::VideoDecoder::new( + let new_decoder = decoder::new_video_decoder( render_context, self.data.clone(), self.decode_hw_acceleration,