diff --git a/crates/viewer/re_renderer/src/lib.rs b/crates/viewer/re_renderer/src/lib.rs index 673e44e56a40..a0634e825a52 100644 --- a/crates/viewer/re_renderer/src/lib.rs +++ b/crates/viewer/re_renderer/src/lib.rs @@ -16,6 +16,7 @@ pub mod mesh; pub mod renderer; pub mod resource_managers; pub mod texture_info; +pub mod video; pub mod view_builder; mod allocator; diff --git a/crates/viewer/re_renderer/src/renderer/mod.rs b/crates/viewer/re_renderer/src/renderer/mod.rs index 339a86b265c5..ccca81217c8a 100644 --- a/crates/viewer/re_renderer/src/renderer/mod.rs +++ b/crates/viewer/re_renderer/src/renderer/mod.rs @@ -31,9 +31,6 @@ pub(crate) use compositor::CompositorDrawData; mod debug_overlay; pub use debug_overlay::{DebugOverlayDrawData, DebugOverlayError, DebugOverlayRenderer}; -mod video; -pub use video::Video; - pub mod gpu_data { pub use super::lines::gpu_data::{LineStripInfo, LineVertex}; pub use super::point_cloud::gpu_data::PositionRadius; diff --git a/crates/viewer/re_renderer/src/renderer/video/decoder/mod.rs b/crates/viewer/re_renderer/src/video/decoder/mod.rs similarity index 100% rename from crates/viewer/re_renderer/src/renderer/video/decoder/mod.rs rename to crates/viewer/re_renderer/src/video/decoder/mod.rs diff --git a/crates/viewer/re_renderer/src/renderer/video/decoder/native.rs b/crates/viewer/re_renderer/src/video/decoder/native.rs similarity index 73% rename from crates/viewer/re_renderer/src/renderer/video/decoder/native.rs rename to crates/viewer/re_renderer/src/video/decoder/native.rs index a587c9b786b6..524ff8d13deb 100644 --- a/crates/viewer/re_renderer/src/renderer/video/decoder/native.rs +++ b/crates/viewer/re_renderer/src/video/decoder/native.rs @@ -1,7 +1,10 @@ #![allow(dead_code, unused_variables, clippy::unnecessary_wraps)] -use crate::resource_managers::GpuTexture2D; -use crate::RenderContext; +use crate::{ + resource_managers::GpuTexture2D, + video::{DecodingError, FrameDecodingResult}, + RenderContext, +}; // TODO(#7298): remove `allow` once we have native video decoding #[allow(unused_imports)] @@ -17,7 +20,10 @@ pub struct VideoDecoder { } impl VideoDecoder { - pub fn new(render_context: &RenderContext, data: re_video::VideoData) -> Option { + pub fn new( + render_context: &RenderContext, + data: re_video::VideoData, + ) -> Result { re_log::warn_once!("Video playback not yet available in the native viewer, try the web viewer instead. See https://github.com/rerun-io/rerun/issues/7298 for more information."); let device = render_context.device.clone(); @@ -27,7 +33,7 @@ impl VideoDecoder { data.config.coded_width as u32, data.config.coded_height as u32, ); - Some(Self { + Ok(Self { data, zeroed_texture, }) @@ -45,7 +51,7 @@ impl VideoDecoder { self.data.config.coded_height as u32 } - pub fn frame_at(&mut self, timestamp: TimeMs) -> GpuTexture2D { - self.zeroed_texture.clone() + pub fn frame_at(&mut self, timestamp: TimeMs) -> FrameDecodingResult { + FrameDecodingResult::Ready(self.zeroed_texture.clone()) } } diff --git a/crates/viewer/re_renderer/src/renderer/video/decoder/web.rs b/crates/viewer/re_renderer/src/video/decoder/web.rs similarity index 81% rename from crates/viewer/re_renderer/src/renderer/video/decoder/web.rs rename to crates/viewer/re_renderer/src/video/decoder/web.rs index 6eeacc9de8b5..c72438854d11 100644 --- a/crates/viewer/re_renderer/src/renderer/video/decoder/web.rs +++ b/crates/viewer/re_renderer/src/video/decoder/web.rs @@ -1,22 +1,21 @@ -// TODO(emilk): proper error handling: pass errors to caller instead of logging them` +use std::sync::Arc; -use super::latest_at_idx; -use crate::resource_managers::GpuTexture2D; -use crate::RenderContext; -use js_sys::Function; -use js_sys::Uint8Array; +use js_sys::{Function, Uint8Array}; use parking_lot::Mutex; -use re_video::TimeMs; -use re_video::VideoData; -use std::ops::Deref; -use std::sync::Arc; -use wasm_bindgen::closure::Closure; -use wasm_bindgen::JsCast as _; -use web_sys::EncodedVideoChunk; -use web_sys::EncodedVideoChunkInit; -use web_sys::EncodedVideoChunkType; -use web_sys::VideoDecoderConfig; -use web_sys::VideoDecoderInit; +use wasm_bindgen::{closure::Closure, JsCast as _}; +use web_sys::{ + EncodedVideoChunk, EncodedVideoChunkInit, EncodedVideoChunkType, VideoDecoderConfig, + VideoDecoderInit, +}; + +use re_video::{TimeMs, VideoData}; + +use super::latest_at_idx; +use crate::{ + resource_managers::GpuTexture2D, + video::{DecodingError, FrameDecodingResult}, + RenderContext, +}; #[derive(Clone)] #[repr(transparent)] @@ -28,7 +27,7 @@ impl Drop for VideoFrame { } } -impl Deref for VideoFrame { +impl std::ops::Deref for VideoFrame { type Target = web_sys::VideoFrame; #[inline] @@ -41,7 +40,6 @@ pub struct VideoDecoder { data: re_video::VideoData, queue: Arc, texture: GpuTexture2D, - zeroed_texture: GpuTexture2D, decoder: web_sys::VideoDecoder, @@ -83,7 +81,7 @@ impl Drop for VideoDecoder { } impl VideoDecoder { - pub fn new(render_context: &RenderContext, data: VideoData) -> Option { + pub fn new(render_context: &RenderContext, data: VideoData) -> Result { let frames = Arc::new(Mutex::new(Vec::with_capacity(16))); let decoder = init_video_decoder({ @@ -105,18 +103,11 @@ impl VideoDecoder { data.config.coded_width as u32, data.config.coded_height as u32, ); - let zeroed_texture = super::alloc_video_frame_texture( - &render_context.device, - &render_context.gpu_resources.textures, - data.config.coded_width as u32, - data.config.coded_height as u32, - ); let mut this = Self { data, queue, texture, - zeroed_texture, decoder, @@ -127,10 +118,10 @@ impl VideoDecoder { }; // immediately enqueue some frames, assuming playback at start - this.reset(); + this.reset()?; let _ = this.frame_at(TimeMs::new(0.0)); - Some(this) + Ok(this) } pub fn duration_ms(&self) -> f64 { @@ -145,16 +136,15 @@ impl VideoDecoder { self.data.config.coded_height as u32 } - pub fn frame_at(&mut self, timestamp: TimeMs) -> GpuTexture2D { + pub fn frame_at(&mut self, timestamp: TimeMs) -> FrameDecodingResult { if timestamp < TimeMs::ZERO { - return self.zeroed_texture.clone(); + return FrameDecodingResult::Error(DecodingError::NegativeTimestamp); } let Some(requested_segment_idx) = latest_at_idx(&self.data.segments, |segment| segment.timestamp, ×tamp) else { - // This should only happen if the video is completely empty. - return self.zeroed_texture.clone(); + return FrameDecodingResult::Error(DecodingError::EmptyVideo); }; let Some(requested_sample_idx) = latest_at_idx( @@ -163,7 +153,7 @@ impl VideoDecoder { ×tamp, ) else { // This should never happen, because segments are never empty. - return self.zeroed_texture.clone(); + return FrameDecodingResult::Error(DecodingError::EmptySegment); }; // Enqueue segments as needed. We maintain a buffer of 2 segments, so we can @@ -179,12 +169,14 @@ impl VideoDecoder { requested_segment_idx as isize - self.current_segment_idx as isize; if segment_distance == 1 { // forward seek to next segment - queue up the one _after_ requested - self.enqueue_all(requested_segment_idx + 1); + self.enqueue_segment(requested_segment_idx + 1); } else { // forward seek by N>1 OR backward seek across segments - reset - self.reset(); - self.enqueue_all(requested_segment_idx); - self.enqueue_all(requested_segment_idx + 1); + if let Err(err) = self.reset() { + return FrameDecodingResult::Error(err); + } + self.enqueue_segment(requested_segment_idx); + self.enqueue_segment(requested_segment_idx + 1); } } else if requested_sample_idx != self.current_sample_idx { // special case: handle seeking backwards within a single segment @@ -192,9 +184,11 @@ impl VideoDecoder { // while maintaining a buffer of 2 segments let sample_distance = requested_sample_idx as isize - self.current_sample_idx as isize; if sample_distance < 0 { - self.reset(); - self.enqueue_all(requested_segment_idx); - self.enqueue_all(requested_segment_idx + 1); + if let Err(err) = self.reset() { + return FrameDecodingResult::Error(err); + } + self.enqueue_segment(requested_segment_idx); + self.enqueue_segment(requested_segment_idx + 1); } } @@ -205,10 +199,10 @@ impl VideoDecoder { let Some(frame_idx) = latest_at_idx(&frames, |(t, _)| *t, ×tamp) else { // no buffered frames - texture will be blank - // not return a zeroed texture, because we may just be behind on decoding + // Don't return a zeroed texture, because we may just be behind on decoding // and showing an old frame is better than showing a blank frame, // because it causes "black flashes" to appear - return self.texture.clone(); + return FrameDecodingResult::Pending(self.texture.clone()); }; // drain up-to (but not including) the frame idx, clearing out any frames @@ -226,9 +220,10 @@ impl VideoDecoder { let frame_duration_ms = frame.duration().map(TimeMs::new).unwrap_or_default(); // This handles the case when we have a buffered frame that's older than the requested timestamp. - // We don't want to show this frame to the user, because it's not actually the one they requested. + // We don't want to show this frame to the user, because it's not actually the one they requested, + // so instead return the last decoded frame. if timestamp - frame_timestamp_ms > frame_duration_ms { - return self.texture.clone(); + return FrameDecodingResult::Pending(self.texture.clone()); } if self.last_used_frame_timestamp != frame_timestamp_ms { @@ -236,25 +231,25 @@ impl VideoDecoder { self.last_used_frame_timestamp = frame_timestamp_ms; } - self.texture.clone() + FrameDecodingResult::Ready(self.texture.clone()) } /// Enqueue all samples in the given segment. /// /// Does nothing if the index is out of bounds. - fn enqueue_all(&self, segment_idx: usize) { + fn enqueue_segment(&self, segment_idx: usize) { let Some(segment) = self.data.segments.get(segment_idx) else { return; }; - self.enqueue(&segment.samples[0], true); + self.enqueue_sample(&segment.samples[0], true); for sample in &segment.samples[1..] { - self.enqueue(sample, false); + self.enqueue_sample(sample, false); } } /// Enqueue the given sample. - fn enqueue(&self, sample: &re_video::Sample, is_key: bool) { + fn enqueue_sample(&self, sample: &re_video::Sample, is_key: bool) { let data = Uint8Array::from( &self.data.data[sample.byte_offset as usize ..sample.byte_offset as usize + sample.byte_length as usize], @@ -268,6 +263,7 @@ impl VideoDecoder { chunk.set_duration(sample.duration.as_f64()); let Some(chunk) = EncodedVideoChunk::new(&chunk) .inspect_err(|err| { + // TODO(#7373): return this error once the decoder tries to return a frame for this sample. how exactly? re_log::error!("failed to create video chunk: {}", js_error_to_string(err)); }) .ok() @@ -276,31 +272,24 @@ impl VideoDecoder { }; if let Err(err) = self.decoder.decode(&chunk) { + // TODO(#7373): return this error once the decoder tries to return a frame for this sample. how exactly? re_log::error!("Failed to decode video chunk: {}", js_error_to_string(&err)); } } /// Reset the video decoder and discard all frames. - fn reset(&mut self) { - if let Err(err) = self.decoder.reset() { - re_log::error!( - "Failed to reset video decoder: {}", - js_error_to_string(&err) - ); - } - - if let Err(err) = self - .decoder + fn reset(&mut self) -> Result<(), DecodingError> { + self.decoder + .reset() + .map_err(|err| DecodingError::ResetFailure(js_error_to_string(&err)))?; + self.decoder .configure(&js_video_decoder_config(&self.data.config)) - { - re_log::error!( - "Failed to configure video decoder: {}", - js_error_to_string(&err) - ); - } + .map_err(|err| DecodingError::ConfigureFailure(js_error_to_string(&err)))?; let mut frames = self.frames.lock(); drop(frames.drain(..)); + + Ok(()) } } @@ -359,11 +348,11 @@ fn copy_video_frame_to_texture( fn init_video_decoder( on_output: impl Fn(web_sys::VideoFrame) + 'static, -) -> Option { +) -> Result { let on_output = Closure::wrap(Box::new(on_output) as Box); let on_error = Closure::wrap(Box::new(|err: js_sys::Error| { + // TODO(#7373): store this error and report during decode let err = std::string::ToString::to_string(&err.to_string()); - re_log::error!("failed to decode video: {err}"); }) as Box); @@ -373,13 +362,8 @@ fn init_video_decoder( let Ok(on_error) = on_error.into_js_value().dyn_into::() else { unreachable!() }; - let decoder = web_sys::VideoDecoder::new(&VideoDecoderInit::new(&on_error, &on_output)) - .inspect_err(|err| { - re_log::error!("failed to create VideoDecoder: {}", js_error_to_string(err)); - }) - .ok()?; - - Some(decoder) + web_sys::VideoDecoder::new(&VideoDecoderInit::new(&on_error, &on_output)) + .map_err(|err| DecodingError::DecoderSetupFailure(js_error_to_string(&err))) } fn js_video_decoder_config(config: &re_video::Config) -> VideoDecoderConfig { diff --git a/crates/viewer/re_renderer/src/renderer/video/mod.rs b/crates/viewer/re_renderer/src/video/mod.rs similarity index 55% rename from crates/viewer/re_renderer/src/renderer/video/mod.rs rename to crates/viewer/re_renderer/src/video/mod.rs index 62a08a9608d9..7b981b48fe28 100644 --- a/crates/viewer/re_renderer/src/renderer/video/mod.rs +++ b/crates/viewer/re_renderer/src/video/mod.rs @@ -1,9 +1,53 @@ mod decoder; -use crate::resource_managers::GpuTexture2D; -use crate::RenderContext; -use re_video::TimeMs; -use re_video::VideoLoadError; +use re_video::{TimeMs, VideoLoadError}; + +use crate::{resource_managers::GpuTexture2D, RenderContext}; + +#[derive(thiserror::Error, Debug)] +pub enum VideoError { + #[error(transparent)] + Load(#[from] VideoLoadError), + + #[error(transparent)] + Init(#[from] DecodingError), +} + +/// Error that can occur during frame decoding. +// TODO(jan, andreas): These errors are for the most part specific to the web decoder right now. +#[derive(thiserror::Error, Debug)] +pub enum DecodingError { + #[error("Failed to create VideoDecoder: {0}")] + DecoderSetupFailure(String), + + #[error("Video seems to be empty, no segments have beem found.")] + EmptyVideo, + + #[error("The current segment is empty.")] + EmptySegment, + + #[error("Failed to reset the decoder: {0}")] + ResetFailure(String), + + #[error("Failed to configure the video decoder: {0}")] + ConfigureFailure(String), + + #[error("The timestamp passed was negative.")] + NegativeTimestamp, +} + +/// Information about the status of a frame decoding. +pub enum FrameDecodingResult { + /// The requested frame got decoded and is ready to be used. + Ready(GpuTexture2D), + + /// The returned texture is from a previous frame or a placeholder, the decoder is still decoding the requested frame. + Pending(GpuTexture2D), + + /// The decoder encountered an error and was not able to produce a texture for the requested timestamp. + /// The returned texture is either a placeholder or the last successfully decoded texture. + Error(DecodingError), +} /// A video file. /// @@ -31,8 +75,7 @@ impl Video { } None => return Err(VideoError::Load(VideoLoadError::UnknownMediaType)), }; - let decoder = - decoder::VideoDecoder::new(render_context, data).ok_or_else(|| VideoError::Init)?; + let decoder = decoder::VideoDecoder::new(render_context, data)?; Ok(Self { decoder }) } @@ -62,17 +105,8 @@ impl Video { /// /// This takes `&mut self` because the decoder maintains a buffer of decoded frames, /// which requires mutation. It is also not thread-safe by default. - pub fn frame_at(&mut self, timestamp_s: f64) -> GpuTexture2D { + pub fn frame_at(&mut self, timestamp_s: f64) -> FrameDecodingResult { re_tracing::profile_function!(); self.decoder.frame_at(TimeMs::new(timestamp_s * 1e3)) } } - -#[derive(thiserror::Error, Debug)] -pub enum VideoError { - #[error("{0}")] - Load(#[from] VideoLoadError), - - #[error("failed to initialize video decoder")] - Init, -} diff --git a/crates/viewer/re_space_view_spatial/src/video_cache.rs b/crates/viewer/re_space_view_spatial/src/video_cache.rs index 370a1464489b..8c6768dee6f4 100644 --- a/crates/viewer/re_space_view_spatial/src/video_cache.rs +++ b/crates/viewer/re_space_view_spatial/src/video_cache.rs @@ -1,5 +1,5 @@ use re_entity_db::VersionedInstancePathHash; -use re_renderer::{renderer::Video, RenderContext}; +use re_renderer::{video::Video, RenderContext}; use re_types::components::MediaType; use re_viewer_context::Cache; diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index dcbe5bbb5755..b89fcebeb5d5 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -1,8 +1,11 @@ use egui::mutex::Mutex; use re_log_types::EntityPath; -use re_renderer::renderer::{ - ColormappedTexture, RectangleOptions, TextureFilterMag, TextureFilterMin, TexturedRect, +use re_renderer::{ + renderer::{ + ColormappedTexture, RectangleOptions, TextureFilterMag, TextureFilterMin, TexturedRect, + }, + video::{FrameDecodingResult, Video}, }; use re_types::{ archetypes::{AssetVideo, VideoFrameReference}, @@ -119,7 +122,7 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { VideoTimeMode::Nanoseconds => video_timestamp.video_time as f64 / 1e9, }; - let (texture, video_width, video_height) = { + let (texture_result, video_width, video_height) = { let mut video = video.lock(); // TODO(andreas): Interior mutability for re_renderer's video would be nice. ( video.frame_at(timestamp_in_seconds), @@ -128,6 +131,21 @@ impl VisualizerSystem for VideoFrameReferenceVisualizer { ) }; + let texture = match texture_result { + FrameDecodingResult::Ready(texture) => texture, + FrameDecodingResult::Pending(texture) => { + ctx.viewer_ctx.egui_ctx.request_repaint(); + texture + } + FrameDecodingResult::Error(err) => { + // TODO(#7373): show this error in the ui + re_log::error_once!( + "Failed to decode video frame for {entity_path}: {err}" + ); + continue; + } + }; + let world_from_entity = spatial_ctx.transform_info.single_entity_transform_required( ctx.target_entity_path, @@ -215,7 +233,7 @@ fn textured_rect_for_video_frame( fn latest_at_query_video_from_datastore( ctx: &ViewerContext<'_>, entity_path: &EntityPath, -) -> Option>> { +) -> Option>> { let query = ctx.current_query(); let results = ctx.recording().query_caches().latest_at(