From 948952c8525cad46bc4e19fa60f5688993de9727 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Tue, 12 Nov 2024 13:42:08 +0100 Subject: [PATCH 01/11] Make `FrameInfo` optional in some places where is is not available --- crates/viewer/re_data_ui/src/video.rs | 2 + .../re_renderer/src/video/chunk_decoder.rs | 11 +++-- crates/viewer/re_renderer/src/video/mod.rs | 11 +---- crates/viewer/re_renderer/src/video/player.rs | 43 ++++++++++--------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/crates/viewer/re_data_ui/src/video.rs b/crates/viewer/re_data_ui/src/video.rs index 7a3447e771ea..0f9e52ee1498 100644 --- a/crates/viewer/re_data_ui/src/video.rs +++ b/crates/viewer/re_data_ui/src/video.rs @@ -166,6 +166,7 @@ pub fn show_decoded_frame_info( }) => { re_ui::list_item::list_item_scope(ui, "decoded_frame_ui", |ui| { let default_open = false; + if let Some(frame_info) = frame_info { ui.list_item_collapsible_noninteractive_label( "Current decoded frame", default_open, @@ -174,6 +175,7 @@ pub fn show_decoded_frame_info( source_image_data_format_ui(ui, &source_pixel_format); }, ); + } }); let response = crate::image::texture_preview_ui( diff --git a/crates/viewer/re_renderer/src/video/chunk_decoder.rs b/crates/viewer/re_renderer/src/video/chunk_decoder.rs index df191cfe7c05..b5122451abb9 100644 --- a/crates/viewer/re_renderer/src/video/chunk_decoder.rs +++ b/crates/viewer/re_renderer/src/video/chunk_decoder.rs @@ -119,9 +119,12 @@ impl VideoChunkDecoder { let frame_time_range = frame.info.presentation_time_range(); - if frame_time_range.contains(&presentation_timestamp) - && video_texture.frame_info.presentation_time_range() != frame_time_range - { + let is_up_to_date = video_texture + .frame_info + .as_ref() + .is_some_and(|info| info.presentation_time_range() == frame_time_range); + + if frame_time_range.contains(&presentation_timestamp) && !is_up_to_date { #[cfg(target_arch = "wasm32")] { video_texture.source_pixel_format = copy_web_video_frame_to_texture( @@ -139,7 +142,7 @@ impl VideoChunkDecoder { )?; } - video_texture.frame_info = frame.info.clone(); + video_texture.frame_info = Some(frame.info.clone()); } Ok(()) diff --git a/crates/viewer/re_renderer/src/video/mod.rs b/crates/viewer/re_renderer/src/video/mod.rs index 67b52d4d2453..15bad868326a 100644 --- a/crates/viewer/re_renderer/src/video/mod.rs +++ b/crates/viewer/re_renderer/src/video/mod.rs @@ -1,7 +1,7 @@ mod chunk_decoder; mod player; -use std::{collections::hash_map::Entry, ops::Range, sync::Arc}; +use std::{collections::hash_map::Entry, sync::Arc}; use ahash::HashMap; use parking_lot::Mutex; @@ -64,14 +64,7 @@ pub struct VideoFrameTexture { pub source_pixel_format: SourceImageDataFormat, /// Meta information about the decoded frame. - pub frame_info: re_video::decode::FrameInfo, -} - -impl VideoFrameTexture { - pub fn presentation_time_range(&self) -> Range { - self.frame_info.presentation_timestamp - ..self.frame_info.presentation_timestamp + self.frame_info.duration - } + pub frame_info: Option, } /// Identifier for an independent video decoding stream. diff --git a/crates/viewer/re_renderer/src/video/player.rs b/crates/viewer/re_renderer/src/video/player.rs index 935d20003ce6..4fa96bf8f7ff 100644 --- a/crates/viewer/re_renderer/src/video/player.rs +++ b/crates/viewer/re_renderer/src/video/player.rs @@ -41,7 +41,7 @@ impl TimedDecodingError { /// A texture of a specific video frame. pub struct VideoTexture { pub texture: GpuTexture2D, - pub frame_info: FrameInfo, + pub frame_info: Option, pub source_pixel_format: SourceImageDataFormat, } @@ -101,7 +101,7 @@ impl VideoPlayer { video_texture: VideoTexture { texture, - frame_info: FrameInfo::default(), + frame_info: None, source_pixel_format: SourceImageDataFormat::WgpuCompatible( wgpu::TextureFormat::Rgba8Unorm, ), @@ -136,28 +136,25 @@ impl VideoPlayer { .min(self.data.duration + self.data.samples_statistics.minimum_presentation_timestamp); // Don't seek past the end of the video. let error_on_last_frame_at = self.last_error.is_some(); - let result = self.frame_at_internal(render_ctx, presentation_timestamp, video_data); + self.frame_at_internal(render_ctx, presentation_timestamp, video_data)?; - match result { - Ok(()) => { - let is_active_frame = self - .video_texture - .frame_info - .presentation_time_range() - .contains(&presentation_timestamp); + let frame_info = self.video_texture.frame_info.clone(); + + if let Some(frame_info) = frame_info { + let time_range = frame_info.presentation_time_range(); + let is_active_frame = time_range.contains(&presentation_timestamp); let is_pending = !is_active_frame; - if is_pending && error_on_last_frame_at { + + let show_spinner = if is_pending && error_on_last_frame_at { // If we switched from error to pending, clear the texture. // This is important to avoid flickering, in particular when switching from // benign errors like DecodingError::NegativeTimestamp. // If we don't do this, we see the last valid texture which can look really weird. clear_texture(render_ctx, &self.video_texture.texture); - self.video_texture.frame_info = FrameInfo::default(); - } - - let time_range = self.video_texture.frame_info.presentation_time_range(); - let show_spinner = if presentation_timestamp < time_range.start { + self.video_texture.frame_info = None; + true + } else if presentation_timestamp < time_range.start { // We're seeking backwards and somehow forgot to reset. true } else if presentation_timestamp < time_range.end { @@ -170,17 +167,21 @@ impl VideoPlayer { true // Very old frame - show spinner } }; - Ok(VideoFrameTexture { texture: self.video_texture.texture.clone(), is_pending, show_spinner, - frame_info: self.video_texture.frame_info.clone(), + frame_info: Some(frame_info), + source_pixel_format: self.video_texture.source_pixel_format, + }) + } else { + Ok(VideoFrameTexture { + texture: self.video_texture.texture.clone(), + is_pending: true, + show_spinner: true, + frame_info: None, source_pixel_format: self.video_texture.source_pixel_format, }) - } - - Err(err) => Err(err), } } From f94c0e2a379588785c52ef822a3d4e8ccfe50127 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Tue, 12 Nov 2024 13:43:26 +0100 Subject: [PATCH 02/11] Remove `FrameInfo::default` --- crates/store/re_video/src/decode/av1.rs | 2 +- crates/store/re_video/src/decode/mod.rs | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/crates/store/re_video/src/decode/av1.rs b/crates/store/re_video/src/decode/av1.rs index c28215ae0f95..77e02c8fd178 100644 --- a/crates/store/re_video/src/decode/av1.rs +++ b/crates/store/re_video/src/decode/av1.rs @@ -255,7 +255,7 @@ fn create_frame(debug_name: &str, picture: &dav1d::Picture) -> Result { info: FrameInfo { presentation_timestamp: Time(picture.timestamp().unwrap_or(0)), duration: Time(picture.duration()), - ..Default::default() + latest_decode_timestamp: None, }, }) } diff --git a/crates/store/re_video/src/decode/mod.rs b/crates/store/re_video/src/decode/mod.rs index e4a46805a929..aabd19124fb1 100644 --- a/crates/store/re_video/src/decode/mod.rs +++ b/crates/store/re_video/src/decode/mod.rs @@ -252,15 +252,9 @@ pub type FrameContent = webcodecs::WebVideoFrame; pub struct FrameInfo { /// The presentation timestamp of the frame. /// - /// Decoders are required to report this. - /// A timestamp of [`Time::MAX`] indicates that the frame is invalid or not yet available. pub presentation_timestamp: Time, /// How long the frame is valid. - /// - /// Decoders are required to report this. - /// A duration of [`Time::MAX`] indicates that the frame is invalid or not yet available. - // Implementation note: unlike with presentation timestamp we may be able fine with making this optional. pub duration: Time, /// The decode timestamp of the last chunk that was needed to decode this frame. @@ -269,16 +263,6 @@ pub struct FrameInfo { pub latest_decode_timestamp: Option