Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make VideoDecoder a trait #7577

Merged
merged 1 commit into from
Oct 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 43 additions & 8 deletions crates/viewer/re_renderer/src/video/decoder/mod.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,52 @@
#[cfg(target_arch = "wasm32")]
mod web;
#[cfg(target_arch = "wasm32")]
pub use web::VideoDecoder;

// TODO(#7298): decode on native
#[cfg(not(target_arch = "wasm32"))]
mod native;
#[cfg(not(target_arch = "wasm32"))]
pub use native::VideoDecoder;
use crate::{
resource_managers::GpuTexture2D,
wgpu_resources::{GpuTexturePool, TextureDesc},
RenderContext,
};

use std::sync::Arc;

use super::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult};

/// Decode video to a texture.
///
/// If you want to sample multiple points in a video simultaneously, use multiple decoders.
pub trait VideoDecoder: 'static + Send {
/// Get the video frame at the given time stamp.
///
/// This will seek in the video if needed.
/// If you want to sample multiple points in a video simultaneously, use multiple decoders.
fn frame_at(
&mut self,
render_ctx: &RenderContext,
presentation_timestamp_s: f64,
) -> FrameDecodingResult;
}

use crate::resource_managers::GpuTexture2D;
use crate::wgpu_resources::GpuTexturePool;
use crate::wgpu_resources::TextureDesc;
#[cfg(target_arch = "wasm32")]
pub fn new_video_decoder(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
hw_acceleration: DecodeHardwareAcceleration,
) -> Result<Box<dyn VideoDecoder>, DecodingError> {
let decoder = web::WebVideoDecoder::new(render_context, data, hw_acceleration)?;
Ok(Box::new(decoder))
}

#[cfg(not(target_arch = "wasm32"))]
pub fn new_video_decoder(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
_hw_acceleration: DecodeHardwareAcceleration,
) -> Result<Box<dyn VideoDecoder>, DecodingError> {
let decoder = native::NoNativeVideoDecoder::new(render_context, data)?;
Ok(Box::new(decoder))
}

fn alloc_video_frame_texture(
device: &wgpu::Device,
Expand Down
16 changes: 10 additions & 6 deletions crates/viewer/re_renderer/src/video/decoder/native.rs
Original file line number Diff line number Diff line change
@@ -1,29 +1,31 @@
// TODO(#7298): decode on native

#![allow(dead_code, unused_variables, clippy::unnecessary_wraps)]

use std::sync::Arc;

use crate::{
resource_managers::GpuTexture2D,
video::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult},
video::{DecodingError, FrameDecodingResult},
RenderContext,
};

// TODO(#7298): remove `allow` once we have native video decoding
#[allow(unused_imports)]
use super::latest_at_idx;

use super::alloc_video_frame_texture;
use super::{alloc_video_frame_texture, VideoDecoder};

pub struct VideoDecoder {
/// A [`VideoDecoder`] that always fails.
pub struct NoNativeVideoDecoder {
data: Arc<re_video::VideoData>,
zeroed_texture: GpuTexture2D,
}

impl VideoDecoder {
impl NoNativeVideoDecoder {
pub fn new(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
_hw_acceleration: DecodeHardwareAcceleration,
) -> Result<Self, DecodingError> {
let device = render_context.device.clone();
let zeroed_texture = alloc_video_frame_texture(
Expand All @@ -37,9 +39,11 @@ impl VideoDecoder {
zeroed_texture,
})
}
}

impl VideoDecoder for NoNativeVideoDecoder {
#[allow(clippy::unused_self)]
pub fn frame_at(
fn frame_at(
&mut self,
_render_ctx: &RenderContext,
_presentation_timestamp_s: f64,
Expand Down
21 changes: 13 additions & 8 deletions crates/viewer/re_renderer/src/video/decoder/web.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ use web_sys::{
};
use web_time::Instant;

use super::latest_at_idx;
use crate::{
resource_managers::GpuTexture2D,
video::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult, VideoFrameTexture},
DebugLabel, RenderContext,
};

use super::{latest_at_idx, VideoDecoder};

#[derive(Clone)]
#[repr(transparent)]
struct VideoFrame(web_sys::VideoFrame);
Expand Down Expand Up @@ -61,7 +62,7 @@ struct DecoderOutput {
/// transient errors without flickering.
const DECODING_ERROR_REPORTING_DELAY: Duration = Duration::from_millis(400);

pub struct VideoDecoder {
pub struct WebVideoDecoder {
data: Arc<re_video::VideoData>,
queue: Arc<wgpu::Queue>,
texture: GpuTexture2D,
Expand All @@ -83,11 +84,11 @@ pub struct VideoDecoder {
#[allow(unsafe_code)]
// Clippy did not recognize a safety comment on these impls no matter what I tried:
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe impl Send for VideoDecoder {}
unsafe impl Send for WebVideoDecoder {}

#[allow(unsafe_code)]
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe impl Sync for VideoDecoder {}
unsafe impl Sync for WebVideoDecoder {}

#[allow(unsafe_code)]
#[allow(clippy::undocumented_unsafe_blocks)]
Expand All @@ -97,9 +98,9 @@ unsafe impl Send for VideoFrame {}
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe impl Sync for VideoFrame {}

impl Drop for VideoDecoder {
impl Drop for WebVideoDecoder {
fn drop(&mut self) {
re_log::debug!("Dropping VideoDecoder");
re_log::debug!("Dropping WebVideoDecoder");
if let Err(err) = self.decoder.close() {
if let Some(dom_exception) = err.dyn_ref::<web_sys::DomException>() {
if dom_exception.code() == web_sys::DomException::INVALID_STATE_ERR
Expand All @@ -118,7 +119,7 @@ impl Drop for VideoDecoder {
}
}

impl VideoDecoder {
impl WebVideoDecoder {
pub fn new(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
Expand Down Expand Up @@ -157,8 +158,10 @@ impl VideoDecoder {
error_on_last_frame_at: false,
})
}
}

pub fn frame_at(
impl VideoDecoder for WebVideoDecoder {
fn frame_at(
&mut self,
render_ctx: &RenderContext,
presentation_timestamp_s: f64,
Expand All @@ -185,7 +188,9 @@ impl VideoDecoder {
}
result
}
}

impl WebVideoDecoder {
fn frame_at_internal(&mut self, presentation_timestamp_s: f64) -> FrameDecodingResult {
if presentation_timestamp_s < 0.0 {
return Err(DecodingError::NegativeTimestamp);
Expand Down
4 changes: 2 additions & 2 deletions crates/viewer/re_renderer/src/video/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ pub enum VideoFrameTexture {
pub struct VideoDecodingStreamId(pub u64);

struct DecoderEntry {
decoder: decoder::VideoDecoder,
decoder: Box<dyn decoder::VideoDecoder>,
frame_index: u64,
}

Expand Down Expand Up @@ -192,7 +192,7 @@ impl Video {
let decoder_entry = match decoders.entry(decoder_stream_id) {
Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
Entry::Vacant(vacant_entry) => {
let new_decoder = decoder::VideoDecoder::new(
let new_decoder = decoder::new_video_decoder(
render_context,
self.data.clone(),
self.decode_hw_acceleration,
Expand Down
Loading