Skip to content

Commit

Permalink
Make VideoDecoder a trait (#7577)
Browse files Browse the repository at this point in the history
### What
* Part of #7298 

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7577?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7577?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7577)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.
  • Loading branch information
emilk authored Oct 3, 2024
1 parent ffc2fd6 commit e6f93d8
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 24 deletions.
51 changes: 43 additions & 8 deletions crates/viewer/re_renderer/src/video/decoder/mod.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,52 @@
#[cfg(target_arch = "wasm32")]
mod web;
#[cfg(target_arch = "wasm32")]
pub use web::VideoDecoder;

// TODO(#7298): decode on native
#[cfg(not(target_arch = "wasm32"))]
mod native;
#[cfg(not(target_arch = "wasm32"))]
pub use native::VideoDecoder;
use crate::{
resource_managers::GpuTexture2D,
wgpu_resources::{GpuTexturePool, TextureDesc},
RenderContext,
};

use std::sync::Arc;

use super::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult};

/// Decode video to a texture.
///
/// If you want to sample multiple points in a video simultaneously, use multiple decoders.
pub trait VideoDecoder: 'static + Send {
/// Get the video frame at the given time stamp.
///
/// This will seek in the video if needed.
/// If you want to sample multiple points in a video simultaneously, use multiple decoders.
fn frame_at(
&mut self,
render_ctx: &RenderContext,
presentation_timestamp_s: f64,
) -> FrameDecodingResult;
}

use crate::resource_managers::GpuTexture2D;
use crate::wgpu_resources::GpuTexturePool;
use crate::wgpu_resources::TextureDesc;
#[cfg(target_arch = "wasm32")]
pub fn new_video_decoder(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
hw_acceleration: DecodeHardwareAcceleration,
) -> Result<Box<dyn VideoDecoder>, DecodingError> {
let decoder = web::WebVideoDecoder::new(render_context, data, hw_acceleration)?;
Ok(Box::new(decoder))
}

#[cfg(not(target_arch = "wasm32"))]
pub fn new_video_decoder(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
_hw_acceleration: DecodeHardwareAcceleration,
) -> Result<Box<dyn VideoDecoder>, DecodingError> {
let decoder = native::NoNativeVideoDecoder::new(render_context, data)?;
Ok(Box::new(decoder))
}

fn alloc_video_frame_texture(
device: &wgpu::Device,
Expand Down
16 changes: 10 additions & 6 deletions crates/viewer/re_renderer/src/video/decoder/native.rs
Original file line number Diff line number Diff line change
@@ -1,29 +1,31 @@
// TODO(#7298): decode on native

#![allow(dead_code, unused_variables, clippy::unnecessary_wraps)]

use std::sync::Arc;

use crate::{
resource_managers::GpuTexture2D,
video::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult},
video::{DecodingError, FrameDecodingResult},
RenderContext,
};

// TODO(#7298): remove `allow` once we have native video decoding
#[allow(unused_imports)]
use super::latest_at_idx;

use super::alloc_video_frame_texture;
use super::{alloc_video_frame_texture, VideoDecoder};

pub struct VideoDecoder {
/// A [`VideoDecoder`] that always fails.
pub struct NoNativeVideoDecoder {
data: Arc<re_video::VideoData>,
zeroed_texture: GpuTexture2D,
}

impl VideoDecoder {
impl NoNativeVideoDecoder {
pub fn new(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
_hw_acceleration: DecodeHardwareAcceleration,
) -> Result<Self, DecodingError> {
let device = render_context.device.clone();
let zeroed_texture = alloc_video_frame_texture(
Expand All @@ -37,9 +39,11 @@ impl VideoDecoder {
zeroed_texture,
})
}
}

impl VideoDecoder for NoNativeVideoDecoder {
#[allow(clippy::unused_self)]
pub fn frame_at(
fn frame_at(
&mut self,
_render_ctx: &RenderContext,
_presentation_timestamp_s: f64,
Expand Down
21 changes: 13 additions & 8 deletions crates/viewer/re_renderer/src/video/decoder/web.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ use web_sys::{
};
use web_time::Instant;

use super::latest_at_idx;
use crate::{
resource_managers::GpuTexture2D,
video::{DecodeHardwareAcceleration, DecodingError, FrameDecodingResult, VideoFrameTexture},
DebugLabel, RenderContext,
};

use super::{latest_at_idx, VideoDecoder};

#[derive(Clone)]
#[repr(transparent)]
struct VideoFrame(web_sys::VideoFrame);
Expand Down Expand Up @@ -61,7 +62,7 @@ struct DecoderOutput {
/// transient errors without flickering.
const DECODING_ERROR_REPORTING_DELAY: Duration = Duration::from_millis(400);

pub struct VideoDecoder {
pub struct WebVideoDecoder {
data: Arc<re_video::VideoData>,
queue: Arc<wgpu::Queue>,
texture: GpuTexture2D,
Expand All @@ -83,11 +84,11 @@ pub struct VideoDecoder {
#[allow(unsafe_code)]
// Clippy did not recognize a safety comment on these impls no matter what I tried:
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe impl Send for VideoDecoder {}
unsafe impl Send for WebVideoDecoder {}

#[allow(unsafe_code)]
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe impl Sync for VideoDecoder {}
unsafe impl Sync for WebVideoDecoder {}

#[allow(unsafe_code)]
#[allow(clippy::undocumented_unsafe_blocks)]
Expand All @@ -97,9 +98,9 @@ unsafe impl Send for VideoFrame {}
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe impl Sync for VideoFrame {}

impl Drop for VideoDecoder {
impl Drop for WebVideoDecoder {
fn drop(&mut self) {
re_log::debug!("Dropping VideoDecoder");
re_log::debug!("Dropping WebVideoDecoder");
if let Err(err) = self.decoder.close() {
if let Some(dom_exception) = err.dyn_ref::<web_sys::DomException>() {
if dom_exception.code() == web_sys::DomException::INVALID_STATE_ERR
Expand All @@ -118,7 +119,7 @@ impl Drop for VideoDecoder {
}
}

impl VideoDecoder {
impl WebVideoDecoder {
pub fn new(
render_context: &RenderContext,
data: Arc<re_video::VideoData>,
Expand Down Expand Up @@ -157,8 +158,10 @@ impl VideoDecoder {
error_on_last_frame_at: false,
})
}
}

pub fn frame_at(
impl VideoDecoder for WebVideoDecoder {
fn frame_at(
&mut self,
render_ctx: &RenderContext,
presentation_timestamp_s: f64,
Expand All @@ -185,7 +188,9 @@ impl VideoDecoder {
}
result
}
}

impl WebVideoDecoder {
fn frame_at_internal(&mut self, presentation_timestamp_s: f64) -> FrameDecodingResult {
if presentation_timestamp_s < 0.0 {
return Err(DecodingError::NegativeTimestamp);
Expand Down
4 changes: 2 additions & 2 deletions crates/viewer/re_renderer/src/video/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ pub enum VideoFrameTexture {
pub struct VideoDecodingStreamId(pub u64);

struct DecoderEntry {
decoder: decoder::VideoDecoder,
decoder: Box<dyn decoder::VideoDecoder>,
frame_index: u64,
}

Expand Down Expand Up @@ -192,7 +192,7 @@ impl Video {
let decoder_entry = match decoders.entry(decoder_stream_id) {
Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
Entry::Vacant(vacant_entry) => {
let new_decoder = decoder::VideoDecoder::new(
let new_decoder = decoder::new_video_decoder(
render_context,
self.data.clone(),
self.decode_hw_acceleration,
Expand Down

0 comments on commit e6f93d8

Please sign in to comment.