From 540e896480f1411f2b68c592625c5e6250e048a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=96R=C3=96K=20Attila?= Date: Thu, 18 Jul 2024 03:22:15 +0200 Subject: [PATCH] video/external: Add WebCodecs H.264 decoder --- Cargo.lock | 3 + video/external/Cargo.toml | 13 ++ video/external/src/backend.rs | 8 +- video/external/src/decoder.rs | 3 + video/external/src/decoder/webcodecs.rs | 251 ++++++++++++++++++++++++ 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 video/external/src/decoder/webcodecs.rs diff --git a/Cargo.lock b/Cargo.lock index e684057c56536..3bca486725c7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4579,6 +4579,7 @@ version = "0.1.0" dependencies = [ "bzip2", "hex", + "js-sys", "libloading", "reqwest", "ruffle_render", @@ -4590,6 +4591,8 @@ dependencies = [ "tempfile", "thiserror", "tracing", + "wasm-bindgen", + "web-sys", ] [[package]] diff --git a/video/external/Cargo.toml b/video/external/Cargo.toml index 7d6e5f1c5eb90..ac194f7b3e434 100644 --- a/video/external/Cargo.toml +++ b/video/external/Cargo.toml @@ -24,5 +24,18 @@ bzip2 = { version = "0.4.4", features = ["static"], optional = true } tempfile = { version = "3.13.0", optional = true } sha2 = { version = "0.10.8", optional = true } +# Needed for WebCodecs: +js-sys = { workspace = true, optional = true } +wasm-bindgen = { workspace = true, optional = true } +[dependencies.web-sys] +workspace = true +optional = true +features = [ + "CodecState", "DomException", "DomRectReadOnly", "EncodedVideoChunk", + "EncodedVideoChunkInit", "EncodedVideoChunkType", "VideoDecoder", + "VideoDecoderConfig", "VideoDecoderInit", "VideoFrame", "VideoPixelFormat", +] + [features] openh264 = ["libloading", "reqwest", "hex", "bzip2", "tempfile", "sha2"] +webcodecs = ["web-sys", "js-sys", "wasm-bindgen"] diff --git a/video/external/src/backend.rs b/video/external/src/backend.rs index e20bb56dc2311..52512c3b58ac4 100644 --- a/video/external/src/backend.rs +++ b/video/external/src/backend.rs @@ -46,7 +46,13 @@ impl ExternalVideoBackend { return Ok(decoder); } - Err(Error::DecoderError("No OpenH264".into())) + #[cfg(feature = "webcodecs")] + { + return Ok(Box::new(crate::decoder::webcodecs::H264Decoder::new())); + } + + #[allow(unreachable_code)] + Err(Error::DecoderError("No H.264 decoder available".into())) } pub fn new() -> Self { diff --git a/video/external/src/decoder.rs b/video/external/src/decoder.rs index d7169e5b894ff..bfec3b002433d 100644 --- a/video/external/src/decoder.rs +++ b/video/external/src/decoder.rs @@ -10,4 +10,7 @@ mod openh264_sys; #[cfg(feature = "openh264")] pub mod openh264; +#[cfg(feature = "webcodecs")] +pub mod webcodecs; + pub use ruffle_video_software::decoder::VideoDecoder; diff --git a/video/external/src/decoder/webcodecs.rs b/video/external/src/decoder/webcodecs.rs new file mode 100644 index 0000000000000..f4eb0a2c55148 --- /dev/null +++ b/video/external/src/decoder/webcodecs.rs @@ -0,0 +1,251 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use crate::decoder::VideoDecoder; + +use ruffle_render::bitmap::BitmapFormat; +use ruffle_video::error::Error; +use ruffle_video::frame::{DecodedFrame, EncodedFrame, FrameDependency}; + +use js_sys::Uint8Array; +use wasm_bindgen::prelude::*; +use web_sys::{ + DomException, EncodedVideoChunk, EncodedVideoChunkInit, EncodedVideoChunkType, + VideoDecoder as WebVideoDecoder, VideoDecoderConfig, VideoDecoderInit, VideoFrame, + VideoPixelFormat, +}; + +/// H264 video decoder. +pub struct H264Decoder { + /// How many bytes are used to store the length of the NALU (1, 2, 3, or 4). + length_size: u8, + + decoder: WebVideoDecoder, + + // Simply keeping these objects alive, as they are used by the JS side. + // See: https://rustwasm.github.io/wasm-bindgen/examples/closures.html + #[allow(dead_code)] + output_callback: Closure, + #[allow(dead_code)] + error_callback: Closure, + + last_frame: Rc>>, +} + +impl H264Decoder { + /// `extradata` should hold "AVCC (MP4) format" decoder configuration, including PPS and SPS. + /// Make sure it has any start code emulation prevention "three bytes" removed. + pub fn new() -> Self { + let last_frame = Rc::new(RefCell::new(None)); + let lf = last_frame.clone(); + // TODO: set up tracing log subscriber into these closures ... somehow + let output = move |output: &VideoFrame| { + let visible_rect = output.visible_rect().unwrap(); + + match output.format().unwrap() { + VideoPixelFormat::I420 => { + let mut data: Vec = + vec![ + 0; + visible_rect.width() as usize * visible_rect.height() as usize * 3 / 2 + ]; + let _ = output.copy_to_with_u8_array(&mut data); + last_frame.replace(Some(DecodedFrame::new( + visible_rect.width() as u32, + visible_rect.height() as u32, + BitmapFormat::Yuv420p, + data, + ))); + } + VideoPixelFormat::Bgrx => { + let mut data: Vec = + vec![0; visible_rect.width() as usize * visible_rect.height() as usize * 4]; + let _ = output.copy_to_with_u8_array(&mut data); + for pixel in data.chunks_mut(4) { + pixel.swap(0, 2); + pixel[3] = 0xff; + } + last_frame.replace(Some(DecodedFrame::new( + visible_rect.width() as u32, + visible_rect.height() as u32, + BitmapFormat::Rgba, + data, + ))); + } + _ => { + panic!("unsupported pixel format: {:?}", output.format().unwrap()); + } + }; + }; + + fn error(error: &DomException) { + tracing::error!("webcodecs error {:}", error.message()); + } + + let output_callback = Closure::new(move |frame| output(&frame)); + let error_callback = Closure::new(move |exception| error(&exception)); + + let decoder = WebVideoDecoder::new(&VideoDecoderInit::new( + error_callback.as_ref().unchecked_ref(), + output_callback.as_ref().unchecked_ref(), + )) + .unwrap(); + + Self { + length_size: 0, + decoder, + output_callback, + error_callback, + last_frame: lf, + } + } +} + +impl Default for H264Decoder { + fn default() -> Self { + Self::new() + } +} + +impl VideoDecoder for H264Decoder { + fn configure_decoder(&mut self, configuration_data: &[u8]) -> Result<(), Error> { + // extradata[0]: configuration version, always 1 + // extradata[1]: profile + // extradata[2]: compatibility + // extradata[3]: level + // extradata[4]: 6 reserved bits | NALU length size - 1 + + // The codec string is the profile, compatibility, and level bytes as hex. + + self.length_size = (configuration_data[4] & 0b0000_0011) + 1; + + tracing::warn!("length_size: {}", self.length_size); + + let codec_string = format!( + "avc1.{:02x}{:02x}{:02x}", + configuration_data[1], configuration_data[2], configuration_data[3] + ); + let config = VideoDecoderConfig::new(&codec_string); + tracing::warn!("configuring decoder: {:?}", &configuration_data[1..4]); + tracing::info!("{:?}", self.decoder.state()); + + let data = Uint8Array::from(configuration_data); + config.set_description(&data); + config.set_optimize_for_latency(true); + let _ = self.decoder.configure(&config); + tracing::info!("{:?}", self.decoder.state()); + Ok(()) + } + + fn preload_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result { + tracing::warn!("preloading frame"); + + let mut is_key = false; + let mut offset = 0; + + while offset < encoded_frame.data.len() { + let mut encoded_len = 0; + + for i in 0..self.length_size { + encoded_len = (encoded_len << 8) | encoded_frame.data[offset + i as usize] as u32; + } + + tracing::warn!( + "encoded_len: {}, chunk length: {}", + encoded_len, + encoded_frame.data.len() + ); + + let nal_unit_type = + encoded_frame.data[offset + self.length_size as usize] & 0b0001_1111; + + tracing::warn!("nal_unit_type: {}", nal_unit_type); + + if nal_unit_type == 5u8 { + is_key = true; + } + + offset += encoded_len as usize + self.length_size as usize; + } + + // 3.62 instantaneous decoding refresh (IDR) picture: + // After the decoding of an IDR picture all following coded pictures in decoding order can + // be decoded without inter prediction from any picture decoded prior to the IDR picture. + if is_key { + // openh264_sys::NAL_SLICE_IDR as u8 + tracing::info!("is key"); + Ok(FrameDependency::None) + } else { + tracing::info!("is not key"); + Ok(FrameDependency::Past) + } + } + + fn decode_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result { + tracing::warn!("decoding frame {}", encoded_frame.frame_id); + tracing::info!("{:?}", self.decoder.state()); + + tracing::warn!("queue size: {}", self.decoder.decode_queue_size()); + + let mut offset = 0; + + while offset < encoded_frame.data.len() { + let mut encoded_len = 0; + + for i in 0..self.length_size { + encoded_len = (encoded_len << 8) | encoded_frame.data[offset + i as usize] as u32; + } + + tracing::warn!( + "encoded_len: {}, chunk length: {}", + encoded_len, + encoded_frame.data.len() + ); + + let nal_unit_type = + encoded_frame.data[offset + self.length_size as usize] & 0b0001_1111; + + tracing::warn!("nal_unit_type: {}", nal_unit_type); + + if nal_unit_type != 6u8 && nal_unit_type != 7u8 && nal_unit_type != 8u8 { + // skipping SEI NALus, SPS NALus, and PPS NALus + // 3.62 instantaneous decoding refresh (IDR) picture: + // After the decoding of an IDR picture all following coded pictures in decoding order can + // be decoded without inter prediction from any picture decoded prior to the IDR picture. + let frame_type = if nal_unit_type == 5u8 { + // openh264_sys::NAL_SLICE_IDR as u8 + tracing::info!("is key"); + EncodedVideoChunkType::Key + } else { + tracing::info!("is not key"); + EncodedVideoChunkType::Delta + }; + let timestamp = (encoded_frame.frame_id as f64 - 1.0) * 1000000.0 * 0.5; + tracing::warn!("timestamp: {}", timestamp); + let data = Uint8Array::from( + &encoded_frame.data + [offset..offset + encoded_len as usize + self.length_size as usize], + ); + let init = EncodedVideoChunkInit::new(&data, timestamp, frame_type); + let chunk = EncodedVideoChunk::new(&init).unwrap(); + + let _ = self.decoder.decode(&chunk); + tracing::info!("{:?}", self.decoder.state()); + } + + offset += encoded_len as usize + self.length_size as usize; + } + + assert!( + offset == encoded_frame.data.len(), + "Incomplete NALu at the end" + ); + + match self.last_frame.borrow_mut().take() { + Some(frame) => Ok(frame), + None => Err(Error::DecoderError( + "No output frame produced by the decoder".into(), + )), + } + } +}