From cac34b5e65bf5e4a3e15eb2b365a0ec533daba92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Thu, 26 Oct 2023 11:39:06 -0700 Subject: [PATCH] wip --- livekit-ffi/protocol/video_frame.proto | 6 +- livekit-ffi/src/livekit.proto.rs | 19 +++- livekit-ffi/src/server/requests.rs | 119 ++++++++++++++++++++++--- 3 files changed, 128 insertions(+), 16 deletions(-) diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index b7438eb81..45faf9c7e 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -49,7 +49,10 @@ message NewVideoSourceResponse { OwnedVideoSource source = 1; } message CaptureVideoFrameRequest { uint64 source_handle = 1; VideoFrameInfo frame = 2; - VideoFrameBufferInfo info = 3; + oneof from { + VideoFrameBufferInfo info = 3; + uint64 handle = 4; + } } message CaptureVideoFrameResponse {} @@ -60,6 +63,7 @@ message ToI420Request { oneof from { ArgbBufferInfo argb = 2; VideoFrameBufferInfo buffer = 3; + uint64 handle = 4; } } message ToI420Response { diff --git a/livekit-ffi/src/livekit.proto.rs b/livekit-ffi/src/livekit.proto.rs index e8fccbbdc..639cafb88 100644 --- a/livekit-ffi/src/livekit.proto.rs +++ b/livekit-ffi/src/livekit.proto.rs @@ -583,8 +583,19 @@ pub struct CaptureVideoFrameRequest { pub source_handle: u64, #[prost(message, optional, tag="2")] pub frame: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub info: ::core::option::Option, + #[prost(oneof="capture_video_frame_request::From", tags="3, 4")] + pub from: ::core::option::Option, +} +/// Nested message and enum types in `CaptureVideoFrameRequest`. +pub mod capture_video_frame_request { + #[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum From { + #[prost(message, tag="3")] + Info(super::VideoFrameBufferInfo), + #[prost(uint64, tag="4")] + Handle(u64), + } } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -597,7 +608,7 @@ pub struct CaptureVideoFrameResponse { pub struct ToI420Request { #[prost(bool, tag="1")] pub flip_y: bool, - #[prost(oneof="to_i420_request::From", tags="2, 3")] + #[prost(oneof="to_i420_request::From", tags="2, 3, 4")] pub from: ::core::option::Option, } /// Nested message and enum types in `ToI420Request`. @@ -609,6 +620,8 @@ pub mod to_i420_request { Argb(super::ArgbBufferInfo), #[prost(message, tag="3")] Buffer(super::VideoFrameBufferInfo), + #[prost(uint64, tag="4")] + Handle(u64), } } #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index ad78fd39d..a524e3c1c 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -335,9 +335,109 @@ fn on_to_i420( } // Convert an arbitrary yuv buffer to I420 // Even if the buffer is already in I420, we're still copying it - proto::to_i420_request::From::YuvHandle(handle) => server + proto::to_i420_request::From::Handle(handle) => server .retrieve_handle::(handle)? .to_i420(), + proto::to_i420_request::From::Buffer(info) => { + let mut i420 = I420Buffer::new(info.width, info.height); + let (sy, su, sv) = i420.strides(); + let (dy, du, dv) = i420.data_mut(); + let (w, h) = (info.width as i32, info.height as i32); + + match info.buffer { + Some(proto::video_frame_buffer_info::Buffer::Yuv(yuv)) => { + #[rustfmt::skip] + let y = unsafe { + slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * info.height) as usize) + }; + + #[rustfmt::skip] + let u = unsafe { + slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize) + }; + + #[rustfmt::skip] + let v = unsafe { + slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize) + }; + + match info.buffer_type() { + proto::VideoFrameBufferType::I420 => { + &dy[..sy as usize].copy_from_slice(y); + &du[..su as usize].copy_from_slice(u); + &dv[..sv as usize].copy_from_slice(v); + } + proto::VideoFrameBufferType::I420a => { + #[rustfmt::skip] + let a = unsafe { + slice::from_raw_parts(yuv.data_a_ptr as *const u8, (yuv.stride_a * info.height) as usize) + }; + + &dy[..sy as usize].copy_from_slice(y); + &du[..su as usize].copy_from_slice(u); + &dv[..sv as usize].copy_from_slice(v); + } + proto::VideoFrameBufferType::I422 => { + #[rustfmt::skip] + yuv_helper::i422_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h); + } + proto::VideoFrameBufferType::I444 => { + #[rustfmt::skip] + yuv_helper::i444_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h); + } + proto::VideoFrameBufferType::I010 => { + #[rustfmt::skip] + let y = unsafe { + slice::from_raw_parts(yuv.data_y_ptr as *const u16, (yuv.stride_y * info.height) as usize / std::mem::size_of::()) + }; + + #[rustfmt::skip] + let u = unsafe { + slice::from_raw_parts(yuv.data_u_ptr as *const u16, (yuv.stride_u * yuv.chroma_height) as usize / std::mem::size_of::()) + }; + + #[rustfmt::skip] + let v = unsafe { + slice::from_raw_parts(yuv.data_v_ptr as *const u16, (yuv.stride_v * yuv.chroma_height) as usize / std::mem::size_of::()) + }; + + #[rustfmt::skip] + yuv_helper::i010_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h); + } + _ => { + return Err(FfiError::InvalidRequest("invalid yuv description".into())) + } + } + } + Some(proto::video_frame_buffer_info::Buffer::BiYuv(biyuv)) => { + #[rustfmt::skip] + let y = unsafe { + slice::from_raw_parts(biyuv.data_y_ptr as *const u8, (biyuv.stride_y * info.height) as usize) + }; + + #[rustfmt::skip] + let uv = unsafe { + slice::from_raw_parts(biyuv.data_uv_ptr as *const u8, (biyuv.stride_uv * biyuv.chroma_height) as usize) + }; + + match info.buffer_type() { + proto::VideoFrameBufferType::Nv12 => { + #[rustfmt::skip] + yuv_helper::nv12_to_i420(y, biyuv.stride_y, uv, biyuv.stride_uv, dy, sy, du, su, dv, sv, w, h); + } + _ => { + return Err(FfiError::InvalidRequest( + "invalid biyuv description".into(), + )) + } + } + } + _ => { + return Err(FfiError::InvalidRequest("conversion not supported".into())); + } + } + i420 + } }; let i420: BoxVideoFrameBuffer = Box::new(i420); @@ -381,25 +481,20 @@ fn on_to_argb( )) }; + #[rustfmt::skip] let src_y = unsafe { - slice::from_raw_parts( - yuv.data_y_ptr as *const u8, - (yuv.stride_y * buffer.height) as usize, - ) + slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * buffer.height) as usize) }; + #[rustfmt::skip] let src_u = unsafe { - slice::from_raw_parts( - yuv.data_u_ptr as *const u8, - (yuv.stride_u * yuv.chroma_height) as usize, + slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize, ) }; + #[rustfmt::skip] let src_v = unsafe { - slice::from_raw_parts( - yuv.data_v_ptr as *const u8, - (yuv.stride_v * yuv.chroma_height) as usize, - ) + slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize) }; match rgba_format {