Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@theomonnom
theomonnom committed Oct 26, 2023
1 parent 41f0441 commit cac34b5
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 16 deletions.
6 changes: 5 additions & 1 deletion livekit-ffi/protocol/video_frame.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ message NewVideoSourceResponse { OwnedVideoSource source = 1; }
message CaptureVideoFrameRequest {
uint64 source_handle = 1;
VideoFrameInfo frame = 2;
VideoFrameBufferInfo info = 3;
oneof from {
VideoFrameBufferInfo info = 3;
uint64 handle = 4;
}
}
message CaptureVideoFrameResponse {}

Expand All @@ -60,6 +63,7 @@ message ToI420Request {
oneof from {
ArgbBufferInfo argb = 2;
VideoFrameBufferInfo buffer = 3;
uint64 handle = 4;
}
}
message ToI420Response {
Expand Down
19 changes: 16 additions & 3 deletions livekit-ffi/src/livekit.proto.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,8 +583,19 @@ pub struct CaptureVideoFrameRequest {
pub source_handle: u64,
#[prost(message, optional, tag="2")]
pub frame: ::core::option::Option<VideoFrameInfo>,
#[prost(message, optional, tag="3")]
pub info: ::core::option::Option<VideoFrameBufferInfo>,
#[prost(oneof="capture_video_frame_request::From", tags="3, 4")]
pub from: ::core::option::Option<capture_video_frame_request::From>,
}
/// Nested message and enum types in `CaptureVideoFrameRequest`.
pub mod capture_video_frame_request {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum From {
#[prost(message, tag="3")]
Info(super::VideoFrameBufferInfo),
#[prost(uint64, tag="4")]
Handle(u64),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
Expand All @@ -597,7 +608,7 @@ pub struct CaptureVideoFrameResponse {
pub struct ToI420Request {
#[prost(bool, tag="1")]
pub flip_y: bool,
#[prost(oneof="to_i420_request::From", tags="2, 3")]
#[prost(oneof="to_i420_request::From", tags="2, 3, 4")]
pub from: ::core::option::Option<to_i420_request::From>,
}
/// Nested message and enum types in `ToI420Request`.
Expand All @@ -609,6 +620,8 @@ pub mod to_i420_request {
Argb(super::ArgbBufferInfo),
#[prost(message, tag="3")]
Buffer(super::VideoFrameBufferInfo),
#[prost(uint64, tag="4")]
Handle(u64),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
Expand Down
119 changes: 107 additions & 12 deletions livekit-ffi/src/server/requests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,9 +335,109 @@ fn on_to_i420(
}
// Convert an arbitrary yuv buffer to I420
// Even if the buffer is already in I420, we're still copying it
proto::to_i420_request::From::YuvHandle(handle) => server
proto::to_i420_request::From::Handle(handle) => server
.retrieve_handle::<BoxVideoFrameBuffer>(handle)?
.to_i420(),
proto::to_i420_request::From::Buffer(info) => {
let mut i420 = I420Buffer::new(info.width, info.height);
let (sy, su, sv) = i420.strides();
let (dy, du, dv) = i420.data_mut();
let (w, h) = (info.width as i32, info.height as i32);

match info.buffer {
Some(proto::video_frame_buffer_info::Buffer::Yuv(yuv)) => {
#[rustfmt::skip]
let y = unsafe {
slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * info.height) as usize)
};

#[rustfmt::skip]
let u = unsafe {
slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize)
};

#[rustfmt::skip]
let v = unsafe {
slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize)
};

match info.buffer_type() {
proto::VideoFrameBufferType::I420 => {
&dy[..sy as usize].copy_from_slice(y);
&du[..su as usize].copy_from_slice(u);
&dv[..sv as usize].copy_from_slice(v);
}
proto::VideoFrameBufferType::I420a => {
#[rustfmt::skip]
let a = unsafe {
slice::from_raw_parts(yuv.data_a_ptr as *const u8, (yuv.stride_a * info.height) as usize)
};

&dy[..sy as usize].copy_from_slice(y);
&du[..su as usize].copy_from_slice(u);
&dv[..sv as usize].copy_from_slice(v);
}
proto::VideoFrameBufferType::I422 => {
#[rustfmt::skip]
yuv_helper::i422_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h);
}
proto::VideoFrameBufferType::I444 => {
#[rustfmt::skip]
yuv_helper::i444_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h);
}
proto::VideoFrameBufferType::I010 => {
#[rustfmt::skip]
let y = unsafe {
slice::from_raw_parts(yuv.data_y_ptr as *const u16, (yuv.stride_y * info.height) as usize / std::mem::size_of::<u16>())
};

#[rustfmt::skip]
let u = unsafe {
slice::from_raw_parts(yuv.data_u_ptr as *const u16, (yuv.stride_u * yuv.chroma_height) as usize / std::mem::size_of::<u16>())
};

#[rustfmt::skip]
let v = unsafe {
slice::from_raw_parts(yuv.data_v_ptr as *const u16, (yuv.stride_v * yuv.chroma_height) as usize / std::mem::size_of::<u16>())
};

#[rustfmt::skip]
yuv_helper::i010_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h);
}
_ => {
return Err(FfiError::InvalidRequest("invalid yuv description".into()))
}
}
}
Some(proto::video_frame_buffer_info::Buffer::BiYuv(biyuv)) => {
#[rustfmt::skip]
let y = unsafe {
slice::from_raw_parts(biyuv.data_y_ptr as *const u8, (biyuv.stride_y * info.height) as usize)
};

#[rustfmt::skip]
let uv = unsafe {
slice::from_raw_parts(biyuv.data_uv_ptr as *const u8, (biyuv.stride_uv * biyuv.chroma_height) as usize)
};

match info.buffer_type() {
proto::VideoFrameBufferType::Nv12 => {
#[rustfmt::skip]
yuv_helper::nv12_to_i420(y, biyuv.stride_y, uv, biyuv.stride_uv, dy, sy, du, su, dv, sv, w, h);
}
_ => {
return Err(FfiError::InvalidRequest(
"invalid biyuv description".into(),
))
}
}
}
_ => {
return Err(FfiError::InvalidRequest("conversion not supported".into()));
}
}
i420
}
};

let i420: BoxVideoFrameBuffer = Box::new(i420);
Expand Down Expand Up @@ -381,25 +481,20 @@ fn on_to_argb(
))
};

#[rustfmt::skip]
let src_y = unsafe {
slice::from_raw_parts(
yuv.data_y_ptr as *const u8,
(yuv.stride_y * buffer.height) as usize,
)
slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * buffer.height) as usize)
};

#[rustfmt::skip]
let src_u = unsafe {
slice::from_raw_parts(
yuv.data_u_ptr as *const u8,
(yuv.stride_u * yuv.chroma_height) as usize,
slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize,
)
};

#[rustfmt::skip]
let src_v = unsafe {
slice::from_raw_parts(
yuv.data_v_ptr as *const u8,
(yuv.stride_v * yuv.chroma_height) as usize,
)
slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize)
};

match rgba_format {
Expand Down

0 comments on commit cac34b5

Please sign in to comment.