diff --git a/Cargo.lock b/Cargo.lock index b22bf5a4..d1f1c465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,6 +132,29 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bindgen" +version = "0.69.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" +dependencies = [ + "bitflags 2.4.1", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.41", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -208,6 +231,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -233,6 +265,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -852,6 +895,15 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "imgproc" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3a9cf5e30b522b828c6dcae37532bf14a61c989b316e9beaa94ed3f7efdc7b" +dependencies = [ + "yuv-sys", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -981,12 +1033,28 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +[[package]] +name = "libloading" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "libwebrtc" version = "0.3.0" @@ -1074,6 +1142,7 @@ dependencies = [ "downcast-rs", "env_logger", "futures-util", + "imgproc", "lazy_static", "livekit", "livekit-api", @@ -1379,6 +1448,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1706,6 +1781,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.28" @@ -1917,6 +1998,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2803,6 +2890,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "yuv-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871c1f3a3b08d3299dbd9b7e2a308d0d92f3c6c662c7678f7f9edab855909a60" +dependencies = [ + "bindgen", + "cc", + "regex", +] + [[package]] name = "zip" version = "0.6.6" diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index aa7a0ca5..9c4efe75 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -32,6 +32,7 @@ dashmap = "5.4" env_logger = "0.10" downcast-rs = "1.2" console-subscriber = { version = "0.1", features = ["parking_lot"], optional = true } +imgproc = "0.3.5" [build-dependencies] webrtc-sys-build = { path = "../webrtc-sys/build", version = "0.3.0" } diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto index ed0dd0a9..f61c74c3 100644 --- a/livekit-ffi/protocol/audio_frame.proto +++ b/livekit-ffi/protocol/audio_frame.proto @@ -19,16 +19,6 @@ option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; -// Allocate a new AudioFrameBuffer -// This is not necessary required because the data structure is fairly simple -// But keep the API consistent with VideoFrame -message AllocAudioBufferRequest { - uint32 sample_rate = 1; - uint32 num_channels = 2; - uint32 samples_per_channel = 3; -} -message AllocAudioBufferResponse { OwnedAudioFrameBuffer buffer = 1; } - // Create a new AudioStream // AudioStream is used to receive audio frames from a track message NewAudioStreamRequest { @@ -158,4 +148,4 @@ message AudioResamplerInfo { } message OwnedAudioResampler { FfiOwnedHandle handle = 1; AudioResamplerInfo info = 2; -} \ No newline at end of file +} diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index e6c2e7f4..bec1f593 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -73,15 +73,12 @@ message FfiRequest { GetStatsRequest get_stats = 14; // Video - AllocVideoBufferRequest alloc_video_buffer = 15; NewVideoStreamRequest new_video_stream = 16; NewVideoSourceRequest new_video_source = 17; CaptureVideoFrameRequest capture_video_frame = 18; - ToI420Request to_i420 = 19; - ToArgbRequest to_argb = 20; + VideoConvertRequest video_convert = 19; // Audio - AllocAudioBufferRequest alloc_audio_buffer = 21; NewAudioStreamRequest new_audio_stream = 22; NewAudioSourceRequest new_audio_source = 23; CaptureAudioFrameRequest capture_audio_frame = 24; @@ -113,15 +110,12 @@ message FfiResponse { GetStatsResponse get_stats = 14; // Video - AllocVideoBufferResponse alloc_video_buffer = 15; NewVideoStreamResponse new_video_stream = 16; NewVideoSourceResponse new_video_source = 17; CaptureVideoFrameResponse capture_video_frame = 18; - ToI420Response to_i420 = 19; - ToArgbResponse to_argb = 20; + VideoConvertResponse video_convert = 19; // Audio - AllocAudioBufferResponse alloc_audio_buffer = 21; NewAudioStreamResponse new_audio_stream = 22; NewAudioSourceResponse new_audio_source = 23; CaptureAudioFrameResponse capture_audio_frame = 24; diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index 6734c299..28016b04 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -19,19 +19,14 @@ option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; -// Allocate a new VideoFrameBuffer -message AllocVideoBufferRequest { - VideoFrameBufferType type = 1; // Only I420 is supported atm - uint32 width = 2; - uint32 height = 3; -} -message AllocVideoBufferResponse { OwnedVideoFrameBuffer buffer = 1; } - // Create a new VideoStream // VideoStream is used to receive video frames from a track message NewVideoStreamRequest { uint64 track_handle = 1; VideoStreamType type = 2; + // Get the frame on a specific format + optional VideoBufferType format = 3; + bool normalize_stride = 4; // if true, stride will be set to width/chroma_width } message NewVideoStreamResponse { OwnedVideoStream stream = 1; } @@ -48,41 +43,23 @@ message NewVideoSourceResponse { OwnedVideoSource source = 1; } // Push a frame to a VideoSource message CaptureVideoFrameRequest { uint64 source_handle = 1; - VideoFrameInfo frame = 2; - oneof from { - VideoFrameBufferInfo info = 3; - uint64 handle = 4; - } + VideoBufferInfo buffer = 2; + int64 timestamp_us = 3; // In microseconds + VideoRotation rotation = 4; } + message CaptureVideoFrameResponse {} -// Convert a RGBA frame to a I420 YUV frame -// Or convert another YUV frame format to I420 -message ToI420Request { +message VideoConvertRequest { bool flip_y = 1; - oneof from { - ArgbBufferInfo argb = 2; - VideoFrameBufferInfo buffer = 3; - uint64 handle = 4; - } + VideoBufferInfo buffer = 2; + VideoBufferType dst_type = 3; } -message ToI420Response { - OwnedVideoFrameBuffer buffer = 1; +message VideoConvertResponse { + optional string error = 1; + OwnedVideoBuffer buffer = 2; } -// Convert a YUV frame to a RGBA frame -// Only I420 is supported atm -message ToArgbRequest { - VideoFrameBufferInfo buffer = 1; - uint64 dst_ptr = 2; - VideoFormatType dst_format = 3; - uint32 dst_stride = 4; - uint32 dst_width = 5; - uint32 dst_height = 6; - bool flip_y = 7; -} -message ToArgbResponse {} - // // VideoFrame buffers // @@ -107,79 +84,38 @@ enum VideoRotation { VIDEO_ROTATION_270 = 3; } -enum VideoFormatType { - FORMAT_ARGB = 0; - FORMAT_BGRA = 1; - FORMAT_ABGR = 2; - FORMAT_RGBA = 3; -} - -enum VideoFrameBufferType { - NATIVE = 0; - I420 = 1; - I420A = 2; - I422 = 3; - I444 = 4; - I010 = 5; - NV12 = 6; -} - -message ArgbBufferInfo { - uint64 ptr = 1; - VideoFormatType format = 2; - uint32 stride = 3; - uint32 width = 4; - uint32 height = 5; -} - -message VideoFrameInfo { - int64 timestamp_us = 1; // In microseconds - VideoRotation rotation = 2; -} - -message VideoFrameBufferInfo { - VideoFrameBufferType buffer_type = 1; +enum VideoBufferType { + RGBA = 0; + ABGR = 1; + ARGB = 2; + BGRA = 3; + RGB24 = 4; + I420 = 5; + I420A = 6; + I422 = 7; + I444 = 8; + I010 = 9; + NV12 = 10; +} + +message VideoBufferInfo { + message ComponentInfo { + uint32 offset = 1; + uint32 stride = 2; + uint32 size = 3; + } + VideoBufferType type = 1; uint32 width = 2; uint32 height = 3; - oneof buffer { - PlanarYuvBufferInfo yuv = 4; - BiplanarYuvBufferInfo bi_yuv = 5; - NativeBufferInfo native = 6; - } + uint64 data_ptr = 4; + uint32 data_len = 5; + uint32 stride = 6; // for packed formats + repeated ComponentInfo components = 7; } -message OwnedVideoFrameBuffer { +message OwnedVideoBuffer { FfiOwnedHandle handle = 1; - VideoFrameBufferInfo info = 2; -} - -message PlanarYuvBufferInfo { - uint32 chroma_width = 1; - uint32 chroma_height = 2; - uint32 stride_y = 3; - uint32 stride_u = 4; - uint32 stride_v = 5; - uint32 stride_a = 6; - - // *const u8 or *const u16 - uint64 data_y_ptr = 7; - uint64 data_u_ptr = 8; - uint64 data_v_ptr = 9; - uint64 data_a_ptr = 10; // nullptr = no alpha -} - -message BiplanarYuvBufferInfo { - uint32 chroma_width = 1; - uint32 chroma_height = 2; - uint32 stride_y = 3; - uint32 stride_uv = 4; - - uint64 data_y_ptr = 5; - uint64 data_uv_ptr = 6; -} - -message NativeBufferInfo { - // TODO(theomonnom): Expose graphic context? + VideoBufferInfo info = 2; } // @@ -210,8 +146,9 @@ message VideoStreamEvent { } message VideoFrameReceived { - VideoFrameInfo frame = 1; - OwnedVideoFrameBuffer buffer = 2; + OwnedVideoBuffer buffer = 1; + int64 timestamp_us = 2; // In microseconds + VideoRotation rotation = 3; } message VideoStreamEOS {} diff --git a/livekit-ffi/src/conversion/video_frame.rs b/livekit-ffi/src/conversion/video_frame.rs index 988562a4..5498c870 100644 --- a/livekit-ffi/src/conversion/video_frame.rs +++ b/livekit-ffi/src/conversion/video_frame.rs @@ -14,7 +14,7 @@ use livekit::{ options::{VideoCodec, VideoResolution}, - webrtc::{prelude::*, video_frame, video_source::VideoResolution as VideoSourceResolution}, + webrtc::{prelude::*, video_source::VideoResolution as VideoSourceResolution}, }; use crate::{ @@ -28,18 +28,6 @@ impl From for VideoSourceResolution { } } -impl proto::VideoFrameInfo { - pub fn from(frame: &VideoFrame) -> Self - where - T: AsRef, - { - Self { - timestamp_us: frame.timestamp_us, - rotation: proto::VideoRotation::from(frame.rotation).into(), - } - } -} - impl From<&FfiVideoSource> for proto::VideoSourceInfo { fn from(source: &FfiVideoSource) -> Self { Self { r#type: source.source_type as i32 } @@ -52,17 +40,6 @@ impl From<&FfiVideoStream> for proto::VideoStreamInfo { } } -impl From for VideoFormatType { - fn from(format: proto::VideoFormatType) -> Self { - match format { - proto::VideoFormatType::FormatArgb => Self::ARGB, - proto::VideoFormatType::FormatBgra => Self::BGRA, - proto::VideoFormatType::FormatAbgr => Self::ABGR, - proto::VideoFormatType::FormatRgba => Self::RGBA, - } - } -} - impl From for proto::VideoRotation { fn from(rotation: VideoRotation) -> proto::VideoRotation { match rotation { @@ -85,21 +62,6 @@ impl From for VideoRotation { } } -impl From for proto::VideoFrameBufferType { - fn from(buffer_type: VideoBufferType) -> Self { - match buffer_type { - VideoBufferType::Native => Self::Native, - VideoBufferType::I420 => Self::I420, - VideoBufferType::I420A => Self::I420a, - VideoBufferType::I422 => Self::I422, - VideoBufferType::I444 => Self::I444, - VideoBufferType::I010 => Self::I010, - VideoBufferType::NV12 => Self::Nv12, - _ => panic!("unsupported buffer type on FFI server"), - } - } -} - impl From for proto::VideoResolution { fn from(resolution: VideoResolution) -> Self { Self { @@ -131,160 +93,3 @@ impl From for VideoCodec { } } } - -macro_rules! impl_yuv_into { - (@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => { - Self { - chroma_width: $buffer.chroma_width(), - chroma_height: $buffer.chroma_height(), - stride_y: $buffer.strides().0, - stride_u: $buffer.strides().1, - stride_v: $buffer.strides().2, - data_y_ptr: $data_y.as_ptr() as u64, - data_u_ptr: $data_u.as_ptr() as u64, - data_v_ptr: $data_v.as_ptr() as u64, - ..Default::default() - } - }; - ($fncname:ident, $buffer:ty, ALPHA) => { - fn $fncname(buffer: $buffer) -> Self { - let (data_y, data_u, data_v, data_a) = buffer.data(); - let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v); - proto.stride_a = buffer.strides().3; - proto.data_a_ptr = data_a.map(|data_a| data_a.as_ptr() as u64).unwrap_or(0); - proto - } - }; - ($fncname:ident, $buffer:ty) => { - fn $fncname(buffer: $buffer) -> Self { - let (data_y, data_u, data_v) = buffer.data(); - impl_yuv_into!(@fields, buffer, data_y, data_u, data_v) - } - }; -} - -macro_rules! impl_biyuv_into { - ($fncname:ident, $buffer:ty) => { - fn $fncname(buffer: $buffer) -> Self { - let (stride_y, stride_uv) = buffer.strides(); - let (data_y, data_uv) = buffer.data(); - Self { - chroma_width: buffer.chroma_width(), - chroma_height: buffer.chroma_height(), - stride_y, - stride_uv, - data_y_ptr: data_y.as_ptr() as u64, - data_uv_ptr: data_uv.as_ptr() as u64, - } - } - }; -} - -impl proto::PlanarYuvBufferInfo { - impl_yuv_into!(from_i420, &I420Buffer); - impl_yuv_into!(from_i420a, &I420ABuffer, ALPHA); - impl_yuv_into!(from_i422, &I422Buffer); - impl_yuv_into!(from_i444, &I444Buffer); - impl_yuv_into!(from_i010, &I010Buffer); -} - -impl proto::BiplanarYuvBufferInfo { - impl_biyuv_into!(from_nv12, &NV12Buffer); -} - -impl proto::VideoFrameBufferInfo { - #[cfg(not(target_arch = "wasm32"))] - fn from_native(buffer: &video_frame::native::NativeBuffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::Native.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::Native( - proto::NativeBufferInfo {}, - )), - } - } - - fn from_i420(buffer: &I420Buffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::I420.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv( - proto::PlanarYuvBufferInfo::from_i420(buffer), - )), - } - } - - fn from_i420a(buffer: &I420ABuffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::I420a.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv( - proto::PlanarYuvBufferInfo::from_i420a(buffer), - )), - } - } - - fn from_i422(buffer: &I422Buffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::I422.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv( - proto::PlanarYuvBufferInfo::from_i422(buffer), - )), - } - } - - fn from_i444(buffer: &I444Buffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::I444.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv( - proto::PlanarYuvBufferInfo::from_i444(buffer), - )), - } - } - - fn from_i010(buffer: &I010Buffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::I010.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv( - proto::PlanarYuvBufferInfo::from_i010(buffer), - )), - } - } - - fn from_nv12(buffer: &NV12Buffer) -> Self { - Self { - buffer_type: proto::VideoFrameBufferType::Nv12.into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(proto::video_frame_buffer_info::Buffer::BiYuv( - proto::BiplanarYuvBufferInfo::from_nv12(buffer), - )), - } - } -} - -impl> From for proto::VideoFrameBufferInfo { - fn from(buffer: B) -> Self { - let buffer = buffer.as_ref(); - match buffer.buffer_type() { - #[cfg(not(target_arch = "wasm32"))] - VideoBufferType::Native => Self::from_native(buffer.as_native().unwrap()), - VideoBufferType::I420 => Self::from_i420(buffer.as_i420().unwrap()), - VideoBufferType::I420A => Self::from_i420a(buffer.as_i420a().unwrap()), - VideoBufferType::I422 => Self::from_i422(buffer.as_i422().unwrap()), - VideoBufferType::I444 => Self::from_i444(buffer.as_i444().unwrap()), - VideoBufferType::I010 => Self::from_i010(buffer.as_i010().unwrap()), - VideoBufferType::NV12 => Self::from_nv12(buffer.as_nv12().unwrap()), - _ => panic!("unsupported buffer type on this platform"), - } - } -} diff --git a/livekit-ffi/src/livekit.proto.rs b/livekit-ffi/src/livekit.proto.rs index 1015b9e5..2a686643 100644 --- a/livekit-ffi/src/livekit.proto.rs +++ b/livekit-ffi/src/livekit.proto.rs @@ -1518,24 +1518,6 @@ pub struct OwnedParticipant { #[prost(message, optional, tag="2")] pub info: ::core::option::Option, } -/// Allocate a new VideoFrameBuffer -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AllocVideoBufferRequest { - /// Only I420 is supported atm - #[prost(enumeration="VideoFrameBufferType", tag="1")] - pub r#type: i32, - #[prost(uint32, tag="2")] - pub width: u32, - #[prost(uint32, tag="3")] - pub height: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AllocVideoBufferResponse { - #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, -} /// Create a new VideoStream /// VideoStream is used to receive video frames from a track #[allow(clippy::derive_partial_eq_without_eq)] @@ -1545,6 +1527,12 @@ pub struct NewVideoStreamRequest { pub track_handle: u64, #[prost(enumeration="VideoStreamType", tag="2")] pub r#type: i32, + /// Get the frame on a specific format + #[prost(enumeration="VideoBufferType", optional, tag="3")] + pub format: ::core::option::Option, + /// if true, stride will be set to width/chroma_width + #[prost(bool, tag="4")] + pub normalize_stride: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1577,77 +1565,34 @@ pub struct CaptureVideoFrameRequest { #[prost(uint64, tag="1")] pub source_handle: u64, #[prost(message, optional, tag="2")] - pub frame: ::core::option::Option, - #[prost(oneof="capture_video_frame_request::From", tags="3, 4")] - pub from: ::core::option::Option, -} -/// Nested message and enum types in `CaptureVideoFrameRequest`. -pub mod capture_video_frame_request { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum From { - #[prost(message, tag="3")] - Info(super::VideoFrameBufferInfo), - #[prost(uint64, tag="4")] - Handle(u64), - } + pub buffer: ::core::option::Option, + /// In microseconds + #[prost(int64, tag="3")] + pub timestamp_us: i64, + #[prost(enumeration="VideoRotation", tag="4")] + pub rotation: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CaptureVideoFrameResponse { } -/// Convert a RGBA frame to a I420 YUV frame -/// Or convert another YUV frame format to I420 #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ToI420Request { +pub struct VideoConvertRequest { #[prost(bool, tag="1")] pub flip_y: bool, - #[prost(oneof="to_i420_request::From", tags="2, 3, 4")] - pub from: ::core::option::Option, -} -/// Nested message and enum types in `ToI420Request`. -pub mod to_i420_request { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum From { - #[prost(message, tag="2")] - Argb(super::ArgbBufferInfo), - #[prost(message, tag="3")] - Buffer(super::VideoFrameBufferInfo), - #[prost(uint64, tag="4")] - Handle(u64), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ToI420Response { - #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, -} -/// Convert a YUV frame to a RGBA frame -/// Only I420 is supported atm -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ToArgbRequest { - #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, - #[prost(uint64, tag="2")] - pub dst_ptr: u64, - #[prost(enumeration="VideoFormatType", tag="3")] - pub dst_format: i32, - #[prost(uint32, tag="4")] - pub dst_stride: u32, - #[prost(uint32, tag="5")] - pub dst_width: u32, - #[prost(uint32, tag="6")] - pub dst_height: u32, - #[prost(bool, tag="7")] - pub flip_y: bool, + #[prost(message, optional, tag="2")] + pub buffer: ::core::option::Option, + #[prost(enumeration="VideoBufferType", tag="3")] + pub dst_type: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ToArgbResponse { +pub struct VideoConvertResponse { + #[prost(string, optional, tag="1")] + pub error: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, optional, tag="2")] + pub buffer: ::core::option::Option, } // // VideoFrame buffers @@ -1665,106 +1610,43 @@ pub struct VideoResolution { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ArgbBufferInfo { - #[prost(uint64, tag="1")] - pub ptr: u64, - #[prost(enumeration="VideoFormatType", tag="2")] - pub format: i32, - #[prost(uint32, tag="3")] - pub stride: u32, - #[prost(uint32, tag="4")] - pub width: u32, - #[prost(uint32, tag="5")] - pub height: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct VideoFrameInfo { - /// In microseconds - #[prost(int64, tag="1")] - pub timestamp_us: i64, - #[prost(enumeration="VideoRotation", tag="2")] - pub rotation: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct VideoFrameBufferInfo { - #[prost(enumeration="VideoFrameBufferType", tag="1")] - pub buffer_type: i32, +pub struct VideoBufferInfo { + #[prost(enumeration="VideoBufferType", tag="1")] + pub r#type: i32, #[prost(uint32, tag="2")] pub width: u32, #[prost(uint32, tag="3")] pub height: u32, - #[prost(oneof="video_frame_buffer_info::Buffer", tags="4, 5, 6")] - pub buffer: ::core::option::Option, + #[prost(uint64, tag="4")] + pub data_ptr: u64, + #[prost(uint32, tag="5")] + pub data_len: u32, + /// for packed formats + #[prost(uint32, tag="6")] + pub stride: u32, + #[prost(message, repeated, tag="7")] + pub components: ::prost::alloc::vec::Vec, } -/// Nested message and enum types in `VideoFrameBufferInfo`. -pub mod video_frame_buffer_info { +/// Nested message and enum types in `VideoBufferInfo`. +pub mod video_buffer_info { #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Buffer { - #[prost(message, tag="4")] - Yuv(super::PlanarYuvBufferInfo), - #[prost(message, tag="5")] - BiYuv(super::BiplanarYuvBufferInfo), - #[prost(message, tag="6")] - Native(super::NativeBufferInfo), +#[derive(Clone, PartialEq, ::prost::Message)] + pub struct ComponentInfo { + #[prost(uint32, tag="1")] + pub offset: u32, + #[prost(uint32, tag="2")] + pub stride: u32, + #[prost(uint32, tag="3")] + pub size: u32, } } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct OwnedVideoFrameBuffer { +pub struct OwnedVideoBuffer { #[prost(message, optional, tag="1")] pub handle: ::core::option::Option, #[prost(message, optional, tag="2")] - pub info: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PlanarYuvBufferInfo { - #[prost(uint32, tag="1")] - pub chroma_width: u32, - #[prost(uint32, tag="2")] - pub chroma_height: u32, - #[prost(uint32, tag="3")] - pub stride_y: u32, - #[prost(uint32, tag="4")] - pub stride_u: u32, - #[prost(uint32, tag="5")] - pub stride_v: u32, - #[prost(uint32, tag="6")] - pub stride_a: u32, - /// *const u8 or *const u16 - #[prost(uint64, tag="7")] - pub data_y_ptr: u64, - #[prost(uint64, tag="8")] - pub data_u_ptr: u64, - #[prost(uint64, tag="9")] - pub data_v_ptr: u64, - /// nullptr = no alpha - #[prost(uint64, tag="10")] - pub data_a_ptr: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BiplanarYuvBufferInfo { - #[prost(uint32, tag="1")] - pub chroma_width: u32, - #[prost(uint32, tag="2")] - pub chroma_height: u32, - #[prost(uint32, tag="3")] - pub stride_y: u32, - #[prost(uint32, tag="4")] - pub stride_uv: u32, - #[prost(uint64, tag="5")] - pub data_y_ptr: u64, - #[prost(uint64, tag="6")] - pub data_uv_ptr: u64, -} -/// TODO(theomonnom): Expose graphic context? -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct NativeBufferInfo { + pub info: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1803,9 +1685,12 @@ pub mod video_stream_event { #[derive(Clone, PartialEq, ::prost::Message)] pub struct VideoFrameReceived { #[prost(message, optional, tag="1")] - pub frame: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub buffer: ::core::option::Option, + pub buffer: ::core::option::Option, + /// In microseconds + #[prost(int64, tag="2")] + pub timestamp_us: i64, + #[prost(enumeration="VideoRotation", tag="3")] + pub rotation: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1903,67 +1788,47 @@ impl VideoRotation { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] -pub enum VideoFormatType { - FormatArgb = 0, - FormatBgra = 1, - FormatAbgr = 2, - FormatRgba = 3, -} -impl VideoFormatType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - VideoFormatType::FormatArgb => "FORMAT_ARGB", - VideoFormatType::FormatBgra => "FORMAT_BGRA", - VideoFormatType::FormatAbgr => "FORMAT_ABGR", - VideoFormatType::FormatRgba => "FORMAT_RGBA", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "FORMAT_ARGB" => Some(Self::FormatArgb), - "FORMAT_BGRA" => Some(Self::FormatBgra), - "FORMAT_ABGR" => Some(Self::FormatAbgr), - "FORMAT_RGBA" => Some(Self::FormatRgba), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum VideoFrameBufferType { - Native = 0, - I420 = 1, - I420a = 2, - I422 = 3, - I444 = 4, - I010 = 5, - Nv12 = 6, -} -impl VideoFrameBufferType { +pub enum VideoBufferType { + Rgba = 0, + Abgr = 1, + Argb = 2, + Bgra = 3, + Rgb24 = 4, + I420 = 5, + I420a = 6, + I422 = 7, + I444 = 8, + I010 = 9, + Nv12 = 10, +} +impl VideoBufferType { /// String value of the enum field names used in the ProtoBuf definition. /// /// The values are not transformed in any way and thus are considered stable /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - VideoFrameBufferType::Native => "NATIVE", - VideoFrameBufferType::I420 => "I420", - VideoFrameBufferType::I420a => "I420A", - VideoFrameBufferType::I422 => "I422", - VideoFrameBufferType::I444 => "I444", - VideoFrameBufferType::I010 => "I010", - VideoFrameBufferType::Nv12 => "NV12", + VideoBufferType::Rgba => "RGBA", + VideoBufferType::Abgr => "ABGR", + VideoBufferType::Argb => "ARGB", + VideoBufferType::Bgra => "BGRA", + VideoBufferType::Rgb24 => "RGB24", + VideoBufferType::I420 => "I420", + VideoBufferType::I420a => "I420A", + VideoBufferType::I422 => "I422", + VideoBufferType::I444 => "I444", + VideoBufferType::I010 => "I010", + VideoBufferType::Nv12 => "NV12", } } /// Creates an enum from field names used in the ProtoBuf definition. pub fn from_str_name(value: &str) -> ::core::option::Option { match value { - "NATIVE" => Some(Self::Native), + "RGBA" => Some(Self::Rgba), + "ABGR" => Some(Self::Abgr), + "ARGB" => Some(Self::Argb), + "BGRA" => Some(Self::Bgra), + "RGB24" => Some(Self::Rgb24), "I420" => Some(Self::I420), "I420A" => Some(Self::I420a), "I422" => Some(Self::I422), @@ -2745,25 +2610,6 @@ impl DataPacketKind { } } } -/// Allocate a new AudioFrameBuffer -/// This is not necessary required because the data structure is fairly simple -/// But keep the API consistent with VideoFrame -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AllocAudioBufferRequest { - #[prost(uint32, tag="1")] - pub sample_rate: u32, - #[prost(uint32, tag="2")] - pub num_channels: u32, - #[prost(uint32, tag="3")] - pub samples_per_channel: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AllocAudioBufferResponse { - #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, -} /// Create a new AudioStream /// AudioStream is used to receive audio frames from a track #[allow(clippy::derive_partial_eq_without_eq)] @@ -3049,7 +2895,7 @@ impl AudioSourceType { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FfiRequest { - #[prost(oneof="ffi_request::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27")] + #[prost(oneof="ffi_request::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 22, 23, 24, 25, 26, 27")] pub message: ::core::option::Option, } /// Nested message and enum types in `FfiRequest`. @@ -3086,8 +2932,6 @@ pub mod ffi_request { #[prost(message, tag="14")] GetStats(super::GetStatsRequest), /// Video - #[prost(message, tag="15")] - AllocVideoBuffer(super::AllocVideoBufferRequest), #[prost(message, tag="16")] NewVideoStream(super::NewVideoStreamRequest), #[prost(message, tag="17")] @@ -3095,12 +2939,8 @@ pub mod ffi_request { #[prost(message, tag="18")] CaptureVideoFrame(super::CaptureVideoFrameRequest), #[prost(message, tag="19")] - ToI420(super::ToI420Request), - #[prost(message, tag="20")] - ToArgb(super::ToArgbRequest), + VideoConvert(super::VideoConvertRequest), /// Audio - #[prost(message, tag="21")] - AllocAudioBuffer(super::AllocAudioBufferRequest), #[prost(message, tag="22")] NewAudioStream(super::NewAudioStreamRequest), #[prost(message, tag="23")] @@ -3119,7 +2959,7 @@ pub mod ffi_request { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FfiResponse { - #[prost(oneof="ffi_response::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27")] + #[prost(oneof="ffi_response::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 22, 23, 24, 25, 26, 27")] pub message: ::core::option::Option, } /// Nested message and enum types in `FfiResponse`. @@ -3156,8 +2996,6 @@ pub mod ffi_response { #[prost(message, tag="14")] GetStats(super::GetStatsResponse), /// Video - #[prost(message, tag="15")] - AllocVideoBuffer(super::AllocVideoBufferResponse), #[prost(message, tag="16")] NewVideoStream(super::NewVideoStreamResponse), #[prost(message, tag="17")] @@ -3165,12 +3003,8 @@ pub mod ffi_response { #[prost(message, tag="18")] CaptureVideoFrame(super::CaptureVideoFrameResponse), #[prost(message, tag="19")] - ToI420(super::ToI420Response), - #[prost(message, tag="20")] - ToArgb(super::ToArgbResponse), + VideoConvert(super::VideoConvertResponse), /// Audio - #[prost(message, tag="21")] - AllocAudioBuffer(super::AllocAudioBufferResponse), #[prost(message, tag="22")] NewAudioStream(super::NewAudioStreamResponse), #[prost(message, tag="23")] diff --git a/livekit-ffi/src/server/colorcvt/cvtimpl.rs b/livekit-ffi/src/server/colorcvt/cvtimpl.rs new file mode 100644 index 00000000..c36503ff --- /dev/null +++ b/livekit-ffi/src/server/colorcvt/cvtimpl.rs @@ -0,0 +1,587 @@ +use super::*; +use crate::proto; +use crate::{FfiError, FfiResult}; +use imgproc::colorcvt; + +pub unsafe fn cvt( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + match buffer.r#type() { + proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y), + proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y), + proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y), + proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y), + proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y), + proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y), + proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y), + proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y), + proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y), + proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y), + proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y), + } +} + +pub unsafe fn cvt_rgba( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); + let proto::VideoBufferInfo { stride, width, height, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + match dst_type { + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dy, du, dv) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::abgr_to_i420( + data, stride, dy, width, du, chroma_w, dv, chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + Err(FfiError::InvalidRequest(format!("rgba to {:?} is not supported", dst_type).into())) + } + } +} + +pub unsafe fn cvt_abgr( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); + let proto::VideoBufferInfo { stride, width, height, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + match dst_type { + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + imgproc::colorcvt::rgba_to_i420( + data, stride, dst_y, width, dst_u, chroma_w, dst_v, chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + Err(FfiError::InvalidRequest(format!("abgr to {:?} is not supported", dst_type).into())) + } + } +} + +pub unsafe fn cvt_argb( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::Argb); + let proto::VideoBufferInfo { stride, width, height, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + match dst_type { + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::bgra_to_i420( + data, stride, dst_y, width, dst_u, chroma_w, dst_v, chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + Err(FfiError::InvalidRequest(format!("argb to {:?} is not supported", dst_type).into())) + } + } +} + +pub unsafe fn cvt_bgra( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::Bgra); + let proto::VideoBufferInfo { stride, width, height, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + match dst_type { + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::argb_to_i420( + data, stride, dst_y, width, dst_u, chroma_w, dst_v, chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + Err(FfiError::InvalidRequest(format!("bgra to {:?} is not supported", dst_type).into())) + } + } +} + +pub unsafe fn cvt_rgb24( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgb24); + let proto::VideoBufferInfo { stride, width, height, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + match dst_type { + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::raw_to_i420( + data, stride, dst_y, width, dst_u, chroma_w, dst_v, chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("rgb24 to {:?} is not supported", dst_type).into(), + )) + } + } +} + +pub unsafe fn cvt_i420( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::I420); + let proto::VideoBufferInfo { width, height, components, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i420(data, c0.stride, c1.stride, c2.stride, height); + + match dst_type { + proto::VideoBufferType::Rgba + | proto::VideoBufferType::Abgr + | proto::VideoBufferType::Argb + | proto::VideoBufferType::Bgra => { + let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let stride = width * 4; + + macro_rules! cvt { + ($rgba:expr, $fnc:ident) => { + if dst_type == $rgba { + colorcvt::$fnc( + y, c0.stride, u, c1.stride, v, c2.stride, &mut dst, stride, width, + height, flip_y, + ); + } + }; + } + + cvt!(proto::VideoBufferType::Rgba, i420_to_abgr); + cvt!(proto::VideoBufferType::Abgr, i420_to_rgba); + cvt!(proto::VideoBufferType::Argb, i420_to_bgra); + cvt!(proto::VideoBufferType::Bgra, i420_to_argb); + + let info = rgba_info(&dst, dst_type, width, height); + Ok((dst, info)) + } + proto::VideoBufferType::Rgb24 => { + let mut dst = vec![0u8; (width * height * 3) as usize].into_boxed_slice(); + let stride = width * 3; + + colorcvt::i420_to_raw( + y, c0.stride, u, c1.stride, v, c2.stride, &mut dst, stride, width, height, flip_y, + ); + + let info = rgb_info(&dst, dst_type, width, height); + Ok((dst, info)) + } + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::i420_copy( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("i420 to {:?} is not supported", dst_type).into(), + )) + } + } +} + +pub unsafe fn cvt_i420a( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::I420a); + let proto::VideoBufferInfo { width, height, components, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + let (c0, c1, c2, c3) = (&components[0], &components[1], &components[2], &components[3]); + let (y, u, v, a) = split_i420a(data, c0.stride, c1.stride, c2.stride, c3.stride, height); + + match dst_type { + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::i420_copy( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + proto::VideoBufferType::I420a => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2 + width * height) as usize] + .into_boxed_slice(); + let (dst_y, dst_u, dst_v, dst_a) = + split_i420a_mut(&mut dst, width, chroma_w, chroma_w, width, height); + + colorcvt::i420a_copy( + y, c0.stride, u, c1.stride, v, c2.stride, a, c3.stride, dst_y, width, dst_u, + chroma_w, dst_v, chroma_w, dst_a, width, width, height, flip_y, + ); + + let info = i420a_info( + dst.as_ptr(), + dst.len(), + width, + height, + width, + chroma_w, + chroma_w, + width, + ); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("i420a to {:?} is not supported", dst_type).into(), + )) + } + } +} + +pub unsafe fn cvt_i422( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::I422); + let proto::VideoBufferInfo { width, height, components, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i422(data, c0.stride, c1.stride, c2.stride, height); + + match dst_type { + proto::VideoBufferType::Rgba + | proto::VideoBufferType::Abgr + | proto::VideoBufferType::Argb => { + let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let stride = buffer.width * 4; + + macro_rules! cvt { + ($rgba:expr, $fnc:ident) => { + if dst_type == $rgba { + colorcvt::$fnc( + y, c0.stride, u, c1.stride, v, c2.stride, &mut dst, stride, width, + height, flip_y, + ); + } + }; + } + + cvt!(proto::VideoBufferType::Rgba, i422_to_abgr); + cvt!(proto::VideoBufferType::Abgr, i422_to_rgba); + cvt!(proto::VideoBufferType::Argb, i422_to_bgra); + + let info = rgba_info(&dst, dst_type, width, height); + Ok((dst, info)) + } + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::i422_to_i420( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + proto::VideoBufferType::I422 => { + let chroma_w = (width + 1) / 2; + let chroma_h = height; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i422_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::i422_copy( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + let info = i422_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("i422 to {:?} is not supported", dst_type).into(), + )) + } + } +} + +pub unsafe fn cvt_i444( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::I444); + let proto::VideoBufferInfo { width, height, components, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i444(data, c0.stride, c1.stride, c2.stride, height); + + match dst_type { + proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { + let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let stride = buffer.width * 4; + + macro_rules! cvt { + ($rgba:expr, $fnc:ident) => { + if dst_type == $rgba { + imgproc::colorcvt::$fnc( + y, c0.stride, u, c1.stride, v, c2.stride, &mut dst, stride, width, + height, flip_y, + ); + } + }; + } + + cvt!(proto::VideoBufferType::Rgba, i444_to_abgr); + cvt!(proto::VideoBufferType::Bgra, i444_to_argb); + + let info = rgba_info(&dst, dst_type, width, height); + Ok((dst, info)) + } + proto::VideoBufferType::I420 => { + let chroma_w = (width + 1) / 2; + let chroma_h = (height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::i444_to_i420( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + proto::VideoBufferType::I444 => { + let chroma_w = width; + let chroma_h = height; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i444_mut(&mut dst, width, chroma_w, chroma_w, height); + + colorcvt::i444_copy( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + + let info = i444_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("i444 to {:?} is not supported", dst_type).into(), + )) + } + } +} + +pub unsafe fn cvt_i010( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::I010); + let proto::VideoBufferInfo { width, height, components, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i010(data, c0.stride, c1.stride, c2.stride, height); + + let (_, y, _) = unsafe { y.align_to::() }; + let (_, u, _) = unsafe { u.align_to::() }; + let (_, v, _) = unsafe { v.align_to::() }; + + match dst_type { + proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { + let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let stride = buffer.width * 4; + + macro_rules! cvt { + ($rgba:expr, $fnc:ident) => { + if dst_type == $rgba { + imgproc::colorcvt::$fnc( + y, c0.stride, u, c1.stride, v, c2.stride, &mut dst, stride, width, + height, flip_y, + ); + } + }; + } + + cvt!(proto::VideoBufferType::Rgba, i010_to_abgr); + cvt!(proto::VideoBufferType::Bgra, i010_to_argb); + + let info = rgba_info(&dst, dst_type, width, height); + Ok((dst, info)) + } + proto::VideoBufferType::I420 => { + let chroma_w = (buffer.width + 1) / 2; + let chroma_h = (buffer.height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + imgproc::colorcvt::i010_to_i420( + y, c0.stride, u, c1.stride, v, c2.stride, dst_y, width, dst_u, chroma_w, dst_v, + chroma_w, width, height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("i010 to {:?} is not supported", dst_type).into(), + )) + } + } +} + +pub unsafe fn cvt_nv12( + buffer: proto::VideoBufferInfo, + dst_type: proto::VideoBufferType, + flip_y: bool, +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + assert_eq!(buffer.r#type(), proto::VideoBufferType::Nv12); + let proto::VideoBufferInfo { width, height, components, data_ptr, data_len, .. } = buffer; + let data = unsafe { slice::from_raw_parts(data_ptr as *const u8, data_len as usize) }; + + let (c0, c1) = (&components[0], &components[1]); + let (y, uv) = split_nv12(data, c0.stride, c1.stride, height); + + match dst_type { + proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { + let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let stride = buffer.width * 4; + + macro_rules! cvt { + ($rgba:expr, $fnc:ident) => { + if dst_type == $rgba { + imgproc::colorcvt::$fnc( + y, c0.stride, uv, c1.stride, &mut dst, stride, width, height, flip_y, + ); + } + }; + } + + cvt!(proto::VideoBufferType::Rgba, nv12_to_abgr); + cvt!(proto::VideoBufferType::Bgra, nv12_to_argb); + + let info = rgba_info(&dst, dst_type, width, height); + Ok((dst, info)) + } + proto::VideoBufferType::I420 => { + let chroma_w = (buffer.width + 1) / 2; + let chroma_h = (buffer.height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_u, dst_v) = split_i420_mut(&mut dst, width, chroma_w, chroma_w, height); + + imgproc::colorcvt::nv12_to_i420( + y, c0.stride, uv, c1.stride, dst_y, width, dst_u, chroma_w, dst_v, chroma_w, width, + height, flip_y, + ); + + let info = i420_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w, chroma_w); + Ok((dst, info)) + } + proto::VideoBufferType::Nv12 => { + let chroma_w = (buffer.width + 1) / 2; + let chroma_h = (buffer.height + 1) / 2; + let mut dst = + vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let (dst_y, dst_uv) = split_nv12_mut(&mut dst, width, chroma_w, height); + + imgproc::colorcvt::nv12_copy( + y, c0.stride, uv, c1.stride, dst_y, width, dst_uv, chroma_w, width, height, flip_y, + ); + + let info = nv12_info(dst.as_ptr(), dst.len(), width, height, width, chroma_w); + Ok((dst, info)) + } + _ => { + return Err(FfiError::InvalidRequest( + format!("nv12 to {:?} is not supported", dst_type).into(), + )) + } + } +} diff --git a/livekit-ffi/src/server/colorcvt/mod.rs b/livekit-ffi/src/server/colorcvt/mod.rs new file mode 100644 index 00000000..117eacae --- /dev/null +++ b/livekit-ffi/src/server/colorcvt/mod.rs @@ -0,0 +1,637 @@ +use crate::{proto, FfiResult}; +use livekit::webrtc::{prelude::*, video_frame::BoxVideoBuffer}; +use std::slice; + +pub mod cvtimpl; + +macro_rules! to_i420 { + ($buffer:ident, $data:expr) => {{ + let proto::VideoBufferInfo { width, height, components, .. } = $buffer; + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i420($data, c0.stride, c1.stride, c2.stride, height); + let mut i420 = I420Buffer::with_strides(width, height, c0.stride, c1.stride, c2.stride); + + let (dy, du, dv) = i420.data_mut(); + dy.copy_from_slice(y); + du.copy_from_slice(u); + dv.copy_from_slice(v); + Box::new(i420) as BoxVideoBuffer + }}; +} + +pub unsafe fn to_libwebrtc_buffer(info: proto::VideoBufferInfo) -> BoxVideoBuffer { + let data = unsafe { slice::from_raw_parts(info.data_ptr as *const u8, info.data_len as usize) }; + let r#type = info.r#type(); + let proto::VideoBufferInfo { width, height, components, .. } = info.clone(); + + match r#type { + // For rgba buffer, automatically convert to I420 + proto::VideoBufferType::Rgba => { + let (data, info) = + cvtimpl::cvt_rgba(info, proto::VideoBufferType::I420, false).unwrap(); + to_i420!(info, &data) + } + proto::VideoBufferType::Abgr => { + let (data, info) = + cvtimpl::cvt_abgr(info, proto::VideoBufferType::I420, false).unwrap(); + to_i420!(info, &data) + } + proto::VideoBufferType::Argb => { + let (data, info) = + cvtimpl::cvt_argb(info, proto::VideoBufferType::I420, false).unwrap(); + to_i420!(info, &data) + } + proto::VideoBufferType::Bgra => { + let (data, info) = + cvtimpl::cvt_bgra(info, proto::VideoBufferType::I420, false).unwrap(); + to_i420!(info, &data) + } + proto::VideoBufferType::Rgb24 => { + let (data, info) = + cvtimpl::cvt_rgb24(info, proto::VideoBufferType::I420, false).unwrap(); + to_i420!(info, &data) + } + proto::VideoBufferType::I420 | proto::VideoBufferType::I420a => { + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i420(data, c0.stride, c1.stride, c2.stride, height); + let mut i420 = I420Buffer::with_strides(width, height, c0.stride, c1.stride, c2.stride); + + let (dy, du, dv) = i420.data_mut(); + dy.copy_from_slice(y); + du.copy_from_slice(u); + dv.copy_from_slice(v); + Box::new(i420) as BoxVideoBuffer + } + proto::VideoBufferType::I422 => { + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i422(data, c0.stride, c1.stride, c2.stride, height); + let mut i422 = I422Buffer::with_strides(width, height, c0.stride, c1.stride, c2.stride); + + let (dy, du, dv) = i422.data_mut(); + dy.copy_from_slice(y); + du.copy_from_slice(u); + dv.copy_from_slice(v); + Box::new(i422) as BoxVideoBuffer + } + proto::VideoBufferType::I444 => { + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i444(data, c0.stride, c1.stride, c2.stride, height); + let mut i444 = I444Buffer::with_strides(width, height, c0.stride, c1.stride, c2.stride); + + let (dy, du, dv) = i444.data_mut(); + dy.copy_from_slice(y); + du.copy_from_slice(u); + dv.copy_from_slice(v); + Box::new(i444) as BoxVideoBuffer + } + proto::VideoBufferType::I010 => { + let (c0, c1, c2) = (&components[0], &components[1], &components[2]); + let (y, u, v) = split_i010(data, c0.stride, c1.stride, c2.stride, height); + + let (_, y, _) = unsafe { y.align_to::() }; + let (_, u, _) = unsafe { u.align_to::() }; + let (_, v, _) = unsafe { v.align_to::() }; + + let mut i010 = I010Buffer::with_strides(width, height, c0.stride, c1.stride, c2.stride); + + let (dy, du, dv) = i010.data_mut(); + dy.copy_from_slice(y); + du.copy_from_slice(u); + dv.copy_from_slice(v); + Box::new(i010) as BoxVideoBuffer + } + proto::VideoBufferType::Nv12 => { + let (c0, c1) = (&components[0], &components[1]); + let (y, uv) = split_nv12(data, c0.stride, c1.stride, info.height); + let mut nv12 = NV12Buffer::with_strides(info.width, info.height, c0.stride, c1.stride); + + let (dy, duv) = nv12.data_mut(); + dy.copy_from_slice(y); + duv.copy_from_slice(uv); + Box::new(nv12) as BoxVideoBuffer + } + } +} + +pub fn to_video_buffer_info( + rtcbuffer: BoxVideoBuffer, + dst_type: Option, + _normalize_stride: bool, // always normalize stride for now.. +) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + match rtcbuffer.buffer_type() { + // Convert Native buffer to I420 + VideoBufferType::Native => { + let i420 = rtcbuffer.to_i420(); + let (stride_y, stride_u, stride_v) = i420.strides(); + let ptr = i420.data().0.as_ptr() as *const u8; + let len = (stride_y * i420.height() + + stride_u * i420.chroma_height() + + stride_v * i420.chroma_height()) as usize; + let info = + i420_info(ptr, len, i420.width(), i420.height(), stride_y, stride_u, stride_v); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + } + VideoBufferType::I420 => { + let i420 = rtcbuffer.as_i420().unwrap(); + let (stride_y, stride_u, stride_v) = i420.strides(); + let ptr = i420.data().0.as_ptr() as *const u8; + let len = (stride_y * i420.height() + + stride_u * i420.chroma_height() + + stride_v * i420.chroma_height()) as usize; + let info = + i420_info(ptr, len, i420.width(), i420.height(), stride_y, stride_u, stride_v); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + } + VideoBufferType::I420A => { + let i420 = rtcbuffer.as_i420a().unwrap(); + let (stride_y, stride_u, stride_v, stride_a) = i420.strides(); + let ptr = i420.data().0.as_ptr() as *const u8; + let len = (stride_y * i420.height() + + stride_u * i420.chroma_height() + + stride_v * i420.chroma_height() + + stride_a * i420.height()) as usize; + + let info = i420a_info( + ptr, + len, + i420.width(), + i420.height(), + stride_y, + stride_u, + stride_v, + stride_a, + ); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false) } + } + VideoBufferType::I422 => { + let i422 = rtcbuffer.as_i422().unwrap(); + let (stride_y, stride_u, stride_v) = i422.strides(); + let ptr = i422.data().0.as_ptr() as *const u8; + let len = (stride_y * i422.height() + + stride_u * i422.chroma_height() + + stride_v * i422.chroma_height()) as usize; + let info = + i422_info(ptr, len, i422.width(), i422.height(), stride_y, stride_u, stride_v); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false) } + } + VideoBufferType::I444 => { + let i444 = rtcbuffer.as_i444().unwrap(); + let (stride_y, stride_u, stride_v) = i444.strides(); + let ptr = i444.data().0.as_ptr() as *const u8; + let len = (stride_y * i444.height() + + stride_u * i444.chroma_height() + + stride_v * i444.chroma_height()) as usize; + let info = + i444_info(ptr, len, i444.width(), i444.height(), stride_y, stride_u, stride_v); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false) } + } + VideoBufferType::I010 => { + let i010 = rtcbuffer.as_i010().unwrap(); + let (stride_y, stride_u, stride_v) = i010.strides(); + let ptr = i010.data().0.as_ptr() as *const u8; + let len = (stride_y * i010.height() + + stride_u * i010.chroma_height() + + stride_v * i010.chroma_height()) as usize; + let info = + i010_info(ptr, len, i010.width(), i010.height(), stride_y, stride_u, stride_v); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false) } + } + VideoBufferType::NV12 => { + let nv12 = rtcbuffer.as_nv12().unwrap(); + let (stride_y, stride_uv) = nv12.strides(); + let ptr = nv12.data().0.as_ptr() as *const u8; + let len = stride_y * nv12.height() + stride_uv * nv12.chroma_height() * 2; + let info = + nv12_info(ptr, len as usize, nv12.width(), nv12.height(), stride_y, stride_uv); + unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false) } + } + _ => todo!(), + } +} + +pub fn split_i420_mut( + src: &mut [u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&mut [u8], &mut [u8], &mut [u8]) { + let chroma_height = (height + 1) / 2; + let (luma, chroma) = src.split_at_mut((stride_y * height) as usize); + let (u, v) = chroma.split_at_mut((stride_u * chroma_height) as usize); + (luma, u, v) +} + +pub fn split_i420( + dst: &[u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&[u8], &[u8], &[u8]) { + let chroma_height = (height + 1) / 2; + let (luma, chroma) = dst.split_at((stride_y * height) as usize); + let (u, v) = chroma.split_at((stride_u * chroma_height) as usize); + (luma, u, v) +} + +pub fn split_i420a( + src: &[u8], + stride_y: u32, + stride_u: u32, + stride_v: u32, + _stride_a: u32, + height: u32, +) -> (&[u8], &[u8], &[u8], &[u8]) { + let chroma_height = (height + 1) / 2; + let (luma, chroma) = src.split_at((stride_y * height) as usize); + let (u, v) = chroma.split_at((stride_u * chroma_height) as usize); + let (v, a) = v.split_at((stride_v * chroma_height) as usize); + (luma, u, v, a) +} + +pub fn split_i420a_mut( + src: &mut [u8], + stride_y: u32, + stride_u: u32, + stride_v: u32, + _stride_a: u32, + height: u32, +) -> (&mut [u8], &mut [u8], &mut [u8], &mut [u8]) { + let chroma_height = (height + 1) / 2; + let (luma, chroma) = src.split_at_mut((stride_y * height) as usize); + let (u, v) = chroma.split_at_mut((stride_u * chroma_height) as usize); + let (v, a) = v.split_at_mut((stride_v * chroma_height) as usize); + (luma, u, v, a) +} + +pub fn split_i422( + src: &[u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&[u8], &[u8], &[u8]) { + let (luma, chroma) = src.split_at((stride_y * height) as usize); + let (u, v) = chroma.split_at((stride_u * height) as usize); + (luma, u, v) +} + +pub fn split_i422_mut( + src: &mut [u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&mut [u8], &mut [u8], &mut [u8]) { + let (luma, chroma) = src.split_at_mut((stride_y * height) as usize); + let (u, v) = chroma.split_at_mut((stride_u * height) as usize); + (luma, u, v) +} + +pub fn split_i444( + src: &[u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&[u8], &[u8], &[u8]) { + let (luma, chroma) = src.split_at((stride_y * height) as usize); + let (u, v) = chroma.split_at((stride_u * height) as usize); + (luma, u, v) +} + +pub fn split_i444_mut( + src: &mut [u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&mut [u8], &mut [u8], &mut [u8]) { + let (luma, chroma) = src.split_at_mut((stride_y * height) as usize); + let (u, v) = chroma.split_at_mut((stride_u * height) as usize); + (luma, u, v) +} + +pub fn split_i010( + src: &[u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&[u8], &[u8], &[u8]) { + let chroma_height = (height + 1) / 2; + let (luma, chroma) = src.split_at((stride_y * height) as usize); + let (u, v) = chroma.split_at((stride_u * chroma_height) as usize); + (luma, u, v) +} + +pub fn split_i010_mut( + src: &mut [u8], + stride_y: u32, + stride_u: u32, + _stride_v: u32, + height: u32, +) -> (&mut [u8], &mut [u8], &mut [u8]) { + let chroma_height = (height + 1) / 2; + let (luma, chroma) = src.split_at_mut((stride_y * height) as usize); + let (u, v) = chroma.split_at_mut((stride_u * chroma_height) as usize); + (luma, u, v) +} + +pub fn split_nv12(src: &[u8], stride_y: u32, _stride_uv: u32, height: u32) -> (&[u8], &[u8]) { + let (luma, chroma) = src.split_at((stride_y * height) as usize); + (luma, chroma) +} + +pub fn split_nv12_mut( + src: &mut [u8], + stride_y: u32, + _stride_uv: u32, + height: u32, +) -> (&mut [u8], &mut [u8]) { + let (luma, chroma) = src.split_at_mut((stride_y * height) as usize); + (luma, chroma) +} + +pub fn i420_info( + data_ptr: *const u8, + data_len: usize, + width: u32, + height: u32, + stride_y: u32, + stride_u: u32, + stride_v: u32, +) -> proto::VideoBufferInfo { + let chroma_height = (height + 1) / 2; + + let mut components = Vec::with_capacity(3); + let c1 = proto::video_buffer_info::ComponentInfo { + offset: 0, + stride: stride_y, + size: stride_y * height, + }; + + let c2 = proto::video_buffer_info::ComponentInfo { + offset: c1.size, + stride: stride_u, + size: stride_u * chroma_height, + }; + + let c3 = proto::video_buffer_info::ComponentInfo { + offset: c1.size + c2.size, + stride: stride_v, + size: stride_v * chroma_height, + }; + components.extend_from_slice(&[c1, c2, c3]); + + proto::VideoBufferInfo { + width, + height, + r#type: proto::VideoBufferType::I420.into(), + components, + data_ptr: data_ptr as u64, + data_len: data_len as u32, + stride: 0, + } +} + +pub fn i420a_info( + data_ptr: *const u8, + data_len: usize, + width: u32, + height: u32, + stride_y: u32, + stride_u: u32, + stride_v: u32, + stride_a: u32, +) -> proto::VideoBufferInfo { + let chroma_height = (height + 1) / 2; + + let mut components = Vec::with_capacity(4); + let c1 = proto::video_buffer_info::ComponentInfo { + offset: 0, + stride: stride_y, + size: stride_y * height, + }; + + let c2 = proto::video_buffer_info::ComponentInfo { + offset: c1.size, + stride: stride_u, + size: stride_u * chroma_height, + }; + + let c3 = proto::video_buffer_info::ComponentInfo { + offset: c1.size + c2.size, + stride: stride_v, + size: stride_v * chroma_height, + }; + + let c4 = proto::video_buffer_info::ComponentInfo { + offset: c1.size + c2.size + c3.size, + stride: stride_a, + size: stride_a * height, + }; + components.extend_from_slice(&[c1, c2, c3, c4]); + + proto::VideoBufferInfo { + width, + height, + r#type: proto::VideoBufferType::I420a.into(), + components, + data_ptr: data_ptr as u64, + data_len: data_len as u32, + stride: 0, + } +} + +pub fn i422_info( + data_ptr: *const u8, + data_len: usize, + width: u32, + height: u32, + stride_y: u32, + stride_u: u32, + stride_v: u32, +) -> proto::VideoBufferInfo { + let mut components = Vec::with_capacity(3); + let c1 = proto::video_buffer_info::ComponentInfo { + offset: 0, + stride: stride_y, + size: stride_y * height, + }; + + let c2 = proto::video_buffer_info::ComponentInfo { + offset: c1.size, + stride: stride_u, + size: stride_u * height, + }; + + let c3 = proto::video_buffer_info::ComponentInfo { + offset: c1.size + c2.size, + stride: stride_v, + size: stride_v * height, + }; + components.extend_from_slice(&[c1, c2, c3]); + + proto::VideoBufferInfo { + width, + height, + r#type: proto::VideoBufferType::I422.into(), + components, + data_ptr: data_ptr as u64, + data_len: data_len as u32, + stride: 0, + } +} + +pub fn i444_info( + data_ptr: *const u8, + data_len: usize, + width: u32, + height: u32, + stride_y: u32, + stride_u: u32, + stride_v: u32, +) -> proto::VideoBufferInfo { + let mut components = Vec::with_capacity(3); + let c1 = proto::video_buffer_info::ComponentInfo { + offset: 0, + stride: stride_y, + size: stride_y * height, + }; + + let c2 = proto::video_buffer_info::ComponentInfo { + offset: c1.size, + stride: stride_u, + size: stride_u * height, + }; + + let c3 = proto::video_buffer_info::ComponentInfo { + offset: c1.size + c2.size, + stride: stride_v, + size: stride_v * height, + }; + components.extend_from_slice(&[c1, c2, c3]); + + proto::VideoBufferInfo { + width, + height, + r#type: proto::VideoBufferType::I444.into(), + components, + data_ptr: data_ptr as u64, + data_len: data_len as u32, + stride: 0, + } +} + +pub fn i010_info( + data_ptr: *const u8, + data_len: usize, + width: u32, + height: u32, + stride_y: u32, + stride_u: u32, + stride_v: u32, +) -> proto::VideoBufferInfo { + let chroma_height = (height + 1) / 2; + + let mut components = Vec::with_capacity(3); + let c1 = proto::video_buffer_info::ComponentInfo { + offset: 0, + stride: stride_y, + size: stride_y * height, + }; + + let c2 = proto::video_buffer_info::ComponentInfo { + offset: c1.size, + stride: stride_u, + size: stride_u * chroma_height, + }; + + let c3 = proto::video_buffer_info::ComponentInfo { + offset: c1.size + c2.size, + stride: stride_v, + size: stride_v * chroma_height, + }; + components.extend_from_slice(&[c1, c2, c3]); + + proto::VideoBufferInfo { + width, + height, + r#type: proto::VideoBufferType::I010.into(), + components, + data_ptr: data_ptr as u64, + data_len: data_len as u32, + stride: 0, + } +} + +pub fn nv12_info( + data_ptr: *const u8, + data_len: usize, + width: u32, + height: u32, + stride_y: u32, + stride_uv: u32, +) -> proto::VideoBufferInfo { + let chroma_height = (height + 1) / 2; + + let mut components = Vec::with_capacity(2); + let c1 = proto::video_buffer_info::ComponentInfo { + offset: 0, + stride: stride_y, + size: stride_y * height, + }; + + let c2 = proto::video_buffer_info::ComponentInfo { + offset: c1.size, + stride: stride_uv, + size: stride_uv * chroma_height * 2, + }; + components.extend_from_slice(&[c1, c2]); + + proto::VideoBufferInfo { + width, + height, + r#type: proto::VideoBufferType::Nv12.into(), + components, + data_ptr: data_ptr as u64, + data_len: data_len as u32, + stride: 0, + } +} + +pub fn rgba_info( + data: &[u8], + r#type: proto::VideoBufferType, + width: u32, + height: u32, +) -> proto::VideoBufferInfo { + proto::VideoBufferInfo { + width, + height, + r#type: r#type.into(), + components: Vec::default(), + data_ptr: data.as_ptr() as u64, + data_len: data.len() as u32, + stride: width * 4, + } +} + +pub fn rgb_info( + data: &[u8], + r#type: proto::VideoBufferType, + width: u32, + height: u32, +) -> proto::VideoBufferInfo { + proto::VideoBufferInfo { + width, + height, + r#type: r#type.into(), + components: Vec::default(), + data_ptr: data.as_ptr() as u64, + data_len: data.len() as u32, + stride: width * 3, + } +} diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index fb0373ec..d5e14b20 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -30,6 +30,7 @@ use crate::{proto, proto::FfiEvent, FfiError, FfiHandleId, FfiResult, INVALID_HA pub mod audio_source; pub mod audio_stream; +pub mod colorcvt; pub mod logger; pub mod requests; pub mod room; @@ -60,6 +61,7 @@ impl FfiHandle for FfiDataBuffer {} impl FfiHandle for Arc> {} impl FfiHandle for AudioFrame<'static> {} impl FfiHandle for BoxVideoBuffer {} +impl FfiHandle for Box<[u8]> {} pub struct FfiServer { /// Store all Ffi handles inside an HashMap, if this isn't efficient enough diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 8d31daa4..e0f549bd 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -14,18 +14,15 @@ use std::{slice, sync::Arc}; +use colorcvt::cvtimpl; use livekit::{ prelude::*, - webrtc::{ - native::{audio_resampler, yuv_helper}, - prelude::*, - video_frame::{BoxVideoBuffer, I420Buffer}, - }, + webrtc::{native::audio_resampler, prelude::*}, }; use parking_lot::Mutex; use super::{ - audio_source, audio_stream, room, + audio_source, audio_stream, colorcvt, room, room::{FfiParticipant, FfiPublication, FfiTrack}, video_source, video_stream, FfiError, FfiResult, FfiServer, }; @@ -237,28 +234,6 @@ fn on_get_stats( Ok(proto::GetStatsResponse { async_id }) } -/// Allocate a new video buffer -fn on_alloc_video_buffer( - server: &'static FfiServer, - alloc: proto::AllocVideoBufferRequest, -) -> FfiResult { - let buffer: BoxVideoBuffer = match alloc.r#type() { - proto::VideoFrameBufferType::I420 => Box::new(I420Buffer::new(alloc.width, alloc.height)), - _ => return Err(FfiError::InvalidRequest("frame type is not supported".into())), - }; - - let handle_id = server.next_id(); - let buffer_info = proto::VideoFrameBufferInfo::from(&buffer); - server.store_handle(handle_id, buffer); - - Ok(proto::AllocVideoBufferResponse { - buffer: Some(proto::OwnedVideoFrameBuffer { - handle: Some(proto::FfiOwnedHandle { id: handle_id }), - info: Some(buffer_info), - }), - }) -} - /// Create a new VideoStream, a video stream is used to receive frames from a Track fn on_new_video_stream( server: &'static FfiServer, @@ -288,198 +263,33 @@ unsafe fn on_capture_video_frame( Ok(proto::CaptureVideoFrameResponse::default()) } -/// Convert a frame to I420 -/// The destination can now be a new buffer or an existing buffer +/// Convert a video frame /// /// # Safety: The user must ensure that the pointers/len provided are valid /// There is no way for us to verify the inputs -unsafe fn on_to_i420( +unsafe fn on_video_convert( server: &'static FfiServer, - to_i420: proto::ToI420Request, -) -> FfiResult { - let from = to_i420.from.ok_or(FfiError::InvalidRequest("from is empty".into()))?; - - #[rustfmt::skip] - let i420 = match from { - proto::to_i420_request::From::Handle(handle) => server - .retrieve_handle::(handle)? - .to_i420(), - proto::to_i420_request::From::Argb(info) => { - let mut i420 = I420Buffer::new(info.width, info.height); - let (w, h) = ( - info.width as i32, - info.height as i32 * if to_i420.flip_y { -1 } else { 1 }, - ); - let (sy, su, sv) = i420.strides(); - let (dy, du, dv) = i420.data_mut(); - - let argb = slice::from_raw_parts(info.ptr as *const u8, (info.stride * info.height) as usize); - - match info.format() { - proto::VideoFormatType::FormatArgb => { - yuv_helper::argb_to_i420(argb, info.stride, dy, sy, du, su, dv, sv, w, h); - } - proto::VideoFormatType::FormatAbgr => { - yuv_helper::abgr_to_i420(argb, info.stride, dy, sy, du, su, dv, sv, w, h); - } - _ => { - return Err(FfiError::InvalidRequest( - "the format is not supported".into(), - )) - } - } - - i420 - } - proto::to_i420_request::From::Buffer(info) => { - let mut i420 = I420Buffer::new(info.width, info.height); - let (w, h) = (info.width as i32, info.height as i32); - let (sy, su, sv) = i420.strides(); - let (dy, du, dv) = i420.data_mut(); - - match &info.buffer { - Some(proto::video_frame_buffer_info::Buffer::Yuv(yuv)) => { - match info.buffer_type() { - proto::VideoFrameBufferType::I420 - | proto::VideoFrameBufferType::I420a - | proto::VideoFrameBufferType::I422 - | proto::VideoFrameBufferType::I444 => { - - let (y, u, v) = ( - slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * info.height) as usize), - slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize), - slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize) - ); - - match info.buffer_type() { - proto::VideoFrameBufferType::I420 | proto::VideoFrameBufferType::I420a => { - dy[..sy as usize].copy_from_slice(y); - du[..su as usize].copy_from_slice(u); - dv[..sv as usize].copy_from_slice(v); - }, - proto::VideoFrameBufferType::I422 => { - yuv_helper::i422_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h); - }, - proto::VideoFrameBufferType::I444 => { - yuv_helper::i444_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h); - } - _ => unreachable!() - } - } - proto::VideoFrameBufferType::I010 => { - let (y, u, v) = ( - slice::from_raw_parts(yuv.data_y_ptr as *const u16, (yuv.stride_y * info.height) as usize / std::mem::size_of::()), - slice::from_raw_parts(yuv.data_u_ptr as *const u16, (yuv.stride_u * yuv.chroma_height) as usize / std::mem::size_of::()), - slice::from_raw_parts(yuv.data_v_ptr as *const u16, (yuv.stride_v * yuv.chroma_height) as usize / std::mem::size_of::()) - ); - - yuv_helper::i010_to_i420(y, yuv.stride_y, u, yuv.stride_u, v, yuv.stride_v, dy, sy, du, su, dv, sv, w, h); - } - _ => return Err(FfiError::InvalidRequest("invalid yuv description".into())) - }; - } - Some(proto::video_frame_buffer_info::Buffer::BiYuv(biyuv)) => { - let (y, uv) = ( - slice::from_raw_parts(biyuv.data_y_ptr as *const u8, (biyuv.stride_y * info.height) as usize), - slice::from_raw_parts(biyuv.data_uv_ptr as *const u8, (biyuv.stride_uv * biyuv.chroma_height) as usize) - ); - - if info.buffer_type() == proto::VideoFrameBufferType::Nv12 { - yuv_helper::nv12_to_i420(y, biyuv.stride_y, uv, biyuv.stride_uv, dy, sy, du, su, dv, sv, w, h); - } - } - _ => return Err(FfiError::InvalidRequest("conversion not supported".into())) - } - i420 - } + video_convert: proto::VideoConvertRequest, +) -> FfiResult { + let Some(ref buffer) = video_convert.buffer else { + return Err(FfiError::InvalidRequest("buffer is empty".into())); }; - let i420: BoxVideoBuffer = Box::new(i420); - let handle_id = server.next_id(); - let buffer_info = proto::VideoFrameBufferInfo::from(&i420); - server.store_handle(handle_id, i420); - Ok(proto::ToI420Response { - buffer: Some(proto::OwnedVideoFrameBuffer { - handle: Some(proto::FfiOwnedHandle { id: handle_id }), - info: Some(buffer_info), - }), - }) -} - -/// Convert a YUY buffer to argb -/// # Safety: the caller must ensure that the buffer is valid -unsafe fn on_to_argb( - _server: &'static FfiServer, - to_argb: proto::ToArgbRequest, -) -> FfiResult { - let buffer = - to_argb.buffer.as_ref().ok_or(FfiError::InvalidRequest("buffer is empty".into()))?; - - let argb = slice::from_raw_parts_mut( - to_argb.dst_ptr as *mut u8, - (to_argb.dst_stride * to_argb.dst_height) as usize, - ); - - let w = to_argb.dst_width as i32; - let h = to_argb.dst_height as i32 * (if to_argb.flip_y { -1 } else { 1 }); - let stride = to_argb.dst_stride; - let rgba_format = to_argb.dst_format(); - - match buffer.buffer_type() { - proto::VideoFrameBufferType::I420 => { - let Some(proto::video_frame_buffer_info::Buffer::Yuv(yuv)) = &buffer.buffer else { - return Err(FfiError::InvalidRequest("invalid i420 buffer description".into())); + let flip_y = video_convert.flip_y; + let dst_type = video_convert.dst_type(); + match cvtimpl::cvt(buffer.clone(), dst_type, flip_y) { + Ok((buffer, info)) => { + let id = server.next_id(); + server.store_handle(id, buffer); + let owned_info = proto::OwnedVideoBuffer { + handle: Some(proto::FfiOwnedHandle { id }), + info: Some(info), }; - - #[rustfmt::skip] - let (src_y, src_u, src_v) = ( - slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * buffer.height) as usize), - slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize), - slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize) - ); - - match rgba_format { - proto::VideoFormatType::FormatArgb => { - #[rustfmt::skip] - yuv_helper::i420_to_bgra(src_y, yuv.stride_y, src_u, yuv.stride_u, src_v, yuv.stride_v, argb, stride, w, h); - } - proto::VideoFormatType::FormatBgra => { - #[rustfmt::skip] - yuv_helper::i420_to_argb(src_y, yuv.stride_y, src_u, yuv.stride_u, src_v, yuv.stride_v, argb, stride, w, h); - } - proto::VideoFormatType::FormatRgba => { - #[rustfmt::skip] - yuv_helper::i420_to_abgr(src_y, yuv.stride_y, src_u, yuv.stride_u, src_v, yuv.stride_v, argb, stride, w, h); - } - proto::VideoFormatType::FormatAbgr => { - #[rustfmt::skip] - yuv_helper::i420_to_rgba(src_y, yuv.stride_y, src_u, yuv.stride_u, src_v, yuv.stride_v, argb, stride, w, h); - } - } + Ok(proto::VideoConvertResponse { buffer: Some(owned_info), error: None }) } - _ => return Err(FfiError::InvalidRequest("to_argb buffer type is not supported".into())), - } - - Ok(proto::ToArgbResponse::default()) -} - -/// Allocate a new audio buffer -fn on_alloc_audio_buffer( - server: &'static FfiServer, - alloc: proto::AllocAudioBufferRequest, -) -> FfiResult { - let frame = AudioFrame::new(alloc.sample_rate, alloc.num_channels, alloc.samples_per_channel); - - let handle_id = server.next_id(); - let buffer_info = proto::AudioFrameBufferInfo::from(&frame); - server.store_handle(handle_id, frame); - Ok(proto::AllocAudioBufferResponse { - buffer: Some(proto::OwnedAudioFrameBuffer { - handle: Some(proto::FfiOwnedHandle { id: handle_id }), - info: Some(buffer_info), - }), - }) + Err(err) => Ok(proto::VideoConvertResponse { buffer: None, error: Some(err.to_string()) }), + } } /// Create a new audio stream (used to receive audio frames from a track) @@ -780,9 +590,6 @@ pub fn handle_request( proto::ffi_request::Message::GetStats(get_stats) => { proto::ffi_response::Message::GetStats(on_get_stats(server, get_stats)?) } - proto::ffi_request::Message::AllocVideoBuffer(alloc) => { - proto::ffi_response::Message::AllocVideoBuffer(on_alloc_video_buffer(server, alloc)?) - } proto::ffi_request::Message::NewVideoStream(new_stream) => { proto::ffi_response::Message::NewVideoStream(on_new_video_stream(server, new_stream)?) } @@ -792,15 +599,9 @@ pub fn handle_request( proto::ffi_request::Message::CaptureVideoFrame(push) => unsafe { proto::ffi_response::Message::CaptureVideoFrame(on_capture_video_frame(server, push)?) }, - proto::ffi_request::Message::ToI420(to_i420) => unsafe { - proto::ffi_response::Message::ToI420(on_to_i420(server, to_i420)?) + proto::ffi_request::Message::VideoConvert(video_convert) => unsafe { + proto::ffi_response::Message::VideoConvert(on_video_convert(server, video_convert)?) }, - proto::ffi_request::Message::ToArgb(to_argb) => unsafe { - proto::ffi_response::Message::ToArgb(on_to_argb(server, to_argb)?) - }, - proto::ffi_request::Message::AllocAudioBuffer(alloc) => { - proto::ffi_response::Message::AllocAudioBuffer(on_alloc_audio_buffer(server, alloc)?) - } proto::ffi_request::Message::NewAudioStream(new_stream) => { proto::ffi_response::Message::NewAudioStream(on_new_audio_stream(server, new_stream)?) } diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs index 67c7c30c..b15c0901 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -12,15 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::slice; - -use livekit::webrtc::{ - prelude::*, - video_frame::{BoxVideoBuffer, VideoFrame}, -}; - -use super::FfiHandle; +use super::{colorcvt, FfiHandle}; use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; +use livekit::webrtc::{prelude::*, video_frame::VideoFrame}; pub struct FfiVideoSource { pub handle_id: FfiHandleId, @@ -63,120 +57,21 @@ impl FfiVideoSource { pub unsafe fn capture_frame( &self, - server: &'static server::FfiServer, + _server: &'static server::FfiServer, capture: proto::CaptureVideoFrameRequest, ) -> FfiResult<()> { match self.source { #[cfg(not(target_arch = "wasm32"))] RtcVideoSource::Native(ref source) => { - let frame_info = - capture.frame.ok_or(FfiError::InvalidRequest("frame is empty".into()))?; - - let from = - capture.from.ok_or(FfiError::InvalidRequest("capture from is empty".into()))?; - - // copy the provided buffer - #[rustfmt::skip] - let buffer: BoxVideoBuffer = match from { - proto::capture_video_frame_request::From::Info(info) => { - match &info.buffer { - Some(proto::video_frame_buffer_info::Buffer::Yuv(yuv)) => { - match info.buffer_type() { - proto::VideoFrameBufferType::I420 - | proto::VideoFrameBufferType::I420a - | proto::VideoFrameBufferType::I422 - | proto::VideoFrameBufferType::I444 => { - - let (y, u, v) = ( - slice::from_raw_parts(yuv.data_y_ptr as *const u8, (yuv.stride_y * info.height) as usize), - slice::from_raw_parts(yuv.data_u_ptr as *const u8, (yuv.stride_u * yuv.chroma_height) as usize), - slice::from_raw_parts(yuv.data_v_ptr as *const u8, (yuv.stride_v * yuv.chroma_height) as usize) - ); - - match info.buffer_type() { - proto::VideoFrameBufferType::I420 | proto::VideoFrameBufferType::I420a => { - let mut i420 = I420Buffer::with_strides(info.width, info.height, yuv.stride_y, yuv.stride_u, yuv.stride_v); - let (dy, du, dv) = i420.data_mut(); - - dy.copy_from_slice(y); - du.copy_from_slice(u); - dv.copy_from_slice(v); - Box::new(i420) as BoxVideoBuffer - }, - proto::VideoFrameBufferType::I422 => { - let mut i422 = I422Buffer::with_strides(info.width, info.height, yuv.stride_y, yuv.stride_u, yuv.stride_v); - let (dy, du, dv) = i422.data_mut(); - - dy.copy_from_slice(y); - du.copy_from_slice(u); - dv.copy_from_slice(v); - Box::new(i422) as BoxVideoBuffer - }, - proto::VideoFrameBufferType::I444 => { - let mut i444 = I444Buffer::with_strides(info.width, info.height, yuv.stride_y, yuv.stride_u, yuv.stride_v); - let (dy, du, dv) = i444.data_mut(); - - dy.copy_from_slice(y); - du.copy_from_slice(u); - dv.copy_from_slice(v); - Box::new(i444) as BoxVideoBuffer - } - _ => unreachable!() - } - } - proto::VideoFrameBufferType::I010 => { - let (y, u, v) = ( - slice::from_raw_parts(yuv.data_y_ptr as *const u16, (yuv.stride_y * info.height) as usize / std::mem::size_of::()), - slice::from_raw_parts(yuv.data_u_ptr as *const u16, (yuv.stride_u * yuv.chroma_height) as usize / std::mem::size_of::()), - slice::from_raw_parts(yuv.data_v_ptr as *const u16, (yuv.stride_v * yuv.chroma_height) as usize / std::mem::size_of::()) - ); - - let mut i010 = I010Buffer::with_strides(info.width, info.height, yuv.stride_y, yuv.stride_u, yuv.stride_v); - let (dy, du, dv) = i010.data_mut(); - - dy.copy_from_slice(y); - du.copy_from_slice(u); - dv.copy_from_slice(v); - Box::new(i010) as BoxVideoBuffer - } - _ => return Err(FfiError::InvalidRequest("invalid yuv description".into())) - } - } - Some(proto::video_frame_buffer_info::Buffer::BiYuv(biyuv)) => { - let (y, uv) = ( - slice::from_raw_parts(biyuv.data_y_ptr as *const u8, (biyuv.stride_y * info.height) as usize), - slice::from_raw_parts(biyuv.data_uv_ptr as *const u8, (biyuv.stride_uv * biyuv.chroma_height) as usize) - ); - - if info.buffer_type() == proto::VideoFrameBufferType::Nv12 { - let mut nv12 = NV12Buffer::with_strides(info.width, info.height, biyuv.stride_y, biyuv.stride_uv); - let (dy, duv) = nv12.data_mut(); - - dy.copy_from_slice(y); - duv.copy_from_slice(uv); - Box::new(nv12) as BoxVideoBuffer - } else { - return Err(FfiError::InvalidRequest("invalid biyuv description".into())) - } - } - _ => return Err(FfiError::InvalidRequest("conversion not supported".into())) - } - } - proto::capture_video_frame_request::From::Handle(handle) => { - let (_, buffer) = server - .ffi_handles - .remove(&handle) - .ok_or(FfiError::InvalidRequest("handle not found".into()))?; - - *(buffer - .downcast::() - .map_err(|_| FfiError::InvalidRequest("handle is not video frame".into()))?) - } - }; + let buffer = capture + .buffer + .as_ref() + .ok_or(FfiError::InvalidRequest("frame is empty".into()))?; + let buffer = colorcvt::to_libwebrtc_buffer(buffer.clone()); let frame = VideoFrame { - rotation: frame_info.rotation().into(), - timestamp_us: frame_info.timestamp_us, + rotation: capture.rotation().into(), + timestamp_us: capture.timestamp_us, buffer, }; diff --git a/livekit-ffi/src/server/video_stream.rs b/livekit-ffi/src/server/video_stream.rs index 23560701..e013fa04 100644 --- a/livekit-ffi/src/server/video_stream.rs +++ b/livekit-ffi/src/server/video_stream.rs @@ -16,7 +16,7 @@ use futures_util::StreamExt; use livekit::webrtc::{prelude::*, video_stream::native::NativeVideoStream}; use tokio::sync::oneshot; -use super::{room::FfiTrack, FfiHandle}; +use super::{colorcvt, room::FfiTrack, FfiHandle}; use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; pub struct FfiVideoStream { @@ -59,6 +59,8 @@ impl FfiVideoStream { server.async_runtime.spawn(Self::native_video_stream_task( server, handle_id, + new_stream.format.and_then(|_| Some(new_stream.format())), + new_stream.normalize_stride, NativeVideoStream::new(rtc_track), close_rx, )); @@ -80,6 +82,8 @@ impl FfiVideoStream { async fn native_video_stream_task( server: &'static server::FfiServer, stream_handle: FfiHandleId, + dst_type: Option, + normalize_stride: bool, mut native_stream: NativeVideoStream, mut close_rx: oneshot::Receiver<()>, ) { @@ -93,22 +97,28 @@ impl FfiVideoStream { break; }; + + let Ok((buffer, info)) = colorcvt::to_video_buffer_info(frame.buffer, dst_type, normalize_stride) else { + log::error!("video stream failed to convert video frame to {:?}", dst_type); + continue; + }; + let handle_id = server.next_id(); - let frame_info = proto::VideoFrameInfo::from(&frame); - let buffer_info = proto::VideoFrameBufferInfo::from(&frame.buffer); - server.store_handle(handle_id, frame.buffer); + server.store_handle(handle_id, buffer); + if let Err(err) = server.send_event(proto::ffi_event::Message::VideoStreamEvent( proto::VideoStreamEvent { stream_handle, message: Some(proto::video_stream_event::Message::FrameReceived( proto::VideoFrameReceived { - frame: Some(frame_info), - buffer: Some(proto::OwnedVideoFrameBuffer { + timestamp_us: frame.timestamp_us, + rotation: proto::VideoRotation::from(frame.rotation).into(), + buffer: Some(proto::OwnedVideoBuffer { handle: Some(proto::FfiOwnedHandle { id: handle_id, }), - info: Some(buffer_info), + info: Some(info), }), } )),