new video buffer api (#297)

livekit · Feb 1, 2024 · b8871e6 · b8871e6
1 parent cf34856
commit b8871e6
Show file tree

Hide file tree

Showing 13 changed files with 1,508 additions and 917 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml
@@ -32,6 +32,7 @@ dashmap = "5.4"
 env_logger = "0.10"
 downcast-rs = "1.2"
 console-subscriber = { version = "0.1", features = ["parking_lot"], optional = true }
+imgproc = "0.3.5"
 
 [build-dependencies]
 webrtc-sys-build = { path = "../webrtc-sys/build", version = "0.3.0" }

diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto
@@ -19,16 +19,6 @@ option csharp_namespace = "LiveKit.Proto";
 
 import "handle.proto";
 
-// Allocate a new AudioFrameBuffer
-// This is not necessary required because the data structure is fairly simple
-// But keep the API consistent with VideoFrame
-message AllocAudioBufferRequest { 
-  uint32 sample_rate = 1;
-  uint32 num_channels = 2;
-  uint32 samples_per_channel = 3;
-}
-message AllocAudioBufferResponse { OwnedAudioFrameBuffer buffer = 1; }
-
 // Create a new AudioStream
 // AudioStream is used to receive audio frames from a track
 message NewAudioStreamRequest {
@@ -158,4 +148,4 @@ message AudioResamplerInfo { }
 message OwnedAudioResampler {
   FfiOwnedHandle handle = 1;
   AudioResamplerInfo info = 2;
-}
+}
diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto
@@ -73,15 +73,12 @@ message FfiRequest {
     GetStatsRequest get_stats = 14;
 
     // Video
-    AllocVideoBufferRequest alloc_video_buffer = 15;
     NewVideoStreamRequest new_video_stream = 16;
     NewVideoSourceRequest new_video_source = 17;
     CaptureVideoFrameRequest capture_video_frame = 18;
-    ToI420Request to_i420 = 19;
-    ToArgbRequest to_argb = 20;
+    VideoConvertRequest video_convert = 19;
 
     // Audio
-    AllocAudioBufferRequest alloc_audio_buffer = 21;
     NewAudioStreamRequest new_audio_stream = 22;
     NewAudioSourceRequest new_audio_source = 23;
     CaptureAudioFrameRequest capture_audio_frame = 24;
@@ -113,15 +110,12 @@ message FfiResponse {
     GetStatsResponse get_stats = 14;
 
     // Video
-    AllocVideoBufferResponse alloc_video_buffer = 15;
     NewVideoStreamResponse new_video_stream = 16;
     NewVideoSourceResponse new_video_source = 17;
     CaptureVideoFrameResponse capture_video_frame = 18;
-    ToI420Response to_i420 = 19;
-    ToArgbResponse to_argb = 20;
+    VideoConvertResponse video_convert = 19;
 
     // Audio
-    AllocAudioBufferResponse alloc_audio_buffer = 21;
     NewAudioStreamResponse new_audio_stream = 22;
     NewAudioSourceResponse new_audio_source = 23;
     CaptureAudioFrameResponse capture_audio_frame = 24;