Custom video frame processing

Sources/LiveKit/Protocols/VideoProcessor.swift

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2024 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+@objc
+public protocol VideoProcessor {
+    func process(frame: VideoFrame) -> VideoFrame?
+}

Sources/LiveKit/Track/Capturers/VideoCapturer.swift

@@ -70,16 +70,22 @@ public class VideoCapturer: NSObject, Loggable, VideoCapturerProtocol {
 
     let dimensionsCompleter = AsyncCompleter<Dimensions>(label: "Dimensions", defaultTimeout: .defaultCaptureStart)
 
-    struct State: Equatable {
+    struct State {
         // Counts calls to start/stopCapturer so multiple Tracks can use the same VideoCapturer.
         var startStopCounter: Int = 0
         var dimensions: Dimensions? = nil
+        weak var processor: VideoProcessor? = nil
     }
 
     var _state = StateSync(State())
 
     public var dimensions: Dimensions? { _state.dimensions }
 
+    public weak var processor: VideoProcessor? {
+        get { _state.processor }
+        set { _state.mutate { $0.processor = newValue } }
+    }
+
     func set(dimensions newValue: Dimensions?) {
         let didUpdate = _state.mutate {
             let oldDimensions = $0.dimensions
@@ -223,17 +229,31 @@ extension VideoCapturer {
                                device: AVCaptureDevice?,
                                options: VideoCaptureOptions)
     {
+        var rtcFrame: LKRTCVideoFrame = frame
+        guard var lkFrame: VideoFrame = frame.toLKType() else {
+            log("Failed to convert a RTCVideoFrame to VideoFrame.", .error)
+            return
+        }
+
+        // Apply processing if we have a processor attached.
+        if let processor = _state.processor {
+            guard let processedFrame = processor.process(frame: lkFrame) else {
+                log("VideoProcessor didn't return a frame, skipping frame.", .warning)
+                return
+            }
+            lkFrame = processedFrame
+            rtcFrame = processedFrame.toRTCType()
+        }
+
         // Resolve real dimensions (apply frame rotation)
-        set(dimensions: Dimensions(width: frame.width, height: frame.height).apply(rotation: frame.rotation))
+        set(dimensions: Dimensions(width: rtcFrame.width, height: rtcFrame.height).apply(rotation: rtcFrame.rotation))
 
-        delegate?.capturer(capturer, didCapture: frame)
+        delegate?.capturer(capturer, didCapture: rtcFrame)
 
         if rendererDelegates.isDelegatesNotEmpty {
-            if let lkVideoFrame = frame.toLKType() {
-                rendererDelegates.notify { renderer in
-                    renderer.render?(frame: lkVideoFrame)
-                    renderer.render?(frame: lkVideoFrame, captureDevice: device, captureOptions: options)
-                }
+            rendererDelegates.notify { renderer in
+                renderer.render?(frame: lkFrame)
+                renderer.render?(frame: lkFrame, captureDevice: device, captureOptions: options)
             }
         }
     }

Sources/LiveKit/Types/VideoFrame.swift

@@ -37,6 +37,11 @@ public class CVPixelVideoBuffer: VideoBuffer, RTCCompatibleVideoBuffer {
         _rtcType.pixelBuffer
     }
 
+    public init(pixelBuffer: CVPixelBuffer) {
+        _rtcType = LKRTCCVPixelBuffer(pixelBuffer: pixelBuffer)
+    }
+
+    // Internal only.
     init(rtcCVPixelBuffer: LKRTCCVPixelBuffer) {
         _rtcType = rtcCVPixelBuffer
     }