Fix multi-cam support (#461)

Sources/LiveKit/Support/DeviceManager.swift

@@ -33,7 +33,7 @@ class DeviceManager: Loggable {
 
     // Async version, waits until inital device fetch is complete
     public func devices() async throws -> [AVCaptureDevice] {
-        try await devicesCompleter.wait()
+        try await _devicesCompleter.wait()
     }
 
     // Sync version
@@ -78,33 +78,74 @@ class DeviceManager: Loggable {
 
     private struct State {
         var devices: [AVCaptureDevice] = []
+        var multiCamDeviceSets: [Set<AVCaptureDevice>] = []
     }
 
     private let _state = StateSync(State())
 
-    private let devicesCompleter = AsyncCompleter<[AVCaptureDevice]>(label: "devices", defaultTimeout: 10)
+    private let _devicesCompleter = AsyncCompleter<[AVCaptureDevice]>(label: "devices", defaultTimeout: 10)
+    private let _multiCamDeviceSetsCompleter = AsyncCompleter<[Set<AVCaptureDevice>]>(label: "multiCamDeviceSets", defaultTimeout: 10)
 
-    private var _observation: NSKeyValueObservation?
+    private var _devicesObservation: NSKeyValueObservation?
+    private var _multiCamDeviceSetsObservation: NSKeyValueObservation?
+
+    /// Find multi-cam compatible devices.
+    func multiCamCompatibleDevices(for devices: Set<AVCaptureDevice>) async throws -> [AVCaptureDevice] {
+        let deviceSets = try await _multiCamDeviceSetsCompleter.wait()
+
+        let compatibleDevices = deviceSets.filter { $0.isSuperset(of: devices) }
+            .reduce(into: Set<AVCaptureDevice>()) { $0.formUnion($1) }
+            .subtracting(devices)
+
+        let devices = try await _devicesCompleter.wait()
+
+        // This ensures the ordering is same as the devices array.
+        return devices.filter { compatibleDevices.contains($0) }
+    }
 
     init() {
         log()
 
         #if os(iOS) || os(macOS)
         DispatchQueue.global(qos: .background).async { [weak self] in
             guard let self else { return }
-            self._observation = self.discoverySession.observe(\.devices, options: [.initial, .new]) { [weak self] _, value in
+            self._devicesObservation = self.discoverySession.observe(\.devices, options: [.initial, .new]) { [weak self] _, value in
                 guard let self else { return }
-                // Sort priority: .front = 2, .back = 1, .unspecified = 3
-                let devices = (value.newValue ?? []).sorted(by: { $0.position.rawValue > $1.position.rawValue })
+                let devices = (value.newValue ?? []).sortedByFacingPositionPriority()
                 self.log("Devices: \(String(describing: devices))")
                 self._state.mutate { $0.devices = devices }
-                self.devicesCompleter.resume(returning: devices)
+                self._devicesCompleter.resume(returning: devices)
+                #if os(macOS)
+                self._multiCamDeviceSetsCompleter.resume(returning: [])
+                #endif
             }
         }
         #elseif os(visionOS)
         // For visionOS, there is no DiscoverySession so return the Persona camera if available.
         let devices: [AVCaptureDevice] = [.systemPreferredCamera].compactMap { $0 }
-        devicesCompleter.resume(returning: devices)
+        _state.mutate { $0.devices = devices }
+        _devicesCompleter.resume(returning: devices)
+        _multiCamDeviceSetsCompleter.resume(returning: [])
+        #endif
+
+        #if os(iOS)
+        DispatchQueue.global(qos: .background).async { [weak self] in
+            guard let self else { return }
+            self._multiCamDeviceSetsObservation = self.discoverySession.observe(\.supportedMultiCamDeviceSets, options: [.initial, .new]) { [weak self] _, value in
+                guard let self else { return }
+                let deviceSets = (value.newValue ?? [])
+                self.log("MultiCam deviceSets: \(String(describing: deviceSets))")
+                self._state.mutate { $0.multiCamDeviceSets = deviceSets }
+                self._multiCamDeviceSetsCompleter.resume(returning: deviceSets)
+            }
+        }
         #endif
     }
 }
+
+extension [AVCaptureDevice] {
+    /// Sort priority: .front = 2, .back = 1, .unspecified = 3.
+    func sortedByFacingPositionPriority() -> [Element] {
+        sorted(by: { $0.facingPosition.rawValue > $1.facingPosition.rawValue })
+    }
+}

Sources/LiveKit/Track/Capturers/CameraCapturer.swift

@@ -90,18 +90,19 @@ public class CameraCapturer: VideoCapturer {
     private lazy var adapter: VideoCapturerDelegateAdapter = .init(cameraCapturer: self)
 
     #if os(iOS)
-    public static let multiCamSession = AVCaptureMultiCamSession()
+    private static let _multiCamSession = AVCaptureMultiCamSession()
     #endif
 
-    // RTCCameraVideoCapturer used internally for now
-    private lazy var capturer: LKRTCCameraVideoCapturer = {
+    public var captureSession: AVCaptureSession {
         #if os(iOS)
-        let result = LKRTCCameraVideoCapturer(delegate: adapter, captureSession: Self.multiCamSession)
+        Self._multiCamSession
         #else
-        let result = LKRTCCameraVideoCapturer(delegate: adapter)
+        AVCaptureSession()
         #endif
-        return result
-    }()
+    }
+
+    // RTCCameraVideoCapturer used internally for now
+    private lazy var capturer: LKRTCCameraVideoCapturer = .init(delegate: adapter, captureSession: captureSession)
 
     init(delegate: LKRTCVideoCapturerDelegate, options: CameraCaptureOptions) {
         _cameraCapturerState = StateSync(State(options: options))
@@ -161,8 +162,22 @@ public class CameraCapturer: VideoCapturer {
         var device: AVCaptureDevice? = options.device
 
         if device == nil {
+            #if os(iOS)
+            let devices: [AVCaptureDevice]
+            if AVCaptureMultiCamSession.isMultiCamSupported {
+                // Get the list of devices already on the shared multi-cam session.
+                let existingDevices = captureSession.inputs.compactMap { $0 as? AVCaptureDeviceInput }.map(\.device)
+                log("Existing devices: \(existingDevices)")
+                // Compute other multi-cam compatible devices.
+                devices = try await DeviceManager.shared.multiCamCompatibleDevices(for: Set(existingDevices))
+            } else {
+                devices = try await CameraCapturer.captureDevices()
+            }
+            #else
             let devices = try await CameraCapturer.captureDevices()
-            device = devices.first(where: { $0.position == self.options.position }) ?? devices.first
+            #endif
+
+            device = devices.first { $0.position == self.options.position } ?? devices.first
         }
 
         guard let device else {
@@ -187,7 +202,7 @@ public class CameraCapturer: VideoCapturer {
             // Use the preferred capture format if specified in options
             selectedFormat = foundFormat
         } else {
-            if let foundFormat = sortedFormats.first(where: { $0.dimensions.area >= self.options.dimensions.area && $0.format.fpsRange().contains(self.options.fps) }) {
+            if let foundFormat = sortedFormats.first(where: { $0.dimensions.area >= self.options.dimensions.area && $0.format.fpsRange().contains(self.options.fps) && $0.format.isMultiCamSupportediOS }) {
                 // Use the first format that satisfies preferred dimensions & fps
                 selectedFormat = foundFormat
             } else if let foundFormat = sortedFormats.first(where: { $0.dimensions.area >= self.options.dimensions.area }) {
@@ -314,4 +329,12 @@ extension AVCaptureDevice.Format {
             result = merge(range: result, with: current)
         }
     }
+
+    var isMultiCamSupportediOS: Bool {
+        #if os(iOS)
+        return AVCaptureMultiCamSession.isMultiCamSupported ? isMultiCamSupported : true
+        #else
+        return true
+        #endif
+    }
 }