Improvements for framecryptor

CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## 1.5.0
+
+* Update default bitrates according to VMAF guide
+* Support multi-codec simulcast.
+* Support SVC publishing with AV1/VP9.
+* More robustness for E2EE.
+* Configurable Audio Modes for Android.
+
 ## 1.4.3
 
 * Fix: remove js_bindings and use the built-in AudioContext for js interop to support flutter 3.13.0.

example/lib/pages/connect.dart

@@ -135,7 +135,7 @@ class _ConnectPageState extends State<ConnectPage> {
         final keyProvider = await BaseKeyProvider.create();
         e2eeOptions = E2EEOptions(keyProvider: keyProvider);
         var sharedKey = _sharedKeyCtrl.text;
-        await keyProvider.setKey(sharedKey);
+        await keyProvider.setSharedKey(sharedKey);
       }
 
       String preferredCodec = 'VP8';

example/lib/widgets/participant.dart

@@ -169,8 +169,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
                         firstAudioPublication?.subscribed == true,
                     connectionQuality: widget.participant.connectionQuality,
                     isScreenShare: widget.isScreenShare,
-                    enabledE2EE: widget.participant.firstTrackEncryptionType !=
-                        EncryptionType.kNone,
+                    enabledE2EE: widget.participant.isEncrypted,
                   ),
                 ],
               ),

ios/livekit_client.podspec

@@ -16,5 +16,5 @@ Pod::Spec.new do |s|
   s.static_framework    = true
 
   s.dependency 'Flutter'
-  s.dependency 'WebRTC-SDK', '114.5735.06'
+  s.dependency 'WebRTC-SDK', '114.5735.07'
 end

lib/src/core/engine.dart

@@ -204,11 +204,12 @@ class Engine extends Disposable with EventsEmittable<EngineEvent> {
     Iterable<lk_models.VideoLayer>? videoLayers,
     Iterable<lk_rtc.SimulcastCodec>? simulcastCodecs,
     String? sid,
+    String? videoCodec,
   }) async {
     // TODO: Check if cid already published
 
     lk_models.Encryption_Type encryptionType = lk_models.Encryption_Type.NONE;
-    if (roomOptions.e2eeOptions != null) {
+    if (roomOptions.e2eeOptions != null && !isSVCCodec(videoCodec ?? '')) {
       switch (roomOptions.e2eeOptions!.encryptionType) {
         case EncryptionType.kNone:
           encryptionType = lk_models.Encryption_Type.NONE;

lib/src/core/room.dart

@@ -255,6 +255,11 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
         _getOrCreateRemoteParticipant(info.sid, info);
       }
 
+      if (e2eeManager != null && event.response.sifTrailer.isNotEmpty) {
+        e2eeManager!.keyProvider
+            .setSifTrailer(Uint8List.fromList(event.response.sifTrailer));
+      }
+
       logger.fine('Room Connect completed');
     })
     ..on<SignalParticipantUpdateEvent>(

lib/src/e2ee/e2ee_manager.dart

@@ -18,15 +18,16 @@ import 'package:flutter_webrtc/flutter_webrtc.dart';
 
 import '../core/room.dart';
 import '../e2ee/events.dart';
+import '../e2ee/options.dart';
 import '../events.dart';
 import '../extensions.dart';
 import '../managers/event.dart';
+import '../utils.dart';
 import 'key_provider.dart';
 
 class E2EEManager {
   Room? _room;
-  final Map<String, FrameCryptor> _frameCryptors = {};
-  final List<FrameCryptor> _senderFrameCryptors = [];
+  final Map<Map<String, String>, FrameCryptor> _frameCryptors = {};
   final BaseKeyProvider _keyProvider;
   final Algorithm _algorithm = Algorithm.kAesGcm;
   bool _enabled = true;
@@ -40,13 +41,15 @@ class E2EEManager {
       _listener = _room!.createListener();
       _listener!
         ..on<LocalTrackPublishedEvent>((event) async {
-          var trackId = event.publication.sid;
-          var participantId = event.participant.sid;
+          if (event.publication.encryptionType == EncryptionType.kNone ||
+              isSVCCodec(event.publication.track?.codec ?? '')) {
+            // no need to setup frame cryptor
+            return;
+          }
           var frameCryptor = await _addRtpSender(
-              event.publication.track!.sender!,
-              participantId,
-              trackId,
-              event.publication.track!.kind.name.toLowerCase());
+              sender: event.publication.track!.sender!,
+              identity: event.participant.identity,
+              sid: event.publication.sid);
           if (kIsWeb && event.publication.track!.codec != null) {
             await frameCryptor.updateCodec(event.publication.track!.codec!);
           }
@@ -63,21 +66,30 @@ class E2EEManager {
               state: _e2eeStateFromFrameCryptoState(state),
             ));
           };
-          _senderFrameCryptors.add(frameCryptor);
         })
         ..on<LocalTrackUnpublishedEvent>((event) async {
-          var trackId = event.publication.sid;
-          var frameCryptor = _frameCryptors.remove(trackId);
-          _senderFrameCryptors.remove(frameCryptor);
-          await frameCryptor?.dispose();
+          for (var key in _frameCryptors.keys.toList()) {
+            if (key.keys.first == event.participant.identity &&
+                key.values.first == event.publication.sid) {
+              var frameCryptor = _frameCryptors.remove(key);
+              await frameCryptor?.setEnabled(false);
+              await frameCryptor?.dispose();
+            }
+          }
         })
         ..on<TrackSubscribedEvent>((event) async {
-          var trackId = event.publication.sid;
-          var participantId = event.participant.sid;
-          var frameCryptor = await _addRtpReceiver(event.track.receiver!,
-              participantId, trackId, event.track.kind.name.toLowerCase());
+          var codec = event.publication.mimeType.split('/')[1];
+          if (event.publication.encryptionType == EncryptionType.kNone ||
+              isSVCCodec(codec)) {
+            // no need to setup frame cryptor
+            return;
+          }
+          var frameCryptor = await _addRtpReceiver(
+            receiver: event.track.receiver!,
+            identity: event.participant.identity,
+            sid: event.publication.sid,
+          );
           if (kIsWeb) {
-            var codec = event.publication.mimeType.split('/')[1];
             await frameCryptor.updateCodec(codec.toLowerCase());
           }
           frameCryptor.onFrameCryptorStateChanged = (trackId, state) {
@@ -95,16 +107,28 @@ class E2EEManager {
           };
         })
         ..on<TrackUnsubscribedEvent>((event) async {
-          var trackId = event.publication.sid;
-          var frameCryptor = _frameCryptors.remove(trackId);
-          await frameCryptor?.dispose();
+          for (var key in _frameCryptors.keys.toList()) {
+            if (key.keys.first == event.participant.identity &&
+                key.values.first == event.publication.sid) {
+              var frameCryptor = _frameCryptors.remove(key);
+              await frameCryptor?.setEnabled(false);
+              await frameCryptor?.dispose();
+            }
+          }
         });
     }
   }
 
-  Future<void> ratchetKey() async {
-    for (var frameCryptor in _senderFrameCryptors) {
-      var newKey = await _keyProvider.ratchetKey(frameCryptor.participantId, 0);
+  BaseKeyProvider get keyProvider => _keyProvider;
+
+  Future<void> ratchetKey({String? participantId, int? keyIndex}) async {
+    if (participantId != null) {
+      var newKey = await _keyProvider.ratchetKey(participantId, keyIndex);
+      if (kDebugMode) {
+        print('newKey: $newKey');
+      }
+    } else {
+      var newKey = await _keyProvider.ratchetSharedKey(keyIndex: keyIndex);
       if (kDebugMode) {
         print('newKey: $newKey');
       }
@@ -121,54 +145,41 @@ class E2EEManager {
     _frameCryptors.clear();
   }
 
-  Future<FrameCryptor> _addRtpSender(RTCRtpSender sender, String participantId,
-      String trackId, String kind) async {
-    var pid = '$kind-sender-$participantId-$trackId';
+  Future<FrameCryptor> _addRtpSender(
+      {required RTCRtpSender sender,
+      required String identity,
+      required String sid}) async {
     var frameCryptor = await frameCryptorFactory.createFrameCryptorForRtpSender(
-        participantId: pid,
+        participantId: identity,
         sender: sender,
         algorithm: _algorithm,
         keyProvider: _keyProvider.keyProvider);
-    _frameCryptors[trackId] = frameCryptor;
+    _frameCryptors[{identity: sid}] = frameCryptor;
     await frameCryptor.setEnabled(_enabled);
-    if (_keyProvider.options.sharedKey) {
-      await _keyProvider.keyProvider
-          .setKey(participantId: pid, index: 0, key: _keyProvider.sharedKey!);
-      await frameCryptor.setKeyIndex(0);
-    }
+    await frameCryptor.setKeyIndex(0);
     return frameCryptor;
   }
 
-  Future<FrameCryptor> _addRtpReceiver(RTCRtpReceiver receiver,
-      String participantId, String trackId, String kind) async {
-    var pid = '$kind-receiver-$participantId-$trackId';
+  Future<FrameCryptor> _addRtpReceiver(
+      {required RTCRtpReceiver receiver,
+      required String identity,
+      required String sid}) async {
     var frameCryptor =
         await frameCryptorFactory.createFrameCryptorForRtpReceiver(
-            participantId: pid,
+            participantId: identity,
             receiver: receiver,
             algorithm: _algorithm,
             keyProvider: _keyProvider.keyProvider);
-    _frameCryptors[trackId] = frameCryptor;
+    _frameCryptors[{identity: sid}] = frameCryptor;
     await frameCryptor.setEnabled(_enabled);
-    if (_keyProvider.options.sharedKey) {
-      await _keyProvider.keyProvider
-          .setKey(participantId: pid, index: 0, key: _keyProvider.sharedKey!);
-      await frameCryptor.setKeyIndex(0);
-    }
+    await frameCryptor.setKeyIndex(0);
     return frameCryptor;
   }
 
   Future<void> setEnabled(bool enabled) async {
     _enabled = enabled;
     for (var frameCryptor in _frameCryptors.entries) {
       await frameCryptor.value.setEnabled(enabled);
-      if (_keyProvider.options.sharedKey) {
-        await _keyProvider.keyProvider.setKey(
-            participantId: frameCryptor.key,
-            index: 0,
-            key: _keyProvider.sharedKey!);
-        await frameCryptor.value.setKeyIndex(0);
-      }
     }
   }
 

lib/src/e2ee/key_provider.dart

@@ -18,7 +18,8 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 
 const defaultRatchetSalt = 'LKFrameEncryptionKey';
 const defaultMagicBytes = 'LK-ROCKS';
-const defaultRatchetWindowSize = 16;
+const defaultRatchetWindowSize = 0;
+const defaultFailureTolerance = -1;
 
 class KeyInfo {
   final String participantId;
@@ -32,8 +33,13 @@ class KeyInfo {
 }
 
 abstract class KeyProvider {
-  Future<void> setKey(String key, {String? participantId, int keyIndex = 0});
-  Future<Uint8List> ratchetKey(String participantId, int index);
+  Future<void> setSharedKey(String key, {int? keyIndex});
+  Future<Uint8List> ratchetSharedKey({int? keyIndex});
+  Future<Uint8List> exportSharedKey({int? keyIndex});
+  Future<void> setKey(String key, {String? participantId, int? keyIndex});
+  Future<Uint8List> ratchetKey(String participantId, int? keyIndex);
+  Future<Uint8List> exportKey(String participantId, int? keyIndex);
+  Future<void> setSifTrailer(Uint8List trailer);
   rtc.KeyProvider get keyProvider;
 }
 
@@ -54,6 +60,7 @@ class BaseKeyProvider implements KeyProvider {
     String? ratchetSalt,
     String? uncryptedMagicBytes,
     int? ratchetWindowSize,
+    int? failureTolerance,
   }) async {
     rtc.KeyProviderOptions options = rtc.KeyProviderOptions(
       sharedKey: sharedKey,
@@ -62,26 +69,56 @@ class BaseKeyProvider implements KeyProvider {
       ratchetWindowSize: ratchetWindowSize ?? defaultRatchetWindowSize,
       uncryptedMagicBytes: Uint8List.fromList(
           (uncryptedMagicBytes ?? defaultMagicBytes).codeUnits),
+      failureTolerance: failureTolerance ?? defaultFailureTolerance,
     );
     final keyProvider =
         await rtc.frameCryptorFactory.createDefaultKeyProvider(options);
     return BaseKeyProvider(keyProvider, options);
   }
 
   @override
-  Future<Uint8List> ratchetKey(String participantId, int index) =>
-      _keyProvider.ratchetKey(participantId: participantId, index: index);
+  Future<void> setSharedKey(String key, {int? keyIndex}) async {
+    _sharedKey = Uint8List.fromList(key.codeUnits);
+    return _keyProvider.setSharedKey(key: _sharedKey!, index: keyIndex ?? 0);
+  }
+
+  @override
+  Future<Uint8List> ratchetSharedKey({int? keyIndex}) async {
+    if (_sharedKey == null) {
+      throw Exception('shared key not set');
+    }
+    _sharedKey = await _keyProvider.ratchetSharedKey(index: keyIndex ?? 0);
+    return _sharedKey!;
+  }
+
+  @override
+  Future<Uint8List> exportSharedKey({int? keyIndex}) async {
+    if (_sharedKey == null) {
+      throw Exception('shared key not set');
+    }
+    return _keyProvider.exportSharedKey(index: keyIndex ?? 0);
+  }
+
+  @override
+  Future<Uint8List> ratchetKey(String participantId, int? keyIndex) =>
+      _keyProvider.ratchetKey(
+          participantId: participantId, index: keyIndex ?? 0);
+
+  @override
+  Future<Uint8List> exportKey(String participantId, int? keyIndex) =>
+      _keyProvider.exportKey(
+          participantId: participantId, index: keyIndex ?? 0);
 
   @override
   Future<void> setKey(String key,
-      {String? participantId, int keyIndex = 0}) async {
+      {String? participantId, int? keyIndex}) async {
     if (options.sharedKey) {
       _sharedKey = Uint8List.fromList(key.codeUnits);
       return;
     }
     final keyInfo = KeyInfo(
       participantId: participantId ?? '',
-      keyIndex: keyIndex,
+      keyIndex: keyIndex ?? 0,
       key: Uint8List.fromList(key.codeUnits),
     );
     return _setKey(keyInfo);
@@ -98,4 +135,9 @@ class BaseKeyProvider implements KeyProvider {
       key: keyInfo.key,
     );
   }
+
+  @override
+  Future<void> setSifTrailer(Uint8List trailer) async {
+    return _keyProvider.setSifTrailer(trailer: trailer);
+  }
 }