Framecryptor decrypting fixes (#520)

* fix: decrypting audio when e2ee * chore: let linter run on web/*.dart files * feat: add voiceIsolation support
livekit · Jun 5, 2024 · 20e6217 · 20e6217
1 parent d9b7823
commit 20e6217
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 87 deletions.
diff --git a/analysis_options.yaml b/analysis_options.yaml
@@ -27,6 +27,8 @@
 include: package:flutter_lints/flutter.yaml
 
 analyzer:
+  errors:
+    constant_identifier_names: ignore
   #
   # Enforce stricter type-checking
   # https://dart.dev/guides/language/analysis-options#enabling-additional-type-checks
@@ -41,7 +43,7 @@ analyzer:
     - '**/*.pbenum.dart'
     - '**/*.pbjson.dart'
     - '**/*.pbserver.dart'
-    - 'web/*.dart'
+    # - 'web/*.dart'
 
 linter:
   rules:

diff --git a/pubspec.lock b/pubspec.lock
@@ -101,18 +101,18 @@ packages:
     dependency: "direct main"
     description:
       name: connectivity_plus
-      sha256: "224a77051d52a11fbad53dd57827594d3bd24f945af28bd70bab376d68d437f0"
+      sha256: db7a4e143dc72cc3cb2044ef9b052a7ebfe729513e6a82943bc3526f784365b8
       url: "https://pub.dev"
     source: hosted
-    version: "5.0.2"
+    version: "6.0.3"
   connectivity_plus_platform_interface:
     dependency: transitive
     description:
       name: connectivity_plus_platform_interface
-      sha256: cf1d1c28f4416f8c654d7dc3cd638ec586076255d407cef3ddbdaf178272a71a
+      sha256: b6a56efe1e6675be240de39107281d4034b64ac23438026355b4234042a35adb
       url: "https://pub.dev"
     source: hosted
-    version: "1.2.4"
+    version: "2.0.0"
   convert:
     dependency: transitive
     description:
@@ -149,10 +149,10 @@ packages:
     dependency: "direct main"
     description:
       name: dart_webrtc
-      sha256: "8bcc17fa56101524e3539a4703f2471f20abf349130ed988430db9914bd38174"
+      sha256: fe4db21dc389b99e04cb7bf43bc927dba2e42768d4c28211b66a4b5a16e4d516
       url: "https://pub.dev"
     source: hosted
-    version: "1.3.2"
+    version: "1.4.5"
   dbus:
     dependency: transitive
     description:
@@ -236,10 +236,10 @@ packages:
     dependency: "direct main"
     description:
       name: flutter_webrtc
-      sha256: "80ee0de0550a60337c728940cd073e9b54b08ea1a2a61ffd8ccee35036f000e1"
+      sha256: "1c61bc08d14be57ac28e9e540c44b8b1b9ab1b25bbdb66a8c658e61a3211cc5d"
       url: "https://pub.dev"
     source: hosted
-    version: "0.10.2"
+    version: "0.10.7"
   glob:
     dependency: transitive
     description:
@@ -284,26 +284,26 @@ packages:
     dependency: transitive
     description:
       name: leak_tracker
-      sha256: "78eb209deea09858f5269f5a5b02be4049535f568c07b275096836f01ea323fa"
+      sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a"
       url: "https://pub.dev"
     source: hosted
-    version: "10.0.0"
+    version: "10.0.4"
   leak_tracker_flutter_testing:
     dependency: transitive
     description:
       name: leak_tracker_flutter_testing
-      sha256: b46c5e37c19120a8a01918cfaf293547f47269f7cb4b0058f21531c2465d6ef0
+      sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8"
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.1"
+    version: "3.0.3"
   leak_tracker_testing:
     dependency: transitive
     description:
       name: leak_tracker_testing
-      sha256: a597f72a664dbd293f3bfc51f9ba69816f84dcd403cdac7066cb3f6003f3ab47
+      sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3"
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.1"
+    version: "3.0.1"
   lints:
     dependency: transitive
     description:
@@ -340,10 +340,10 @@ packages:
     dependency: "direct main"
     description:
       name: meta
-      sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04
+      sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136"
       url: "https://pub.dev"
     source: hosted
-    version: "1.11.0"
+    version: "1.12.0"
   mockito:
     dependency: "direct dev"
     description:
@@ -553,10 +553,10 @@ packages:
     dependency: transitive
     description:
       name: test_api
-      sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b"
+      sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.1"
+    version: "0.7.0"
   tint:
     dependency: transitive
     description:
@@ -593,10 +593,10 @@ packages:
     dependency: transitive
     description:
       name: vm_service
-      sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957
+      sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec"
       url: "https://pub.dev"
     source: hosted
-    version: "13.0.0"
+    version: "14.2.1"
   watcher:
     dependency: transitive
     description:
@@ -606,7 +606,7 @@ packages:
     source: hosted
     version: "1.1.0"
   web:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: web
       sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27"
@@ -663,4 +663,4 @@ packages:
     version: "3.1.2"
 sdks:
   dart: ">=3.3.0 <4.0.0"
-  flutter: ">=3.10.0"
+  flutter: ">=3.18.0-18.0.pre.54"
diff --git a/web/e2ee.cryptor.dart b/web/e2ee.cryptor.dart
@@ -422,9 +422,8 @@ class FrameCryptor {
         // skip for encryption for empty dtx frames
         buffer.isEmpty) {
       sifGuard.recordUserFrame();
-      if (keyOptions.discardFrameWhenCryptorNotReady) {
-        return;
-      }
+      if (keyOptions.discardFrameWhenCryptorNotReady) return;
+      logger.fine('enqueing empty frame');
       controller.enqueue(frame);
       return;
     }
@@ -435,7 +434,7 @@ class FrameCryptor {
         var magicBytesBuffer = buffer.sublist(
             buffer.length - magicBytes.length - 1, buffer.length - 1);
         logger.finer(
-            'magicBytesBuffer $magicBytesBuffer, magicBytes $magicBytes, ');
+            'magicBytesBuffer $magicBytesBuffer, magicBytes $magicBytes');
         if (magicBytesBuffer.toString() == magicBytes.toString()) {
           sifGuard.recordSif();
           if (sifGuard.isSifAllowed()) {
@@ -445,6 +444,7 @@ class FrameCryptor {
             finalBuffer.add(Uint8List.fromList(
                 buffer.sublist(0, buffer.length - (magicBytes.length + 1))));
             frame.data = crypto.jsArrayBufferFrom(finalBuffer.toBytes());
+            logger.fine('enqueing silent frame');
             controller.enqueue(frame);
           } else {
             logger.finer('SIF limit reached, dropping frame');
@@ -469,6 +469,12 @@ class FrameCryptor {
       initialKeySet = keyHandler.getKeySet(keyIndex);
       initialKeyIndex = keyIndex;
 
+      /// missingKey flow:
+      /// tries to decrypt once, fails, tries to ratchet once and decrypt again,
+      /// fails (does not save ratcheted key), bumps _decryptionFailureCount,
+      /// if higher than failuretolerance hasValidKey is set to false, on next
+      /// frame it fires a missingkey
+      /// to throw missingkeys faster lower your failureTolerance
       if (initialKeySet == null || !keyHandler.hasValidKey) {
         if (lastError != CryptorError.kMissingKey) {
           lastError = CryptorError.kMissingKey;
@@ -482,14 +488,14 @@ class FrameCryptor {
             'error': 'Missing key for track $trackId'
           });
         }
-        controller.enqueue(frame);
+        // controller.enqueue(frame);
         return;
       }
-      var endDecLoop = false;
       var currentkeySet = initialKeySet;
-      while (!endDecLoop) {
-        try {
-          decrypted = await jsutil.promiseToFuture<ByteBuffer>(crypto.decrypt(
+
+      Future<void> decryptFrameInternal() async {
+        decrypted = await jsutil.promiseToFuture<ByteBuffer>(
+          crypto.decrypt(
             crypto.AesGcmParams(
               name: 'AES-GCM',
               iv: crypto.jsArrayBufferFrom(iv),
@@ -498,56 +504,78 @@ class FrameCryptor {
             ),
             currentkeySet.encryptionKey,
             crypto.jsArrayBufferFrom(
-                buffer.sublist(headerLength, buffer.length - ivLength - 2)),
-          ));
-
-          if (currentkeySet != initialKeySet) {
-            logger.fine(
-                'ratchetKey: decryption ok, reset state to kKeyRatcheted');
-            await keyHandler.setKeySetFromMaterial(
-                currentkeySet, initialKeyIndex);
-          }
+              buffer.sublist(headerLength, buffer.length - ivLength - 2),
+            ),
+          ),
+        );
+        if (decrypted == null) {
+          throw Exception('[decryptFrameInternal] could not decrypt');
+        }
+
+        if (currentkeySet != initialKeySet) {
+          logger.fine('ratchetKey: decryption ok, newState: kKeyRatcheted');
+          await keyHandler.setKeySetFromMaterial(
+              currentkeySet, initialKeyIndex);
+        }
 
-          endDecLoop = true;
+        if (lastError != CryptorError.kOk &&
+            lastError != CryptorError.kKeyRatcheted &&
+            ratchetCount > 0) {
+          logger.finer(
+              'KeyRatcheted: ssrc ${metaData.synchronizationSource} timestamp ${frame.timestamp} ratchetCount $ratchetCount  participantId: $participantIdentity');
+          logger.finer(
+              'ratchetKey: lastError != CryptorError.kKeyRatcheted, reset state to kKeyRatcheted');
 
-          if (lastError != CryptorError.kOk &&
-              lastError != CryptorError.kKeyRatcheted &&
-              ratchetCount > 0) {
-            logger.finer(
-                'KeyRatcheted: ssrc ${metaData.synchronizationSource} timestamp ${frame.timestamp} ratchetCount $ratchetCount  participantId: $participantIdentity');
-            logger.finer(
-                'ratchetKey: lastError != CryptorError.kKeyRatcheted, reset state to kKeyRatcheted');
-
-            lastError = CryptorError.kKeyRatcheted;
-            postMessage({
-              'type': 'cryptorState',
-              'msgType': 'event',
-              'participantId': participantIdentity,
-              'trackId': trackId,
-              'kind': kind,
-              'state': 'keyRatcheted',
-              'error': 'Key ratcheted ok'
-            });
-          }
-        } catch (e) {
-          lastError = CryptorError.kInternalError;
-          endDecLoop = ratchetCount >= keyOptions.ratchetWindowSize ||
-              keyOptions.ratchetWindowSize <= 0;
-          if (endDecLoop) {
-            rethrow;
-          }
-          var newKeyBuffer = crypto.jsArrayBufferFrom(await keyHandler.ratchet(
-              currentkeySet.material, keyOptions.ratchetSalt));
-          var newMaterial = await keyHandler.ratchetMaterial(
-              currentkeySet.material, newKeyBuffer);
-          currentkeySet =
-              await keyHandler.deriveKeys(newMaterial, keyOptions.ratchetSalt);
-          ratchetCount++;
+          lastError = CryptorError.kKeyRatcheted;
+          postMessage({
+            'type': 'cryptorState',
+            'msgType': 'event',
+            'participantId': participantIdentity,
+            'trackId': trackId,
+            'kind': kind,
+            'state': 'keyRatcheted',
+            'error': 'Key ratcheted ok'
+          });
         }
       }
 
+      Future<void> ratchedKeyInternal() async {
+        if (ratchetCount >= keyOptions.ratchetWindowSize ||
+            keyOptions.ratchetWindowSize <= 0) {
+          throw Exception('[ratchedKeyInternal] cannot ratchet anymore');
+        }
+
+        var newKeyBuffer = crypto.jsArrayBufferFrom(await keyHandler.ratchet(
+            currentkeySet.material, keyOptions.ratchetSalt));
+        var newMaterial = await keyHandler.ratchetMaterial(
+            currentkeySet.material, newKeyBuffer);
+        currentkeySet =
+            await keyHandler.deriveKeys(newMaterial, keyOptions.ratchetSalt);
+        ratchetCount++;
+        await decryptFrameInternal();
+      }
+
+      try {
+        /// gets frame -> tries to decrypt -> tries to ratchet (does this failureTolerance
+        /// times, then says missing key)
+        /// we only save the new key after ratcheting if we were able to decrypt something
+        await decryptFrameInternal();
+      } catch (e) {
+        lastError = CryptorError.kInternalError;
+        await ratchedKeyInternal();
+      }
+
+      if (decrypted == null) {
+        throw Exception(
+            '[decodeFunction] decryption failed even after ratchting');
+      }
+
+      // we can now be sure that decryption was a success
+      keyHandler.decryptionSuccess();
+
       logger.finer(
-          'buffer: ${buffer.length}, decrypted: ${decrypted?.asUint8List().length ?? 0}');
+          'buffer: ${buffer.length}, decrypted: ${decrypted!.asUint8List().length}');
+
       var finalBuffer = BytesBuilder();
 
       finalBuffer.add(Uint8List.fromList(buffer.sublist(0, headerLength)));
@@ -584,15 +612,6 @@ class FrameCryptor {
         });
       }
 
-      /// Since the key it is first send and only afterwards actually used for encrypting, there were
-      /// situations when the decrypting failed due to the fact that the received frame was not encrypted
-      /// yet and ratcheting, of course, did not solve the problem. So if we fail RATCHET_WINDOW_SIZE times,
-      ///  we come back to the initial key.
-      if (initialKeySet != null) {
-        logger.warning(
-            'decryption failed, ratcheting back to initial key, keyIndex: $initialKeyIndex');
-        await keyHandler.setKeySetFromMaterial(initialKeySet, initialKeyIndex);
-      }
       keyHandler.decryptionFailure();
     }
   }

diff --git a/web/e2ee.keyhandler.dart b/web/e2ee.keyhandler.dart
@@ -38,6 +38,8 @@ class KeyOptions {
   Uint8List ratchetSalt;
   int ratchetWindowSize = 0;
   int failureTolerance;
+
+  /// usually automatically set by whatever livekit sends you in the JoinResponse
   Uint8List? uncryptedMagicBytes;
   int keyRingSze;
   bool discardFrameWhenCryptorNotReady;

diff --git a/web/e2ee.logger.dart b/web/e2ee.logger.dart
@@ -27,7 +27,7 @@ enum LoggerLevel {
   kOFF
 }
 
-final logger = Logger('E2EE.Worker');
+final logger = Logger('VOIP E2EE.Worker');
 
 /// disable logging
 void disableLogging() {

diff --git a/web/e2ee.worker.dart b/web/e2ee.worker.dart
@@ -70,8 +70,9 @@ void unsetCryptorParticipant(String trackId) {
 
 void main() async {
   // configure logs for debugging
-  Logger.root.level = Level.WARNING;
+  Logger.root.level = Level.FINE;
   Logger.root.onRecord.listen((record) {
+    // ignore: avoid_print
     print('[${record.loggerName}] ${record.level.name}: ${record.message}');
   });