Merge branch 'main' into feat/changes-for-framecryptor

example/macos/Podfile

@@ -1,4 +1,4 @@
-platform :osx, '10.11'
+platform :osx, '10.14'
 
 # CocoaPods analytics sends network stats synchronously affecting flutter build latency.
 ENV['COCOAPODS_DISABLE_STATS'] = 'true'

lib/src/core/transport.dart

@@ -15,6 +15,7 @@
 import 'dart:async';
 
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:sdp_transform/sdp_transform.dart' as sdp_transform;
 
 import '../exceptions.dart';
 import '../extensions.dart';
@@ -26,6 +27,30 @@ import '../support/platform.dart';
 import '../types/other.dart';
 import '../utils.dart';
 
+const ddExtensionURI =
+    'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension';
+
+/* The svc codec (av1/vp9) would use a very low bitrate at the begining and
+increase slowly by the bandwidth estimator until it reach the target bitrate. The
+process commonly cost more than 10 seconds cause subscriber will get blur video at
+the first few seconds. So we use a 70% of target bitrate here as the start bitrate to
+eliminate this issue.
+*/
+const startBitrateForSVC = 0.7;
+
+class TrackBitrateInfo {
+  String? cid;
+  rtc.RTCRtpTransceiver? transceiver;
+  String codec;
+  int maxbr;
+  TrackBitrateInfo({
+    required this.cid,
+    required this.transceiver,
+    required this.codec,
+    required this.maxbr,
+  });
+}
+
 typedef TransportOnOffer = void Function(rtc.RTCSessionDescription offer);
 typedef PeerConnectionCreate = Future<rtc.RTCPeerConnection> Function(
     Map<String, dynamic> configuration,
@@ -35,6 +60,7 @@ typedef PeerConnectionCreate = Future<rtc.RTCPeerConnection> Function(
 class Transport extends Disposable {
   final rtc.RTCPeerConnection pc;
   final List<rtc.RTCIceCandidate> _pendingCandidates = [];
+  final List<TrackBitrateInfo> _bitrateTrackers = [];
   bool restartingIce = false;
   bool renegotiate = false;
   TransportOnOffer? onOffer;
@@ -150,8 +176,67 @@ class Transport extends Disposable {
     // actually negotiate
     logger.fine('starting to negotiate');
     final offer = await pc.createOffer(options?.toMap() ?? <String, dynamic>{});
+
+    final sdpParsed = sdp_transform.parse(offer.sdp ?? '');
+    sdpParsed['media']?.forEach((media) {
+      if (media['type'] == 'video') {
+        ensureVideoDDExtensionForSVC(media, media['type'], media['port'],
+            media['protocol'], media['payloads']);
+
+        // mung sdp for codec bitrate setting that can't apply by sendEncoding
+        for (var trackbr in _bitrateTrackers) {
+          if (media['msid'] == null ||
+              trackbr.cid == null ||
+              !(media['msid'] as String).contains(trackbr.cid!)) {
+            continue;
+          }
+
+          var codecPayload = 0;
+          for (var rtp in media['rtp']) {
+            if (rtp['codec']?.toUpperCase() == trackbr.codec.toUpperCase()) {
+              codecPayload = rtp['payload'];
+              continue;
+            }
+            continue;
+          }
+
+          if (codecPayload == 0) {
+            continue;
+          }
+
+          var fmtpFound = false;
+          for (var fmtp in media['fmtp']) {
+            if (fmtp['payload'] == codecPayload) {
+              if (!(fmtp['config'] as String)
+                  .contains('x-google-start-bitrate')) {
+                fmtp['config'] +=
+                    ';x-google-start-bitrate=${(trackbr.maxbr * startBitrateForSVC).toInt()}';
+              }
+              if (!(fmtp['config'] as String)
+                  .contains('x-google-max-bitrate')) {
+                fmtp['config'] += ';x-google-max-bitrate=${trackbr.maxbr}';
+              }
+              fmtpFound = true;
+              break;
+            }
+          }
+
+          if (!fmtpFound) {
+            media['fmtp']?.add({
+              'payload': codecPayload,
+              'config':
+                  'x-google-start-bitrate=${(trackbr.maxbr * startBitrateForSVC).toInt()};x-google-max-bitrate=${trackbr.maxbr}',
+            });
+          }
+
+          continue;
+        }
+      }
+    });
+
     try {
-      await pc.setLocalDescription(offer);
+      await setMungedSDP(
+          sd: offer, munged: sdp_transform.write(sdpParsed, null));
     } catch (e) {
       throw NegotiationError(e.toString());
     }
@@ -192,4 +277,81 @@ class Transport extends Disposable {
     }
     return null;
   }
+
+  void setTrackBitrateInfo(TrackBitrateInfo info) {
+    _bitrateTrackers.add(info);
+  }
+
+  bool ensureVideoDDExtensionForSVC(
+    Map<String, dynamic> media,
+    String? type,
+    num port,
+    String protocol,
+    String? payloads,
+  ) {
+    final codec = media['rtp']?[0]?['codec']?.toLowerCase();
+    if (!isSVCCodec(codec)) {
+      return false;
+    }
+
+    var maxID = 0;
+    bool ddFound = false;
+    List<dynamic>? ext = media['ext'];
+    if (ext != null) {
+      for (var e in ext) {
+        if (e['uri'] == ddExtensionURI) {
+          ddFound = true;
+          continue;
+        }
+        if (e['value'] > maxID) {
+          maxID = e['value'];
+        }
+      }
+    }
+
+    if (!ddFound) {
+      ext?.add({
+        'value': maxID + 1,
+        'uri': ddExtensionURI,
+      });
+    }
+
+    return ddFound;
+  }
+
+  Future<void> setMungedSDP(
+      {required rtc.RTCSessionDescription sd,
+      String? munged,
+      bool? remote}) async {
+    if (munged != null) {
+      final originalSdp = sd.sdp;
+      sd.sdp = munged;
+      try {
+        logger.fine(
+            'setting munged ${remote == true ? 'remote' : 'local'} description munged: $munged ');
+        if (remote == true) {
+          await pc.setRemoteDescription(sd);
+        } else {
+          await pc.setLocalDescription(sd);
+        }
+        return;
+      } catch (e) {
+        logger.warning(
+            'not able to set ${sd.type}, falling back to unmodified sdp error: $e, sdp: $munged ');
+        sd.sdp = originalSdp;
+      }
+    }
+
+    try {
+      if (remote == true) {
+        await pc.setRemoteDescription(sd);
+      } else {
+        await pc.setLocalDescription(sd);
+      }
+    } catch (e) {
+      // this error cannot always be caught.ght
+      logger.warning('unable to set ${sd.type}, error: $e, sdp: ${sd.sdp}');
+      rethrow;
+    }
+  }
 }

lib/src/participant/local.dart

@@ -20,6 +20,7 @@ import 'package:meta/meta.dart';
 import '../core/engine.dart';
 import '../core/room.dart';
 import '../core/signal_client.dart';
+import '../core/transport.dart';
 import '../events.dart';
 import '../exceptions.dart';
 import '../extensions.dart';
@@ -265,6 +266,19 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
       await sender.setParameters(parameters);
     }
 
+    if (kIsWeb &&
+        lkBrowser() == BrowserType.firefox &&
+        track.kind == lk_models.TrackType.AUDIO) {
+      //TOOD:
+    } else if (isSVCCodec(publishOptions.videoCodec) &&
+        encodings?.first.maxBitrate != null) {
+      room.engine.publisher?.setTrackBitrateInfo(TrackBitrateInfo(
+          cid: track.getCid(),
+          transceiver: track.transceiver,
+          codec: publishOptions.videoCodec,
+          maxbr: encodings![0].maxBitrate! ~/ 1000));
+    }
+
     await room.engine.negotiate();
 
     final pub = LocalTrackPublication<LocalVideoTrack>(

lib/src/utils.dart

@@ -433,10 +433,10 @@ class Utils {
     if (scalabilityMode != null && isSVCCodec(options.videoCodec)) {
       logger.info('using svc with scalabilityMode ${scalabilityMode}');
 
-      final sm = ScalabilityMode(scalabilityMode);
-
-      List<rtc.RTCRtpEncoding> encodings = [];
+      //final sm = ScalabilityMode(scalabilityMode);
 
+      List<rtc.RTCRtpEncoding> encodings = [videoEncoding.toRTCRtpEncoding()];
+      /*
       if (sm.spatial > 3) {
         throw Exception('unsupported scalabilityMode: ${scalabilityMode}');
       }
@@ -448,7 +448,7 @@ class Utils {
           scaleResolutionDownBy: null,
           numTemporalLayers: sm.temporal.toInt(),
         ));
-      }
+      }*/
       encodings[0].scalabilityMode = scalabilityMode;
       logger.fine('encodings $encodings');
       return encodings;