diff --git a/example/macos/Podfile b/example/macos/Podfile index dade8dfa..049abe29 100644 --- a/example/macos/Podfile +++ b/example/macos/Podfile @@ -1,4 +1,4 @@ -platform :osx, '10.11' +platform :osx, '10.14' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' diff --git a/lib/src/core/transport.dart b/lib/src/core/transport.dart index 4f314b84..e2c990ae 100644 --- a/lib/src/core/transport.dart +++ b/lib/src/core/transport.dart @@ -15,6 +15,7 @@ import 'dart:async'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:sdp_transform/sdp_transform.dart' as sdp_transform; import '../exceptions.dart'; import '../extensions.dart'; @@ -26,6 +27,30 @@ import '../support/platform.dart'; import '../types/other.dart'; import '../utils.dart'; +const ddExtensionURI = + 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension'; + +/* The svc codec (av1/vp9) would use a very low bitrate at the begining and +increase slowly by the bandwidth estimator until it reach the target bitrate. The +process commonly cost more than 10 seconds cause subscriber will get blur video at +the first few seconds. So we use a 70% of target bitrate here as the start bitrate to +eliminate this issue. +*/ +const startBitrateForSVC = 0.7; + +class TrackBitrateInfo { + String? cid; + rtc.RTCRtpTransceiver? transceiver; + String codec; + int maxbr; + TrackBitrateInfo({ + required this.cid, + required this.transceiver, + required this.codec, + required this.maxbr, + }); +} + typedef TransportOnOffer = void Function(rtc.RTCSessionDescription offer); typedef PeerConnectionCreate = Future Function( Map configuration, @@ -35,6 +60,7 @@ typedef PeerConnectionCreate = Future Function( class Transport extends Disposable { final rtc.RTCPeerConnection pc; final List _pendingCandidates = []; + final List _bitrateTrackers = []; bool restartingIce = false; bool renegotiate = false; TransportOnOffer? onOffer; @@ -150,8 +176,67 @@ class Transport extends Disposable { // actually negotiate logger.fine('starting to negotiate'); final offer = await pc.createOffer(options?.toMap() ?? {}); + + final sdpParsed = sdp_transform.parse(offer.sdp ?? ''); + sdpParsed['media']?.forEach((media) { + if (media['type'] == 'video') { + ensureVideoDDExtensionForSVC(media, media['type'], media['port'], + media['protocol'], media['payloads']); + + // mung sdp for codec bitrate setting that can't apply by sendEncoding + for (var trackbr in _bitrateTrackers) { + if (media['msid'] == null || + trackbr.cid == null || + !(media['msid'] as String).contains(trackbr.cid!)) { + continue; + } + + var codecPayload = 0; + for (var rtp in media['rtp']) { + if (rtp['codec']?.toUpperCase() == trackbr.codec.toUpperCase()) { + codecPayload = rtp['payload']; + continue; + } + continue; + } + + if (codecPayload == 0) { + continue; + } + + var fmtpFound = false; + for (var fmtp in media['fmtp']) { + if (fmtp['payload'] == codecPayload) { + if (!(fmtp['config'] as String) + .contains('x-google-start-bitrate')) { + fmtp['config'] += + ';x-google-start-bitrate=${(trackbr.maxbr * startBitrateForSVC).toInt()}'; + } + if (!(fmtp['config'] as String) + .contains('x-google-max-bitrate')) { + fmtp['config'] += ';x-google-max-bitrate=${trackbr.maxbr}'; + } + fmtpFound = true; + break; + } + } + + if (!fmtpFound) { + media['fmtp']?.add({ + 'payload': codecPayload, + 'config': + 'x-google-start-bitrate=${(trackbr.maxbr * startBitrateForSVC).toInt()};x-google-max-bitrate=${trackbr.maxbr}', + }); + } + + continue; + } + } + }); + try { - await pc.setLocalDescription(offer); + await setMungedSDP( + sd: offer, munged: sdp_transform.write(sdpParsed, null)); } catch (e) { throw NegotiationError(e.toString()); } @@ -192,4 +277,81 @@ class Transport extends Disposable { } return null; } + + void setTrackBitrateInfo(TrackBitrateInfo info) { + _bitrateTrackers.add(info); + } + + bool ensureVideoDDExtensionForSVC( + Map media, + String? type, + num port, + String protocol, + String? payloads, + ) { + final codec = media['rtp']?[0]?['codec']?.toLowerCase(); + if (!isSVCCodec(codec)) { + return false; + } + + var maxID = 0; + bool ddFound = false; + List? ext = media['ext']; + if (ext != null) { + for (var e in ext) { + if (e['uri'] == ddExtensionURI) { + ddFound = true; + continue; + } + if (e['value'] > maxID) { + maxID = e['value']; + } + } + } + + if (!ddFound) { + ext?.add({ + 'value': maxID + 1, + 'uri': ddExtensionURI, + }); + } + + return ddFound; + } + + Future setMungedSDP( + {required rtc.RTCSessionDescription sd, + String? munged, + bool? remote}) async { + if (munged != null) { + final originalSdp = sd.sdp; + sd.sdp = munged; + try { + logger.fine( + 'setting munged ${remote == true ? 'remote' : 'local'} description munged: $munged '); + if (remote == true) { + await pc.setRemoteDescription(sd); + } else { + await pc.setLocalDescription(sd); + } + return; + } catch (e) { + logger.warning( + 'not able to set ${sd.type}, falling back to unmodified sdp error: $e, sdp: $munged '); + sd.sdp = originalSdp; + } + } + + try { + if (remote == true) { + await pc.setRemoteDescription(sd); + } else { + await pc.setLocalDescription(sd); + } + } catch (e) { + // this error cannot always be caught.ght + logger.warning('unable to set ${sd.type}, error: $e, sdp: ${sd.sdp}'); + rethrow; + } + } } diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 86849fb9..0a7fc78e 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -20,6 +20,7 @@ import 'package:meta/meta.dart'; import '../core/engine.dart'; import '../core/room.dart'; import '../core/signal_client.dart'; +import '../core/transport.dart'; import '../events.dart'; import '../exceptions.dart'; import '../extensions.dart'; @@ -265,6 +266,19 @@ class LocalParticipant extends Participant { await sender.setParameters(parameters); } + if (kIsWeb && + lkBrowser() == BrowserType.firefox && + track.kind == lk_models.TrackType.AUDIO) { + //TOOD: + } else if (isSVCCodec(publishOptions.videoCodec) && + encodings?.first.maxBitrate != null) { + room.engine.publisher?.setTrackBitrateInfo(TrackBitrateInfo( + cid: track.getCid(), + transceiver: track.transceiver, + codec: publishOptions.videoCodec, + maxbr: encodings![0].maxBitrate! ~/ 1000)); + } + await room.engine.negotiate(); final pub = LocalTrackPublication( diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 35454b89..7c2cc255 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -433,10 +433,10 @@ class Utils { if (scalabilityMode != null && isSVCCodec(options.videoCodec)) { logger.info('using svc with scalabilityMode ${scalabilityMode}'); - final sm = ScalabilityMode(scalabilityMode); - - List encodings = []; + //final sm = ScalabilityMode(scalabilityMode); + List encodings = [videoEncoding.toRTCRtpEncoding()]; + /* if (sm.spatial > 3) { throw Exception('unsupported scalabilityMode: ${scalabilityMode}'); } @@ -448,7 +448,7 @@ class Utils { scaleResolutionDownBy: null, numTemporalLayers: sm.temporal.toInt(), )); - } + }*/ encodings[0].scalabilityMode = scalabilityMode; logger.fine('encodings $encodings'); return encodings; diff --git a/pubspec.lock b/pubspec.lock index 2c2bdd2a..f95b4969 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -61,10 +61,10 @@ packages: dependency: transitive description: name: built_value - sha256: "598a2a682e2a7a90f08ba39c0aaa9374c5112340f0a2e275f61b59389543d166" + sha256: ff627b645b28fb8bdb69e645f910c2458fd6b65f6585c3a53e0626024897dedf url: "https://pub.dev" source: hosted - version: "8.6.1" + version: "8.6.2" characters: dependency: transitive description: @@ -85,10 +85,10 @@ packages: dependency: transitive description: name: code_builder - sha256: "4ad01d6e56db961d29661561effde45e519939fdaeb46c351275b182eac70189" + sha256: "315a598c7fbe77f22de1c9da7cfd6fd21816312f16ffa124453b4fc679e540f1" url: "https://pub.dev" source: hosted - version: "4.5.0" + version: "4.6.0" collection: dependency: "direct main" description: @@ -218,10 +218,10 @@ packages: dependency: "direct dev" description: name: flutter_lints - sha256: "2118df84ef0c3ca93f96123a616ae8540879991b8b57af2f81b76a7ada49b2a4" + sha256: a25a15ebbdfc33ab1cd26c63a6ee519df92338a9c10f122adda92938253bef04 url: "https://pub.dev" source: hosted - version: "2.0.2" + version: "2.0.3" flutter_test: dependency: "direct dev" description: flutter @@ -364,50 +364,50 @@ packages: dependency: transitive description: name: path_provider - sha256: "909b84830485dbcd0308edf6f7368bc8fd76afa26a270420f34cabea2a6467a0" + sha256: a1aa8aaa2542a6bc57e381f132af822420216c80d4781f7aa085ca3229208aaa url: "https://pub.dev" source: hosted - version: "2.1.0" + version: "2.1.1" path_provider_android: dependency: transitive description: name: path_provider_android - sha256: "5d44fc3314d969b84816b569070d7ace0f1dea04bd94a83f74c4829615d22ad8" + sha256: "6b8b19bd80da4f11ce91b2d1fb931f3006911477cec227cce23d3253d80df3f1" url: "https://pub.dev" source: hosted - version: "2.1.0" + version: "2.2.0" path_provider_foundation: dependency: transitive description: name: path_provider_foundation - sha256: "1b744d3d774e5a879bb76d6cd1ecee2ba2c6960c03b1020cd35212f6aa267ac5" + sha256: "19314d595120f82aca0ba62787d58dde2cc6b5df7d2f0daf72489e38d1b57f2d" url: "https://pub.dev" source: hosted - version: "2.3.0" + version: "2.3.1" path_provider_linux: dependency: transitive description: name: path_provider_linux - sha256: ba2b77f0c52a33db09fc8caf85b12df691bf28d983e84cf87ff6d693cfa007b3 + sha256: f7a1fe3a634fe7734c8d3f2766ad746ae2a2884abe22e241a8b301bf5cac3279 url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.2.1" path_provider_platform_interface: dependency: transitive description: name: path_provider_platform_interface - sha256: bced5679c7df11190e1ddc35f3222c858f328fff85c3942e46e7f5589bf9eb84 + sha256: "94b1e0dd80970c1ce43d5d4e050a9918fce4f4a775e6142424c30a29a363265c" url: "https://pub.dev" source: hosted - version: "2.1.0" + version: "2.1.1" path_provider_windows: dependency: transitive description: name: path_provider_windows - sha256: ee0e0d164516b90ae1f970bdf29f726f1aa730d7cfc449ecc74c495378b705da + sha256: "8bc9f22eee8690981c22aa7fc602f5c85b497a6fb2ceb35ee5a5e5ed85ad8170" url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.2.1" petitparser: dependency: transitive description: @@ -420,10 +420,10 @@ packages: dependency: transitive description: name: platform - sha256: "57c07bf82207aee366dfaa3867b3164e4f03a238a461a11b0e8a3a510d51203d" + sha256: ae68c7bfcd7383af3629daafb32fb4e8681c7154428da4febcff06200585f102 url: "https://pub.dev" source: hosted - version: "3.1.1" + version: "3.1.2" platform_detect: dependency: "direct main" description: @@ -436,10 +436,10 @@ packages: dependency: transitive description: name: plugin_platform_interface - sha256: "43798d895c929056255600343db8f049921cbec94d31ec87f1dc5c16c01935dd" + sha256: da3fdfeccc4d4ff2da8f8c556704c08f912542c5fb3cf2233ed75372384a034d url: "https://pub.dev" source: hosted - version: "2.1.5" + version: "2.1.6" protobuf: dependency: "direct main" description: @@ -456,6 +456,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.4" + sdp_transform: + dependency: "direct main" + description: + name: sdp_transform + sha256: "73e412a5279a5c2de74001535208e20fff88f225c9a4571af0f7146202755e45" + url: "https://pub.dev" + source: hosted + version: "0.3.2" sky_engine: dependency: transitive description: flutter @@ -577,10 +585,10 @@ packages: dependency: transitive description: name: win32 - sha256: f2add6fa510d3ae152903412227bda57d0d5a8da61d2c39c1fb022c9429a41c0 + sha256: "9e82a402b7f3d518fb9c02d0e9ae45952df31b9bf34d77baf19da2de03fc2aaa" url: "https://pub.dev" source: hosted - version: "5.0.6" + version: "5.0.7" win32_registry: dependency: transitive description: @@ -593,10 +601,10 @@ packages: dependency: transitive description: name: xdg_directories - sha256: f0c26453a2d47aa4c2570c6a033246a3fc62da2fe23c7ffdd0a7495086dc0247 + sha256: "589ada45ba9e39405c198fe34eb0f607cddb2108527e658136120892beac46d2" url: "https://pub.dev" source: hosted - version: "1.0.2" + version: "1.0.3" xml: dependency: transitive description: @@ -615,4 +623,4 @@ packages: version: "3.1.2" sdks: dart: ">=3.0.0 <4.0.0" - flutter: ">=3.3.0" + flutter: ">=3.7.0" diff --git a/pubspec.yaml b/pubspec.yaml index f1ac7477..9da584d5 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -44,6 +44,7 @@ dependencies: js: ^0.6.4 platform_detect: ^2.0.7 dart_webrtc: 1.1.3 + sdp_transform: ^0.3.2 dev_dependencies: flutter_test: