diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 37dae3d1fb..922f97f1ba 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio, toggleDtx } from '../sdp-munging'; +import { + enableHighQualityAudio, + preserveCodec, + toggleDtx, +} from '../sdp-munging'; import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; describe('sdp-munging', () => { @@ -21,4 +25,167 @@ a=maxptime:40`; expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); expect(sdpWithHighQualityAudio).toContain('stereo=1'); }); + + it('preserves the preferred codec', () => { + const sdp = `v=0 +o=- 8608371809202407637 2 IN IP4 127.0.0.1 +s=- +t=0 0 +a=extmap-allow-mixed +a=msid-semantic: WMS 52fafc21-b8bb-4f4f-8072-86a29cb6590e +a=group:BUNDLE 0 +m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101 +c=IN IP4 0.0.0.0 +a=rtpmap:98 VP9/90000 +a=rtpmap:99 rtx/90000 +a=rtpmap:100 VP9/90000 +a=rtpmap:101 rtx/90000 +a=fmtp:98 profile-id=0 +a=fmtp:99 apt=98 +a=fmtp:100 profile-id=2 +a=fmtp:101 apt=100 +a=rtcp:9 IN IP4 0.0.0.0 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00 +a=setup:actpass +a=mid:0 +a=msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232 +a=sendonly +a=ice-ufrag:LvRk +a=ice-pwd:IpBRr2Rrg9TkOgayjYqALhPY +a=fingerprint:sha-256 18:DE:8F:ED:E6:A2:0C:99:A8:25:AB:C9:F8:3D:91:4C:3E:9F:B4:1F:22:87:A7:3C:85:8F:F3:51:09:A7:E3:FA +a=ice-options:trickle +a=ssrc:3192778601 cname:yYSN5R+RG2j3luO7 +a=ssrc:3192778601 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232 +a=ssrc:283365205 cname:yYSN5R+RG2j3luO7 +a=ssrc:283365205 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232 +a=ssrc-group:FID 3192778601 283365205 +a=rtcp-mux +a=rtcp-rsize`; + const target = preserveCodec(sdp, '0', { + mimeType: 'video/VP9', + clockRate: 90000, + sdpFmtpLine: 'profile-id=0', + }); + expect(target).toContain('VP9'); + expect(target).not.toContain('profile-id=2'); + }); + + it('handles ios munging', () => { + const sdp = `v=0 +o=- 525780719364332676 2 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 +a=extmap-allow-mixed +a=msid-semantic: WMS BF3AFE62-88F8-4189-99D7-7CAE159205E3 +m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:SAkq +a=ice-pwd:FYHHro0VWRO8CjI/M1VG5vRw +a=ice-options:trickle renomination +a=fingerprint:sha-256 03:5B:16:0E:E1:7B:FE:4F:9A:5C:AC:CF:08:21:4B:49:CE:53:79:E6:97:AE:4E:73:F8:43:34:C3:11:F7:6D:E7 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00 +a=sendonly +a=msid:BF3AFE62-88F8-4189-99D7-7CAE159205E3 6013DC02-A0A5-43A9-9D41-9D4A89648A42 +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 H264/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29 +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:98 H264/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP8/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:127 VP9/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=127 +a=rtpmap:35 AV1/90000 +a=rtcp-fb:35 goog-remb +a=rtcp-fb:35 transport-cc +a=rtcp-fb:35 ccm fir +a=rtcp-fb:35 nack +a=rtcp-fb:35 nack pli +a=rtpmap:36 rtx/90000 +a=fmtp:36 apt=35 +a=rtpmap:104 red/90000 +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 ulpfec/90000 +a=rid:q send +a=rid:h send +a=rid:f send +a=simulcast:send q;h;f`; + const target = preserveCodec(sdp, '0', { + mimeType: 'video/H264', + clockRate: 90000, + sdpFmtpLine: + 'profile-level-id=42e029;packetization-mode=1;level-asymmetry-allowed=1', + }); + expect(target).toContain('H264'); + expect(target).toContain('profile-level-id=42e029'); + expect(target).not.toContain('profile-level-id=640c29'); + expect(target).not.toContain('VP9'); + expect(target).not.toContain('AV1'); + }); }); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index b8bdfd66af..317c1a8e7a 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -129,6 +129,61 @@ export const toggleDtx = (sdp: string, enable: boolean): string => { return sdp.replace(opusFmtp.original, newFmtp); }; +/** + * Returns and SDP with all the codecs except the given codec removed. + */ +export const preserveCodec = ( + sdp: string, + mid: string, + codec: RTCRtpCodec, +): string => { + const [kind, codecName] = codec.mimeType.toLowerCase().split('/'); + + const toSet = (fmtpLine: string) => + new Set(fmtpLine.split(';').map((f) => f.trim().toLowerCase())); + + const equal = (a: Set, b: Set) => { + if (a.size !== b.size) return false; + for (const item of a) if (!b.has(item)) return false; + return true; + }; + + const codecFmtp = toSet(codec.sdpFmtpLine || ''); + const parsedSdp = SDP.parse(sdp); + for (const media of parsedSdp.media) { + if (media.type !== kind || String(media.mid) !== mid) continue; + + // find the payload id of the desired codec + const payloads = new Set(); + for (const rtp of media.rtp) { + if ( + rtp.codec.toLowerCase() === codecName && + media.fmtp.some( + (f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp), + ) + ) { + payloads.add(rtp.payload); + } + } + + // find the corresponding rtx codec by matching apt= + for (const fmtp of media.fmtp) { + const match = fmtp.config.match(/(apt)=(\d+)/); + if (!match) continue; + const [, , preservedCodecPayload] = match; + if (payloads.has(Number(preservedCodecPayload))) { + payloads.add(fmtp.payload); + } + } + + media.rtp = media.rtp.filter((r) => payloads.has(r.payload)); + media.fmtp = media.fmtp.filter((f) => payloads.has(f.payload)); + media.rtcpFb = media.rtcpFb?.filter((f) => payloads.has(f.payload)); + media.payloads = Array.from(payloads).join(' '); + } + return SDP.write(parsedSdp); +}; + /** * Enables high-quality audio through SDP munging for the given trackMid. * diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index b223dfc4e0..f479101e53 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -20,6 +20,7 @@ import { PublishOptions } from '../types'; import { enableHighQualityAudio, extractMid, + preserveCodec, toggleDtx, } from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; @@ -530,6 +531,12 @@ export class Publisher { if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { offer.sdp = this.enableHighQualityAudio(offer.sdp); } + if (this.isPublishing(TrackType.VIDEO)) { + // Hotfix for platforms that don't respect the ordered codec list + // (Firefox, Android, Linux, etc...). + // We remove all the codecs from the SDP except the one we want to use. + offer.sdp = this.removeUnpreferredCodecs(offer.sdp, TrackType.VIDEO); + } } const trackInfos = this.getAnnouncedTracks(offer.sdp); @@ -564,6 +571,23 @@ export class Publisher { ); }; + private removeUnpreferredCodecs(sdp: string, trackType: TrackType): string { + const opts = this.publishOptsForTrack.get(trackType); + if (!opts || !opts.forceSingleCodec) return sdp; + + const codec = opts.forceCodec || opts.preferredCodec; + const orderedCodecs = this.getCodecPreferences(trackType, codec); + if (!orderedCodecs || orderedCodecs.length === 0) return sdp; + + const transceiver = this.transceiverCache.get(trackType); + if (!transceiver) return sdp; + + const index = this.transceiverInitOrder.indexOf(trackType); + const mid = extractMid(transceiver, index, sdp); + const [codecToPreserve] = orderedCodecs; + return preserveCodec(sdp, mid, codecToPreserve); + } + private enableHighQualityAudio = (sdp: string) => { const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); if (!transceiver) return sdp; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 8c498c029c..01a8707f9d 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -50,7 +50,7 @@ export const getPreferredCodecs = ( } const sdpFmtpLine = codec.sdpFmtpLine; - if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42e01f')) { + if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42')) { // this is not the baseline h264 codec, prioritize it lower partiallyPreferred.push(codec); continue; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 3cb8a59858..9d951b3ba2 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -167,6 +167,12 @@ export type PublishOptions = { * Use with caution. */ forceCodec?: PreferredCodec; + /** + * When using a preferred codec, force the use of a single codec. + * Enabling this, it will remove all other supported codecs from the SDP. + * Defaults to false. + */ + forceSingleCodec?: boolean; /** * The preferred scalability to use when publishing the video stream. * Applicable only for SVC codecs. diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 7b74d416d6..34176ee561 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -50,6 +50,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { | PreferredCodec | undefined; const bitrateOverride = router.query['bitrate'] as string | undefined; + const forceSingleCodec = router.query['force_single_codec'] === 'true'; const bitrateFactorOverride = router.query['bitrate_factor'] as | string | undefined; @@ -72,6 +73,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { call.updatePublishOptions({ preferredCodec: 'vp9', forceCodec: videoCodecOverride, + forceSingleCodec, scalabilityMode, preferredBitrate, bitrateDownscaleFactor: bitrateFactorOverride @@ -94,6 +96,8 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { bitrateFactorOverride, bitrateOverride, call, + forceSingleCodec, + maxSimulcastLayers, scalabilityMode, videoCodecOverride, ],