fix(ios): use vp8 when h264 constrainted baseline isn't available (#1597

) - Use VP8 on RN iOS when constrained h264 (`42e01f`) isn't available. - Favor `packetization-mode=1` for h264 --------- Co-authored-by: Santhosh Vaiyapuri <[email protected]>
GetStream · Nov 27, 2024 · 6281216 · 6281216
1 parent da6461b
commit 6281216
Show file tree

Hide file tree

Showing 7 changed files with 140 additions and 26 deletions.
diff --git a/.github/workflows/react-native-workflow.yml b/.github/workflows/react-native-workflow.yml
@@ -130,7 +130,7 @@ jobs:
     name: Deploy iOS
     needs: build_ios
     timeout-minutes: 60
-    if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/PBE-5855-feat/react-native-video-design-v2' }}
+    if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/use-vp8-on-ios' }}
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4

diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts
@@ -188,4 +188,96 @@ a=simulcast:send q;h;f`;
     expect(target).not.toContain('VP9');
     expect(target).not.toContain('AV1');
   });
+
+  it('works with iOS RN vp8', () => {
+    const sdp = `v=0
+o=- 2055959380019004946 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=extmap-allow-mixed
+a=msid-semantic: WMS FE2B3B06-61D7-4ACC-A4EF-76441C116E47
+m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=ice-ufrag:gCgh
+a=ice-pwd:bz18EOLBL9+kSJfLiVOyU4RP
+a=ice-options:trickle renomination
+a=fingerprint:sha-256 6B:04:36:6D:E6:92:B5:68:DA:30:CF:53:46:14:49:5B:48:3E:B9:F7:06:B4:E8:85:B1:8C:B3:1C:EB:E8:F8:16
+a=setup:actpass
+a=mid:0
+a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=extmap:3 urn:3gpp:video-orientation
+a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
+a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
+a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
+a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
+a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
+a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
+a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
+a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00
+a=sendonly
+a=msid:FE2B3B06-61D7-4ACC-A4EF-76441C116E47 93FCE555-1DA2-4721-901C-5D263E11DF23
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:96 H264/90000
+a=rtcp-fb:96 goog-remb
+a=rtcp-fb:96 transport-cc
+a=rtcp-fb:96 ccm fir
+a=rtcp-fb:96 nack
+a=rtcp-fb:96 nack pli
+a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29
+a=rtpmap:97 rtx/90000
+a=fmtp:97 apt=96
+a=rtpmap:98 H264/90000
+a=rtcp-fb:98 goog-remb
+a=rtcp-fb:98 transport-cc
+a=rtcp-fb:98 ccm fir
+a=rtcp-fb:98 nack
+a=rtcp-fb:98 nack pli
+a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029
+a=rtpmap:99 rtx/90000
+a=fmtp:99 apt=98
+a=rtpmap:100 VP8/90000
+a=rtcp-fb:100 goog-remb
+a=rtcp-fb:100 transport-cc
+a=rtcp-fb:100 ccm fir
+a=rtcp-fb:100 nack
+a=rtcp-fb:100 nack pli
+a=rtpmap:101 rtx/90000
+a=fmtp:101 apt=100
+a=rtpmap:127 VP9/90000
+a=rtcp-fb:127 goog-remb
+a=rtcp-fb:127 transport-cc
+a=rtcp-fb:127 ccm fir
+a=rtcp-fb:127 nack
+a=rtcp-fb:127 nack pli
+a=rtpmap:103 rtx/90000
+a=fmtp:103 apt=127
+a=rtpmap:35 AV1/90000
+a=rtcp-fb:35 goog-remb
+a=rtcp-fb:35 transport-cc
+a=rtcp-fb:35 ccm fir
+a=rtcp-fb:35 nack
+a=rtcp-fb:35 nack pli
+a=rtpmap:36 rtx/90000
+a=fmtp:36 apt=35
+a=rtpmap:104 red/90000
+a=rtpmap:105 rtx/90000
+a=fmtp:105 apt=104
+a=rtpmap:106 ulpfec/90000
+a=rid:q send
+a=rid:h send
+a=rid:f send
+a=simulcast:send q;h;f`;
+    const target = preserveCodec(sdp, '0', {
+      clockRate: 90000,
+      mimeType: 'video/VP8',
+    });
+    expect(target).toContain('VP8');
+    expect(target).not.toContain('VP9');
+  });
 });
diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts
@@ -156,12 +156,16 @@ export const preserveCodec = (
     // find the payload id of the desired codec
     const payloads = new Set<number>();
     for (const rtp of media.rtp) {
-      if (
-        rtp.codec.toLowerCase() === codecName &&
-        media.fmtp.some(
-          (f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp),
-        )
-      ) {
+      if (rtp.codec.toLowerCase() !== codecName) continue;
+      const match =
+        // vp8 doesn't have any fmtp, we preserve it without any additional checks
+        codecName === 'vp8'
+          ? true
+          : media.fmtp.some(
+              (f) =>
+                f.payload === rtp.payload && equal(toSet(f.config), codecFmtp),
+            );
+      if (match) {
         payloads.add(rtp.payload);
       }
     }

diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts
@@ -29,6 +29,8 @@ import { Dispatcher } from './Dispatcher';
 import { VideoLayerSetting } from '../gen/video/sfu/event/events';
 import { TargetResolutionResponse } from '../gen/shims';
 import { withoutConcurrency } from '../helpers/concurrency';
+import { isReactNative } from '../helpers/platforms';
+import { isFirefox } from '../helpers/browsers';
 
 export type PublisherConstructorOpts = {
   sfuClient: StreamSfuClient;
@@ -256,6 +258,7 @@ export class Publisher {
     const codecPreferences = this.getCodecPreferences(
       trackType,
       trackType === TrackType.VIDEO ? codecInUse : undefined,
+      'receiver',
     );
     if (!codecPreferences) return;
 
@@ -458,13 +461,14 @@ export class Publisher {
 
   private getCodecPreferences = (
     trackType: TrackType,
-    preferredCodec?: string,
-    codecPreferencesSource?: 'sender' | 'receiver',
+    preferredCodec: string | undefined,
+    codecPreferencesSource: 'sender' | 'receiver',
   ) => {
     if (trackType === TrackType.VIDEO) {
       return getPreferredCodecs(
         'video',
         preferredCodec || 'vp8',
+        undefined,
         codecPreferencesSource,
       );
     }
@@ -475,6 +479,7 @@ export class Publisher {
         'audio',
         preferredCodec ?? defaultAudioCodec,
         codecToRemove,
+        codecPreferencesSource,
       );
     }
   };
@@ -578,7 +583,9 @@ export class Publisher {
 
   private removeUnpreferredCodecs(sdp: string, trackType: TrackType): string {
     const opts = this.publishOptsForTrack.get(trackType);
-    if (!opts || !opts.forceSingleCodec) return sdp;
+    const forceSingleCodec =
+      !!opts?.forceSingleCodec || isReactNative() || isFirefox();
+    if (!opts || !forceSingleCodec) return sdp;
 
     const codec = opts.forceCodec || getOptimalVideoCodec(opts.preferredCodec);
     const orderedCodecs = this.getCodecPreferences(trackType, codec, 'sender');

diff --git a/packages/client/src/rtc/__tests__/codecs.test.ts b/packages/client/src/rtc/__tests__/codecs.test.ts
@@ -5,7 +5,7 @@ import './mocks/webrtc.mocks';
 describe('codecs', () => {
   it('should return preferred audio codec', () => {
     RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(audioCodecs);
-    const codecs = getPreferredCodecs('audio', 'red');
+    const codecs = getPreferredCodecs('audio', 'red', undefined, 'receiver');
     expect(codecs).toBeDefined();
     expect(codecs?.map((c) => c.mimeType)).toEqual([
       'audio/red',
@@ -20,7 +20,7 @@ describe('codecs', () => {
 
   it('should return preferred video codec', () => {
     RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(videoCodecs);
-    const codecs = getPreferredCodecs('video', 'vp8');
+    const codecs = getPreferredCodecs('video', 'vp8', undefined, 'receiver');
     expect(codecs).toBeDefined();
     // prettier-ignore
     expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([
@@ -40,12 +40,12 @@ describe('codecs', () => {
 
   it('should pick the baseline H264 codec', () => {
     RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(videoCodecs);
-    const codecs = getPreferredCodecs('video', 'h264');
+    const codecs = getPreferredCodecs('video', 'h264', undefined, 'receiver');
     expect(codecs).toBeDefined();
     // prettier-ignore
     expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([
-      ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'],
       ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'],
+      ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'],
       ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f'],
       ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640c1f'],
       ['video/rtx', undefined],
@@ -62,12 +62,12 @@ describe('codecs', () => {
     RTCRtpReceiver.getCapabilities = vi
       .fn()
       .mockReturnValue(videoCodecsFirefox);
-    const codecs = getPreferredCodecs('video', 'h264');
+    const codecs = getPreferredCodecs('video', 'h264', undefined, 'receiver');
     expect(codecs).toBeDefined();
     // prettier-ignore
     expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([
-      ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1'],
       ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1;packetization-mode=1'],
+      ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1'],
       ['video/VP8', 'max-fs=12288;max-fr=60'],
       ['video/rtx', undefined],
       ['video/VP9', 'max-fs=12288;max-fr=60'],

diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts
@@ -14,8 +14,8 @@ import type { PreferredCodec } from '../types';
 export const getPreferredCodecs = (
   kind: 'audio' | 'video',
   preferredCodec: string,
-  codecToRemove?: string,
-  codecPreferencesSource: 'sender' | 'receiver' = 'receiver',
+  codecToRemove: string | undefined,
+  codecPreferencesSource: 'sender' | 'receiver',
 ): RTCRtpCodec[] | undefined => {
   const source =
     codecPreferencesSource === 'receiver' ? RTCRtpReceiver : RTCRtpSender;
@@ -60,12 +60,7 @@ export const getPreferredCodecs = (
       continue;
     }
 
-    // packetization-mode mode is optional; when not present it defaults to 0:
-    // https://datatracker.ietf.org/doc/html/rfc6184#section-6.2
-    if (
-      sdpFmtpLine.includes('packetization-mode=0') ||
-      !sdpFmtpLine.includes('packetization-mode')
-    ) {
+    if (sdpFmtpLine.includes('packetization-mode=1')) {
       preferred.unshift(codec);
     } else {
       preferred.push(codec);
@@ -107,7 +102,9 @@ export const getOptimalVideoCodec = (
   if (isReactNative()) {
     const os = getOSInfo()?.name.toLowerCase();
     if (os === 'android') return preferredOr(preferredCodec, 'vp8');
-    if (os === 'ios' || os === 'ipados') return 'h264';
+    if (os === 'ios' || os === 'ipados') {
+      return supportsH264Baseline() ? 'h264' : 'vp8';
+    }
     return preferredOr(preferredCodec, 'h264');
   }
   if (isSafari()) return 'h264';
@@ -139,6 +136,20 @@ const preferredOr = (
     : fallback;
 };
 
+/**
+ * Returns whether the platform supports the H264 baseline codec.
+ */
+const supportsH264Baseline = (): boolean => {
+  if (!('getCapabilities' in RTCRtpSender)) return false;
+  const capabilities = RTCRtpSender.getCapabilities('video');
+  if (!capabilities) return false;
+  return capabilities.codecs.some(
+    (c) =>
+      c.mimeType.toLowerCase() === 'video/h264' &&
+      c.sdpFmtpLine?.includes('profile-level-id=42e01f'),
+  );
+};
+
 /**
  * Returns whether the codec is an SVC codec.
  *

diff --git a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx
@@ -61,7 +61,7 @@ export const MeetingUI = ({ callId, navigation, route }: Props) => {
     try {
       call?.updatePublishOptions({
         preferredCodec: 'vp9',
-        preferredBitrate: 1500000,
+        forceSingleCodec: true,
       });
       await call?.join({ create: true });
       appStoreSetState({ chatLabelNoted: false });