Fast track publication (#1228)

* Speed up track publication Add fastPublish option to establish publisher peerconnection when joining. Neogtiate new track without AddTrackRequest response * changeset * don't do parallel negotiate if no enabled codecs return * Update src/options.ts Co-authored-by: lukasIO <[email protected]> * negotiate error * use isImmediate option of ts-debounce * Use fastPublish option in JoinResponse Also reduce the negotiate debounce interval * Update protocol dependency --------- Co-authored-by: lukasIO <[email protected]>
livekit · Aug 28, 2024 · 10b3b68 · 10b3b68
1 parent 2d2e92b
commit 10b3b68
Show file tree

Hide file tree

Showing 8 changed files with 147 additions and 98 deletions.
diff --git a/.changeset/smart-wolves-thank.md b/.changeset/smart-wolves-thank.md
@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+fast track publication
diff --git a/example/sample.ts b/example/sample.ts
@@ -107,6 +107,7 @@ const appActions = {
       e2ee: e2eeEnabled
         ? { keyProvider: state.e2eeKeyProvider, worker: new E2EEWorker() }
         : undefined,
+      fastPublish: true,
     };
     if (
       roomOpts.publishDefaults?.videoCodec === 'av1' ||

diff --git a/package.json b/package.json
@@ -53,7 +53,7 @@
     "size-limit": "size-limit"
   },
   "dependencies": {
-    "@livekit/protocol": "1.20.0",
+    "@livekit/protocol": "1.20.1",
     "events": "^3.3.0",
     "loglevel": "^1.8.0",
     "sdp-transform": "^2.14.1",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts
@@ -23,6 +23,8 @@ eliminate this issue.
 */
 const startBitrateForSVC = 0.7;
 
+const debounceInterval = 20;
+
 export const PCEvents = {
   NegotiationStarted: 'negotiationStarted',
   NegotiationComplete: 'negotiationComplete',
@@ -217,18 +219,22 @@ export default class PCTransport extends EventEmitter {
   }
 
   // debounced negotiate interface
-  negotiate = debounce(async (onError?: (e: Error) => void) => {
-    this.emit(PCEvents.NegotiationStarted);
-    try {
-      await this.createAndSendOffer();
-    } catch (e) {
-      if (onError) {
-        onError(e as Error);
-      } else {
-        throw e;
+  negotiate = debounce(
+    async (onError?: (e: Error) => void) => {
+      this.emit(PCEvents.NegotiationStarted);
+      try {
+        await this.createAndSendOffer();
+      } catch (e) {
+        if (onError) {
+          onError(e as Error);
+        } else {
+          throw e;
+        }
       }
-    }
-  }, 100);
+    },
+    debounceInterval,
+    { isImmediate: true },
+  );
 
   async createAndSendOffer(options?: RTCOfferOptions) {
     if (this.onOffer === undefined) {

diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts
@@ -232,7 +232,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       }
 
       // create offer
-      if (!this.subscriberPrimary) {
+      if (!this.subscriberPrimary || joinResponse.fastPublish) {
         this.negotiate();
       }
 

diff --git a/src/room/Room.ts b/src/room/Room.ts
@@ -632,6 +632,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
     this.localParticipant.sid = pi.sid;
     this.localParticipant.identity = pi.identity;
+    this.localParticipant.setEnabledPublishCodecs(joinResponse.enabledPublishCodecs);
 
     if (this.options.e2ee && this.e2eeManager) {
       try {

diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts
@@ -1,5 +1,6 @@
 import {
   AddTrackRequest,
+  Codec,
   DataPacket,
   DataPacket_Kind,
   Encryption_Type,
@@ -9,6 +10,7 @@ import {
   RequestResponse_Reason,
   SimulcastCodec,
   SubscribedQualityUpdate,
+  TrackInfo,
   TrackUnpublishedResponse,
   UserPacket,
 } from '@livekit/protocol';
@@ -110,6 +112,8 @@ export default class LocalParticipant extends Participant {
     }
   >;
 
+  private enabledPublishVideoCodecs: Codec[] = [];
+
   /** @internal */
   constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions) {
     super(sid, identity, undefined, undefined, {
@@ -775,6 +779,17 @@ export default class LocalParticipant extends Participant {
     if (opts.videoCodec === undefined) {
       opts.videoCodec = defaultVideoCodec;
     }
+    if (this.enabledPublishVideoCodecs.length > 0) {
+      // fallback to a supported codec if it is not supported
+      if (
+        !this.enabledPublishVideoCodecs.some(
+          (c) => opts.videoCodec === mimeTypeToVideoCodecString(c.mime),
+        )
+      ) {
+        opts.videoCodec = mimeTypeToVideoCodecString(this.enabledPublishVideoCodecs[0].mime);
+      }
+    }
+
     const videoCodec = opts.videoCodec;
 
     // handle track actions
@@ -908,33 +923,87 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('cannot publish track when not connected');
     }
 
-    const ti = await this.engine.addTrack(req);
-    // server might not support the codec the client has requested, in that case, fallback
-    // to a supported codec
-    let primaryCodecMime: string | undefined;
-    ti.codecs.forEach((codec) => {
-      if (primaryCodecMime === undefined) {
-        primaryCodecMime = codec.mimeType;
+    const negotiate = async () => {
+      if (!this.engine.pcManager) {
+        throw new UnexpectedConnectionState('pcManager is not ready');
       }
-    });
-    if (primaryCodecMime && track.kind === Track.Kind.Video) {
-      const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
-      if (updatedCodec !== videoCodec) {
-        this.log.debug('falling back to server selected codec', {
-          ...this.logContext,
-          ...getLogContextFromTrack(track),
-          codec: updatedCodec,
-        });
-        opts.videoCodec = updatedCodec;
-
-        // recompute encodings since bitrates/etc could have changed
-        encodings = computeVideoEncodings(
-          track.source === Track.Source.ScreenShare,
-          req.width,
-          req.height,
-          opts,
-        );
+
+      track.sender = await this.engine.createSender(track, opts, encodings);
+
+      if (track instanceof LocalVideoTrack) {
+        opts.degradationPreference ??= getDefaultDegradationPreference(track);
+        track.setDegradationPreference(opts.degradationPreference);
+      }
+
+      if (encodings) {
+        if (isFireFox() && track.kind === Track.Kind.Audio) {
+          /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
+             livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
+             publish high quality audio track. But firefox always uses this value as the actual
+             bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
+             So the client need to modify maxaverragebitrates in answer sdp to user provided value to
+             fix the issue.
+           */
+          let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
+          for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
+            if (transceiver.sender === track.sender) {
+              trackTransceiver = transceiver;
+              break;
+            }
+          }
+          if (trackTransceiver) {
+            this.engine.pcManager.publisher.setTrackCodecBitrate({
+              transceiver: trackTransceiver,
+              codec: 'opus',
+              maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
+            });
+          }
+        } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
+          this.engine.pcManager.publisher.setTrackCodecBitrate({
+            cid: req.cid,
+            codec: track.codec,
+            maxbr: encodings[0].maxBitrate / 1000,
+          });
+        }
+      }
+
+      await this.engine.negotiate();
+    };
+
+    let ti: TrackInfo;
+    if (this.enabledPublishVideoCodecs.length > 0) {
+      const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+      ti = rets[0];
+    } else {
+      ti = await this.engine.addTrack(req);
+      // server might not support the codec the client has requested, in that case, fallback
+      // to a supported codec
+      let primaryCodecMime: string | undefined;
+      ti.codecs.forEach((codec) => {
+        if (primaryCodecMime === undefined) {
+          primaryCodecMime = codec.mimeType;
+        }
+      });
+      if (primaryCodecMime && track.kind === Track.Kind.Video) {
+        const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
+        if (updatedCodec !== videoCodec) {
+          this.log.debug('falling back to server selected codec', {
+            ...this.logContext,
+            ...getLogContextFromTrack(track),
+            codec: updatedCodec,
+          });
+          opts.videoCodec = updatedCodec;
+
+          // recompute encodings since bitrates/etc could have changed
+          encodings = computeVideoEncodings(
+            track.source === Track.Source.ScreenShare,
+            req.width,
+            req.height,
+            opts,
+          );
+        }
       }
+      await negotiate();
     }
 
     const publication = new LocalTrackPublication(track.kind, ti, track, {
@@ -945,56 +1014,12 @@ export default class LocalParticipant extends Participant {
     publication.options = opts;
     track.sid = ti.sid;
 
-    if (!this.engine.pcManager) {
-      throw new UnexpectedConnectionState('pcManager is not ready');
-    }
     this.log.debug(`publishing ${track.kind} with encodings`, {
       ...this.logContext,
       encodings,
       trackInfo: ti,
     });
 
-    track.sender = await this.engine.createSender(track, opts, encodings);
-
-    if (track instanceof LocalVideoTrack) {
-      opts.degradationPreference ??= getDefaultDegradationPreference(track);
-      track.setDegradationPreference(opts.degradationPreference);
-    }
-
-    if (encodings) {
-      if (isFireFox() && track.kind === Track.Kind.Audio) {
-        /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
-           livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
-           publish high quality audio track. But firefox always uses this value as the actual
-           bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
-           So the client need to modify maxaverragebitrates in answer sdp to user provided value to
-           fix the issue.
-         */
-        let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
-        for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
-          if (transceiver.sender === track.sender) {
-            trackTransceiver = transceiver;
-            break;
-          }
-        }
-        if (trackTransceiver) {
-          this.engine.pcManager.publisher.setTrackCodecBitrate({
-            transceiver: trackTransceiver,
-            codec: 'opus',
-            maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
-          });
-        }
-      } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
-        this.engine.pcManager.publisher.setTrackCodecBitrate({
-          cid: req.cid,
-          codec: track.codec,
-          maxbr: encodings[0].maxBitrate / 1000,
-        });
-      }
-    }
-
-    await this.engine.negotiate();
-
     if (track instanceof LocalVideoTrack) {
       track.startMonitor(this.engine.client);
     } else if (track instanceof LocalAudioTrack) {
@@ -1081,15 +1106,19 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('cannot publish track when not connected');
     }
 
-    const ti = await this.engine.addTrack(req);
+    const negotiate = async () => {
+      const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+      if (encodings) {
+        transceiverInit.sendEncodings = encodings;
+      }
+      await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
 
-    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-    if (encodings) {
-      transceiverInit.sendEncodings = encodings;
-    }
-    await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
+      await this.engine.negotiate();
+    };
+
+    const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+    const ti = rets[0];
 
-    await this.engine.negotiate();
     this.log.debug(`published ${videoCodec} for track ${track.sid}`, {
       ...this.logContext,
       encodings,
@@ -1309,6 +1338,13 @@ export default class LocalParticipant extends Participant {
     }
   }
 
+  /** @internal */
+  setEnabledPublishCodecs(codecs: Codec[]) {
+    this.enabledPublishVideoCodecs = codecs.filter(
+      (c) => c.mime.split('/')[0].toLowerCase() === 'video',
+    );
+  }
+
   /** @internal */
   updateInfo(info: ParticipantInfo): boolean {
     if (info.sid !== this.sid) {