diff --git a/.changeset/smart-wolves-thank.md b/.changeset/smart-wolves-thank.md new file mode 100644 index 0000000000..f93c07d11d --- /dev/null +++ b/.changeset/smart-wolves-thank.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +fast track publication diff --git a/example/sample.ts b/example/sample.ts index 013c6f72ec..06564475b4 100644 --- a/example/sample.ts +++ b/example/sample.ts @@ -107,6 +107,7 @@ const appActions = { e2ee: e2eeEnabled ? { keyProvider: state.e2eeKeyProvider, worker: new E2EEWorker() } : undefined, + fastPublish: true, }; if ( roomOpts.publishDefaults?.videoCodec === 'av1' || diff --git a/package.json b/package.json index 99e58983e2..45089aa9a1 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "size-limit": "size-limit" }, "dependencies": { - "@livekit/protocol": "1.20.0", + "@livekit/protocol": "1.20.1", "events": "^3.3.0", "loglevel": "^1.8.0", "sdp-transform": "^2.14.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09d4e09a31..6a0d6a1b1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@livekit/protocol': - specifier: 1.20.0 - version: 1.20.0 + specifier: 1.20.1 + version: 1.20.1 events: specifier: ^3.3.0 version: 3.3.0 @@ -1115,8 +1115,8 @@ packages: '@livekit/changesets-changelog-github@0.0.4': resolution: {integrity: sha512-MXaiLYwgkYciZb8G2wkVtZ1pJJzZmVx5cM30Q+ClslrIYyAqQhRbPmZDM79/5CGxb1MTemR/tfOM25tgJgAK0g==} - '@livekit/protocol@1.20.0': - resolution: {integrity: sha512-2RJQwzBa+MfUoy0zBWuyj8S2MTBxeTgREeG0r/1bNmkAFiBhsdgr87gIvblyqJxffUxJpALMu1Ee0M1XHX+9Ug==} + '@livekit/protocol@1.20.1': + resolution: {integrity: sha512-TgyuwOx+XJn9inEYT9OKfFNs9YIPS4BdLa4pF5FDf9MhWRnahKwPe7jxr/+sVdWxYbZmy9hRrH58jSAFu0ONHw==} '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -3278,8 +3278,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.6.0-dev.20240813: - resolution: {integrity: sha512-McNeLEHipa5j8HBxMoN8geftDXyyMHKiN0fGefa+4KzUTR/ZhDbIskwMhYxB159cFZ/sXfy7mzyPhLWyN5IQ5Q==} + typescript@5.7.0-dev.20240827: + resolution: {integrity: sha512-qNwNQBg18O4Z5RRGb07O562OpDlAVlytNcKfqcx8JQRJcs3p/KLHXjr0FbUbJ3SKoxA2vaQ3Zt89YLWHuCXzUw==} engines: {node: '>=14.17'} hasBin: true @@ -4772,7 +4772,7 @@ snapshots: transitivePeerDependencies: - encoding - '@livekit/protocol@1.20.0': + '@livekit/protocol@1.20.1': dependencies: '@bufbuild/protobuf': 1.10.0 @@ -5554,7 +5554,7 @@ snapshots: dependencies: semver: 7.6.0 shelljs: 0.8.5 - typescript: 5.6.0-dev.20240813 + typescript: 5.7.0-dev.20240827 electron-to-chromium@1.4.724: {} @@ -7097,7 +7097,7 @@ snapshots: typescript@5.5.4: {} - typescript@5.6.0-dev.20240813: {} + typescript@5.7.0-dev.20240827: {} uc.micro@2.1.0: {} diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index ba88dcbbbf..f20e607a48 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -23,6 +23,8 @@ eliminate this issue. */ const startBitrateForSVC = 0.7; +const debounceInterval = 20; + export const PCEvents = { NegotiationStarted: 'negotiationStarted', NegotiationComplete: 'negotiationComplete', @@ -217,18 +219,22 @@ export default class PCTransport extends EventEmitter { } // debounced negotiate interface - negotiate = debounce(async (onError?: (e: Error) => void) => { - this.emit(PCEvents.NegotiationStarted); - try { - await this.createAndSendOffer(); - } catch (e) { - if (onError) { - onError(e as Error); - } else { - throw e; + negotiate = debounce( + async (onError?: (e: Error) => void) => { + this.emit(PCEvents.NegotiationStarted); + try { + await this.createAndSendOffer(); + } catch (e) { + if (onError) { + onError(e as Error); + } else { + throw e; + } } - } - }, 100); + }, + debounceInterval, + { isImmediate: true }, + ); async createAndSendOffer(options?: RTCOfferOptions) { if (this.onOffer === undefined) { diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index de026d51c2..f63acf9a55 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -232,7 +232,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } // create offer - if (!this.subscriberPrimary) { + if (!this.subscriberPrimary || joinResponse.fastPublish) { this.negotiate(); } diff --git a/src/room/Room.ts b/src/room/Room.ts index 50d1009c8f..1392739d82 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -632,6 +632,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.localParticipant.sid = pi.sid; this.localParticipant.identity = pi.identity; + this.localParticipant.setEnabledPublishCodecs(joinResponse.enabledPublishCodecs); if (this.options.e2ee && this.e2eeManager) { try { diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 67a19e2f78..425a71ab46 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -1,5 +1,6 @@ import { AddTrackRequest, + Codec, DataPacket, DataPacket_Kind, Encryption_Type, @@ -9,6 +10,7 @@ import { RequestResponse_Reason, SimulcastCodec, SubscribedQualityUpdate, + TrackInfo, TrackUnpublishedResponse, UserPacket, } from '@livekit/protocol'; @@ -110,6 +112,8 @@ export default class LocalParticipant extends Participant { } >; + private enabledPublishVideoCodecs: Codec[] = []; + /** @internal */ constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions) { super(sid, identity, undefined, undefined, { @@ -775,6 +779,17 @@ export default class LocalParticipant extends Participant { if (opts.videoCodec === undefined) { opts.videoCodec = defaultVideoCodec; } + if (this.enabledPublishVideoCodecs.length > 0) { + // fallback to a supported codec if it is not supported + if ( + !this.enabledPublishVideoCodecs.some( + (c) => opts.videoCodec === mimeTypeToVideoCodecString(c.mime), + ) + ) { + opts.videoCodec = mimeTypeToVideoCodecString(this.enabledPublishVideoCodecs[0].mime); + } + } + const videoCodec = opts.videoCodec; // handle track actions @@ -908,33 +923,87 @@ export default class LocalParticipant extends Participant { throw new UnexpectedConnectionState('cannot publish track when not connected'); } - const ti = await this.engine.addTrack(req); - // server might not support the codec the client has requested, in that case, fallback - // to a supported codec - let primaryCodecMime: string | undefined; - ti.codecs.forEach((codec) => { - if (primaryCodecMime === undefined) { - primaryCodecMime = codec.mimeType; + const negotiate = async () => { + if (!this.engine.pcManager) { + throw new UnexpectedConnectionState('pcManager is not ready'); } - }); - if (primaryCodecMime && track.kind === Track.Kind.Video) { - const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime); - if (updatedCodec !== videoCodec) { - this.log.debug('falling back to server selected codec', { - ...this.logContext, - ...getLogContextFromTrack(track), - codec: updatedCodec, - }); - opts.videoCodec = updatedCodec; - - // recompute encodings since bitrates/etc could have changed - encodings = computeVideoEncodings( - track.source === Track.Source.ScreenShare, - req.width, - req.height, - opts, - ); + + track.sender = await this.engine.createSender(track, opts, encodings); + + if (track instanceof LocalVideoTrack) { + opts.degradationPreference ??= getDefaultDegradationPreference(track); + track.setDegradationPreference(opts.degradationPreference); + } + + if (encodings) { + if (isFireFox() && track.kind === Track.Kind.Audio) { + /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1, + livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to + publish high quality audio track. But firefox always uses this value as the actual + bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly. + So the client need to modify maxaverragebitrates in answer sdp to user provided value to + fix the issue. + */ + let trackTransceiver: RTCRtpTransceiver | undefined = undefined; + for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) { + if (transceiver.sender === track.sender) { + trackTransceiver = transceiver; + break; + } + } + if (trackTransceiver) { + this.engine.pcManager.publisher.setTrackCodecBitrate({ + transceiver: trackTransceiver, + codec: 'opus', + maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0, + }); + } + } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) { + this.engine.pcManager.publisher.setTrackCodecBitrate({ + cid: req.cid, + codec: track.codec, + maxbr: encodings[0].maxBitrate / 1000, + }); + } + } + + await this.engine.negotiate(); + }; + + let ti: TrackInfo; + if (this.enabledPublishVideoCodecs.length > 0) { + const rets = await Promise.all([this.engine.addTrack(req), negotiate()]); + ti = rets[0]; + } else { + ti = await this.engine.addTrack(req); + // server might not support the codec the client has requested, in that case, fallback + // to a supported codec + let primaryCodecMime: string | undefined; + ti.codecs.forEach((codec) => { + if (primaryCodecMime === undefined) { + primaryCodecMime = codec.mimeType; + } + }); + if (primaryCodecMime && track.kind === Track.Kind.Video) { + const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime); + if (updatedCodec !== videoCodec) { + this.log.debug('falling back to server selected codec', { + ...this.logContext, + ...getLogContextFromTrack(track), + codec: updatedCodec, + }); + opts.videoCodec = updatedCodec; + + // recompute encodings since bitrates/etc could have changed + encodings = computeVideoEncodings( + track.source === Track.Source.ScreenShare, + req.width, + req.height, + opts, + ); + } } + await negotiate(); } const publication = new LocalTrackPublication(track.kind, ti, track, { @@ -945,56 +1014,12 @@ export default class LocalParticipant extends Participant { publication.options = opts; track.sid = ti.sid; - if (!this.engine.pcManager) { - throw new UnexpectedConnectionState('pcManager is not ready'); - } this.log.debug(`publishing ${track.kind} with encodings`, { ...this.logContext, encodings, trackInfo: ti, }); - track.sender = await this.engine.createSender(track, opts, encodings); - - if (track instanceof LocalVideoTrack) { - opts.degradationPreference ??= getDefaultDegradationPreference(track); - track.setDegradationPreference(opts.degradationPreference); - } - - if (encodings) { - if (isFireFox() && track.kind === Track.Kind.Audio) { - /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1, - livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to - publish high quality audio track. But firefox always uses this value as the actual - bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly. - So the client need to modify maxaverragebitrates in answer sdp to user provided value to - fix the issue. - */ - let trackTransceiver: RTCRtpTransceiver | undefined = undefined; - for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) { - if (transceiver.sender === track.sender) { - trackTransceiver = transceiver; - break; - } - } - if (trackTransceiver) { - this.engine.pcManager.publisher.setTrackCodecBitrate({ - transceiver: trackTransceiver, - codec: 'opus', - maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0, - }); - } - } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) { - this.engine.pcManager.publisher.setTrackCodecBitrate({ - cid: req.cid, - codec: track.codec, - maxbr: encodings[0].maxBitrate / 1000, - }); - } - } - - await this.engine.negotiate(); - if (track instanceof LocalVideoTrack) { track.startMonitor(this.engine.client); } else if (track instanceof LocalAudioTrack) { @@ -1081,15 +1106,19 @@ export default class LocalParticipant extends Participant { throw new UnexpectedConnectionState('cannot publish track when not connected'); } - const ti = await this.engine.addTrack(req); + const negotiate = async () => { + const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' }; + if (encodings) { + transceiverInit.sendEncodings = encodings; + } + await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings); - const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' }; - if (encodings) { - transceiverInit.sendEncodings = encodings; - } - await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings); + await this.engine.negotiate(); + }; + + const rets = await Promise.all([this.engine.addTrack(req), negotiate()]); + const ti = rets[0]; - await this.engine.negotiate(); this.log.debug(`published ${videoCodec} for track ${track.sid}`, { ...this.logContext, encodings, @@ -1309,6 +1338,13 @@ export default class LocalParticipant extends Participant { } } + /** @internal */ + setEnabledPublishCodecs(codecs: Codec[]) { + this.enabledPublishVideoCodecs = codecs.filter( + (c) => c.mime.split('/')[0].toLowerCase() === 'video', + ); + } + /** @internal */ updateInfo(info: ParticipantInfo): boolean { if (info.sid !== this.sid) {