From 1e8465ab3202be0542580648c6443846a39ccfe4 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Fri, 17 May 2024 21:58:17 +0300 Subject: [PATCH] Support SIP DTMF data messages (#1130) * Support SIP DTMF data messages --- .changeset/fifty-steaks-fly.md | 5 +++++ src/room/RTCEngine.ts | 29 +++++++++++++++++++----- src/room/Room.ts | 35 ++++++++++++++++++++++++----- src/room/events.ts | 15 ++++++++++++- src/room/participant/Participant.ts | 2 ++ 5 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 .changeset/fifty-steaks-fly.md diff --git a/.changeset/fifty-steaks-fly.md b/.changeset/fifty-steaks-fly.md new file mode 100644 index 0000000000..502ec928c7 --- /dev/null +++ b/.changeset/fifty-steaks-fly.md @@ -0,0 +1,5 @@ +--- +"livekit-client": minor +--- + +Support SIP DTMF data messages. diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 269ebbb71e..6f46be3ad5 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -26,7 +26,7 @@ import { TrackUnpublishedResponse, Transcription, UpdateSubscription, - UserPacket, + type UserPacket, } from '@livekit/protocol'; import { EventEmitter } from 'events'; import type { MediaAttributes } from 'sdp-transform'; @@ -648,10 +648,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit if (dp.value?.case === 'speaker') { // dispatch speaker updates this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers); - } else if (dp.value?.case === 'user') { - this.emit(EngineEvent.DataPacketReceived, dp.value.value, dp.kind); - } else if (dp.value?.case === 'transcription') { - this.emit(EngineEvent.TranscriptionReceived, dp.value.value); + } else { + if (dp.value?.case === 'user') { + // compatibility + applyUserDataCompat(dp, dp.value.value); + } + this.emit(EngineEvent.DataPacketReceived, dp); } } finally { unlock(); @@ -1392,7 +1394,7 @@ export type EngineEventCallbacks = { receiver?: RTCRtpReceiver, ) => void; activeSpeakersUpdate: (speakers: Array) => void; - dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void; + dataPacketReceived: (packet: DataPacket) => void; transcriptionReceived: (transcription: Transcription) => void; transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void; /** @internal */ @@ -1415,3 +1417,18 @@ export type EngineEventCallbacks = { function supportOptionalDatachannel(protocol: number | undefined): boolean { return protocol !== undefined && protocol > 13; } + +function applyUserDataCompat(newObj: DataPacket, oldObj: UserPacket) { + const participantIdentity = newObj.participantIdentity + ? newObj.participantIdentity + : oldObj.participantIdentity; + newObj.participantIdentity = participantIdentity; + oldObj.participantIdentity = participantIdentity; + + const destinationIdentities = + newObj.destinationIdentities.length !== 0 + ? newObj.destinationIdentities + : oldObj.destinationIdentities; + newObj.destinationIdentities = destinationIdentities; + oldObj.destinationIdentities = destinationIdentities; +} diff --git a/src/room/Room.ts b/src/room/Room.ts index 4bf2bed3ec..f51b38c72f 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -1,5 +1,6 @@ import { ConnectionQualityUpdate, + type DataPacket, DataPacket_Kind, DisconnectReason, JoinResponse, @@ -11,6 +12,7 @@ import { Room as RoomModel, ServerInfo, SimulateScenario, + SipDTMF, SpeakerInfo, StreamStateUpdate, SubscriptionError, @@ -334,7 +336,6 @@ class Room extends (EventEmitter as new () => TypedEmitter) }) .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate) .on(EngineEvent.DataPacketReceived, this.handleDataPacket) - .on(EngineEvent.TranscriptionReceived, this.handleTranscription) .on(EngineEvent.Resuming, () => { this.clearConnectionReconcile(); this.isResuming = true; @@ -1472,24 +1473,47 @@ class Room extends (EventEmitter as new () => TypedEmitter) pub.setSubscriptionError(update.err); }; - private handleDataPacket = (userPacket: UserPacket, kind: DataPacket_Kind) => { + private handleDataPacket = (packet: DataPacket) => { // find the participant - const participant = this.remoteParticipants.get(userPacket.participantIdentity); + const participant = this.remoteParticipants.get(packet.participantIdentity); + if (packet.value.case === 'user') { + this.handleUserPacket(participant, packet.value.value, packet.kind); + } else if (packet.value.case === 'transcription') { + this.handleTranscription(participant, packet.value.value); + } else if (packet.value.case === 'sipDtmf') { + this.handleSipDtmf(participant, packet.value.value); + } + }; + private handleUserPacket = ( + participant: RemoteParticipant | undefined, + userPacket: UserPacket, + kind: DataPacket_Kind, + ) => { this.emit(RoomEvent.DataReceived, userPacket.payload, participant, kind, userPacket.topic); // also emit on the participant participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind); }; + private handleSipDtmf = (participant: RemoteParticipant | undefined, dtmf: SipDTMF) => { + this.emit(RoomEvent.SipDTMFReceived, dtmf, participant); + + // also emit on the participant + participant?.emit(ParticipantEvent.SipDTMFReceived, dtmf); + }; + bufferedSegments: Map = new Map(); - private handleTranscription = (transcription: TranscriptionModel) => { + private handleTranscription = ( + remoteParticipant: RemoteParticipant | undefined, + transcription: TranscriptionModel, + ) => { // find the participant const participant = transcription.participantIdentity === this.localParticipant.identity ? this.localParticipant - : this.remoteParticipants.get(transcription.participantIdentity); + : remoteParticipant; const publication = participant?.trackPublications.get(transcription.trackId); const segments = extractTranscriptionSegments(transcription); @@ -2099,6 +2123,7 @@ export type RoomEventCallbacks = { kind?: DataPacket_Kind, topic?: string, ) => void; + sipDTMFReceived: (dtmf: SipDTMF, participant?: RemoteParticipant) => void; transcriptionReceived: ( transcription: TranscriptionSegment[], participant?: Participant, diff --git a/src/room/events.ts b/src/room/events.ts index 0e9a12845e..5a13073b50 100644 --- a/src/room/events.ts +++ b/src/room/events.ts @@ -197,6 +197,13 @@ export enum RoomEvent { */ DataReceived = 'dataReceived', + /** + * SIP DTMF tones received from another participant. + * + * args: (participant: [[Participant]], dtmf: [[DataPacket_Kind]]) + */ + SipDTMFReceived = 'sipDTMFReceived', + /** * Transcription received from a participant's track. * @beta @@ -408,6 +415,13 @@ export enum ParticipantEvent { */ DataReceived = 'dataReceived', + /** + * SIP DTMF tones received from this participant as sender. + * + * args: (dtmf: [[DataPacket_Kind]]) + */ + SipDTMFReceived = 'sipDTMFReceived', + /** * Transcription received from this participant as data source. * @beta @@ -491,7 +505,6 @@ export enum EngineEvent { MediaTrackAdded = 'mediaTrackAdded', ActiveSpeakersUpdate = 'activeSpeakersUpdate', DataPacketReceived = 'dataPacketReceived', - TranscriptionReceived = 'transcriptionReceived', RTPVideoMapUpdate = 'rtpVideoMapUpdate', DCBufferStatusChanged = 'dcBufferStatusChanged', ParticipantUpdate = 'participantUpdate', diff --git a/src/room/participant/Participant.ts b/src/room/participant/Participant.ts index 530f6b9ff6..cb81ecf53c 100644 --- a/src/room/participant/Participant.ts +++ b/src/room/participant/Participant.ts @@ -3,6 +3,7 @@ import { ParticipantInfo, ParticipantPermission, ConnectionQuality as ProtoQuality, + type SipDTMF, SubscriptionError, } from '@livekit/protocol'; import { EventEmitter } from 'events'; @@ -329,6 +330,7 @@ export type ParticipantEventCallbacks = { participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void; participantNameChanged: (name: string) => void; dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void; + sipDTMFReceived: (dtmf: SipDTMF) => void; transcriptionReceived: ( transcription: TranscriptionSegment[], publication?: TrackPublication,