From 287d10638dbf8a682555de39d2a6814213e595fb Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 18 Jul 2024 10:43:27 +0800 Subject: [PATCH] Match header extensions to remote media sections Firefox would send updated header extension in renegotiation, e.g. publish a track without simucalst then renegotiate second track with simucalst, the two media secontions will have different rtp header extensions in offer. Need to match remote header extentions for each media sections to avoid second track publish failed. --- mediaengine.go | 54 ++++++++++++++++++++++++++++-------- mediaengine_test.go | 33 ++++++++++++++++++++++ peerconnection.go | 4 ++- peerconnection_media_test.go | 7 +++-- sdp.go | 14 +++++++--- 5 files changed, 93 insertions(+), 19 deletions(-) diff --git a/mediaengine.go b/mediaengine.go index cbdcfb6d7d5..47a08d2f24c 100644 --- a/mediaengine.go +++ b/mediaengine.go @@ -455,6 +455,30 @@ func (m *MediaEngine) matchRemoteCodec(remoteCodec RTPCodecParameters, typ RTPCo return matchType, nil } +// Update header extensions from a remote media section +func (m *MediaEngine) updateHeaderExtensionFromMediaSection(media *sdp.MediaDescription) error { + var typ RTPCodecType + switch { + case strings.EqualFold(media.MediaName.Media, "audio"): + typ = RTPCodecTypeAudio + case strings.EqualFold(media.MediaName.Media, "video"): + typ = RTPCodecTypeVideo + default: + return nil + } + extensions, err := rtpExtensionsFromMediaDescription(media) + if err != nil { + return err + } + + for extension, id := range extensions { + if err = m.updateHeaderExtension(id, extension, typ); err != nil { + return err + } + } + return nil +} + // Look up a header extension and enable if it exists func (m *MediaEngine) updateHeaderExtension(id int, extension string, typ RTPCodecType) error { if m.negotiatedHeaderExtensions == nil { @@ -498,14 +522,27 @@ func (m *MediaEngine) updateFromRemoteDescription(desc sdp.SessionDescription) e for _, media := range desc.MediaDescriptions { var typ RTPCodecType + switch { - case !m.negotiatedAudio && strings.EqualFold(media.MediaName.Media, "audio"): - m.negotiatedAudio = true + case strings.EqualFold(media.MediaName.Media, "audio"): typ = RTPCodecTypeAudio - case !m.negotiatedVideo && strings.EqualFold(media.MediaName.Media, "video"): - m.negotiatedVideo = true + case strings.EqualFold(media.MediaName.Media, "video"): typ = RTPCodecTypeVideo + } + + switch { + case !m.negotiatedAudio && typ == RTPCodecTypeAudio: + m.negotiatedAudio = true + case !m.negotiatedVideo && typ == RTPCodecTypeVideo: + m.negotiatedVideo = true default: + // update header extesions from remote sdp if codec is negotiated, Firefox + // would send updated header extension in renegotiation. + // e.g. publish first track without simucalst ->negotiated-> publish second track with simucalst + // then the two media secontions have different rtp header extensions in offer + if err := m.updateHeaderExtensionFromMediaSection(media); err != nil { + return err + } continue } @@ -541,16 +578,9 @@ func (m *MediaEngine) updateFromRemoteDescription(desc sdp.SessionDescription) e continue } - extensions, err := rtpExtensionsFromMediaDescription(media) - if err != nil { + if err := m.updateHeaderExtensionFromMediaSection(media); err != nil { return err } - - for extension, id := range extensions { - if err = m.updateHeaderExtension(id, extension, typ); err != nil { - return err - } - } } return nil } diff --git a/mediaengine_test.go b/mediaengine_test.go index d9a9e7b1ff4..28f76069fbe 100644 --- a/mediaengine_test.go +++ b/mediaengine_test.go @@ -212,6 +212,39 @@ a=rtpmap:111 opus/48000/2 assert.False(t, midVideoEnabled) }) + t.Run("Different Header Extensions on same codec", func(t *testing.T) { + const headerExtensions = `v=0 +o=- 4596489990601351948 2 IN IP4 127.0.0.1 +s=- +t=0 0 +m=audio 9 UDP/TLS/RTP/SAVPF 111 +a=rtpmap:111 opus/48000/2 +m=audio 9 UDP/TLS/RTP/SAVPF 111 +a=extmap:7 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=rtpmap:111 opus/48000/2 +` + + m := MediaEngine{} + assert.NoError(t, m.RegisterDefaultCodecs()) + assert.NoError(t, m.RegisterHeaderExtension(RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, RTPCodecTypeAudio)) + assert.NoError(t, m.RegisterHeaderExtension(RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, RTPCodecTypeAudio)) + assert.NoError(t, m.updateFromRemoteDescription(mustParse(headerExtensions))) + + assert.False(t, m.negotiatedVideo) + assert.True(t, m.negotiatedAudio) + + absID, absAudioEnabled, absVideoEnabled := m.getHeaderExtensionID(RTPHeaderExtensionCapability{sdp.ABSSendTimeURI}) + assert.Equal(t, absID, 0) + assert.False(t, absAudioEnabled) + assert.False(t, absVideoEnabled) + + midID, midAudioEnabled, midVideoEnabled := m.getHeaderExtensionID(RTPHeaderExtensionCapability{sdp.SDESMidURI}) + assert.Equal(t, midID, 7) + assert.True(t, midAudioEnabled) + assert.False(t, midVideoEnabled) + }) + t.Run("Prefers exact codec matches", func(t *testing.T) { const profileLevels = `v=0 o=- 4596489990601351948 2 IN IP4 127.0.0.1 diff --git a/peerconnection.go b/peerconnection.go index 3ae8b732c66..7a7e4835fba 100644 --- a/peerconnection.go +++ b/peerconnection.go @@ -2465,7 +2465,9 @@ func (pc *PeerConnection) generateMatchedSDP(transceivers []*RTPTransceiver, use sender.setNegotiated() } mediaTransceivers := []*RTPTransceiver{t} - mediaSections = append(mediaSections, mediaSection{id: midValue, transceivers: mediaTransceivers, ridMap: getRids(media)}) + + extensions, _ := rtpExtensionsFromMediaDescription(media) + mediaSections = append(mediaSections, mediaSection{id: midValue, transceivers: mediaTransceivers, matchExtensions: extensions, ridMap: getRids(media)}) } } diff --git a/peerconnection_media_test.go b/peerconnection_media_test.go index 1a56dfb4bb7..0e29d90f400 100644 --- a/peerconnection_media_test.go +++ b/peerconnection_media_test.go @@ -1080,6 +1080,9 @@ func TestPeerConnection_Simulcast_Probe(t *testing.T) { return })) + peerConnectionConnected := untilConnectionState(PeerConnectionStateConnected, pcOffer, pcAnswer) + peerConnectionConnected.Wait() + sequenceNumber := uint16(0) sendRTPPacket := func() { sequenceNumber++ @@ -1097,13 +1100,13 @@ func TestPeerConnection_Simulcast_Probe(t *testing.T) { sendRTPPacket() } - assert.NoError(t, signalPair(pcOffer, pcAnswer)) - trackRemoteChan := make(chan *TrackRemote, 1) pcAnswer.OnTrack(func(trackRemote *TrackRemote, _ *RTPReceiver) { trackRemoteChan <- trackRemote }) + assert.NoError(t, signalPair(pcOffer, pcAnswer)) + trackRemote := func() *TrackRemote { for { select { diff --git a/sdp.go b/sdp.go index 7ab0fd89807..f1b59940b10 100644 --- a/sdp.go +++ b/sdp.go @@ -487,6 +487,11 @@ func addTransceiverSDP( parameters := mediaEngine.getRTPParametersByKind(t.kind, directions) for _, rtpExtension := range parameters.HeaderExtensions { + if mediaSection.matchExtensions != nil { + if _, enabled := mediaSection.matchExtensions[rtpExtension.URI]; !enabled { + continue + } + } extURL, err := url.Parse(rtpExtension.URI) if err != nil { return false, err @@ -533,10 +538,11 @@ type simulcastRid struct { } type mediaSection struct { - id string - transceivers []*RTPTransceiver - data bool - ridMap map[string]*simulcastRid + id string + transceivers []*RTPTransceiver + data bool + matchExtensions map[string]int + ridMap map[string]*simulcastRid } func bundleMatchFromRemote(matchBundleGroup *string) func(mid string) bool {