diff --git a/internal/rtpbuffer/rtpbuffer.go b/internal/rtpbuffer/rtpbuffer.go
index 9253507..1deefee 100644
--- a/internal/rtpbuffer/rtpbuffer.go
+++ b/internal/rtpbuffer/rtpbuffer.go
@@ -17,10 +17,10 @@ const (
 
 // RTPBuffer stores RTP packets and allows custom logic around the lifetime of them via the PacketFactory
 type RTPBuffer struct {
-	packets   []*RetainablePacket
-	size      uint16
-	lastAdded uint16
-	started   bool
+	packets      []*RetainablePacket
+	size         uint16
+	highestAdded uint16
+	started      bool
 }
 
 // NewRTPBuffer constructs a new RTPBuffer
@@ -50,16 +50,16 @@ func (r *RTPBuffer) Add(packet *RetainablePacket) {
 	seq := packet.sequenceNumber
 	if !r.started {
 		r.packets[seq%r.size] = packet
-		r.lastAdded = seq
+		r.highestAdded = seq
 		r.started = true
 		return
 	}
 
-	diff := seq - r.lastAdded
+	diff := seq - r.highestAdded
 	if diff == 0 {
 		return
 	} else if diff < Uint16SizeHalf {
-		for i := r.lastAdded + 1; i != seq; i++ {
+		for i := r.highestAdded + 1; i != seq; i++ {
 			idx := i % r.size
 			prevPacket := r.packets[idx]
 			if prevPacket != nil {
@@ -67,6 +67,7 @@ func (r *RTPBuffer) Add(packet *RetainablePacket) {
 			}
 			r.packets[idx] = nil
 		}
+		r.highestAdded = seq
 	}
 
 	idx := seq % r.size
@@ -75,12 +76,11 @@ func (r *RTPBuffer) Add(packet *RetainablePacket) {
 		prevPacket.Release()
 	}
 	r.packets[idx] = packet
-	r.lastAdded = seq
 }
 
 // Get returns the RetainablePacket for the requested sequence number
 func (r *RTPBuffer) Get(seq uint16) *RetainablePacket {
-	diff := r.lastAdded - seq
+	diff := r.highestAdded - seq
 	if diff >= Uint16SizeHalf {
 		return nil
 	}
diff --git a/internal/rtpbuffer/rtpbuffer_test.go b/internal/rtpbuffer/rtpbuffer_test.go
index f1cac81..aa59aec 100644
--- a/internal/rtpbuffer/rtpbuffer_test.go
+++ b/internal/rtpbuffer/rtpbuffer_test.go
@@ -124,6 +124,10 @@ func TestRTPBuffer_WithRTX(t *testing.T) {
 		assertGet(10)
 		assertNOTGet(1, 2, 9)
 
+		// A late packet coming in (such as due to RTX) shouldn't invalidate other packets.
+		add(9)
+		assertGet(3, 4, 5, 6, 7, 8, 9, 10)
+
 		add(22)
 		assertGet(22)
 		assertNOTGet(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)