Add Retransmission and FEC to TrackLocal

If the MediaEngine contains support for them a SSRC will be generated
appropriately

Co-authored-by: aggresss <[email protected]>
Co-authored-by: Kevin Wang <[email protected]>

Resolves #1989
Resolves #1675

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/pion/datachannel v1.5.9
 	github.com/pion/dtls/v3 v3.0.2
 	github.com/pion/ice/v4 v4.0.1
-	github.com/pion/interceptor v0.1.31
+	github.com/pion/interceptor v0.1.32
 	github.com/pion/logging v0.2.2
 	github.com/pion/randutil v0.1.0
 	github.com/pion/rtcp v1.2.14

go.sum

@@ -41,8 +41,8 @@ github.com/pion/dtls/v3 v3.0.2 h1:425DEeJ/jfuTTghhUDW0GtYZYIwwMtnKKJNMcWccTX0=
 github.com/pion/dtls/v3 v3.0.2/go.mod h1:dfIXcFkKoujDQ+jtd8M6RgqKK3DuaUilm3YatAbGp5k=
 github.com/pion/ice/v4 v4.0.1 h1:2d3tPoTR90F3TcGYeXUwucGlXI3hds96cwv4kjZmb9s=
 github.com/pion/ice/v4 v4.0.1/go.mod h1:2dpakjpd7+74L5j3TAe6gvkbI5UIzOgAnkimm9SuHvA=
-github.com/pion/interceptor v0.1.31 h1:9enhHjP1fDfMI8sqvpO5c/9QuTQnCf2dzPHwwIH4x5w=
-github.com/pion/interceptor v0.1.31/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
+github.com/pion/interceptor v0.1.32 h1:DYbusOBhWKjPMiA5ifyczW03Tnh12gCaYn4VOvLMGk4=
+github.com/pion/interceptor v0.1.32/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
 github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
 github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=

interceptor.go

@@ -159,7 +159,7 @@ func (i *interceptorToTrackLocalWriter) Write(b []byte) (int, error) {
 	return i.WriteRTP(&packet.Header, packet.Payload)
 }
 
-func createStreamInfo(id string, ssrc SSRC, payloadType PayloadType, codec RTPCodecCapability, webrtcHeaderExtensions []RTPHeaderExtensionParameter) *interceptor.StreamInfo {
+func createStreamInfo(id string, ssrc, ssrcFEC, ssrcRTX SSRC, payloadType PayloadType, codec RTPCodecCapability, webrtcHeaderExtensions []RTPHeaderExtensionParameter) *interceptor.StreamInfo {
 	headerExtensions := make([]interceptor.RTPHeaderExtension, 0, len(webrtcHeaderExtensions))
 	for _, h := range webrtcHeaderExtensions {
 		headerExtensions = append(headerExtensions, interceptor.RTPHeaderExtension{ID: h.ID, URI: h.URI})
@@ -171,15 +171,17 @@ func createStreamInfo(id string, ssrc SSRC, payloadType PayloadType, codec RTPCo
 	}
 
 	return &interceptor.StreamInfo{
-		ID:                  id,
-		Attributes:          interceptor.Attributes{},
-		SSRC:                uint32(ssrc),
-		PayloadType:         uint8(payloadType),
-		RTPHeaderExtensions: headerExtensions,
-		MimeType:            codec.MimeType,
-		ClockRate:           codec.ClockRate,
-		Channels:            codec.Channels,
-		SDPFmtpLine:         codec.SDPFmtpLine,
-		RTCPFeedback:        feedbacks,
+		ID:                         id,
+		Attributes:                 interceptor.Attributes{},
+		SSRC:                       uint32(ssrc),
+		SSRCRetransmission:         uint32(ssrcRTX),
+		SSRCForwardErrorCorrection: uint32(ssrcFEC),
+		PayloadType:                uint8(payloadType),
+		RTPHeaderExtensions:        headerExtensions,
+		MimeType:                   codec.MimeType,
+		ClockRate:                  codec.ClockRate,
+		Channels:                   codec.Channels,
+		SDPFmtpLine:                codec.SDPFmtpLine,
+		RTCPFeedback:               feedbacks,
 	}
 }

interceptor_test.go

@@ -196,18 +196,18 @@ func Test_Interceptor_BindUnbind(t *testing.T) {
 	if cnt := atomic.LoadUint32(&cntUnbindLocalStream); cnt != 1 {
 		t.Errorf("UnbindLocalStreamFn is expected to be called once, but called %d times", cnt)
 	}
-	if cnt := atomic.LoadUint32(&cntBindRemoteStream); cnt != 1 {
+	if cnt := atomic.LoadUint32(&cntBindRemoteStream); cnt != 2 {
 		t.Errorf("BindRemoteStreamFn is expected to be called once, but called %d times", cnt)
 	}
-	if cnt := atomic.LoadUint32(&cntUnbindRemoteStream); cnt != 1 {
+	if cnt := atomic.LoadUint32(&cntUnbindRemoteStream); cnt != 2 {
 		t.Errorf("UnbindRemoteStreamFn is expected to be called once, but called %d times", cnt)
 	}
 
 	// BindRTCPWriter/Reader and Close should be called from both side.
 	if cnt := atomic.LoadUint32(&cntBindRTCPWriter); cnt != 2 {
 		t.Errorf("BindRTCPWriterFn is expected to be called twice, but called %d times", cnt)
 	}
-	if cnt := atomic.LoadUint32(&cntBindRTCPReader); cnt != 2 {
+	if cnt := atomic.LoadUint32(&cntBindRTCPReader); cnt != 3 {
 		t.Errorf("BindRTCPReaderFn is expected to be called twice, but called %d times", cnt)
 	}
 	if cnt := atomic.LoadUint32(&cntClose); cnt != 2 {

mediaengine.go

@@ -47,6 +47,12 @@ const (
 	// MimeTypePCMA PCMA MIME type
 	// Note: Matching should be case insensitive.
 	MimeTypePCMA = "audio/PCMA"
+	// MimeTypeRTX RTX MIME type
+	// Note: Matching should be case insensitive.
+	MimeTypeRTX = "video/rtx"
+	// MimeTypeFlexFEC FEC MIME Type
+	// Note: Matching should be case insensitive.
+	MimeTypeFlexFEC = "video/flexfec"
 )
 
 type mediaEngineHeaderExtension struct {
@@ -106,7 +112,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        96,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=96", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=96", nil},
 			PayloadType:        97,
 		},
 
@@ -115,7 +121,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        102,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=102", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=102", nil},
 			PayloadType:        103,
 		},
 
@@ -124,7 +130,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        104,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=104", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=104", nil},
 			PayloadType:        105,
 		},
 
@@ -133,7 +139,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        106,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=106", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=106", nil},
 			PayloadType:        107,
 		},
 
@@ -142,7 +148,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        108,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=108", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=108", nil},
 			PayloadType:        109,
 		},
 
@@ -151,7 +157,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        127,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=127", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=127", nil},
 			PayloadType:        125,
 		},
 
@@ -160,7 +166,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        39,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=39", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=39", nil},
 			PayloadType:        40,
 		},
 
@@ -169,7 +175,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        45,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=45", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=45", nil},
 			PayloadType:        46,
 		},
 
@@ -178,7 +184,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        98,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=98", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=98", nil},
 			PayloadType:        99,
 		},
 
@@ -187,7 +193,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        100,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=100", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=100", nil},
 			PayloadType:        101,
 		},
 
@@ -196,7 +202,7 @@ func (m *MediaEngine) RegisterDefaultCodecs() error {
 			PayloadType:        112,
 		},
 		{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=112", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=112", nil},
 			PayloadType:        113,
 		},
 	} {
@@ -702,3 +708,23 @@ func payloaderForCodec(codec RTPCodecCapability) (rtp.Payloader, error) {
 		return nil, ErrNoPayloaderForCodec
 	}
 }
+
+func (m *MediaEngine) isRTXEnabled(typ RTPCodecType, directions []RTPTransceiverDirection) bool {
+	for _, p := range m.getRTPParametersByKind(typ, directions).Codecs {
+		if p.MimeType == MimeTypeRTX {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (m *MediaEngine) isFECEnabled(typ RTPCodecType, directions []RTPTransceiverDirection) bool {
+	for _, p := range m.getRTPParametersByKind(typ, directions).Codecs {
+		if strings.Contains(p.MimeType, MimeTypeFlexFEC) {
+			return true
+		}
+	}
+
+	return false
+}

mediaengine_test.go

@@ -364,31 +364,31 @@ a=fmtp:97 apt=96
 			PayloadType:        96,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=96", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=96", nil},
 			PayloadType:        97,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
 			RTPCodecCapability: RTPCodecCapability{MimeTypeH264, 90000, 0, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", nil},
 			PayloadType:        102,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=102", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=102", nil},
 			PayloadType:        103,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
 			RTPCodecCapability: RTPCodecCapability{MimeTypeH264, 90000, 0, "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f", nil},
 			PayloadType:        104,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=104", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=104", nil},
 			PayloadType:        105,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
 			RTPCodecCapability: RTPCodecCapability{MimeTypeVP9, 90000, 0, "profile-id=2", nil},
 			PayloadType:        98,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=98", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=98", nil},
 			PayloadType:        99,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.updateFromRemoteDescription(mustParse(profileLevels)))
@@ -400,15 +400,15 @@ a=fmtp:97 apt=96
 		assert.Equal(t, vp9Codec.MimeType, MimeTypeVP9)
 		vp9RTX, _, err := m.getCodecByPayload(97)
 		assert.NoError(t, err)
-		assert.Equal(t, vp9RTX.MimeType, "video/rtx")
+		assert.Equal(t, vp9RTX.MimeType, MimeTypeRTX)
 
 		h264P1Codec, _, err := m.getCodecByPayload(106)
 		assert.NoError(t, err)
 		assert.Equal(t, h264P1Codec.MimeType, MimeTypeH264)
 		assert.Equal(t, h264P1Codec.SDPFmtpLine, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f")
 		h264P1RTX, _, err := m.getCodecByPayload(107)
 		assert.NoError(t, err)
-		assert.Equal(t, h264P1RTX.MimeType, "video/rtx")
+		assert.Equal(t, h264P1RTX.MimeType, MimeTypeRTX)
 		assert.Equal(t, h264P1RTX.SDPFmtpLine, "apt=106")
 
 		h264P0Codec, _, err := m.getCodecByPayload(108)
@@ -417,7 +417,7 @@ a=fmtp:97 apt=96
 		assert.Equal(t, h264P0Codec.SDPFmtpLine, "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f")
 		h264P0RTX, _, err := m.getCodecByPayload(109)
 		assert.NoError(t, err)
-		assert.Equal(t, h264P0RTX.MimeType, "video/rtx")
+		assert.Equal(t, h264P0RTX.MimeType, MimeTypeRTX)
 		assert.Equal(t, h264P0RTX.SDPFmtpLine, "apt=108")
 	})
 
@@ -443,7 +443,7 @@ a=fmtp:97 apt=96
 			PayloadType:        96,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.RegisterCodec(RTPCodecParameters{
-			RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=96", nil},
+			RTPCodecCapability: RTPCodecCapability{MimeTypeRTX, 90000, 0, "apt=96", nil},
 			PayloadType:        97,
 		}, RTPCodecTypeVideo))
 		assert.NoError(t, m.updateFromRemoteDescription(mustParse(profileLevels)))

peerconnection.go

@@ -1048,6 +1048,11 @@ func (pc *PeerConnection) SetRemoteDescription(desc SessionDescription) error {
 		return err
 	}
 
+	// Disable RTX/FEC on RTPSenders if the remote didn't support it
+	for _, sender := range pc.GetSenders() {
+		sender.configureRTXAndFEC()
+	}
+
 	var t *RTPTransceiver
 	localTransceivers := append([]*RTPTransceiver{}, pc.GetTransceivers()...)
 	detectedPlanB := descriptionIsPlanB(pc.RemoteDescription(), pc.log)
@@ -1616,7 +1621,7 @@ func (pc *PeerConnection) handleIncomingSSRC(rtpStream io.Reader, ssrc SSRC) err
 		return err
 	}
 
-	streamInfo := createStreamInfo("", ssrc, params.Codecs[0].PayloadType, params.Codecs[0].RTPCodecCapability, params.HeaderExtensions)
+	streamInfo := createStreamInfo("", ssrc, 0, 0, params.Codecs[0].PayloadType, params.Codecs[0].RTPCodecCapability, params.HeaderExtensions)
 	readStream, interceptor, rtcpReadStream, rtcpInterceptor, err := pc.dtlsTransport.streamsForSSRC(ssrc, *streamInfo)
 	if err != nil {
 		return err

peerconnection_media_test.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/pion/logging"
-	"github.com/pion/randutil"
 	"github.com/pion/rtcp"
 	"github.com/pion/rtp"
 	"github.com/pion/sdp/v3"
@@ -778,7 +777,7 @@ func TestAddTransceiverFromTrackFailsRecvOnly(t *testing.T) {
 func TestPlanBMediaExchange(t *testing.T) {
 	runTest := func(trackCount int, t *testing.T) {
 		addSingleTrack := func(p *PeerConnection) *TrackLocalStaticSample {
-			track, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, fmt.Sprintf("video-%d", randutil.NewMathRandomGenerator().Uint32()), fmt.Sprintf("video-%d", randutil.NewMathRandomGenerator().Uint32()))
+			track, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, fmt.Sprintf("video-%d", util.RandUint32()), fmt.Sprintf("video-%d", util.RandUint32()))
 			assert.NoError(t, err)
 
 			_, err = p.AddTrack(track)
@@ -1020,7 +1019,7 @@ func TestPeerConnection_Simulcast_Probe(t *testing.T) {
 					if len(track.bindings) == 1 {
 						_, err = track.bindings[0].writeStream.WriteRTP(&rtp.Header{
 							Version: 2,
-							SSRC:    randutil.NewMathRandomGenerator().Uint32(),
+							SSRC:    util.RandUint32(),
 						}, []byte{0, 1, 2, 3, 4, 5})
 						assert.NoError(t, err)
 					}

pkg/media/oggwriter/oggwriter.go

@@ -10,9 +10,9 @@ import (
 	"io"
 	"os"
 
-	"github.com/pion/randutil"
 	"github.com/pion/rtp"
 	"github.com/pion/rtp/codecs"
+	"github.com/pion/webrtc/v4/internal/util"
 )
 
 const (
@@ -68,7 +68,7 @@ func NewWith(out io.Writer, sampleRate uint32, channelCount uint16) (*OggWriter,
 		stream:        out,
 		sampleRate:    sampleRate,
 		channelCount:  channelCount,
-		serial:        randutil.NewMathRandomGenerator().Uint32(),
+		serial:        util.RandUint32(),
 		checksumTable: generateChecksumTable(),
 
 		// Timestamp and Granule MUST start from 1

rtpcodec.go

@@ -4,6 +4,7 @@
 package webrtc
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/pion/webrtc/v4/internal/fmtp"
@@ -123,3 +124,15 @@ func codecParametersFuzzySearch(needle RTPCodecParameters, haystack []RTPCodecPa
 
 	return RTPCodecParameters{}, codecMatchNone
 }
+
+// Given a CodecParameters find the RTX CodecParameters if one exists
+func findRTXCodecParameters(needle PayloadType, haystack []RTPCodecParameters) (RTPCodecParameters, bool) {
+	aptStr := fmt.Sprintf("apt=%d", needle)
+	for _, c := range haystack {
+		if aptStr == c.SDPFmtpLine {
+			return c, true
+		}
+	}
+
+	return RTPCodecParameters{}, false
+}

rtpcodingparameters.go

@@ -9,6 +9,12 @@ type RTPRtxParameters struct {
 	SSRC SSRC `json:"ssrc"`
 }
 
+// RTPFecParameters dictionary contains information relating to forward error correction (FEC) settings.
+// https://draft.ortc.org/#dom-rtcrtpfecparameters
+type RTPFecParameters struct {
+	SSRC SSRC `json:"ssrc"`
+}
+
 // RTPCodingParameters provides information relating to both encoding and decoding.
 // This is a subset of the RFC since Pion WebRTC doesn't implement encoding/decoding itself
 // http://draft.ortc.org/#dom-rtcrtpcodingparameters
@@ -17,4 +23,5 @@ type RTPCodingParameters struct {
 	SSRC        SSRC             `json:"ssrc"`
 	PayloadType PayloadType      `json:"payloadType"`
 	RTX         RTPRtxParameters `json:"rtx"`
+	FEC         RTPFecParameters `json:"fec"`
 }