[feat! apps/stream-whip]: Audio (+Video as well) streaming

apps/stream_whip/cmd/index.html

@@ -27,6 +27,7 @@ <h3> ICE Connection States </h3>
 
   window.doWHEP = () => {
     peerConnection.addTransceiver('video', { direction: 'recvonly' })
+    peerConnection.addTransceiver('audio', { direction: 'recvonly' })
 
     peerConnection.ontrack = function (event) {
       document.getElementById('videoPlayer').srcObject = event.streams[0]

apps/stream_whip/internal/server/routers/whip-whep.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/dehwyy/mugen/libraries/go/logg"
 	"github.com/gin-gonic/gin"
-	"github.com/pion/interceptor"
-	"github.com/pion/interceptor/pkg/intervalpli"
 	"github.com/pion/webrtc/v4"
 )
 
@@ -19,6 +17,7 @@ const (
 
 var (
 	videoTrack *webrtc.TrackLocalStaticRTP
+	audioTrack *webrtc.TrackLocalStaticRTP
 
 	peerConnectionConfiguration = webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
@@ -43,6 +42,12 @@ func (r *WhipWhepRouter) RegisterRoutes(baseRouter *gin.RouterGroup) {
 	}, "video", "pion"); err != nil {
 		panic(err)
 	}
+
+	if audioTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{
+		MimeType: webrtc.MimeTypeOpus,
+	}, "audio", "pion"); err != nil {
+		panic(err)
+	}
 }
 
 func (r *WhipWhepRouter) handleWhip(ctx *gin.Context) {
@@ -62,39 +67,56 @@ func (r *WhipWhepRouter) handleWhip(ctx *gin.Context) {
 		panic(err)
 	}
 
-	interceptorRegistry := &interceptor.Registry{}
+	r.log.Info().Msg(string(offer))
 
-	intervalPliFactory, err := intervalpli.NewReceiverInterceptor()
-	if err != nil {
-		panic(err)
-	}
-	interceptorRegistry.Add(intervalPliFactory)
-
-	if err = webrtc.RegisterDefaultInterceptors(mediaEngine, interceptorRegistry); err != nil {
+	if err = mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "", RTCPFeedback: nil,
+		},
+		PayloadType: 111,
+	}, webrtc.RTPCodecTypeAudio); err != nil {
 		panic(err)
 	}
 
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(interceptorRegistry))
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
 
 	peerConnection, err := api.NewPeerConnection(peerConnectionConfiguration)
 	if err != nil {
 		panic(err)
 	}
 
+	// Allow us to receive 1 video trac
 	if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil {
 		panic(err)
 	}
 
+	if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio); err != nil {
+		panic(err)
+	}
+
 	peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { //nolint: revive
 		for {
+			// r.log.Debug().Msgf("track: %+v", track)
 			pkt, _, err := track.ReadRTP()
 			if err != nil {
 				panic(err)
 			}
 
-			if err = videoTrack.WriteRTP(pkt); err != nil {
-				panic(err)
+			switch pkt.PayloadType {
+			// video
+			case 96:
+
+				if err = videoTrack.WriteRTP(pkt); err != nil {
+					panic(err)
+				}
+
+			case 111:
+
+				if err = audioTrack.WriteRTP(pkt); err != nil {
+					panic(err)
+				}
 			}
+
 		}
 	})
 	// !
@@ -112,6 +134,7 @@ func (r *WhipWhepRouter) handleWhip(ctx *gin.Context) {
 		panic(err)
 	}
 
+	// Create channel that is blocked until ICE Gathering is complete
 	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
 
 	// Create answer
@@ -125,8 +148,8 @@ func (r *WhipWhepRouter) handleWhip(ctx *gin.Context) {
 	<-gatherComplete
 
 	ctx.Header("Location", "/whip")
-	ctx.Status(http.StatusCreated)
-	fmt.Fprintf(ctx.Writer, peerConnection.LocalDescription().SDP)
+	ctx.String(http.StatusCreated, "%s", peerConnection.LocalDescription().SDP)
+
 }
 
 func (r *WhipWhepRouter) handleWhep(ctx *gin.Context) {
@@ -142,15 +165,29 @@ func (r *WhipWhepRouter) handleWhep(ctx *gin.Context) {
 	}
 
 	// Add Video Track that is being written to from WHIP Session
-	rtpSender, err := peerConnection.AddTrack(videoTrack)
+	rtpVideoSender, err := peerConnection.AddTrack(videoTrack)
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		rtcpBuf := make([]byte, 1500)
+		for {
+			if _, _, rtcpErr := rtpVideoSender.Read(rtcpBuf); rtcpErr != nil {
+				return
+			}
+		}
+	}()
+
+	rtpAudioSender, err := peerConnection.AddTrack(audioTrack)
 	if err != nil {
 		panic(err)
 	}
 
 	go func() {
 		rtcpBuf := make([]byte, 1500)
 		for {
-			if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
+			if _, _, rtcpErr := rtpAudioSender.Read(rtcpBuf); rtcpErr != nil {
 				return
 			}
 		}
@@ -172,6 +209,7 @@ func (r *WhipWhepRouter) handleWhep(ctx *gin.Context) {
 
 	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
 
+	// Create answer
 	answer, err := peerConnection.CreateAnswer(nil)
 	if err != nil {
 		panic(err)
@@ -182,6 +220,6 @@ func (r *WhipWhepRouter) handleWhep(ctx *gin.Context) {
 	<-gatherComplete
 
 	ctx.Header("Location", "/whep")
-	ctx.Status(http.StatusCreated)
-	fmt.Fprintf(ctx.Writer, peerConnection.LocalDescription().SDP)
+	ctx.String(http.StatusAccepted, "%s", peerConnection.LocalDescription().SDP)
+
 }