Add shell command media source drivers + misc improvements

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "go.testFlags": [
+        "-buildmode=pie"
+    ],
+    "go.testEnvVars": {
+        "PION_LOG_DEBUG": "all"
+    },
+}

examples/cmdsource/.gitignore

@@ -0,0 +1,2 @@
+cmdsource
+recorded.h264

examples/cmdsource/README.md

@@ -0,0 +1,45 @@
+## Instructions
+
+This example is nearly the same as the archive example, but uses the output of a shell command (in this case ffmpeg) as a video input instead of a camera. See the other examples for how to take this track and use or stream it.
+
+### Install required codecs
+
+In this example, we'll be using x264 as our video codec. We also use FFMPEG to generate the test video stream. (Note that cmdsource does not requre ffmpeg to run, it is just what this example uses) Therefore, we need to make sure that these are installed within our system.
+
+Installation steps:
+
+* [ffmpeg](https://ffmpeg.org/)
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download cmdsource example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Run cmdsource example
+
+Run `cd mediadevices/examples/cmdsource && go build && ./cmdsource recorded.h264`
+
+To stop recording, press `Ctrl+c` or send a SIGINT signal.
+
+### Playback recorded video
+
+Use ffplay (part of the ffmpeg project):
+```
+ffplay -f h264 recorded.h264
+```
+
+Or install GStreamer and run:
+```
+gst-launch-1.0 playbin uri=file://${PWD}/recorded.h264
+```
+
+Or run VLC media plyer:
+```
+vlc recorded.h264
+```
+
+A video should start playing in a window.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/cmdsource/main.go

@@ -0,0 +1,139 @@
+package main
+
+// !!!! This example requires ffmpeg to be installed !!!!
+
+import (
+	"errors"
+	"fmt"
+	"image"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
+	"github.com/pion/mediadevices/pkg/driver"
+	"github.com/pion/mediadevices/pkg/driver/cmdsource"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+// handy correlation between the names of frame formats in pion media devices and the same -pix_fmt as passed to ffmpeg
+var ffmpegFrameFormatMap = map[frame.Format]string{
+	frame.FormatI420: "yuv420p",
+	frame.FormatNV21: "nv21",
+	frame.FormatNV12: "nv12",
+	frame.FormatYUY2: "yuyv422",
+	frame.FormatUYVY: "uyvy422",
+	frame.FormatZ16:  "gray",
+}
+
+func ffmpegTestPatternCmd(width int, height int, frameRate float32, frameFormat frame.Format) string {
+	// returns the (command-line) command to tell the ffmpeg program to output a test video stream with the given pixel format, size and framerate to stdout:
+	return fmt.Sprintf("ffmpeg -hide_banner -f lavfi -i testsrc=size=%dx%d:rate=%f -vf realtime -f rawvideo -pix_fmt %s -", width, height, frameRate, ffmpegFrameFormatMap[frameFormat])
+}
+
+func getMediaDevicesDriverId(label string) (string, error) {
+	drivers := driver.GetManager().Query(func(d driver.Driver) bool {
+		return d.Info().Label == label
+	})
+	if len(drivers) == 0 {
+		return "", errors.New("Failed to find the media devices driver for device label: " + label)
+	}
+	return drivers[0].ID(), nil
+}
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s <path/to/output_file.h264>\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT)
+
+	x264Params, err := x264.NewParams()
+	must(err)
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	// configure source video properties (raw video stream format that we should expect the command to output)
+	label := "My Cool Video"
+	videoProps := prop.Media{
+		Video: prop.Video{
+			Width:       640,
+			Height:      480,
+			FrameFormat: frame.FormatI420,
+			FrameRate:   30,
+		},
+		// OR Audio: prop.Audio{}
+	}
+
+	// Add the command source:
+	cmdString := ffmpegTestPatternCmd(videoProps.Video.Width, videoProps.Video.Height, videoProps.Video.FrameRate, videoProps.FrameFormat)
+	err = cmdsource.AddVideoCmdSource(label, cmdString, []prop.Media{videoProps}, 10, true)
+	must(err)
+
+	// Now your video command source will be a driver in mediaDevices:
+	driverId, err := getMediaDevicesDriverId(label)
+	must(err)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.DeviceID = prop.String(driverId)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+	//// --- OR (if the track was setup as audio) --
+	// audioTrack := mediaStream.GetAudioTracks()[0].(*mediadevices.AudioTrack)
+	// defer audioTrack.Close()
+
+	// Do whatever you want with the track, the rest of this example is the same as the archive example:
+	// =================================================================================================
+
+	videoTrack.Transform(video.TransformFunc(func(r video.Reader) video.Reader {
+		return video.ReaderFunc(func() (img image.Image, release func(), err error) {
+			// we send io.EOF signal to the encoder reader to stop reading. Therefore, io.Copy
+			// will finish its execution and the program will finish
+			select {
+			case <-sigs:
+				return nil, func() {}, io.EOF
+			default:
+			}
+
+			return r.Read()
+		})
+	}))
+
+	reader, err := videoTrack.NewEncodedIOReader(x264Params.RTPCodec().MimeType)
+	must(err)
+	defer reader.Close()
+
+	out, err := os.Create(dest)
+	must(err)
+
+	fmt.Println("Recording... Press Ctrl+c to stop")
+	_, err = io.Copy(out, reader)
+	must(err)
+	videoTrack.Close()                   // Ideally we should close the track before the io.Copy is done to save every last frame
+	<-time.After(100 * time.Millisecond) // Give a bit of time for the ffmpeg stream to stop cleanly before the program exits
+	fmt.Println("Your video has been recorded to", dest)
+}

examples/go.sum

@@ -27,6 +27,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165
 	github.com/gen2brain/malgo v0.11.10
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.3.0
 	github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329
 	github.com/pion/interceptor v0.1.12

go.sum

@@ -23,6 +23,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

pkg/codec/codec.go

@@ -29,7 +29,7 @@ func NewRTPH264Codec(clockrate uint32) *RTPCodec {
 				ClockRate:    90000,
 				Channels:     0,
 				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
-				RTCPFeedback: nil,
+				RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack", Parameter: "pli"}, {Type: "ccm", Parameter: "fir"}, {Type: "goog-remb", Parameter: ""}},
 			},
 			PayloadType: 125,
 		},
@@ -46,7 +46,7 @@ func NewRTPVP8Codec(clockrate uint32) *RTPCodec {
 				ClockRate:    90000,
 				Channels:     0,
 				SDPFmtpLine:  "",
-				RTCPFeedback: nil,
+				RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack", Parameter: "pli"}, {Type: "ccm", Parameter: "fir"}, {Type: "goog-remb", Parameter: ""}},
 			},
 			PayloadType: 96,
 		},
@@ -63,7 +63,7 @@ func NewRTPVP9Codec(clockrate uint32) *RTPCodec {
 				ClockRate:    90000,
 				Channels:     0,
 				SDPFmtpLine:  "",
-				RTCPFeedback: nil,
+				RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack", Parameter: "pli"}, {Type: "ccm", Parameter: "fir"}, {Type: "goog-remb", Parameter: ""}},
 			},
 			PayloadType: 98,
 		},

pkg/codec/vpx/vpx.go

@@ -74,6 +74,7 @@ type encoder struct {
 	frame           []byte
 	deadline        int
 	requireKeyFrame bool
+	targetBitrate   int
 	isKeyFrame      bool
 
 	mu     sync.Mutex
@@ -200,14 +201,15 @@ func newEncoder(r video.Reader, p prop.Media, params Params, codecIface *C.vpx_c
 	}
 	t0 := time.Now().Nanosecond() / 1000000
 	return &encoder{
-		r:          video.ToI420(r),
-		codec:      codec,
-		raw:        rawNoBuffer,
-		cfg:        cfg,
-		tStart:     t0,
-		tLastFrame: t0,
-		deadline:   int(params.Deadline / time.Microsecond),
-		frame:      make([]byte, 1024),
+		r:             video.ToI420(r),
+		codec:         codec,
+		raw:           rawNoBuffer,
+		cfg:           cfg,
+		tStart:        t0,
+		tLastFrame:    t0,
+		deadline:      int(params.Deadline / time.Microsecond),
+		frame:         make([]byte, 1024),
+		targetBitrate: params.BitRate,
 	}, nil
 }
 
@@ -260,6 +262,16 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	if duration == 0 {
 		duration = 1
 	}
+
+	targetVpxBitrate := C.uint(float32(e.targetBitrate/1000) * 0.93) // 0.93 takes into account IP/UDP/RTP overhead
+	if e.cfg.rc_target_bitrate != targetVpxBitrate && targetVpxBitrate >= 1 {
+		e.cfg.rc_target_bitrate = targetVpxBitrate
+		rc := C.vpx_codec_enc_config_set(e.codec, e.cfg)
+		if rc != C.VPX_CODEC_OK {
+			return nil, func() {}, fmt.Errorf("vpx_codec_enc_config_set failed (%d)", rc)
+		}
+	}
+
 	var flags int
 	if e.requireKeyFrame {
 		flags = flags | C.VPX_EFLAG_FORCE_KF
@@ -302,6 +314,13 @@ func (e *encoder) ForceKeyFrame() error {
 	return nil
 }
 
+func (e *encoder) SetBitRate(bitrate int) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.targetBitrate = bitrate
+	return nil
+}
+
 func (e *encoder) Controller() codec.EncoderController {
 	return e
 }

pkg/codec/x264/bridge.h

@@ -8,6 +8,7 @@
 #define ERR_ALLOC_PICTURE -3
 #define ERR_OPEN_ENGINE -4
 #define ERR_ENCODE -5
+#define ERR_BITRATE_RECONFIG -6
 
 typedef struct Slice {
   unsigned char *data;
@@ -78,6 +79,23 @@ Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
   return NULL;
 }
 
+#define RC_MARGIN 10000 /* 1kilobits / second*/
+static int apply_target_bitrate(Encoder *e, int target_bitrate) {
+  float adjusted_bitrate = (float)target_bitrate * 0.93; // 0.93 takes into account IP/UDP/RTP overhead
+  int target_encoder_bitrate = (int)adjusted_bitrate / 1000;
+  if (e->param.rc.i_bitrate == target_encoder_bitrate || target_encoder_bitrate <= 1) {
+    return 0; // no change or too small, no error
+  }
+
+  e->param.rc.i_bitrate = target_encoder_bitrate;
+  e->param.rc.f_rate_tolerance = 0.1;
+  e->param.rc.i_vbv_max_bitrate = target_encoder_bitrate + RC_MARGIN / 2;
+  e->param.rc.i_vbv_buffer_size = e->param.rc.i_vbv_max_bitrate;
+  e->param.rc.f_vbv_buffer_init = 0.6;
+  int success = x264_encoder_reconfig(e->h, &e->param);
+  return success; // 0 on success or negative on error
+}
+
 Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
   x264_nal_t *nal;
   int i_nal;

pkg/codec/x264/x264.go

@@ -39,6 +39,8 @@ func (e cerror) Error() string {
 		return errOpenEngine.Error()
 	case C.ERR_ENCODE:
 		return errEncode.Error()
+	case C.ERR_BITRATE_RECONFIG:
+		return errSetBitrate.Error()
 	default:
 		return "unknown error"
 	}
@@ -58,6 +60,7 @@ var (
 	errAllocPicture  = fmt.Errorf("failed to alloc picture")
 	errOpenEngine    = fmt.Errorf("failed to open x264")
 	errEncode        = fmt.Errorf("failed to encode")
+	errSetBitrate    = fmt.Errorf("failed to change x264 encoder bitrate")
 )
 
 func newEncoder(r video.Reader, p prop.Media, params Params) (codec.ReadCloser, error) {
@@ -132,6 +135,16 @@ func (e *encoder) ForceKeyFrame() error {
 	return nil
 }
 
+func (e *encoder) SetBitRate(bitrate int) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	errNum := C.apply_target_bitrate(e.engine, C.int(bitrate))
+	if err := errFromC(errNum); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (e *encoder) Controller() codec.EncoderController {
 	return e
 }

pkg/codec/x264/x264_test.go

@@ -69,17 +69,13 @@ func TestEncoder(t *testing.T) {
 }
 
 func TestShouldImplementKeyFrameControl(t *testing.T) {
-	t.SkipNow() // TODO: Implement key frame control
-
 	e := &encoder{}
 	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
 		t.Error()
 	}
 }
 
 func TestShouldImplementBitRateControl(t *testing.T) {
-	t.SkipNow() // TODO: Implement bit rate control
-
 	e := &encoder{}
 	if _, ok := e.Controller().(codec.BitRateController); !ok {
 		t.Error()