Skip to content

Source Sink Design

Jump to bottom
Lukas Herman edited this page Oct 11, 2020 · 63 revisions

Goal

  • Provide a standard way of reading media data
  • Provide an extension to non-webrtc applications
  • Easy to use

Problem

  • The current design is tightly coupled to WebRTC, it feels awkward if being used for other use cases
  • The simple example is not actually that simple 😓

Design

Location: pkg/io/video and pkg/io/audio

type Broadcaster struct {}

func (broadcaster *Broadcaster) Source() Reader {}
func (broadcaster *Broadcaster) ReplaceSource(source Reader) {}
func (broadcaster *Broadcaster) NewReader() Reader {}

Location: root

package mediadevices

type videoRef struct {
  *video.Broadcaster
  counter uint
}

type audioRef struct {
  *audio.Broadcaster
  counter uint
}

// refCounters is being used for letting us know when to Close drivers. For example, if 1 driver is being shared to
// multiple tracks, we don't want to close the driver when there's another track that's still consuming the driver.
var videoRefCounters map[string]*videoRef
var audioRefCounters map[string]*audioRef

func GetUserMedia(c MediaStreamConstraints) MediaStream {}

type Track interface {
  ID() string
  Stop() error
  OnEnded(func(error))
  Bind(LocalTrackContext)
  Unbind(LocalTrackContext)
}

type VideoTrack interface {
  Track
  *video.Broadcaster
  TransformSource(...video.TransformFunc)
}

type AudioTrack interface {
  Track
  *audio.Broadcaster
  TransformSource(...video.TransformFunc)
}

type VideoSource interface {
  video.Reader
  Close() error
}

type AudioSource interface {
  audio.Reader
  Close() error
}

func NewVideoTrack(source VideoSource) Track {
  // 1. shim source to capture error in the stream and report it through the handler
  //    that's set from OnEnded. 
  // 2. Wrap source with broadcaster
  // 3. Fill all necessary fields
}

// internal video track creation from driver
func newVideoTrackFromDriver(d driver.Driver) Track {
  // 1. Retrieve video.Reader from the driver
  // 2. Wrap the retrieved video.Reader and the driver into a VideoSource
  // 3. Create track by calling NewVideoTrack
}

// CodecSelector is preinitialized with a list of video/audio encoder builders.
// It is responsible to find desired codec and create a generic encoded Reader
type CodecSelector struct {
}

func (selector *CodecSelector) Populate(...) {}

func (selector *CodecSelector) selectVideoCodec(name string, track *VideoTrack) (codec.ReadCloser, err) {}
func (selector *CodecSelector) selectVideoCodec(name string, track *AudioTrack) (codec.ReadCloser, err) {}

Example

WebRTC

package main

import (
	"github.com/pion/mediadevices"
	"github.com/pion/mediadevices/pkg/codec/openh264"
	"github.com/pion/mediadevices/pkg/codec/x264"
	"github.com/pion/mediadevices/pkg/prop"

	// Note: If you don't have a camera or microphone or your adapters are not supported,
	//       you can always swap your adapters with our dummy adapters below.
	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
)

func main() {
	var setting webrtc.SettingEngine

	x264Params, _ := x264.NewParams()
	openh264Params, _ := openh264.NewParams()
	codecSelector, _ := mediadevices.NewCodecSelector(x264Params, openh264Params)
	codecSelector.Populate(&setting)

	// User can still modify setting to their needs here

	api := webrtc.NewAPI(webrtc.WithSettingEngine(&setting))

	// Assume that we have configured the api and have proper config
	pc, _ := api.NewPeerConnection(peerConnectionConfig)

	mediaStream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
		Video: func(constraint *mediadevices.MediaTrackConstraints) {
			constraint.Width = prop.Int(600)
			constraint.Height = prop.Int(400)
		},
                Codec: codecSelector,
	})

        // Add a custom video transformation
        videoTrack := mediaStream.GetVideoTracks()[0]
        videoTrack.TransformSource(transformFn1, transformFn2)

	pc.AddTransceiverFromTrack(videoTrack,
		webrtc.RtpTransceiverInit{
			Direction: webrtc.RTPTransceiverDirectionSendonly,
		},
	)

	// peerconnection should negotiate with the other peer
	// peerconnection should set the negotiated codec with LocalRTPTrack.setParameters()

	// mediadevices then will set the codec and attach the encoder to the pipeline. PeerConnection now can start
	// pulling encoded frames from mediadevices.
}

Non WebRTC

package main

import (
       "github.com/pion/webrtc/v2"
       "github.com/pion/mediadevices"
)

func main() {
       s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
		Video: func(c *mediadevices.MediaTrackConstraints) {
			c.Width = 640
			c.Height = 480
		},
       })
       
       videoTrack := s.GetVideoTracks()[0]
       videoReader := videoTrack.NewReader()
       // Do whatever with this videoReader
}

Reference

Usage

Developer Guide

Clone this wiki locally