Archives for June 2009

GStreamer, RTP and live streaming

June 14, 2009 at 11:08 AM

I wanted to stream a live video feed to the Internet, and have some Flash player available on the web that would make my live stream available.

I've searched services, or ways to do that. I've found ustream.tv and justin.tv.

After trying to use my Ubuntu machine to stream video out there (with a simple V4L2 camera), I've had much trouble (ustream.tv still doesn't work) but worked out a way to effectively stream a 2 hours ceremony directly on the web, with 2 computers, a wireless router, a Mini-DV camera, a simple sound card, GStreamer and the justin.tv web service.

Here are the pieces I've used (and the references that helped me out):

GStreamer 0.10
- dv1394src and plenty of other useful elements
- RTP support through GstRtpBin
Python and the GStreamer python bindings
WebCamStudio: for testing, finally I didn't use it
vloopback: to pipe some video content into a virtual V4L camera device (which Flash apps can read). You can take it from the WebCamStudio website.
mjpegtools_yuv_to_v4l-0.2.tgz at http://panteltje.com/panteltje/mcamip/
PulseAudio to stream audio to the Flash app
http://www.justin.tv/ web service

Website references I've found useful to understand the whole GStreamer thing and especially the RTP parts:

http://www.jejik.com/articles/2007/01/streaming_audio_over_tcp_with_python-gstreamer/
http://blog.nicolargo.com/2009/02/jai-streame-avec-gstreamer.html
Very useful (thanks wtay):
- http://webcvs.freedesktop.org/gstreamer/gst-plugins-good/gst/rtp/README?view=markup
Absolutely not useful (even though Google will make you think it's the best result):
- http://gstreamer.freedesktop.org/documentation/rtp.html
http://www.jonobacon.org/2006/11/03/gstreamer-dynamic-pads-explained/
http://gstreamer.freedesktop.org/data/doc/gstreamer/head/manual/html/section-caps-api.html
http://marlenehe.wordpress.com/2009/06/03/mercredi-3-juin-2009/
And finally, especially for the PulseAudio part: http://ronnietucker.co.uk/blog/broadcasting-video-from-ubuntu-to-ustream-with-webcamstudio/

Here are the scripts I've used. First the video_streamer.py file:

#!/usr/bin/env python
########### VIDEO_STREAMER 

import gobject, pygst
pygst.require("0.10")
import gst
import gobject
import sys
import os
import readline

REMOTE_HOST = '192.168.33.153'
WRITE_VIDEO_CAPS = 'video.caps'

mainloop = gobject.MainLoop()
pipeline = gst.Pipeline('server')
bus = pipeline.get_bus()

dv1394src = gst.element_factory_make("dv1394src", "dv1394src")
dvdemux = gst.element_factory_make("dvdemux", "dvdemux")
q1 = gst.element_factory_make("queue", "q1")
q2 = gst.element_factory_make("queue", "q2")
dvdec = gst.element_factory_make("dvdec", "dvdec")
videoscale = gst.element_factory_make('videoscale')
ffmpegcs = gst.element_factory_make("ffmpegcolorspace", "ffmpegcs")
capsfilter = gst.element_factory_make('capsfilter')
capsfilter.set_property('caps', gst.caps_from_string('video/x-raw-yuv, width=320, height=240'))
tcpsrc = gst.element_factory_make("tcpserversrc", "source")
x264enc = gst.element_factory_make("x264enc", "x264enc")
x264enc.set_property('qp-min', 18)
rtph264pay = gst.element_factory_make("rtph264pay", "rtph264pay")
udpsink_rtpout = gst.element_factory_make("udpsink", "udpsink0")
udpsink_rtpout.set_property('host', REMOTE_HOST)
udpsink_rtpout.set_property('port', 10000)
udpsink_rtcpout = gst.element_factory_make("udpsink", "udpsink1")
udpsink_rtcpout.set_property('host', REMOTE_HOST)
udpsink_rtcpout.set_property('port', 10001)
udpsrc_rtcpin = gst.element_factory_make("udpsrc", "udpsrc0")
udpsrc_rtcpin.set_property('port', 10002)

rtpbin = gst.element_factory_make('gstrtpbin', 'gstrtpbin')

# Add elements
pipeline.add(dv1394src, dvdemux, q1, dvdec, videoscale, ffmpegcs, capsfilter, x264enc, rtph264pay, rtpbin, udpsink_rtpout, udpsink_rtcpout, udpsrc_rtcpin)

# Link them
dv1394src.link(dvdemux)
def dvdemux_padded(dbin, pad):
    print "dvdemux got pad %s" % pad.get_name()
    if pad.get_name() == 'video':
        print "Linking dvdemux to queue1"
        dvdemux.link(q1)

# Create links
dvdemux.connect('pad-added', dvdemux_padded)

gst.element_link_many(q1, dvdec, videoscale, capsfilter, ffmpegcs, x264enc, rtph264pay)

rtph264pay.link_pads('src', rtpbin, 'send_rtp_sink_0')
rtpbin.link_pads('send_rtp_src_0', udpsink_rtpout, 'sink')
rtpbin.link_pads('send_rtcp_src_0', udpsink_rtcpout, 'sink')
udpsrc_rtcpin.link_pads('src', rtpbin, 'recv_rtcp_sink_0')

def go():
    print "Setting locked state for udpsink"
    print udpsink_rtcpout.set_locked_state(gst.STATE_PLAYING)
    print "Setting pipeline to PLAYING"
    print pipeline.set_state(gst.STATE_PLAYING)
    print "Waiting pipeline to settle"
    print pipeline.get_state()
    print "Final caps written to", WRITE_VIDEO_CAPS
    open(WRITE_VIDEO_CAPS, 'w').write(str(udpsink_rtpout.get_pad('sink').get_property('caps')))
    mainloop.run()

go()

And for the audio part, in the file audio_streamer.py:

#!/usr/bin/env python
# -=- encoding: utf-8 -=-

import gobject, pygst
pygst.require("0.10")
import gst
import gobject
import sys
import os
import readline


# To the laptop that will catch everything
REMOTE_HOST = '192.168.33.153'
WRITE_AUDIO_CAPS = 'audio.caps'

mainloop = gobject.MainLoop()
pipeline = gst.Pipeline('server')
bus = pipeline.get_bus()

#alsasrc = gst.element_factory_make("autoaudiosrc")
alsasrc = gst.element_factory_make("alsasrc")
alsasrc.set_property('device', 'plughw:1,0')
q1 = gst.element_factory_make("queue", "q1")
q2 = gst.element_factory_make("queue", "q2")
audioconvert1 = gst.element_factory_make("audioconvert")
audioconvert2 = gst.element_factory_make("audioconvert")
vorbisenc = gst.element_factory_make("vorbisenc")
rtpvorbispay = gst.element_factory_make("rtpvorbispay")
udpsink_rtpout = gst.element_factory_make("udpsink", "udpsink0")
udpsink_rtpout.set_property('host', REMOTE_HOST)
udpsink_rtpout.set_property('port', 11000)
udpsink_rtcpout = gst.element_factory_make("udpsink", "udpsink1")
udpsink_rtcpout.set_property('host', REMOTE_HOST)
udpsink_rtcpout.set_property('port', 11001)
udpsrc_rtcpin = gst.element_factory_make("udpsrc", "udpsrc0")
udpsrc_rtcpin.set_property('port', 11002)

rtpbin = gst.element_factory_make('gstrtpbin', 'gstrtpbin')

# Add elements
pipeline.add(alsasrc, q1, audioconvert1, audioconvert2, vorbisenc, rtpvorbispay, rtpbin, udpsink_rtpout, udpsink_rtcpout, udpsrc_rtcpin)

# Link them
alsasrc.link(audioconvert1)
audioconvert1.link(vorbisenc)
vorbisenc.link(rtpvorbispay)
rtpvorbispay.link_pads('src', rtpbin, 'send_rtp_sink_0')
rtpbin.link_pads('send_rtp_src_0', udpsink_rtpout, 'sink')
rtpbin.link_pads('send_rtcp_src_0', udpsink_rtcpout, 'sink')
udpsrc_rtcpin.link_pads('src', rtpbin, 'recv_rtcp_sink_0')

def go():
    print "Setting locked state for udpsink"
    print udpsink_rtcpout.set_locked_state(gst.STATE_PLAYING)
    print "Setting pipeline to PLAYING"
    print pipeline.set_state(gst.STATE_PLAYING)
    print "Waiting pipeline to settle"
    print pipeline.get_state()
    print "Final caps writte to", WRITE_AUDIO_CAPS
    open(WRITE_AUDIO_CAPS, 'w').write(str(udpsink_rtpout.get_pad('sink').get_property('caps')))
    mainloop.run()

go()

Note that the video.caps and audio.caps file must be shared between hosts, using Samba, NFS, or sshfs for example

For testing purposes, I've used video_receiver.py

#!/usr/bin/env python
# -=- encoding: utf-8 -=-
################ VIDEO RECEIVER

import gobject, pygst
pygst.require("0.10")
import gst


# TODO: detect from the RTPSource element inside the GstRtpBin
REMOTE_HOST = '192.168.34.150'
READ_VIDEO_CAPS = 'video.caps'

pipeline = gst.Pipeline('server')

caps = open(READ_VIDEO_CAPS).read().replace('\\', '')
rtpbin = gst.element_factory_make('gstrtpbin', 'rtpbin')
rtpbin.set_property('latency', 400)
udpsrc_rtpin = gst.element_factory_make('udpsrc', 'udpsrc0')
udpsrc_rtpin.set_property('port', 10000)
udpsrc_caps = gst.caps_from_string(caps)
udpsrc_rtpin.set_property('caps', udpsrc_caps)
udpsrc_rtcpin = gst.element_factory_make('udpsrc', 'udpsrc1')
udpsrc_rtcpin.set_property('port', 10001)
udpsink_rtcpout = gst.element_factory_make('udpsink', 'udpsink0')
udpsink_rtcpout.set_property('host', REMOTE_HOST)
udpsink_rtcpout.set_property('port', 10002)

rtph264depay = gst.element_factory_make('rtph264depay', 'rtpdepay')
q1 = gst.element_factory_make("queue", "q1")
q2 = gst.element_factory_make("queue", "q2")
avimux = gst.element_factory_make('avimux', 'avimux')
filesink = gst.element_factory_make('filesink', 'filesink')
filesink.set_property('location', '/tmp/go.avi')

ffmpegcs = gst.element_factory_make("ffmpegcolorspace", "ffmpegcs")
ffdec264 = gst.element_factory_make('ffdec_h264', 'ffdec264')
autovideosink = gst.element_factory_make('autovideosink')

pipeline.add(rtpbin, udpsrc_rtpin, udpsrc_rtcpin, udpsink_rtcpout,
             rtph264depay, q1, avimux, ffdec264, autovideosink)

# Receive the RTP and RTCP streams
udpsrc_rtpin.link_pads('src', rtpbin, 'recv_rtp_sink_0')
udpsrc_rtcpin.link_pads('src', rtpbin, 'recv_rtcp_sink_0')
# reply with RTCP stream
rtpbin.link_pads('send_rtcp_src_0', udpsink_rtcpout, 'sink')
# Plus the RTP into the rest of the pipe...
def rtpbin_pad_added(obj, pad):
    print "PAD ADDED"
    print "  obj", obj
    print "  pad", pad
    rtpbin.link(rtph264depay)
rtpbin.connect('pad-added', rtpbin_pad_added)
gst.element_link_many(rtph264depay, q1, ffdec264, autovideosink)

def start():
    pipeline.set_state(gst.STATE_PLAYING)
    udpsink_rtcpout.set_locked_state(gst.STATE_PLAYING)
    print "Started..."

def loop():
    print "Running..."
    gobject.MainLoop().run()

if __name__ == '__main__':
    start()
    loop()

Here is the actual script to forward the incoming video to the virtual vloopback device. In video_forwarder.py:

#!/usr/bin/env python
# -=- encoding: utf-8 -=-
############### VIDEO FORWARDER

import gobject, pygst
pygst.require("0.10")
import gst


# TODO: detect from RTPSource
REMOTE_HOST = '192.168.34.150'
READ_VIDEO_CAPS = 'video.caps'

pipeline = gst.Pipeline('server')

caps = open(READ_VIDEO_CAPS).read().replace('\\', '')
rtpbin = gst.element_factory_make('gstrtpbin', 'rtpbin')
rtpbin.set_property('latency', 400)
udpsrc_rtpin = gst.element_factory_make('udpsrc', 'udpsrc0')
udpsrc_rtpin.set_property('port', 10000)
udpsrc_caps = gst.caps_from_string(caps)
udpsrc_rtpin.set_property('caps', udpsrc_caps)
udpsrc_rtcpin = gst.element_factory_make('udpsrc', 'udpsrc1')
udpsrc_rtcpin.set_property('port', 10001)
udpsink_rtcpout = gst.element_factory_make('udpsink', 'udpsink0')
udpsink_rtcpout.set_property('host', REMOTE_HOST)
udpsink_rtcpout.set_property('port', 10002)

rtph264depay = gst.element_factory_make('rtph264depay', 'rtpdepay')
q1 = gst.element_factory_make("queue", "q1")
q2 = gst.element_factory_make("queue", "q2")
avimux = gst.element_factory_make('avimux', 'avimux')

ffmpegcs = gst.element_factory_make("ffmpegcolorspace", "ffmpegcs")
ffdec264 = gst.element_factory_make('ffdec_h264', 'ffdec264')
autovideosink = gst.element_factory_make('autovideosink')
y4menc = gst.element_factory_make('y4menc')
filesink = gst.element_factory_make('filesink', 'filesink')
filesink.set_property('location', '/tmp/go.pipe')

pipeline.add(rtpbin, udpsrc_rtpin, udpsrc_rtcpin, udpsink_rtcpout,
             rtph264depay, q1, avimux, ffdec264, y4menc, filesink)

# Receive the RTP and RTCP streams
udpsrc_rtpin.link_pads('src', rtpbin, 'recv_rtp_sink_0')
udpsrc_rtcpin.link_pads('src', rtpbin, 'recv_rtcp_sink_0')
# reply with RTCP stream
rtpbin.link_pads('send_rtcp_src_0', udpsink_rtcpout, 'sink')
# Plus the RTP into the rest of the pipe...
def rtpbin_pad_added(obj, pad):
    print "PAD ADDED"
    print "  obj", obj
    print "  pad", pad
    rtpbin.link(rtph264depay)
rtpbin.connect('pad-added', rtpbin_pad_added)
gst.element_link_many(rtph264depay, q1, ffdec264, y4menc, filesink)

def start():
    pipeline.set_state(gst.STATE_PLAYING)
    udpsink_rtcpout.set_locked_state(gst.STATE_PLAYING)
    print "Started..."

def loop():
    print "Running..."
    gobject.MainLoop().run()

if __name__ == '__main__':
    import os
    os.system('rm /tmp/go.pipe')
    os.system('mkfifo /tmp/go.pipe')
    pipeline.get_state()
    os.system('cat /tmp/go.pipe | mjpegtools_yuv_to_v4l /dev/video2 &')
    start()
    loop()

The last bit is to pipe audio through the PulseAudio daemon, so that Flash thinks it's coming from the microphone. In audio_receiver.py:

#!/usr/bin/env python
# -=- encoding: utf-8 -=-
########### AUDIO RECEIVER

import gobject, pygst
pygst.require("0.10")
import gst


# Stream to:
REMOTE_HOST = '192.168.34.150'
READ_AUDIO_CAPS = 'audio.caps'

caps = open(READ_AUDIO_CAPS).read().replace('\\', '')

pipeline = gst.Pipeline('audio-receiver')

rtpbin = gst.element_factory_make('gstrtpbin')
rtpbin.set_property('latency', 1000)
udpsrc_rtpin = gst.element_factory_make('udpsrc')
udpsrc_rtpin.set_property('port', 11000)
udpsrc_caps = gst.caps_from_string(caps)
udpsrc_rtpin.set_property('caps', udpsrc_caps)
udpsrc_rtcpin = gst.element_factory_make('udpsrc')
udpsrc_rtcpin.set_property('port', 11001)
udpsink_rtcpout = gst.element_factory_make('udpsink')
udpsink_rtcpout.set_property('host', REMOTE_HOST)
udpsink_rtcpout.set_property('port', 11002)

rtpvorbisdepay = gst.element_factory_make('rtpvorbisdepay')
q1 = gst.element_factory_make("queue", "q1")
q2 = gst.element_factory_make("queue", "q2")

audioconvert = gst.element_factory_make("audioconvert")
vorbisdec = gst.element_factory_make('vorbisdec')
autoaudiosink = gst.element_factory_make('pulsesink')

pipeline.add(rtpbin, udpsrc_rtpin, udpsrc_rtcpin, udpsink_rtcpout, audioconvert,
             rtpvorbisdepay, q1, vorbisdec, autoaudiosink)

# Receive the RTP and RTCP streams
udpsrc_rtpin.link_pads('src', rtpbin, 'recv_rtp_sink_0')
udpsrc_rtcpin.link_pads('src', rtpbin, 'recv_rtcp_sink_0')
# reply with RTCP stream
rtpbin.link_pads('send_rtcp_src_0', udpsink_rtcpout, 'sink')
# Plus the RTP into the rest of the pipe...

def rtpbin_pad_added(obj, pad):
    print "PAD ADDED"
    print "  obj", obj
    print "  pad", pad
    rtpbin.link(rtpvorbisdepay)
rtpbin.connect('pad-added', rtpbin_pad_added)

gst.element_link_many(rtpvorbisdepay, q1, vorbisdec, audioconvert,
                      autoaudiosink)

def start():
    pipeline.set_state(gst.STATE_PLAYING)
    udpsink_rtcpout.set_locked_state(gst.STATE_PLAYING)
    print "Started..."

def loop():
    print "Running..."
    gobject.MainLoop().run()

if __name__ == '__main__':
    start()
    loop()

Here are pitfalls about RTP I've fell in:

the CAPS had to be set and exchanged between the different RTP parts. That's what video.caps and audio.caps are used in those scripts.

Read and Post Comments

Alexandre Bourget

Archives for June 2009

GStreamer, RTP and live streaming

Latest Posts

Categories

Archives