using raspberry pi camera with ffmpeg (hardware accelerated)

rpi_camera_v4l2_ffmpeg.md

Using Raspberry Pi Camera with ffmpeg

Capturing video from the rpi camera with ffmpeg can vary from less than 5% to 100% of the CPU (rpi zero) depending on ffmpeg using the hardware acceleration or not.

On many github issues one finds the suggestion of using h264_omx codec to use the gpu - but it does not ship with the default ffmpeg on Raspbian.

Instead I found that one can use the v4l2 driver provided by raspbian to get hardware accelerated h264 output. Also setting the video size will save one from using a (cpu) scale filter.

ffmpeg

capture h264 video from rpi camera

ffmpeg -f video4linux2 -input_format h264 -video_size 1280x720 -framerate 30 -i /dev/video0 -vcodec copy -an test.h264

capture jpg image from rpi camera

ffmpeg -f video4linux2 -input_format mjpeg -video_size 1280x720 -i /dev/video0 -vframes 1 -f mjpeg -

Note: ffmpeg currently does not support setting the bitrate of v4l2 devices see bug report - call v4l2-ctl --set-ctrl video_bitrate=300000 instead (setting to 300k)

v4l2 driver swithing to "stills mode" modprobe bcm2835-v4l2 max_video_width=2592 max_video_height=1944 source

nodejs example for use with homebridge

Use homebridge-camera-rpi:

npm install -g homebridge-camera-rpi

Code

handleSnapshotRequest and handleStreamRequest for hap-nodejs or homebridge-camera-ffmpeg

var spawn = require('child_process').spawn

...

handleSnapshotRequest = function (request, callback) {
  let ffmpegCommand = `\
-f video4linux2 -input_format mjpeg -video_size ${request.width}x${request.height} -i /dev/video0 \
-vframes 1 -f mjpeg -`
  console.log(ffmpegCommand)
  let ffmpeg = spawn('ffmpeg', ffmpegCommand.split(' '), {env: process.env})
  var imageBuffer = Buffer(0)
  ffmpeg.stdout.on('data', function (data) { imageBuffer = Buffer.concat([imageBuffer, data]) })
  ffmpeg.stderr.on('data', function (data) { console.log('ffmpeg', String(data)) })
  ffmpeg.on('close', function (code) { callback(undefined, imageBuffer) })
}

...

handleStreamRequest = function (request) {
  var sessionID = request['sessionID']
  var requestType = request['type']
  if (!sessionID) return
  let sessionIdentifier = uuid.unparse(sessionID)

  if (requestType === 'start' && this.pendingSessions[sessionIdentifier]) {
    var width = 1280
    var height = 720
    var fps = 30
    var bitrate = 300

    if (request['video']) {
      width = request['video']['width']
      height = request['video']['height']
      fps = Math.min(fps, request['video']['fps'])
      bitrate = request['video']['max_bit_rate']
    }

    let srtp = this.pendingSessions[sessionIdentifier]['video_srtp'].toString('base64')
    let address = this.pendingSessions[sessionIdentifier]['address']
    let port = this.pendingSessions[sessionIdentifier]['video_port']

    let ffmpegCommand = `\
-f video4linux2 -input_format h264 -video_size ${width}x${height} -framerate ${fps} -i /dev/video0 \
-vcodec copy -an -payload_type 99 -ssrc 1 -f rtp \
-srtp_out_suite AES_CM_128_HMAC_SHA1_80 -srtp_out_params ${srtp} \
srtp://${address}:${port}?rtcpport=${port}&localrtcpport=${port}&pkt_size=1378`
    console.log(ffmpegCommand)
    let ffmpeg = spawn('ffmpeg', ffmpegCommand.split(' '), {env: process.env})
    ffmpeg.stderr.on('data', function (data) { console.log('ffmpeg', String(data)) })
    this.ongoingSessions[sessionIdentifier] = ffmpeg

    delete this.pendingSessions[sessionIdentifier]
  }
  if (requestType === 'stop' && this.ongoingSessions[sessionIdentifier]) {
    this.ongoingSessions[sessionIdentifier].kill('SIGKILL')
    delete this.ongoingSessions[sessionIdentifier]
  }
}

Works only with Buster, not with the 'new' Bullseye OS.