This script is created to handle stalled ffmpeg processes

ffmpeg_monitor.sh

#!/bin/bash
# This script is created to handle stalled ffmpeg processes

PROCESS=ffmpeg
USER=hightail
TIME=+5
#LOGSERVER=local-hightailspaces.com
LOGSERVERPORT=6379
KEY=logstash
KEYTYPE=RPUSH
TYPE=ffmpeg-monitor
KILL=false

logstash_process_details () { # msg, "pid,user,stime,cputime,pcpu,rss,args"
  LOG_PID=$(echo $2 | awk '{print $1}')
  LOG_USER=$(echo $2 | awk '{print $2}')
  LOG_STIME=$(echo $2 | awk '{print $3}')
  LOG_CPUTIME=$(echo $2 | awk '{print $4}')
  LOG_PCPU=$(echo $2 | awk '{print $5}')
  LOG_RSS=$(echo $2 | awk '{print $6}')
  LOG_ARGS=$(echo $2 | awk '{for (i=7; i<=NF; i++) print $i}' | tr '\n' ' ' | sed -e 's/[[:space:]]*$//')

  if [ -z "${LOGSERVER}" ] ; then
    echo "{\"message\": \"${1}\", \"type\": \"${TYPE}\", \"pid\": \"${LOG_PID}\", \"user\": \"${LOG_USER}\", \"stime\": \"${LOG_STIME}\", \"cputime\": \"${LOG_CPUTIME}\", \"pcpu\": \"${LOG_PCPU}\", \"rss\": \"${LOG_RSS}\", \"args\": \"${LOG_ARGS}\"}"
  else
    (printf "%s %s \"%s\"\r\n" "${KEYTYPE}" "${KEY}" "{\\\"message\\\": \\\"${1}\\\", \\\"type\\\": \\\"${TYPE}\\\", \\\"pid\\\": \\\"${LOG_PID}\\\", \\\"user\\\": \\\"${LOG_USER}\\\", \\\"stime\\\": \\\"${LOG_STIME}\\\", \\\"cputime\\\": \\\"${LOG_CPUTIME}\\\", \\\"pcpu\\\": \\\"${LOG_PCPU}\\\", \\\"rss\\\": \\\"${LOG_RSS}\\\", \\\"args\\\": \\\"${LOG_ARGS}\\\"}"; sleep 1) # | nc ${LOGSERVER} ${LOGSERVERPORT} > /dev/null
  fi
}

logstash () {
  if [ -z "${LOGSERVER}" ] ; then
    echo "${1}"
  else
    (printf "%s %s \"%s\"\r\n" "${KEYTYPE}" "${KEY}" "{\\\"message\\\": \\\"${1}\\\", \\\"type\\\": \\\"${TYPE}\\\"}"; sleep 1) | nc ${LOGSERVER} ${LOGSERVERPORT} > /dev/null
  fi
}

PROCESSES=($(find /proc -maxdepth 1 -user ${USER} -type d -mmin ${TIME} -exec basename -z {} \; | xargs -0 ps -o pid,comm | grep ${PROCESS} | awk '{print $1;}' ))
PROCESSCOUNT=$(echo ${PROCESSES[*]} | wc -w)
logstash "Number of ${PROCESS} processes running longer than ${TIME} minutes ${PROCESSCOUNT}"

for PID in "${PROCESSES[@]}"; do
  if [ "${KILL}" = true ] ; then
    logstash_process_details "Killing ${PROCESS} ${PID}" "$(ps --no-headers -o pid,user,stime,cputime,pcpu,rss,args ${PID})"
    kill "${PID}" 2> /dev/null
    kill -9 "${PID}" 2> /dev/null
  else
    logstash_process_details "TEST: Killing ${PROCESS} ${PID}" "$(ps --no-headers -o pid,user,stime,cputime,pcpu,rss,args ${PID})"
  fi
done