check_recorder.sh

#! /bin/bash
logger Recorder Check Running
SERVER=localhost
PORT=2003
TOTAL_SENT_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Sending ' | wc -l)
NO_STREAM_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'No Stream for ' | wc -l)
DECODE_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep  'Control Channel Message Decode Rate: 0/sec' | wc -l)
DURATION_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep  'Deleting this call as it has a duration less than minimum duration of' | wc -l)
ENCRYPTED_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep  'Not Recording: ENCRYPTED' | wc -l)
OLDER1=$(find /tmp/streamthis-*-lastrun -not -newermt '-1 minutes' | wc -l)
OLDER5=$(find /tmp/streamthis-*-lastrun -not -newermt '-5 minutes' | wc -l)
OLDER10=$(find /tmp/streamthis-*-lastrun -not -newermt '-10 minutes' | wc -l)
AGE_COUNT=$(find /tmp/streamthis-scclaw-lastrun -not -newermt '-5 minutes' | wc -l)

find /tmp/streamthis-*-lastrun | while read file; do
  DATE=`date +%s`;
  UTIME=`stat -L --format %Y $file`;
  AGE=`expr $DATE - $UTIME`;
  STREAM=`echo $file | awk -F '-' '{print $2}'`;
  METRIC="recorder.recorder3.stream.$STREAM.age $AGE"
  echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

  SENT_COUNT=$(sed -n "/^$(date --date='1 minutes ago' '+%b %_d %H:%M')/,\$p" /var/log/syslog | grep 'Sending ' | grep $STREAM | wc -l)
  METRIC="recorder.recorder3.streams.$STREAM.sent.count $SENT_COUNT"
  echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
done
HISTORY=$1
LIMIT=$2

METRIC="recorder.recorder3.streams.older.1 $OLDER1"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

METRIC="recorder.recorder3.streams.older.5 $OLDER5"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

METRIC="recorder.recorder3.streams.older.10 $OLDER10"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

METRIC="recorder.recorder3.streams.sent.count $TOTAL_SENT_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

METRIC="recorder.recorder3.streams.none.count $NO_STREAM_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

METRIC="recorder.recorder3.streams.encrypted.count $ENCRYPTED_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

METRIC="recorder.recorder3.recorder.errors.decoder.count $DECODE_COUNT"
echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;

if [ "$DECODE_COUNT" -gt "$LIMIT" ]; then
       logger Too many control channel decode failures, count: $DECODE_COUNT is higher than limit: $LIMIT restarting recorder;
        METRIC="recorder.recorder3.recorder.restart.decoder.count 1"
        echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
        systemctl restart recorder;
        RETVAL=1;
else
        METRIC="recorder.recorder3.recorder.restart.decoder.count 0"
        echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
        RETVAL=0;
fi

METRIC="recorder.recorder3.recorder.errors.duration.count $DURATION_COUNT"
if [ "$DURATION_COUNT" -gt "$LIMIT" ]; then
        logger Too many 0 duration calls, count: $DURATION_COUNT is higher than limit: $LIMIT restarting recorder;
        METRIC="recorder.recorder3.recorder.restart.duration.count 1"
        echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
        systemctl restart recorder;
        RETVAL=1;
else
        METRIC="recorder.recorder3.recorder.restart.duration.count 0"
        echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
        RETVAL=0;
fi

if [ "$AGE_COUNT" -gt "0" ]; then
        logger The SCCLAW Stream is too old, restarting recorder;
        METRIC="recorder.recorder3.recorder.restart.age.count 1"
        echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
	      systemctl restart recorder;
        RETVAL=1;
else
	METRIC="recorder.recorder3.recorder.restart.age.count 0"
        echo "${METRIC} `date +%s`" | nc ${SERVER} ${PORT} -q0;
	      RETVAL=0;
fi
exit $RETVAL;