skyline analyzer watcher

snype_skyline_analyzer.sh

#!/bin/bash
####    snype_skyline_analyzer.sh    ####
#
####
# DESCRIPTION
# This mitigates against analyzer running but it connection timing out
#

# THIS IS JUST A GIST
# THIS IS JUST A GIST
# Deal with logging and variables such as SERVER, statsd_node and if you have
# monit in the mix, etc yourself and your own skyline log paths etc these are
# are just bits for reference for a pattern to run via cron every minute
# Example script log is just flushed here on every run

LOGFILE=/var/log/skyline/snype_skyline_analyzer.log
> $LOGFILE

function log () {
  local log_string="$1"
  local output_type="$2"
  echo "$(date +%Y%m%d%H%M%S) - $output_type: $log_string" >> $LOGFILE
}

# Handle Connection timed out
ANALYZER_RESTART=0
HORIZON_RESTART=0
LOGFILE_EPOCH=$(stat --format=%Y /var/log/skyline/analyzer.log)
SECONDS_DIFFERENCE=$(( $TIMESTAMP - $LOGFILE_EPOCH ))
if [ $SECONDS_DIFFERENCE -gt 180 ]; then
  log "The analyzer log was last modified $SECONDS_DIFFERENCE seconds ago" warn
  log "analyzer set to restart" notice
  ANALYZER_RESTART=1
fi
CONNECTION_TIMEOUT=$(tail -n 100 /var/log/skyline/analyzer.log | grep -c "Connection timed out")
if [ $CONNECTION_TIMEOUT -gt 0 ]; then
  log "The analyzer log reports connection timeout" warn
  log "horizon and analyzer set to restart" notice
  ANALYZER_RESTART=1
  HORIZON_RESTART=1
fi

if [ $ANALYZER_RESTART -eq 0 ]; then
  LOG_ENTRY_DATE=$(tail -n 1 /var/log/skyline/analyzer.log | cut -d' ' -f1-2)
  LOG_EPOCH=$(date -d "$LOG_ENTRY_DATE" +%s)
  SECONDS_DIFF=$(( $TIMESTAMP - $LOG_EPOCH ))
  if [ $SECONDS_DIFF -gt 180 ]; then
    log "The analyzer log was last modified $SECONDS_DIFF seconds ago" warn
    log "analyzer set to restart" notice
    ANALYZER_RESTART=1
  fi
fi

if [ $HORIZON_RESTART -eq 1 ]; then
  log "Restarting horizon" notice
  /sbin/service horizon stop >> $LOGFILE
  /sbin/service horizon start >> $LOGFILE
  echo "$HOSTNAME.skyline.horizon.restarted:$TIMESTAMP|g"| nc -w 5 -u $statsd_node 8125
  log "Submitted $HOSTNAME.skyline.horizon.restarted:$TIMESTAMP|g to $statsd_node" notice
fi

if [ $ANALYZER_RESTART -eq 1 ]; then
  log "Restarting analyzer" notice
  /sbin/service analyzer stop >> $LOGFILE
  /sbin/service analyzer start >> $LOGFILE
  echo "$HOSTNAME.skyline.analyzer.stalled:$TIMESTAMP|g"| nc -w 5 -u $statsd_node 8125
  log "Submitted $HOSTNAME.skyline.analyzer.stalled:$TIMESTAMP|g to $statsd_node" notice
fi