sh.sh

#!/bin/bash

function kill_failed () {
  sudo monit status | awk -F "[' ]" '/  pid/ { x = $NF } /status/ { if ($NF == "failed") { system("kill -9 "x);system("echo \"killing failed process \""x);exit(3)} }'
}

function start_and_wait_for_queues () {
  echo "Restarting the workers"
  sudo monit restart -g resque_workers
  sudo monit status | awk -F "[' ]" '/  status/ { if (($NF != "running") && ($NF != "failed")) { system("sleep 15"); exit(2)} }'
  WAIT=$?
  while [[ $WAIT == 2 ]]; do
    echo "waiting for everything to start..."
    sudo monit status | awk -F "[' ]" '/  status/ { if (($NF != "running") && ($NF != "failed")) { system("sleep 15"); exit(2)} }'
    WAIT=$?
  done
}

function check_queues () {
  sudo monit status | awk -F "[' ]" '/Process/ { p = $3; } /  pid/ { printf("%s\t%d\n", p, $NF); }' > procs.log
  awk 'FILENAME == "old_procs.log" { p[$1] = $2; } FILENAME == "procs.log" { if (p[$1] == $2) { system("kill -9 "$2);system("echo \"Duplicate process at\"$1");exit(1); } }' old_procs.log procs.log
}


touch procs.log old_procs.log

sudo monit status | awk -F "[' ]" '/Process/ { p = $3; } /  pid/ { printf("%s\t%d\n", p, $NF); }' > old_procs.log
start_and_wait_for_queues
kill_failed
KILLED=$?
while [[ $KILLED == 3 ]]; do
  start_and_wait_for_queues
  kill_failed
  killed=$?
done

check_queues
CHECK=$?
while [[ $CHECK == 1 ]]; do
  start_and_wait_for_queues
  check_queues
  CHECK=$?
done

rm procs.log old_procs.log