This script runs health checks in a set of hosts and updates their status on an nginx server. It also sends email notifications with the linux mail command. All other dependencies come natively with linux. It is a great alternative to the comercial nginx health_heck feature.

healthcheck.sh

#!/bin/bash
# By @jacmkno - Activisual S.A.S. - http://activisual.net
# Script summary
# This script runs health cehcks in a set of hosts and updates their status on an nginx server

HOSTS=(201.208.65.152 46.56.115.234 42.79.216.211)
HOSTS_COUNT=3 # Number of hosts in the array
MAIL_SUBJECT='ACTIVISUAL - Health check status change'
MAIL_TO='[email protected]'
NGINX_CONF=/etc/nginx/nginx.conf

#Expression used in grep to accept a proper response. Server response is stripped from linebreaks and the response headers are included at the begining
TEST_EXPRESSION='HTTP/1.1 200 OK.*region-footer'

SCRIPT_DURATION=60 # Keep sending proves until this.
MAX_ERRORS=3 # Flag server as down if the last MAX_ERRORS tests are all errors
CURL_TIMEOUT=2 # Maximum time culr will wait for an answer
SLEEP_SECS=2 # Wait this number of seconds after each prove
PROTOCOL='http' # All servers will be tested with HTTP protocol on port 80

let MIN_LINES=HOSTS_COUNT*MAX_ERRORS*5
let MAX_TIME=`date +%s`+SCRIPT_DURATION-CURL_TIMEOUT-SLEEP_SECS

last_lines=`tr [:space:] '\n' < /tmp/healthchecks|grep \.|tail -n $MIN_LINES`
echo $last_lines > /tmp/healthchecks

while [ `date +%s` -lt $MAX_TIME ]
do
  for host in "${HOSTS[@]}"
  do
    ans=`curl -is -m $CURL_TIMEOUT "$PROTOCOL://$host"|tr -d "\n"|grep -e "$TEST_EXPRESSION"|wc -l`
    echo $host:$ans >> /tmp/healthchecks
    errorsCnt=`tr [:space:] '\n' < /tmp/healthchecks|grep "$host"|tail -n $MAX_ERRORS|grep ":0"|wc -l`
    let serverDown=errorsCnt/MAX_ERRORS
    oldStatus=`cat /tmp/health.$host`
    status="$host is down? $serverDown"
    if [ -n "$oldStatus" ]
    then
      if [ "$oldStatus" != "$status" ]
      then
        if [ "$serverDown" -eq 0 ] && [ "$errorsCnt" -gt 0 ]; then
          status="$oldStatus"
        else
          if [ "$serverDown" -eq 0 ]
          then
            sed 's/'"$host"'\(:[0-9]\+ \| \)down /'$host'\1/' $NGINX_CONF > /tmp/health.newconf
          else
            sed '/down/! s/'"$host"'\(:[0-9]\+ \| \)/&down /' $NGINX_CONF > /tmp/health.newconf
          fi
          echo "status change: $status, errors: $errorsCnt" | mail -s "$MAIL_SUBJECT" $MAIL_TO
          #echo "DEBUG INFO: status change - $status, errors: $errorsCnt"
          mv /tmp/health.newconf $NGINX_CONF
          service nginx restart
        fi
      fi
    fi
    #echo "DEBUG INFO: $status, errors: $errorsCnt"
    echo $status > /tmp/health.$host
    if [ `date +%s` -gt $MAX_TIME ]
    then
      exit 1;
    fi
    sleep $SLEEP_SECS
  done
done