AWS HA Proxy failover script. Uses two nodes with autofailover by assuming a "VIP."

crontab

@reboot /path/to/ha_monitor.sh >> /var/log/ha_monitor.log 2>&1

ha_monitor.sh

#!/bin/sh
# This script will monitor another HA node and take over a Virtual IP (VIP)
# if communication with the other node fails.
#
# This script was written with a few assumptions:
# * This is running CentOS 7 or similar
# * This instance has a single network interface (eth0)
# * This instance can ping the other instance (HA_NODE_IP). Open ICMP in sec groups.
#
# Some helpful articles used in developing this script:
# https://aws.amazon.com/articles/2127188135977316 (<- original script from here)
# http://www.joshwieder.net/2015/08/assigning-multiple-ip-addresses-to.html
# https://codepen.io/tsabat/post/assign-a-floating-secondary-ip-address-in-aws-vpc
#
# This script requires at least these permissions in AWS IAM:
# {
#  "Statement": [
#    {
#      "Action": [
#        "ec2:AssignPrivateIpAddresses",
#        "ec2:DescribeInstances"
#      ],
#      "Effect": "Allow",
#      "Resource": "*"
#    }
#  ]
# }
#
######## CONFIG SECTION ########
# This is the Virtal IP (a.k.a. "Floating" IP)
VIP=172.5.5.5
# The IP of the other node in this setup
HA_NODE_IP=172.5.6.6
# The EC2 region this instance is running within
REGION=us-east-1
######## END CONFIG SECTION ########
function logger {
  echo `date` "-- $1"
}
logger "Started HA monitor"
# Grabs the instance ID
INSTANCE_ID=`/usr/bin/curl --silent http://169.254.169.254/latest/meta-data/instance-id`
logger "Running on instance $INSTANCE_ID"
# Determines the network interface ID
logger "Determining the Elastic Network Interface ID... (you may see errors about credentials, which is normal)"
while [ "$ENI_ID" == "" ]; do
  ENI_ID=`aws ec2 describe-instances --output=text --instance-id $INSTANCE_ID --region $REGION --query 'Reservations[0].Instances[0].NetworkInterfaces[0].NetworkInterfaceId'`
done
logger "Found ENI $ENI_ID"
# Check to see if the VIP is already applied to the interface
ipconfigured=`/sbin/ip addr show | grep $VIP -m 1 | wc -l`
if [ "$ipconfigured" == "0" ]; then
  # Add the VIP to the interface
  logger "Adding $VIP/20 as a floating IP on eth0"
  /sbin/ip addr add $VIP/20 dev eth0
fi
logger "Starting heartbeat detection. This will remain quiet until a heartbeat failure is detected."
while [ . ]; do
  # Checks to see if this instance already has the VIP
  hasvip=`aws ec2 describe-instances --instance-id $INSTANCE_ID --region $REGION | grep $VIP -m 1 | wc -l`
  # Pings other instance to see if it's up and running
  pingresult=`ping -c 3 -W 1 $HA_NODE_IP | grep time= | wc -l`
  if [ "$hasvip" == "0" ] && [ "$pingresult" == "0" ]; then
    logger "HA heartbeat failed, taking over VIP"
    # This command has no output
    aws ec2 assign-private-ip-addresses --network-interface-id $ENI_ID --private-ip-addresses $VIP --allow-reassignment --region $REGION
    if [ $? -eq 0 ]; then
      logger "assign private ip address succeeded!"
    else
      logger "assign private ip address failed.";
    fi
    # try restarting network service if still not responding
    pingresult=`ping -c 1 -W 1 $VIP | grep time= | wc -l`
    if [ "$pingresult" == "0" ]; then
      logger "Restarting eth0"
      ifdown eth0 && ifup eth0
    fi
    # wait a minute after re-assigning IP
    sleep 60
  fi
  # wait 2 seconds between pings
  sleep 2
done