repave_nomad_servers.sh

#!/usr/bin/env bash

set -e

sanitize_nomad () {
  nomad system gc
  nomad system reconcile summaries
}

deregister_instance () {
  local dia=${1}    # arn
  local dii=${2}    # instance
  echo -e "deregistering instance ${dii} from ${dia}\n"
  aws elbv2 deregister-targets \
  --target-group-arn "${dia}" \
  --targets Id="${dii}"
  echo -e "\n\n"
}

wait_until_terminated () {
  local wuti=${1}   # instance
  while true; do
    sleep 10

    local isn=$(aws ec2 describe-instances --instance-id "${wuti}" \
    | jq -r '.Reservations[].Instances[0].State.Name')

    echo -e "instance ${wuti} state name is ${isn}\n"

    sanitize_nomad

    if [[ "${isn}" == "terminated" ]]; then
      sleep 30
      break
    fi
  done
}

INSTANCES=($(aws ec2 describe-instances \
--filters "Name=tag:Name,Values=my-${ENVIRONMENT}-${SERVER_NAME}" \
"Name=instance-state-name,Values=running" | \
jq -r '.Reservations[].Instances[] | [.LaunchTime, .InstanceId, .PrivateDnsName] | @csv' | \
sed 's/"//g' | \
sort))

echo """
AWS EC2 Instances (CSV order is LaunchTime, InstanceId, PrivateDnsName):

$(printf '%s\n' "${INSTANCES[@]}")
"""

TARGET_GROUP_ARNS=($(aws elbv2 describe-target-groups \
--names my-${ENVIRONMENT}-nomad-server | \
jq -r '.TargetGroups[].TargetGroupArn'))

echo """
Target Group: ${TARGET_GROUP_ARNS[0]}
"""

# number of alive nomad servers
NOMAD_SERVERS=3

for i in "${INSTANCES[@]}"; do
  a=(${i//,/ })
  lt=${a[0]}    # LaunchTime
  ii=${a[1]}    # InstanceId
  pdn=${a[2]}   # PrivateDnsName

  echo """
LaunchTime: ${lt}
InstanceId: ${ii}
PrivateDnsName: ${pdn}
"""

  # number of eligible and ready nomad nodes
  nnc=$(nomad node status | grep " eligible " | grep -c ready | xargs)

  # nomad node id
  nni=$(nomad node status | grep "${pdn}" | awk '{print $1}' | xargs)

  echo """
Nomad Server Count: ${NOMAD_SERVERS}
Nomad Node Count: ${nnc}
Nomad Node ID: ${nni}
"""

  deregister_instance ${TARGET_GROUP_ARNS[0]} ${ii}

  sleep 70

  echo -e "disabling eligibility for ${nni}/${pdn}\n"
  nomad node eligibility -disable "${nni}"
  echo -e "\n\n"

  sanitize_nomad
  sleep 30
  sanitize_nomad

  echo -e "draining allocations for ${nni}/${pdn}\n"
  nomad node drain -yes -m "repaving" -deadline 3m -enable "${nni}"
  echo -e "\n\n"

  sanitize_nomad
  sleep 30
  sanitize_nomad

  echo -e "terminating ec2 ${ii}/${pdn}\n"
  aws ec2 terminate-instances --instance-ids "${ii}"
  echo -e "\n\n"

  wait_until_terminated ${ii}

  while true; do
    sleep 10

    sanitize_nomad

    # number of eligible and ready nomad nodes
    tnsc=$(nomad server members | grep -c " alive " | xargs)

    nomad server members
    echo -e "\n\n"

    # compare previous number of alive nomad servers to current
    if [[ ${NOMAD_SERVERS} -eq ${tnsc} ]]; then
      sleep 30
      break
    fi
  done
done