rolling-drain.sh

#!/usr/bin/env bash

if [[ "$1" == '-h' ]] || [[ $# != 1 ]]; then
  echo "Usage: $(basename "$0") <nodepool>"
  echo
  echo "<nodepool> can be a partial match to grep for, e.g. '2016-08-19'"
  echo
  echo "NOTE: Before draining all nodes in the old pool should be cordoned!"
  exit 1;
fi

declare -r nodepool="$1"

current_pod_count() {
  kubectl get pods --all-namespaces prod 2> /dev/null | grep -v kube-system | grep -v infra | grep -v default | wc -l | tr -d ' '
}

# only watch prod pod counts - other envs are not critical
running_prod_pod_count() {
  kubectl get pods --namespace prod 2> /dev/null | \
    grep Running | wc -l | tr -d ' '
}

init_pod_count=`running_prod_pod_count`

init_pod_threshold=`printf "%.0f" $(echo $init_pod_count \* .99 | bc)`

# we don't use this anymore since all nodes will be unschedulable
# after cordoning them
# might be useful in the future to select nodes that have pods on them (i.e.
# haven't already been drained)
schedulable='{{range .items}}{{if not .spec.unschedulable}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'
all_nodes='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'


echo
echo "--"
echo "Starting a rolling drain of schedulable nodes matching $nodepool"
echo "--"
echo

echo "Initial pod count: $init_pod_count"
echo "Threshold: $init_pod_threshold"
echo


# Future improvement: keep track of index in the loop vs total count of all
# nodes so we can report percentage complete.
while read node; do
  echo "[$(date)] Draining ${node}..."
  echo "[$(date)] Running: kubectl drain --force --ignore-daemonsets $node"
  kubectl drain --force --ignore-daemonsets $node
  echo "[$(date)] Sleeping for 10 seconds..."
  sleep 10

  echo "[$(date)] Waiting for prod pod count of `running_prod_pod_count` to surpass $init_pod_threshold..."
  while [ `running_prod_pod_count` -lt "$init_pod_threshold" ]; do
    echo "[$(date)] Waiting for prod pod count of `running_prod_pod_count` to surpass $init_pod_threshold..."
    sleep 2
  done
  echo "[$(date)] Finished draining ${node}. Prod pod count is at `running_prod_pod_count`."
  echo
done << EOF
  $(kubectl get nodes -o go-template="$all_nodes" | grep $nodepool)
EOF