nodes-with-failing-pods.sh

  while true; do
    nodesWithFailingPods=""
    if failedPods=$(kubectl get pods -n default --no-headers | awk  '$4 > 1 {print $1}') && [[ "$failedPods" != "" ]]; then
      for failedPod in $failedPods; do
        echo "$(date) pod failed too many times: '$failedPod'"
        if failedPodNodeName=$(kubectl get pod -n default "$failedPod" -ojsonpath='{.spec.nodeName'}); then
          nodesWithFailingPods="$nodesWithFailingPods $failedPodNodeName"
        fi
      done

      if kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints --no-headers | grep "effect:NoSchedule key:node.kubernetes.io/unschedulable"; then
        echo "$(date) already one node(s) draining"
      else
        for nodeWithFailingPods in $nodesWithFailingPods; do
          while true; do
            kubectl get node "$nodeWithFailingPods" || break

            echo "$(date) node drain: '$nodeWithFailingPods'"
            kubectl drain "$nodeWithFailingPods" --delete-emptydir-data --ignore-daemonsets --force || true
            sleep 1
          done
          # only drain one node at a time
          break
        done
      fi
    else
      echo "$(date) no failed pods"
    fi

    sleep 20
  done