Script to gracefully shutdown a compact OpenShift cluster.

shutdown-compact-openshift.sh

#!/bin/bash

# This script tries to gracefully shutdown a 3-node / "compact" OCP cluster
# There seems to be some coordination required when shutting down
# when ODF, Logging, and Virtualization are deployed.

CLUSTER_NAME="your-cluster-name-here"
OC_BIN=/usr/local/bin/oc
KUBECONFIG=~/.kube/config
OC_CMD="$OC_BIN --kubeconfig=$KUBECONFIG"

# check if we're connected to the correct cluster
CONNECTED_CLUSTER=$($OC_CMD whoami --show-console)
if [[ $CONNECTED_CLUSTER =~ $CLUSTER_NAME ]]; then
  CERT_EXPIRE=$($OC_CMD -n openshift-kube-apiserver-operator get secret/kube-apiserver-to-kubelet-signer -o jsonpath='{.metadata.annotations.auth\.openshift\.io/certificate-not-after}')
  echo "Please restart the cluster before the certificates expire: $(date -d $CERT_EXPIRE)"
else
  echo "Error: Not connected to $CLUSTER_NAME"
  echo "found $CONNECTED_CLUSTER instead!"
  exit 1
fi

echo "Marking all nodes as Unschedulable"
for i in $($OC_CMD get nodes -o name); do
  $OC_CMD adm cordon $i
done

echo ; echo "Shut down things that are using ODF storage..."
echo "  Shutting down all VirtualMachines..."
$OC_CMD delete virtualmachineinstances --all --all-namespaces

echo ; echo "  Shutting down the monitoring stack"
$OC_CMD delete pods -n openshift-monitoring -l app.kubernetes.io/name=prometheus
$OC_CMD delete pods -n openshift-monitoring -l app.kubernetes.io/name=alertmanager

echo ; echo "  Shutting down NooBaa and it's postgres that likes to get stuck..."
$OC_CMD delete pods -n openshift-storage -l app=noobaa

# nothing should be using ODF at this point, move on to shutting down the nodes
echo "Telling the nodes to shutdown..."
for node in $($OC_CMD get nodes -o name); do
  oc debug $node -- chroot /host shutdown -h 1
done

echo ; echo "Please remember to \"uncordon\" the nodes when the cluster is restarted!"
#for i in $(oc get nodes -o name); do oc adm uncordon $i; done

Completely stolen from:

https://hackmd.io/@johnsimcall/HJXwaZkJC

Completely stolen from:

https://hackmd.io/@johnsimcall/HJXwaZkJC

❤️