Skip to content

Instantly share code, notes, and snippets.

@hexfusion
Last active October 25, 2021 15:18
Show Gist options
  • Save hexfusion/479a28fcdb83de166d7c6e5de3a4bae9 to your computer and use it in GitHub Desktop.
Save hexfusion/479a28fcdb83de166d7c6e5de3a4bae9 to your computer and use it in GitHub Desktop.
#!/bin/bash
read -p "Number of etcd to remain: " ETCD_MEMBERS_KEEP
# scale down CVO and CEO
NODES=($(oc get node -l 'node-role.kubernetes.io/master=' -o 'jsonpath={.items[*].metadata.name}'))
REMAINING_ETCD_NODE=${NODES[-1]}
for (( i=0; i<$ETCD_MEMBERS_KEEP; i++ )); do
unset NODES[-1]
done
# populate 1mb data file to pipe into etcdctl
oc debug "node/$REMAINING_ETCD_NODE" -- chroot /host /bin/bash -c "dd if=/dev/urandom bs=1000000 count=1 | base64 -w0| sed 's/ //g'| tr -d '\n' > /var/lib/etcd/1mb-data-file"
oc rsh -n openshift-etcd etcd-$REMAINING_ETCD_NODE bash -c 'for key in {1..3000};do etcdctl put /fat/$key < /var/lib/etcd/1mb-data-file;done'
for node in ${NODES[@]}; do
# get the member id for this node and scale it down.
id=$(oc rsh -c etcdctl -n openshift-etcd etcd-$node etcdctl member list | grep $node | awk '{print $1}' | tr -d ',')
echo "scaling down etcd $id on $node"
oc rsh -c etcdctl -n openshift-etcd etcd-$node etcdctl member remove $id
# stop etcd static pod.
echo "stopping etcd on $node";
oc debug "node/$node" -- chroot /host /bin/bash -c 'sudo -i; mv /etc/kubernetes/manifests/etcd-pod.yaml /tmp'
sleep 5
echo "removing etcd data-dir on $node";
oc debug "node/$node" -- chroot /host /bin/bash -c 'sudo -i;rm -rf /var/lib/etcd'
done
oc rsh -n openshift-etcd etcd-${REMAINING_ETCD_NODE} etcdctl member list -w table
# let cluster settle
sleep 120
# scale back up
oc scale --replicas 1 -n openshift-cluster-version deployments/cluster-version-operator || true
oc scale --replicas 1 -n openshift-etcd-operator deployments/etcd-operator || true
# roll out new etcd
oc patch etcd cluster -p='{"spec": {"forceRedeploymentReason": "rollout-new-'"$( date --rfc-3339=ns )"'"}}' --type=merge
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment