Last active
October 25, 2021 15:18
-
-
Save hexfusion/479a28fcdb83de166d7c6e5de3a4bae9 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
read -p "Number of etcd to remain: " ETCD_MEMBERS_KEEP | |
# scale down CVO and CEO | |
NODES=($(oc get node -l 'node-role.kubernetes.io/master=' -o 'jsonpath={.items[*].metadata.name}')) | |
REMAINING_ETCD_NODE=${NODES[-1]} | |
for (( i=0; i<$ETCD_MEMBERS_KEEP; i++ )); do | |
unset NODES[-1] | |
done | |
# populate 1mb data file to pipe into etcdctl | |
oc debug "node/$REMAINING_ETCD_NODE" -- chroot /host /bin/bash -c "dd if=/dev/urandom bs=1000000 count=1 | base64 -w0| sed 's/ //g'| tr -d '\n' > /var/lib/etcd/1mb-data-file" | |
oc rsh -n openshift-etcd etcd-$REMAINING_ETCD_NODE bash -c 'for key in {1..3000};do etcdctl put /fat/$key < /var/lib/etcd/1mb-data-file;done' | |
for node in ${NODES[@]}; do | |
# get the member id for this node and scale it down. | |
id=$(oc rsh -c etcdctl -n openshift-etcd etcd-$node etcdctl member list | grep $node | awk '{print $1}' | tr -d ',') | |
echo "scaling down etcd $id on $node" | |
oc rsh -c etcdctl -n openshift-etcd etcd-$node etcdctl member remove $id | |
# stop etcd static pod. | |
echo "stopping etcd on $node"; | |
oc debug "node/$node" -- chroot /host /bin/bash -c 'sudo -i; mv /etc/kubernetes/manifests/etcd-pod.yaml /tmp' | |
sleep 5 | |
echo "removing etcd data-dir on $node"; | |
oc debug "node/$node" -- chroot /host /bin/bash -c 'sudo -i;rm -rf /var/lib/etcd' | |
done | |
oc rsh -n openshift-etcd etcd-${REMAINING_ETCD_NODE} etcdctl member list -w table | |
# let cluster settle | |
sleep 120 | |
# scale back up | |
oc scale --replicas 1 -n openshift-cluster-version deployments/cluster-version-operator || true | |
oc scale --replicas 1 -n openshift-etcd-operator deployments/etcd-operator || true | |
# roll out new etcd | |
oc patch etcd cluster -p='{"spec": {"forceRedeploymentReason": "rollout-new-'"$( date --rfc-3339=ns )"'"}}' --type=merge | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment