Check etcd status on OCP 4.6.1

check_etcd.sh

#!/bin/bash

# Ref: https://docs.openshift.com/container-platform/4.6/backup_and_restore/backing-up-etcd.html

etcd_node=`oc get pods -n openshift-etcd -l app=etcd -o=jsonpath='{.items[0].spec.nodeName}'`
ssh -i ~/.ssh/dmz-ocp4-rsa core@$etcd_node

id=$(sudo crictl ps --name etcdctl | awk 'FNR==2{ print $1}') && sudo crictl exec -it $id /bin/bash
etcdctl member list -w table
etcdctl endpoint health --cluster
etcdctl endpoint health -w table
etcdctl endpoint status -w table --cluster
exit

gistfile1.txt

# root @ bastion.dmz.ocp4.local in ~/etcd_backup
$ etcd_node=`oc get pods -n openshift-etcd -l app=etcd -o=jsonpath='{.items[0].spec.nodeName}'`

# root @ bastion.dmz.ocp4.local in ~/etcd_backup
$ ssh -i ~/.ssh/dmz-ocp4-rsa core@$etcd_node
Red Hat Enterprise Linux CoreOS 46.82.202010091720-0
  Part of OpenShift 4.6, RHCOS is a Kubernetes native operating system
  managed by the Machine Config Operator (`clusteroperator/machine-config`).

WARNING: Direct SSH access to machines is not recommended; instead,
make configuration changes via `machineconfig` objects:
  https://docs.openshift.com/container-platform/4.6/architecture/architecture-rhcos.html

---
Last login: Fri Feb 19 02:09:34 2021 from 192.168.76.100

id=$(sudo crictl ps --name etcdctl | awk 'FNR==2{ print $1}') && sudo crictl exec -it $id /bin/bash
etcdctl member list -w table
etcdctl endpoint health --cluster
etcdctl endpoint health -w table
exit
exit[core@master0 ~]$
[core@master0 ~]$ id=$(sudo crictl ps --name etcdctl | awk 'FNR==2{ print $1}') && sudo crictl exec -it $id /bin/bash
etcdctl member list -w table
etcdctl endpoint health --cluster
etcdctl endpoint health -w table
exit
exitetcdctl member list -w table
etcdctl endpoint health --cluster
etcdctl endpoint health -w table
exit
exit[root@master0 /]# etcdctl member list -w table
+------------------+---------+---------+---------------------------+---------------------------+------------+
|        ID        | STATUS  |  NAME   |        PEER ADDRS         |       CLIENT ADDRS        | IS LEARNER |
+------------------+---------+---------+---------------------------+---------------------------+------------+
| 5d9ec24d066fd4f1 | started | master0 | https://192.168.76.1:2380 | https://192.168.76.1:2379 |      false |
| 9716d0ced1aa6dd8 | started | master2 | https://192.168.76.3:2380 | https://192.168.76.3:2379 |      false |
| b56b2984143eb82f | started | master1 | https://192.168.76.2:2380 | https://192.168.76.2:2379 |      false |
+------------------+---------+---------+---------------------------+---------------------------+------------+
[root@master0 /]# etcdctl endpoint health --cluster
https://192.168.76.3:2379 is healthy: successfully committed proposal: took = 16.186327ms
https://192.168.76.1:2379 is healthy: successfully committed proposal: took = 16.511055ms
https://192.168.76.2:2379 is healthy: successfully committed proposal: took = 17.676371ms
[root@master0 /]# etcdctl endpoint health -w table
+---------------------------+--------+-------------+-------+
|         ENDPOINT          | HEALTH |    TOOK     | ERROR |
+---------------------------+--------+-------------+-------+
| https://192.168.76.3:2379 |   true | 16.544665ms |       |
| https://192.168.76.2:2379 |   true |  17.19241ms |       |
| https://192.168.76.1:2379 |   true | 18.328995ms |       |
+---------------------------+--------+-------------+-------+
[root@master0 /]# etcdctl endpoint status -w table --cluster
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.76.1:2379 | 5d9ec24d066fd4f1 |   3.4.9 |   42 MB |     false |      false |         6 |      38994 |              38994 |        |
| https://192.168.76.3:2379 | 9716d0ced1aa6dd8 |   3.4.9 |   78 MB |      true |      false |         6 |      38994 |              38994 |        |
| https://192.168.76.2:2379 | b56b2984143eb82f |   3.4.9 |   78 MB |     false |      false |         6 |      38994 |              38994 |        |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@master0 /]# exit
exit
[core@master0 ~]$

Very nice gist, thanks!

I came up with something similar:

check_etcd_status() {
    local ETCD_QUERIES="
        etcdctl member list -w table && \\
        etcdctl endpoint health --cluster && \\
        etcdctl endpoint health -w table && \\
        etcdctl endpoint status -w table --cluster
    "
    local ETCD_POD=$(oc get pods -n openshift-etcd \
        -l app=etcd \
        --no-headers \
        | head -n1 \
        | awk '{print $1;}')

    warn "Check ETCD status"
    oc exec -n openshift-etcd \
        -c etcdctl \
        "$ETCD_POD" -- /bin/bash -c "$ETCD_QUERIES"
}