Monitor VMI counts per node, node status

monitor-vmis

#!/bin/bash

function indent() {
    local -i level=${1:-4}
    if [[ -n "$*" ]] ; then
	shift
    fi
    local istr="%${level}s %s\n"
    local LINE
    if [[ -n "$*" ]] ; then
	printf "$istr" "" "$*"
    else
	while IFS= read -r LINE ; do
	    printf "$istr" "" "$LINE"
	done
    fi
}

function meta_distribution() {
    local count
    local -a nodes=("$@")
    local -A nodecounts=()
    local -A counts
    local -i total=0
    local -i min=0
    local -i max=0
    local -a lines
    local -i grand_total=0
    local -a x
    local LINE
    while IFS= read -r LINE ; do
	if [[ -z "$LINE" ]] ; then
	    continue
	fi
	lines+=("$LINE")
	local -i x
	local node
	read -r x node <<< "$LINE"
	nodecounts[${node:-unknown}]=$x
	counts[$x]=$((${counts[$x]:-0} + 1))
	total=$((total+1))
	grand_total=$((grand_total + x))
    done
    if ((total <= 0)) ; then
	return
    fi
    local -i count
    for count in "${!counts[@]}" ; do
	if ((min == 0)) ; then
	    min=$count
	    max=$count
	elif ((count < min)) ; then
	    min=$count
	elif ((count > max)) ; then
	    max=$count
	fi
    done
    indent 4 "VMI count per node distribution:"
    for ((count=min; count <= max; count++)) ; do
	if ((${counts[$count]:-0} > 0)) ; then
	    indent 8 "$(printf "%8i %3i\n" "$count" "${counts[$count]:-0}")"
	fi
    done
    indent 8 "$(printf "%8s %3i\n" nodes "$total")"
    echo
    indent 4 "VMI detail distribution:"
    local n
    for n in "${nodes[@]}" ; do
	printf "%7d %s\n" "${nodecounts[$n]:-0}" "$n"
    done | sort -k1n -k2 | indent 8
    #(IFS=$'\n'; echo "${lines[*]}" | indent 8)
    indent 8 "$(printf "%7d total VMIs\n" "$grand_total")"
}

while : ; do
    date '+%Y-%m-%d %H:%M:%S'
    declare -a nodes=()
    declare -A nodestatus=()
    while read -r node status ; do
	nodes+=("$node")
	nodestatus[$status]=$((${nodestatus[$status]:-0} + 1))
    done <<< "$(oc get node --no-headers | grep worker | sort | awk '{print $1, $2}')"
    indent 4 "By VM status:"
    oc get vm -A --no-headers | awk '{print $4}' | sort | uniq -c | indent 8
    echo
    indent 4 MCP:
    oc get mcp worker | indent 8
    echo
    indent 4 "Node status:"
    for a in "${!nodestatus[@]}" ; do
	printf "%5d %s\n" "${nodestatus[$a]}" "$a"
    done | sort -k 2 | indent 8
    echo
    indent 4 "oc adm top node:"
    oc adm top node | indent 8
    echo
    oc get vmi -A -owide --no-headers 2>/dev/null | awk '{print $6}' | sort | uniq -c |sort -n| meta_distribution "${nodes[@]}"
    echo
    echo
    sleep 60
done

The usual way I run this is

(./monitor-vmis </dev/null > "vmi-monitor-$(date '+%Y-%m-%dT%T').log" 2>&1 &)