cycle-monitor-pods-4-10-target-down-workaround.sh

#/bin/bash
set -euo pipefail

monitoringPodList="$(oc -n openshift-monitoring get pods)"

firstDelete=(
  "openshift-monitoring/alertmanager-main-0"
  "openshift-monitoring/prometheus-k8s-0"
  "openshift-monitoring/$(echo "$monitoringPodList" | grep "grafana" | awk '{print $1}')"
  "openshift-monitoring/$(echo "$monitoringPodList" | grep "telemeter-client" | awk '{print $1}')"
  "openshift-monitoring/$(echo "$monitoringPodList" | grep "thanos-querier" | awk '{print $1}' | head -n 1)"
  "openshift-user-workload-monitoring/prometheus-user-workload-0"
)

secondDelete=(
  "openshift-monitoring/alertmanager-main-1"
  "openshift-monitoring/prometheus-k8s-1"
  "openshift-monitoring/$(echo "$monitoringPodList" | grep "thanos-querier" | awk '{print $1}' | tail -n 1)"
  "openshift-user-workload-monitoring/prometheus-user-workload-1"
)

printf "Deleting the following pods:\n"
for pod in ${firstDelete[@]}; do
  printf "\t$pod\n"
done
for pod in ${secondDelete[@]}; do
  printf "\t$pod\n"
done

read -p "Are you sure? [y/n]"  -n 1 -r
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
    [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
fi


for pod in ${firstDelete[@]}; do
  echo "Deleting pod $pod"
  oc -n "$(cut -d / -f 1 <<< $pod)" delete pod "$(cut -d / -f 2 <<< $pod)"
done

echo "Waiting for userworkloadmonitoring-0 and prometheus-0 to come back up..."
oc wait --for=condition=ready -n openshift-user-workload-monitoring pod prometheus-user-workload-0
oc wait --for=condition=ready -n openshift-monitoring pod prometheus-k8s-0

for pod in ${secondDelete[@]}; do
  echo "Deleting pod $pod"
  oc -n "$(cut -d / -f 1 <<< $pod)" delete pod "$(cut -d / -f 2 <<< $pod)"
done

Workaround for known issue https://bugzilla.redhat.com/show_bug.cgi?id=2033575 in Openshift 4.10. Run this after connected to your cluster.