K8s-CronJob-Silence-Alerts-Without-Duplicates

ConfigMap-silence-alerts-script.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: silence-alerts-script
  namespace: openshift-monitoring
data:
  runScript.sh: |
    #!/bin/bash
    oc -n openshift-monitoring exec alertmanager-main-0 -- amtool silence --alertmanager.url="http://localhost:9093" -o json | jq -r '.[].comment' >> /tmp/existing-silences.txt

    while read -r line;
    do
      #Exact match
      if grep -Fx "$line" /tmp/existing-silences.txt
      then
        echo "silence $line exists"
      else
       case $line in
         openshift-admins-team-custom-silence-DisconnectedInstallation)
          # non-interactive mode otherwise our loop will break after execution
          oc -n openshift-monitoring exec alertmanager-main-0 -c alertmanager -- amtool silence add 'alertname=~APIRemovedInNextEUSReleaseInUse|CannotRetrieveUpdates|ClusterNotUpgradeable|InsightsDisabled' --alertmanager.url="http://localhost:9093" --duration="10y" --comment="openshift-admins-team-custom-silence-DisconnectedInstallation"
          ;;
         openshift-admins-team-custom-silence-UserWorkloadsAlerts)
          # non-interactive mode otherwise our loop will break after execution
          oc -n openshift-monitoring exec alertmanager-main-0 -c alertmanager -- amtool silence add 'namespace!~trident|openshift-.*|kube-.*' --alertmanager.url="http://localhost:9093" --duration="10y" --comment="openshift-admins-team-custom-silence-UserWorkloadsAlerts"
          ;;
         openshift-admins-team-custom-silence-LowSeverity)
          # non-interactive mode otherwise our loop will break after execution
          oc -n openshift-monitoring exec alertmanager-main-0 -c alertmanager -- amtool silence add 'severity=~info|none' --alertmanager.url="http://localhost:9093" --duration="10y" --comment="openshift-admins-team-custom-silence-LowSeverity"
          ;;
       esac
      fi
    done < /script/whitelist.txt

  whitelist.txt: |
    openshift-admins-team-custom-silence-DisconnectedInstallation
    openshift-admins-team-custom-silence-UserWorkloadsAlerts
    openshift-admins-team-custom-silence-LowSeverity

CronJob-silence-alerts.yaml

apiVersion: batch/v1
kind: CronJob
metadata:
  name: silence-alerts
  namespace: openshift-monitoring
spec:
  # Run once a day
  schedule: "0 9 * * *"
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        metadata:
          labels:
            name: silence-alerts
        spec:
          # Privileged (exec commands in pod in openshift-monitoring namespace)
          serviceAccountName: cronjob-sa
          containers:
          - name: silence-alerts
            # https://medium.com/@tamber/mini-howto-the-ultimate-container-tool-image-for-openshift-8e338094822e
            image: quay.io/tamber/container-tools:latest
            command:
            - /script/runScript.sh
            volumeMounts:
            - name: silence-alerts-script
              mountPath: /script
          restartPolicy: Never
          volumes:
          - name: silence-alerts-script
            configMap:
              name: silence-alerts-script
              defaultMode: 493

ServiceAccount-and-RBAC.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cronjob-sa
  namespace: openshift-monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cronjob-sa-crb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: cronjob-sa
  namespace: openshift-monitoring