Check Project Monitoring

check-mon-cron.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: fix-project-monitoring
data:
  fix-project-monitoring.sh: |
    #!/usr/bin/env bash
    
    set -e

    namespaces=$(kubectl get namespaces -o name)

    for namespace in $namespaces
    do
    	if [[ $namespace =~ ^namespace/cattle-project-p-[a-z0-9]+$ ]]; then
    		namespace_name=$(echo -n $namespace | cut -d '/' -f 2)
    		if kubectl get namespace "${namespace_name}-monitoring" &>/dev/null; then
    			echo Project $namespace_name has a monitoring namespace
    			if ! kubectl -n "${namespace_name}-monitoring" get rolebindings.rbac.authorization.k8s.io "${namespace_name}-mon-admin" &>/dev/null; then
    				echo Project monitoring $namespace_name is missing the mon-admin role binding
    				echo Removing project annotation from namespace
    				projectId=$(kubectl get namespace $namespace_name -o jsonpath='{.metadata.annotations.field\.cattle\.io/projectId}')
    				echo Project id is $projectId
    				kubectl annotate namespace $namespace_name field.cattle.io/projectId-
                                sleep 5
    				echo Adding it again
    				kubectl annotate namespace $namespace_name field.cattle.io/projectId=${projectId}
    			fi
    		fi
    	fi
    done
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fix-project-monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fix-project-monitoring
rules:
  - apiGroups: [ "" ]
    resources: [ "namespaces" ]
    verbs: [ "get", "list", "post", "patch" ]
  - apiGroups: [ "rbac.authorization.k8s.io" ]
    resources: [ "rolebindings" ]
    verbs: [ "get", "list" ]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fix-project-monitoring
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fix-project-monitoring
subjects:
  - kind: ServiceAccount
    name: fix-project-monitoring
    namespace: cattle-monitoring-system
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: fix-project-monitoring
spec:
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          securityContext:
            fsGroup: 1000
          serviceAccountName: fix-project-monitoring
          containers:
            - name: fix-project-monitoring
              image: mtr.devops.telekom.de/mcsps/mcsps-tools:latest
              command:
                - /bin/bash
                - /scripts/fix-project-monitoring.sh
              resources:
                requests:
                  cpu: 100m
                  memory: 128Mi
                limits:
                  cpu: 500m
                  memory: 512Mi
              securityContext:
                readOnlyRootFilesystem: true
                capabilities:
                  drop:
                    - ALL
                privileged: false
                allowPrivilegeEscalation: false
                runAsUser: 1000
              volumeMounts:
                - mountPath: /scripts
                  name: scripts
          volumes:
            - name: scripts
              configMap:
                name: fix-project-monitoring
  schedule: "*/20 * * * *"

check-mon.sh

#!/bin/sh
# check for project admin role binding in all project monitoring namespaces

CHECK=()
# cluster to compute
CLUSTER=c-fxzb9
# create a temporary project for transfer
TMPPROJECT=p-5fx59
for i in $(kubectl get namespaces -l helm.cattle.io/helm-project-operated=true -l field.cattle.io/projectId=cattle-monitoring --no-headers -o=custom-columns=NAME:.metadata.labels.'helm\.cattle\.io\/projectId'); do
    if [[ "$i" =~ ^p-[a-z0-9-] ]]; then
        kubectl -n cattle-project-$i-monitoring get rolebindings.rbac.authorization.k8s.io cattle-project-$i-mon-admin && {
          echo "ok"
    } || {
          echo "not ok"
          CHECK+=("$i")
    }
    fi;
done

echo
if [ ${#CHECK[@]} -eq 0 ]; then
    echo "No errors, hooray"
else
    echo "please repair"
    echo "temp project is: ${TMPPROJECT}"
    for b in "${CHECK[@]}"; do
        echo "kubectl annotate ns cattle-project-$b field.cattle.io/projectId=${CLUSTER}:${TMPPROJECT} --overwrite"
        echo "sleep 10"
        echo "kubectl annotate ns cattle-project-$b field.cattle.io/projectId=${CLUSTER}:$b --overwrite"
    done
fi