Federate prometheus metrics to another prom instance

Federation-Quick-n-Dirty.md

Prom Operator

oc apply -f -<<EOF
apiVersion: v1
kind: Namespace
metadata:
  creationTimestamp: null
  name: prometheus
spec: {}
status: {}
---
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
  name: prometheus-j6qbp
  namespace: prometheus
spec:
  targetNamespaces:
  - prometheus
---
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: prometheus
  namespace: prometheus
spec:
  channel: beta
  installPlanApproval: Automatic
  name: prometheus
  source: community-operators
  sourceNamespace: openshift-marketplace
  startingCSV: prometheusoperator.0.47.0
EOF

Deploy Blue

kubectl apply -f -<<EOF
apiVersion: v1
kind: Namespace
metadata:
  creationTimestamp: null
  name: blue
spec: {}
status: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: blue
    version: v1
  name: blue
  namespace: blue
spec:
  ports:
    - port: 9000
      name: http
  selector:
    app: blue
  type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: blue
    version: v1
  name: blue
  namespace: blue
spec:
  selector:
    matchLabels:
      app: blue
      version: v1
  replicas: 1
  template:
    metadata:
      labels:
        app: blue
        version: v1
    spec:
      serviceAccountName: blue
      containers:
        - image: docker.io/cmwylie19/metrics-demo
          name: blue
          resources:
            requests:
              memory: "64Mi"
              cpu: "250m"
            limits:
              memory: "128Mi"
              cpu: "500m"
          ports:
            - containerPort: 9000
              name: http
          imagePullPolicy: Always
      restartPolicy: Always
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: blue
  namespace: blue
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
  creationTimestamp: null
  labels:
    app: blue
    version: v1
  name: blue
  namespace: blue
spec:
  port:
    targetPort: http
  to:
    kind: ""
    name: blue
    weight: null
EOF

Deploy Prom Instance

k apply -f -<<EOF
kind: Prometheus
apiVersion: monitoring.coreos.com/v1
metadata:
  name: k8s
  namespace: prometheus
spec:
  serviceMonitorSelector: {}
  serviceMonitorNamespaceSelector: {}
  logLevel: debug
  logFormat: json
  replicas: 1
  image: quay.io/prometheus/prometheus:v2.35.0
  serviceAccountName: prometheus-k8s
  remoteWrite:
    - url: http://$(kubectl get route blue -n blue -ojsonpath='{.status.ingress[0].host}')/alert # My app to read the metrics
  resources:
    requests:
      memory: 400Mi
EOF

RBAC

kubectl apply -f -<<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  creationTimestamp: null
  name: scrape-resources
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - pods/status
  - endpoints
  - services
  verbs:
  - list
  - get
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  creationTimestamp: null
  name: scrape-resources-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: scrape-resources
subjects:
- kind: ServiceAccount
  name: prometheus-k8s
  namespace: openshift-monitoring
EOF

Service Monitor

kubectl apply -f -<<EOF
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    app: blue
  name: blue
  namespace: prometheus
spec:
  endpoints:
  - port: http
  namespaceSelector:
    matchNames:
    - blue
  selector:
    matchLabels:
      app: blue
EOF

ServiceMonitor Federate

k apply -f -<<EOF
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: blue-federated
  namespace: openshift-monitoring
  labels:
    app: blue
spec:
  jobLabel: blue-federated
  namespaceSelector:
    matchNames:
      - prometheus
  selector:
    matchLabels:
       operated-prometheus: "true"
  endpoints:
    - interval: 3s
      scrapeTimeout: 2s 
      params:
        'match[]':
          - up{container="blue"}
          - response_status{container="blue"}
          - http_requests_total{container="blue"}
      path: /federate
      port: web
      targetPort: 9090
      scheme: http
      honorLabels: true
EOF