prometheus custom metrics server kubernetes resource yamls

custom-metrics-apiserver-tls-keys.certificate.yaml

apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  name: custom-metrics-apiserver-tls-keys
  labels:
    app: prometheus-custom-metrics
  namespace: prometheus
spec:
  secretName: custom-metrics-apiserver-tls-keys
  commonName: custom-metrics-apiserver.prometheus.svc
  organization:
    - custom-metrics-apiserver
  duration: 9999h
  usages:
    - server auth
  # Issuer references are always required.
  issuerRef:
    name: cluster-ca
    kind: ClusterIssuer

custom-metrics-apiserver.deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: custom-metrics-apiserver
  annotations:
    reloader.stakater.com/auto: "true"
  name: custom-metrics-apiserver
  namespace: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: custom-metrics-apiserver
  template:
    metadata:
      labels:
        app: custom-metrics-apiserver
      name: custom-metrics-apiserver
    spec:
      serviceAccountName: custom-metrics-apiserver
      containers:
        - name: custom-metrics-apiserver
          image: directxman12/k8s-prometheus-adapter-amd64:v0.7.0
          args:
            - --secure-port=6443
            - --tls-cert-file=/var/run/serving-cert/tls.crt
            - --tls-private-key-file=/var/run/serving-cert/tls.key
            - --logtostderr=true
            - --prometheus-url=http://prometheus:9090/
            - --metrics-relist-interval=1m
            - --config=/etc/adapter/config.yaml
            # - --v=8
          ports:
            - containerPort: 6443
          volumeMounts:
            - mountPath: /var/run/serving-cert
              name: volume-serving-cert
              readOnly: true
            - mountPath: /etc/adapter/
              name: config
              readOnly: true
            - mountPath: /tmp
              name: tmp-vol
      volumes:
        - name: volume-serving-cert
          secret:
            secretName: custom-metrics-apiserver-tls-keys
        - name: config
          configMap:
            name: custom-metrics-config
        - name: tmp-vol
          emptyDir: {}
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule

custom-metrics-apiserver.pod-disruption-budget.yaml

apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: custom-metrics-apiserver
  namespace: prometheus
spec:
  minAvailable: 0
  selector:
    matchLabels:
      app: custom-metrics-apiserver

custom-metrics-apiserver.service.yaml

apiVersion: v1
kind: Service
metadata:
  name: custom-metrics-apiserver
  namespace: prometheus
spec:
  ports:
    - port: 443
      targetPort: 6443
  selector:
    app: custom-metrics-apiserver

custom-metrics-auth-reader.role-binding.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: custom-metrics-auth-reader
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
subjects:
  - kind: ServiceAccount
    name: custom-metrics-apiserver
    namespace: prometheus

custom-metrics-config.configMap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: custom-metrics-config
  namespace: prometheus
data:
  config.yaml: |
    rules:
    - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}'
      seriesFilters: []
      resources:
        overrides:
          namespace:
            resource: namespace
          pod_name:
            resource: pod
      name:
        matches: ^container_(.*)_seconds_total$
        as: ""
      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[1m])) by (<<.GroupBy>>)
    - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}'
      seriesFilters:
      - isNot: ^container_.*_seconds_total$
      resources:
        overrides:
          namespace:
            resource: namespace
          pod_name:
            resource: pod
      name:
        matches: ^container_(.*)_total$
        as: ""
      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[1m])) by (<<.GroupBy>>)
    - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}'
      seriesFilters:
      - isNot: ^container_.*_total$
      resources:
        overrides:
          namespace:
            resource: namespace
          pod_name:
            resource: pod
      name:
        matches: ^container_(.*)$
        as: ""
      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>)
    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
      seriesFilters:
      - isNot: .*_total$
      resources:
        template: <<.Resource>>
      name:
        matches: ""
        as: ""
      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
      seriesFilters:
      - isNot: .*_seconds_total
      resources:
        template: <<.Resource>>
      name:
        matches: ^(.*)_total$
        as: ""
      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
    - seriesQuery: '{namespace="elasticsearch",__name__="elasticsearch_thread_pool_completed_count"}'
      seriesFilters: []
      resources:
        template: <<.Resource>>
      name:
        matches: "^(elasticsearch_thread_pool_completed)_count$"
        as: "${1}_per_second"
      metricsQuery: sum(rate(elasticsearch_thread_pool_completed_count{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
      seriesFilters: []
      resources:
        template: <<.Resource>>
      name:
        matches: ^(.*)_seconds_total$
        as: ""
      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
    resourceRules:
      cpu:
        containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
        nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
        resources:
          overrides:
            instance:
              resource: node
            namespace:
              resource: namespace
            pod_name:
              resource: pod
        containerLabel: container_name
      memory:
        containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
        nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
        resources:
          overrides:
            instance:
              resource: node
            namespace:
              resource: namespace
            pod_name:
              resource: pod
        containerLabel: container_name
      window: 1m
    externalRules:
    - seriesQuery: '{__name__=~"^.*_queue_(length|size)$",namespace!=""}'
      resources:
        overrides:
          namespace:
            resource: namespace
      name:
        matches: ^.*_queue_(length|size)$
        as: "$0"
      metricsQuery: max(<<.Series>>{<<.LabelMatchers>>})
    - seriesQuery: '{__name__=~"^.*_queue$",namespace!=""}'
      resources:
        overrides:
          namespace:
            resource: namespace
      name:
        matches: ^.*_queue$
        as: "$0"
      metricsQuery: max(<<.Series>>{<<.LabelMatchers>>})

custom-metrics-resource-reader.cluster-role-binding.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: custom-metrics-resource-reader
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: custom-metrics-resource-reader
subjects:
  - kind: ServiceAccount
    name: custom-metrics-apiserver
    namespace: prometheus

custom-metrics-resource-reader.cluster-role.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: custom-metrics-resource-reader
rules:
  - apiGroups:
      - ""
    resources:
      - pods
      - nodes
      - nodes/stats
    verbs:
      - get
      - list
      - watch

custom-metrics-server-resources.clusterrole.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: custom-metrics-server-resources
rules:
  - apiGroups:
      - custom.metrics.k8s.io
      - external.metrics.k8s.io
    resources: ["*"]
    verbs: ["*"]

custom-metrics-system-auth-delegator.cluster-role-binding.yaml

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: custom-metrics:system:auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
  - kind: ServiceAccount
    name: custom-metrics-apiserver
    namespace: prometheus

hpa-controller-custom-metrics.cluster-role-binding.yaml

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: hpa-controller-custom-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: custom-metrics-server-resources
subjects:
  - kind: ServiceAccount
    name: horizontal-pod-autoscaler
    namespace: kube-system

kustomization.yaml

resources:
  - custom-metrics-apiserver.deployment.yaml
  - custom-metrics-apiserver.pod-disruption-budget.yaml
  - custom-metrics-apiserver.service.yaml
  - custom-metrics-apiserver-tls-keys.certificate.yaml
  - custom-metrics-auth-reader.role-binding.yaml
  - custom-metrics-resource-reader.cluster-role.yaml
  - custom-metrics-resource-reader.cluster-role-binding.yaml
  - custom-metrics-server-resources.clusterrole.yaml
  - custom-metrics-system-auth-delegator.cluster-role-binding.yaml
  - hpa-controller-custom-metrics.cluster-role-binding.yaml
  - prometheus-custom-metrics.serviceaccount.yaml
  - v1beta1.custom.metrics.k8s.io.apiservice.yaml
  - v1beta1.external.metrics.k8s.io.apiservice.yaml
  - v1beta2.custom.metrics.k8s.io.apiservice.yaml
  - custom-metrics-config.configMap.yaml

prometheus-custom-metrics.serviceaccount.yaml

kind: ServiceAccount
apiVersion: v1
metadata:
  name: custom-metrics-apiserver
  namespace: prometheus

v1beta1.custom.metrics.k8s.io.apiservice.yaml

apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta1.custom.metrics.k8s.io
  annotations:
    cert-manager.io/inject-ca-from: cert-manager/cluster-ca
spec:
  service:
    name: custom-metrics-apiserver
    namespace: prometheus
  group: custom.metrics.k8s.io
  version: v1beta1
  groupPriorityMinimum: 100
  versionPriority: 100

v1beta1.external.metrics.k8s.io.apiservice.yaml

apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta1.external.metrics.k8s.io
  annotations:
    cert-manager.io/inject-ca-from: cert-manager/cluster-ca
spec:
  service:
    name: custom-metrics-apiserver
    namespace: prometheus
  group: external.metrics.k8s.io
  version: v1beta1
  groupPriorityMinimum: 100
  versionPriority: 100

v1beta2.custom.metrics.k8s.io.apiservice.yaml

apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta2.custom.metrics.k8s.io
  annotations:
    cert-manager.io/inject-ca-from: cert-manager/cluster-ca
spec:
  service:
    name: custom-metrics-apiserver
    namespace: prometheus
  group: custom.metrics.k8s.io
  version: v1beta2
  groupPriorityMinimum: 100
  versionPriority: 200