Modified yaml files for Prometheus+Thanos setup

1-thanos-manifests.md

This gist contains modified YAML files used in this post: https://www.infracloud.io/thanos-ha-scalable-prometheus/

Forked from: https://github.com/improbable-eng/thanos/tree/master/kube/manifests

mockmetrics-deployment.yaml

apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: mockmetrics
spec:
  replicas: 1
  template:
    metadata:
      name: mockmetrics
      labels:
        app: mockmetrics
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
    spec:
      containers:
      - name: mockmetrics
        image: docker.io/bhavin192/mockmetrics:prom-bulk
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 50m
            memory: 28Mi
          requests:
            cpu: 50m
            memory: 28Mi

prometheus-gcs.yaml

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: prometheus-gcs
  labels:
    app: prometheus-gcs
    thanos-peer: "true"
spec:
  serviceName: "prometheus"
  replicas: 2
  selector:
    matchLabels:
      app: prometheus-gcs
      thanos-peer: "true"
  template:
    metadata:
      labels:
        app: prometheus-gcs
        thanos-peer: "true"
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "10902"
    spec:
## Commented out because Minikube has only one node, should be commented in for any production setup
#      affinity:
#        podAntiAffinity:
#          requiredDuringSchedulingIgnoredDuringExecution:
#          - labelSelector:
#              matchExpressions:
#              - key: app
#                operator: In
#                values:
#                - prometheus
#            topologyKey: kubernetes.io/hostname
      serviceAccountName: prometheus-server
      containers:
      - name: prometheus
        image: quay.io/prometheus/prometheus:v2.5.0
        args:
        - "--storage.tsdb.retention=24h"
        - "--config.file=/etc/prometheus-shared/prometheus.yml"
        - "--storage.tsdb.path=/var/prometheus"
        - "--storage.tsdb.min-block-duration=2h"
        - "--storage.tsdb.max-block-duration=2h"
        - "--web.enable-lifecycle"
        ports:
        - name: prom-http
          containerPort: 9090
        volumeMounts:
        - name: config-shared
          mountPath: /etc/prometheus-shared
        - name: data
          mountPath: /var/prometheus
      - name: thanos-sidecar
        # Always use explicit image tags (release or master-<date>-sha) instead of ambigous `latest` or `master`.
        image: improbable/thanos:v0.1.0
        env:
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: /creds/gcs-credentials.json
        args:
        - "sidecar"
        - "--log.level=debug"
        - "--tsdb.path=/var/prometheus"
        - "--prometheus.url=http://127.0.0.1:9090"
        - "--cluster.peers=thanos-peers.monitoring.svc.cluster.local:10900"
       # NOTE: This is required to be added in GCS prior startup of this.
        - "--gcs.bucket=thanos-store"
        - "--reloader.config-file=/etc/prometheus/prometheus.yml.tmpl"
        - "--reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yml"
        ports:
        - name: sidecar-http
          containerPort: 10902
        - name: grpc
          containerPort: 10901
        - name: cluster
          containerPort: 10900
        volumeMounts:
        - name: data
          mountPath: /var/prometheus
        - name: gcs-credentials
          mountPath: /creds/
        - name: config-shared
          mountPath: /etc/prometheus-shared
        - name: config
          mountPath: /etc/prometheus
      volumes:
      - name: config
        configMap:
          name: prometheus-config-gcs
      - name: config-shared
        emptyDir: {}
      - name: data
        emptyDir: {}
      - name: gcs-credentials
        secret:
          defaultMode: 420
          # NOTE: gcs-credentials secret with single file gcs-credentials.json (GCS service account) is required.
          secretName: gcs-credentials
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config-gcs
data:
  prometheus.yml.tmpl: |-
    global:
      external_labels:
        monitor: prometheus
        replica: '$(HOSTNAME)'

    scrape_configs:
    - job_name: prometheus
      static_configs:
        - targets:
          - "127.0.0.1:9090"

    - job_name: kubelets
      kubernetes_sd_configs:
      - role: node

    - job_name: kube_pods
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: keep
        regex: mockmetrics
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: (.+?)(?::\d+)?;(\d+)
        replacement: ${1}:${2}
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_pod_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

    # Scrapes the endpoint lists for the main Prometheus endpoints
    - job_name: kube_endpoints
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        source_labels: [__meta_kubernetes_service_label_app]
        regex: prometheus
      - action: replace
        source_labels: [__meta_kubernetes_service_label_app]
        target_label: job
      - action: replace
        target_label: prometheus
        source_labels: [__meta_kubernetes_service_label_prometheus]
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: prometheus-gcs
  name: prometheus-gcs
spec:
  externalTrafficPolicy: Cluster
  ports:
  - port: 9090
    protocol: TCP
    targetPort: prom-http
    name: http-prometheus
  - port: 10902
    protocol: TCP
    targetPort: sidecar-http
    name: http-sidecar-metrics
  selector:
    app: prometheus-gcs
  sessionAffinity: None
  type: NodePort
status:
  loadBalancer: {}

---
apiVersion: v1
kind: Service
metadata:
  name: thanos-peers
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: cluster
    port: 10900
    targetPort: cluster
  selector:
    # Useful endpoint for gathering all thanos components for common gossip cluster.
    thanos-peer: "true"

prometheus-rbac.yaml

# Source: prometheus/templates/server-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: prometheus
  name: prometheus-server
---
# Source: prometheus/templates/server-clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  labels:
    app: prometheus
  name: prometheus-server
rules:
  - apiGroups:
      - ""
    resources:
      - nodes
      - nodes/proxy
      - services
      - endpoints
      - pods
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - get
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
      - ingresses
    verbs:
      - get
      - list
      - watch
  - nonResourceURLs:
      - "/metrics"
    verbs:
      - get
---
# Source: prometheus/templates/server-clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  labels:
    app: prometheus
  name: prometheus-server
subjects:
  - kind: ServiceAccount
    name: prometheus-server
    namespace: monitoring
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus-server
---

thanos-query.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-query
  labels:
    app: thanos-query
    thanos-peer: "true"
spec:
  replicas: 2
  selector:
    matchLabels:
      app: thanos-query
      thanos-peer: "true"
  template:
    metadata:
      labels:
        app: thanos-query
        thanos-peer: "true"
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "10902"
    spec:
      containers:
      - name: thanos-query
        # Always use explicit image tags (release or master-<date>-sha) instead of ambigous `latest` or `master`.
        image: improbable/thanos:v0.1.0
        args:
        - "query"
        - "--log.level=debug"
        - "--cluster.peers=thanos-peers.monitoring.svc.cluster.local:10900"
        - "--query.replica-label=replica"
        ports:
        - name: http
          containerPort: 10902
        - name: grpc
          containerPort: 10901
        - name: cluster
          containerPort: 10900
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: http
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: thanos-query
  name: thanos-query
spec:
  externalTrafficPolicy: Cluster
  ports:
  - port: 9090
    nodePort: 30909
    protocol: TCP
    targetPort: http
    name: http-query
  selector:
    app: thanos-query
  sessionAffinity: None
  type: NodePort
  # type: LoadBalancer

thanos-store.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-store
spec:
  serviceName: "thanos-store"
  replicas: 1
  selector:
    matchLabels:
      app: thanos
      thanos-peer: "true"
  template:
    metadata:
      labels:
        app: thanos
        thanos-peer: "true"
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "10902"
    spec:
      containers:
      - name: thanos-store
        # Always use explicit image tags (release or master-<date>-sha) instead of ambigous `latest` or `master`.
        image: improbable/thanos:v0.1.0
        env:
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: /creds/gcs-credentials.json
        args:
        - "store"
        - "--log.level=debug"
        - "--data-dir=/var/thanos/store"
        - "--cluster.peers=thanos-peers.monitoring.svc.cluster.local:10900"
        # NOTE: This is required to be added in GCS prior startup of this.
        - "--gcs.bucket=thanos-store"
        ports:
        - name: http
          containerPort: 10902
        - name: grpc
          containerPort: 10901
        - name: cluster
          containerPort: 10900
        volumeMounts:
        - mountPath: /creds/
          name: gcs-credentials
          readOnly: true
        - name: data
          mountPath: /var/thanos/store
      volumes:
      - name: data
        emptyDir: {}
      - name: gcs-credentials
        secret:
          defaultMode: 420
          # NOTE: gcs-credentials secret with single file gcs-credentials.json is required.
          secretName: gcs-credentials