Prometheus Setup

config.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus
  namespace: monitoring

data:
  prometheus.yml: |
    # A scrape configuration for running Prometheus on a Kubernetes cluster.
    # This uses separate scrape configs for cluster components (i.e. API server, node)
    # and services to allow each to use different authentication configs.
    #
    # Kubernetes labels will be added as Prometheus labels on metrics via the
    # `labelmap` relabeling action.

    # Scrape config for cluster components.
    scrape_configs:
    - job_name: 'kubernetes-cluster'

      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration (`in_cluster` below) because discovery & scraping are two
      # separate concerns in Prometheus.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default'
        in_cluster: true

      relabel_configs:
        - source_labels: [__meta_kubernetes_role]
          action: keep
          regex: (?:apiserver|node)
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
          replacement: $1$2
        - source_labels: [__meta_kubernetes_role]
          action: replace
          target_label: kubernetes_role
          regex: (.+)
          replacement: $1

      metric_relabel_configs:
        - source_labels: [io_kubernetes_pod_namespace]
          regex:  'serve-hostnames-.*'
          action: drop

    # Scrape config for service endpoints.
    #]
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
    # to set this to `https` & most likely set the `tls_config` of the scrape config.
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: If the metrics are exposed on a different port to the
    # service then set this appropriately.
    - job_name: 'kubernetes-service-endpoints'

      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default'
        in_cluster: true

      relabel_configs:
      - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: endpoint;true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_role]
        action: replace
        target_label: kubernetes_role
      - source_labels: [__meta_kubernetes_service_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

    - job_name: 'prometheus'
      target_groups:
        - targets: ['localhost:9090']

deployment.yaml

apiVersion: extensions/v1beta1
kind: Deployment

metadata:
  name: prometheus 
  namespace: monitoring 

spec:
  replicas: 1 
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      terminationGracePeriodSeconds: 300
      containers:
        - name: prometheus 
          image: prom/prometheus:0.18.0
          args:
            - -config.file=/etc/prometheus/prometheus.yml
            - -storage.local.path=/prometheus
            - -storage.local.memory-chunks=4194304
            - -storage.local.max-chunks-to-persist=2097152
            - -storage.local.retention=24h0m0s
            - -web.console.libraries=/etc/prometheus/console_libraries
            - -web.console.templates=/etc/prometheus/consoles
          ports:
            - name: http 
              containerPort: 9090
          volumeMounts:
            - name: data
              mountPath: /prometheus
            - name: config
              mountPath: /etc/prometheus
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: prometheus 
        - name: config
          configMap:
            name: prometheus

prometheus_metrics.txt

# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0.0008621760000000001
go_gc_duration_seconds{quantile="0.25"} 0.011776117000000001
go_gc_duration_seconds{quantile="0.5"} 0.017683455
go_gc_duration_seconds{quantile="0.75"} 0.019927779
go_gc_duration_seconds{quantile="1"} 0.026388614
go_gc_duration_seconds_sum 1.2564161280000001
go_gc_duration_seconds_count 82
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 224
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 1.398762072e+09
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 6.4766704712e+10
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 2.500197e+06
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 3.45969333e+08
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 8.4081664e+07
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 1.398762072e+09
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 8.22525952e+08
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 1.796816896e+09
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 1.1136782e+07
# HELP go_memstats_heap_released_bytes_total Total number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes_total counter
go_memstats_heap_released_bytes_total 1.6826368e+07
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 2.619342848e+09
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.4610541604613532e+19
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 61456
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 3.57106115e+08
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 96640
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 114688
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 1.8884208e+07
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 2.6886144e+07
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 1.841789367e+09
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 2.2147371e+07
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 4.456448e+06
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 4.456448e+06
# HELP go_memstats_sys_bytes Number of bytes obtained by system. Sum of all system allocations.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 2.75952936e+09
# HELP http_request_duration_microseconds The HTTP request latencies in microseconds.
# TYPE http_request_duration_microseconds summary
http_request_duration_microseconds{handler="alerts",quantile="0.5"} NaN
http_request_duration_microseconds{handler="alerts",quantile="0.9"} NaN
http_request_duration_microseconds{handler="alerts",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="alerts"} 0
http_request_duration_microseconds_count{handler="alerts"} 0
http_request_duration_microseconds{handler="consoles",quantile="0.5"} NaN
http_request_duration_microseconds{handler="consoles",quantile="0.9"} NaN
http_request_duration_microseconds{handler="consoles",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="consoles"} 0
http_request_duration_microseconds_count{handler="consoles"} 0
http_request_duration_microseconds{handler="drop_series",quantile="0.5"} NaN
http_request_duration_microseconds{handler="drop_series",quantile="0.9"} NaN
http_request_duration_microseconds{handler="drop_series",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="drop_series"} 0
http_request_duration_microseconds_count{handler="drop_series"} 0
http_request_duration_microseconds{handler="federate",quantile="0.5"} NaN
http_request_duration_microseconds{handler="federate",quantile="0.9"} NaN
http_request_duration_microseconds{handler="federate",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="federate"} 0
http_request_duration_microseconds_count{handler="federate"} 0
http_request_duration_microseconds{handler="graph",quantile="0.5"} NaN
http_request_duration_microseconds{handler="graph",quantile="0.9"} NaN
http_request_duration_microseconds{handler="graph",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="graph"} 2662.548
http_request_duration_microseconds_count{handler="graph"} 1
http_request_duration_microseconds{handler="heap",quantile="0.5"} NaN
http_request_duration_microseconds{handler="heap",quantile="0.9"} NaN
http_request_duration_microseconds{handler="heap",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="heap"} 0
http_request_duration_microseconds_count{handler="heap"} 0
http_request_duration_microseconds{handler="label_values",quantile="0.5"} NaN
http_request_duration_microseconds{handler="label_values",quantile="0.9"} NaN
http_request_duration_microseconds{handler="label_values",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="label_values"} 29093.34
http_request_duration_microseconds_count{handler="label_values"} 2
http_request_duration_microseconds{handler="metrics",quantile="0.5"} NaN
http_request_duration_microseconds{handler="metrics",quantile="0.9"} NaN
http_request_duration_microseconds{handler="metrics",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="metrics"} 0
http_request_duration_microseconds_count{handler="metrics"} 0
http_request_duration_microseconds{handler="options",quantile="0.5"} NaN
http_request_duration_microseconds{handler="options",quantile="0.9"} NaN
http_request_duration_microseconds{handler="options",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="options"} 0
http_request_duration_microseconds_count{handler="options"} 0
http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 15812.62
http_request_duration_microseconds{handler="prometheus",quantile="0.9"} 20279.455
http_request_duration_microseconds{handler="prometheus",quantile="0.99"} 20279.455
http_request_duration_microseconds_sum{handler="prometheus"} 651634.807
http_request_duration_microseconds_count{handler="prometheus"} 35
http_request_duration_microseconds{handler="query",quantile="0.5"} NaN
http_request_duration_microseconds{handler="query",quantile="0.9"} NaN
http_request_duration_microseconds{handler="query",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="query"} 5497.66
http_request_duration_microseconds_count{handler="query"} 3
http_request_duration_microseconds{handler="query_range",quantile="0.5"} NaN
http_request_duration_microseconds{handler="query_range",quantile="0.9"} NaN
http_request_duration_microseconds{handler="query_range",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="query_range"} 7.050216937099999e+07
http_request_duration_microseconds_count{handler="query_range"} 221
http_request_duration_microseconds{handler="series",quantile="0.5"} NaN
http_request_duration_microseconds{handler="series",quantile="0.9"} NaN
http_request_duration_microseconds{handler="series",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="series"} 30529.703999999994
http_request_duration_microseconds_count{handler="series"} 6
http_request_duration_microseconds{handler="static",quantile="0.5"} NaN
http_request_duration_microseconds{handler="static",quantile="0.9"} NaN
http_request_duration_microseconds{handler="static",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="static"} 647.091
http_request_duration_microseconds_count{handler="static"} 1
http_request_duration_microseconds{handler="status",quantile="0.5"} NaN
http_request_duration_microseconds{handler="status",quantile="0.9"} NaN
http_request_duration_microseconds{handler="status",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="status"} 0
http_request_duration_microseconds_count{handler="status"} 0
http_request_duration_microseconds{handler="version",quantile="0.5"} NaN
http_request_duration_microseconds{handler="version",quantile="0.9"} NaN
http_request_duration_microseconds{handler="version",quantile="0.99"} NaN
http_request_duration_microseconds_sum{handler="version"} 0
http_request_duration_microseconds_count{handler="version"} 0
# HELP http_request_size_bytes The HTTP request sizes in bytes.
# TYPE http_request_size_bytes summary
http_request_size_bytes{handler="alerts",quantile="0.5"} NaN
http_request_size_bytes{handler="alerts",quantile="0.9"} NaN
http_request_size_bytes{handler="alerts",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="alerts"} 0
http_request_size_bytes_count{handler="alerts"} 0
http_request_size_bytes{handler="consoles",quantile="0.5"} NaN
http_request_size_bytes{handler="consoles",quantile="0.9"} NaN
http_request_size_bytes{handler="consoles",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="consoles"} 0
http_request_size_bytes_count{handler="consoles"} 0
http_request_size_bytes{handler="drop_series",quantile="0.5"} NaN
http_request_size_bytes{handler="drop_series",quantile="0.9"} NaN
http_request_size_bytes{handler="drop_series",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="drop_series"} 0
http_request_size_bytes_count{handler="drop_series"} 0
http_request_size_bytes{handler="federate",quantile="0.5"} NaN
http_request_size_bytes{handler="federate",quantile="0.9"} NaN
http_request_size_bytes{handler="federate",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="federate"} 0
http_request_size_bytes_count{handler="federate"} 0
http_request_size_bytes{handler="graph",quantile="0.5"} NaN
http_request_size_bytes{handler="graph",quantile="0.9"} NaN
http_request_size_bytes{handler="graph",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="graph"} 597
http_request_size_bytes_count{handler="graph"} 1
http_request_size_bytes{handler="heap",quantile="0.5"} NaN
http_request_size_bytes{handler="heap",quantile="0.9"} NaN
http_request_size_bytes{handler="heap",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="heap"} 0
http_request_size_bytes_count{handler="heap"} 0
http_request_size_bytes{handler="label_values",quantile="0.5"} NaN
http_request_size_bytes{handler="label_values",quantile="0.9"} NaN
http_request_size_bytes{handler="label_values",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="label_values"} 1156
http_request_size_bytes_count{handler="label_values"} 2
http_request_size_bytes{handler="metrics",quantile="0.5"} NaN
http_request_size_bytes{handler="metrics",quantile="0.9"} NaN
http_request_size_bytes{handler="metrics",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="metrics"} 0
http_request_size_bytes_count{handler="metrics"} 0
http_request_size_bytes{handler="options",quantile="0.5"} NaN
http_request_size_bytes{handler="options",quantile="0.9"} NaN
http_request_size_bytes{handler="options",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="options"} 0
http_request_size_bytes_count{handler="options"} 0
http_request_size_bytes{handler="prometheus",quantile="0.5"} 305
http_request_size_bytes{handler="prometheus",quantile="0.9"} 305
http_request_size_bytes{handler="prometheus",quantile="0.99"} 305
http_request_size_bytes_sum{handler="prometheus"} 10675
http_request_size_bytes_count{handler="prometheus"} 35
http_request_size_bytes{handler="query",quantile="0.5"} NaN
http_request_size_bytes{handler="query",quantile="0.9"} NaN
http_request_size_bytes{handler="query",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="query"} 2008
http_request_size_bytes_count{handler="query"} 3
http_request_size_bytes{handler="query_range",quantile="0.5"} NaN
http_request_size_bytes{handler="query_range",quantile="0.9"} NaN
http_request_size_bytes{handler="query_range",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="query_range"} 156370
http_request_size_bytes_count{handler="query_range"} 221
http_request_size_bytes{handler="series",quantile="0.5"} NaN
http_request_size_bytes{handler="series",quantile="0.9"} NaN
http_request_size_bytes{handler="series",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="series"} 3401
http_request_size_bytes_count{handler="series"} 6
http_request_size_bytes{handler="static",quantile="0.5"} NaN
http_request_size_bytes{handler="static",quantile="0.9"} NaN
http_request_size_bytes{handler="static",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="static"} 574
http_request_size_bytes_count{handler="static"} 1
http_request_size_bytes{handler="status",quantile="0.5"} NaN
http_request_size_bytes{handler="status",quantile="0.9"} NaN
http_request_size_bytes{handler="status",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="status"} 0
http_request_size_bytes_count{handler="status"} 0
http_request_size_bytes{handler="version",quantile="0.5"} NaN
http_request_size_bytes{handler="version",quantile="0.9"} NaN
http_request_size_bytes{handler="version",quantile="0.99"} NaN
http_request_size_bytes_sum{handler="version"} 0
http_request_size_bytes_count{handler="version"} 0
# HELP http_requests_total Total number of HTTP requests made.
# TYPE http_requests_total counter
http_requests_total{code="200",handler="graph",method="get"} 1
http_requests_total{code="200",handler="label_values",method="get"} 2
http_requests_total{code="200",handler="prometheus",method="get"} 35
http_requests_total{code="200",handler="query",method="get"} 3
http_requests_total{code="200",handler="query_range",method="get"} 204
http_requests_total{code="200",handler="series",method="get"} 6
http_requests_total{code="200",handler="static",method="get"} 1
http_requests_total{code="400",handler="query_range",method="get"} 17
# HELP http_response_size_bytes The HTTP response sizes in bytes.
# TYPE http_response_size_bytes summary
http_response_size_bytes{handler="alerts",quantile="0.5"} NaN
http_response_size_bytes{handler="alerts",quantile="0.9"} NaN
http_response_size_bytes{handler="alerts",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="alerts"} 0
http_response_size_bytes_count{handler="alerts"} 0
http_response_size_bytes{handler="consoles",quantile="0.5"} NaN
http_response_size_bytes{handler="consoles",quantile="0.9"} NaN
http_response_size_bytes{handler="consoles",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="consoles"} 0
http_response_size_bytes_count{handler="consoles"} 0
http_response_size_bytes{handler="drop_series",quantile="0.5"} NaN
http_response_size_bytes{handler="drop_series",quantile="0.9"} NaN
http_response_size_bytes{handler="drop_series",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="drop_series"} 0
http_response_size_bytes_count{handler="drop_series"} 0
http_response_size_bytes{handler="federate",quantile="0.5"} NaN
http_response_size_bytes{handler="federate",quantile="0.9"} NaN
http_response_size_bytes{handler="federate",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="federate"} 0
http_response_size_bytes_count{handler="federate"} 0
http_response_size_bytes{handler="graph",quantile="0.5"} NaN
http_response_size_bytes{handler="graph",quantile="0.9"} NaN
http_response_size_bytes{handler="graph",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="graph"} 2928
http_response_size_bytes_count{handler="graph"} 1
http_response_size_bytes{handler="heap",quantile="0.5"} NaN
http_response_size_bytes{handler="heap",quantile="0.9"} NaN
http_response_size_bytes{handler="heap",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="heap"} 0
http_response_size_bytes_count{handler="heap"} 0
http_response_size_bytes{handler="label_values",quantile="0.5"} NaN
http_response_size_bytes{handler="label_values",quantile="0.9"} NaN
http_response_size_bytes{handler="label_values",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="label_values"} 1621
http_response_size_bytes_count{handler="label_values"} 2
http_response_size_bytes{handler="metrics",quantile="0.5"} NaN
http_response_size_bytes{handler="metrics",quantile="0.9"} NaN
http_response_size_bytes{handler="metrics",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="metrics"} 0
http_response_size_bytes_count{handler="metrics"} 0
http_response_size_bytes{handler="options",quantile="0.5"} NaN
http_response_size_bytes{handler="options",quantile="0.9"} NaN
http_response_size_bytes{handler="options",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="options"} 0
http_response_size_bytes_count{handler="options"} 0
http_response_size_bytes{handler="prometheus",quantile="0.5"} 4223
http_response_size_bytes{handler="prometheus",quantile="0.9"} 4305
http_response_size_bytes{handler="prometheus",quantile="0.99"} 4305
http_response_size_bytes_sum{handler="prometheus"} 146151
http_response_size_bytes_count{handler="prometheus"} 35
http_response_size_bytes{handler="query",quantile="0.5"} NaN
http_response_size_bytes{handler="query",quantile="0.9"} NaN
http_response_size_bytes{handler="query",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="query"} 511
http_response_size_bytes_count{handler="query"} 3
http_response_size_bytes{handler="query_range",quantile="0.5"} NaN
http_response_size_bytes{handler="query_range",quantile="0.9"} NaN
http_response_size_bytes{handler="query_range",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="query_range"} 321889
http_response_size_bytes_count{handler="query_range"} 221
http_response_size_bytes{handler="series",quantile="0.5"} NaN
http_response_size_bytes{handler="series",quantile="0.9"} NaN
http_response_size_bytes{handler="series",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="series"} 2647
http_response_size_bytes_count{handler="series"} 6
http_response_size_bytes{handler="static",quantile="0.5"} NaN
http_response_size_bytes{handler="static",quantile="0.9"} NaN
http_response_size_bytes{handler="static",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="static"} 6134
http_response_size_bytes_count{handler="static"} 1
http_response_size_bytes{handler="status",quantile="0.5"} NaN
http_response_size_bytes{handler="status",quantile="0.9"} NaN
http_response_size_bytes{handler="status",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="status"} 0
http_response_size_bytes_count{handler="status"} 0
http_response_size_bytes{handler="version",quantile="0.5"} NaN
http_response_size_bytes{handler="version",quantile="0.9"} NaN
http_response_size_bytes{handler="version",quantile="0.99"} NaN
http_response_size_bytes_sum{handler="version"} 0
http_response_size_bytes_count{handler="version"} 0
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 1826.45
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 54
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 2.76619264e+09
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.46105205624e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 7.119745024e+09
# HELP prometheus_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which Prometheus was built.
# TYPE prometheus_build_info gauge
prometheus_build_info{branch="HEAD",goversion="go1.5.3",revision="f12ebd6",version="0.18.0"} 1
# HELP prometheus_config_last_reload_success_timestamp_seconds Timestamp of the last successful configuration reload.
# TYPE prometheus_config_last_reload_success_timestamp_seconds gauge
prometheus_config_last_reload_success_timestamp_seconds 1.461052056e+09
# HELP prometheus_config_last_reload_successful Whether the last configuration reload attempt was successful.
# TYPE prometheus_config_last_reload_successful gauge
prometheus_config_last_reload_successful 1
# HELP prometheus_dns_sd_lookup_failures_total The number of DNS-SD lookup failures.
# TYPE prometheus_dns_sd_lookup_failures_total counter
prometheus_dns_sd_lookup_failures_total 0
# HELP prometheus_dns_sd_lookups_total The number of DNS-SD lookups.
# TYPE prometheus_dns_sd_lookups_total counter
prometheus_dns_sd_lookups_total 0
# HELP prometheus_evaluator_duration_seconds The duration of rule group evaluations.
# TYPE prometheus_evaluator_duration_seconds summary
prometheus_evaluator_duration_seconds{quantile="0.01"} 1.49e-06
prometheus_evaluator_duration_seconds{quantile="0.05"} 1.49e-06
prometheus_evaluator_duration_seconds{quantile="0.5"} 1.945e-06
prometheus_evaluator_duration_seconds{quantile="0.9"} 2.93e-06
prometheus_evaluator_duration_seconds{quantile="0.99"} 2.93e-06
prometheus_evaluator_duration_seconds_sum 7.448800000000001e-05
prometheus_evaluator_duration_seconds_count 35
# HELP prometheus_evaluator_iterations_skipped_total The total number of rule group evaluations skipped due to throttled metric storage.
# TYPE prometheus_evaluator_iterations_skipped_total counter
prometheus_evaluator_iterations_skipped_total 0
# HELP prometheus_local_storage_checkpoint_duration_milliseconds The duration (in milliseconds) it took to checkpoint in-memory metrics and head chunks.
# TYPE prometheus_local_storage_checkpoint_duration_milliseconds gauge
prometheus_local_storage_checkpoint_duration_milliseconds 9453.371854
# HELP prometheus_local_storage_chunk_ops_total The total number of chunk operations by their type.
# TYPE prometheus_local_storage_chunk_ops_total counter
prometheus_local_storage_chunk_ops_total{type="create"} 277009
prometheus_local_storage_chunk_ops_total{type="load"} 193
prometheus_local_storage_chunk_ops_total{type="persist"} 28720
prometheus_local_storage_chunk_ops_total{type="pin"} 4574
prometheus_local_storage_chunk_ops_total{type="transcode"} 89012
prometheus_local_storage_chunk_ops_total{type="unpin"} 4574
# HELP prometheus_local_storage_chunks_to_persist The current number of chunks waiting for persistence.
# TYPE prometheus_local_storage_chunks_to_persist gauge
prometheus_local_storage_chunks_to_persist 77439
# HELP prometheus_local_storage_fingerprint_mappings_total The total number of fingerprints being mapped to avoid collisions.
# TYPE prometheus_local_storage_fingerprint_mappings_total counter
prometheus_local_storage_fingerprint_mappings_total 6116
# HELP prometheus_local_storage_inconsistencies_total A counter incremented each time an inconsistency in the local storage is detected. If this is greater zero, restart the server as soon as possible.
# TYPE prometheus_local_storage_inconsistencies_total counter
prometheus_local_storage_inconsistencies_total 0
# HELP prometheus_local_storage_indexing_batch_duration_milliseconds Quantiles for batch indexing duration in milliseconds.
# TYPE prometheus_local_storage_indexing_batch_duration_milliseconds summary
prometheus_local_storage_indexing_batch_duration_milliseconds{quantile="0.5"} 212.476966
prometheus_local_storage_indexing_batch_duration_milliseconds{quantile="0.9"} 702.743505
prometheus_local_storage_indexing_batch_duration_milliseconds{quantile="0.99"} 859.979022
prometheus_local_storage_indexing_batch_duration_milliseconds_sum 30743.789275000003
prometheus_local_storage_indexing_batch_duration_milliseconds_count 107
# HELP prometheus_local_storage_indexing_batch_sizes Quantiles for indexing batch sizes (number of metrics per batch).
# TYPE prometheus_local_storage_indexing_batch_sizes summary
prometheus_local_storage_indexing_batch_sizes{quantile="0.5"} 102
prometheus_local_storage_indexing_batch_sizes{quantile="0.9"} 429
prometheus_local_storage_indexing_batch_sizes{quantile="0.99"} 646
prometheus_local_storage_indexing_batch_sizes_sum 14297
prometheus_local_storage_indexing_batch_sizes_count 107
# HELP prometheus_local_storage_indexing_queue_capacity The capacity of the indexing queue.
# TYPE prometheus_local_storage_indexing_queue_capacity gauge
prometheus_local_storage_indexing_queue_capacity 16384
# HELP prometheus_local_storage_indexing_queue_length The number of metrics waiting to be indexed.
# TYPE prometheus_local_storage_indexing_queue_length gauge
prometheus_local_storage_indexing_queue_length 0
# HELP prometheus_local_storage_ingested_samples_total The total number of samples ingested.
# TYPE prometheus_local_storage_ingested_samples_total counter
prometheus_local_storage_ingested_samples_total 3.745187e+06
# HELP prometheus_local_storage_maintain_series_duration_milliseconds The duration (in milliseconds) it took to perform maintenance on a series.
# TYPE prometheus_local_storage_maintain_series_duration_milliseconds summary
prometheus_local_storage_maintain_series_duration_milliseconds{location="memory",quantile="0.5"} 0.535956
prometheus_local_storage_maintain_series_duration_milliseconds{location="memory",quantile="0.9"} 1.787545
prometheus_local_storage_maintain_series_duration_milliseconds{location="memory",quantile="0.99"} 23.764302
prometheus_local_storage_maintain_series_duration_milliseconds_sum{location="memory"} 135021.51834800062
prometheus_local_storage_maintain_series_duration_milliseconds_count{location="memory"} 97172
# HELP prometheus_local_storage_max_chunks_to_persist The maximum number of chunks that can be waiting for persistence before sample ingestion will stop.
# TYPE prometheus_local_storage_max_chunks_to_persist gauge
prometheus_local_storage_max_chunks_to_persist 2.097152e+06
# HELP prometheus_local_storage_memory_chunkdescs The current number of chunk descriptors in memory.
# TYPE prometheus_local_storage_memory_chunkdescs gauge
prometheus_local_storage_memory_chunkdescs 791087
# HELP prometheus_local_storage_memory_chunks The current number of chunks in memory, excluding cloned chunks (i.e. chunks without a descriptor).
# TYPE prometheus_local_storage_memory_chunks gauge
prometheus_local_storage_memory_chunks 277202
# HELP prometheus_local_storage_memory_series The current number of series in memory.
# TYPE prometheus_local_storage_memory_series gauge
prometheus_local_storage_memory_series 382000
# HELP prometheus_local_storage_non_existent_series_matches_total How often a non-existent series was referred to during label matching or chunk preloading. This is an indication of outdated label indexes.
# TYPE prometheus_local_storage_non_existent_series_matches_total counter
prometheus_local_storage_non_existent_series_matches_total 0
# HELP prometheus_local_storage_out_of_order_samples_total The total number of samples that were discarded because their timestamps were at or before the last received sample for a series.
# TYPE prometheus_local_storage_out_of_order_samples_total counter
prometheus_local_storage_out_of_order_samples_total 0
# HELP prometheus_local_storage_persist_errors_total The total number of errors while persisting chunks.
# TYPE prometheus_local_storage_persist_errors_total counter
prometheus_local_storage_persist_errors_total 0
# HELP prometheus_local_storage_persistence_urgency_score A score of urgency to persist chunks, 0 is least urgent, 1 most.
# TYPE prometheus_local_storage_persistence_urgency_score gauge
prometheus_local_storage_persistence_urgency_score 0.03692626953125
# HELP prometheus_local_storage_rushed_mode 1 if the storage is in rushed mode, 0 otherwise. In rushed mode, the system behaves as if the persistence_urgency_score is 1.
# TYPE prometheus_local_storage_rushed_mode gauge
prometheus_local_storage_rushed_mode 0
# HELP prometheus_local_storage_series_ops_total The total number of series operations by their type.
# TYPE prometheus_local_storage_series_ops_total counter
prometheus_local_storage_series_ops_total{type="archive"} 58958
prometheus_local_storage_series_ops_total{type="create"} 14297
prometheus_local_storage_series_ops_total{type="maintenance_in_memory"} 97172
# HELP prometheus_notifications_dropped_total Total number of alerts dropped due to alert manager missing in configuration.
# TYPE prometheus_notifications_dropped_total counter
prometheus_notifications_dropped_total 0
# HELP prometheus_notifications_errors_total Total number of errors sending alert notifications.
# TYPE prometheus_notifications_errors_total counter
prometheus_notifications_errors_total 0
# HELP prometheus_notifications_latency_seconds Latency quantiles for sending alert notifications (not including dropped notifications).
# TYPE prometheus_notifications_latency_seconds summary
prometheus_notifications_latency_seconds{quantile="0.5"} NaN
prometheus_notifications_latency_seconds{quantile="0.9"} NaN
prometheus_notifications_latency_seconds{quantile="0.99"} NaN
prometheus_notifications_latency_seconds_sum 0
prometheus_notifications_latency_seconds_count 0
# HELP prometheus_notifications_queue_capacity The capacity of the alert notifications queue.
# TYPE prometheus_notifications_queue_capacity gauge
prometheus_notifications_queue_capacity 10000
# HELP prometheus_notifications_queue_length The number of alert notifications in the queue.
# TYPE prometheus_notifications_queue_length gauge
prometheus_notifications_queue_length 0
# HELP prometheus_notifications_sent_total Total number of alerts successfully sent.
# TYPE prometheus_notifications_sent_total counter
prometheus_notifications_sent_total 0
# HELP prometheus_rule_evaluation_failures_total The total number of rule evaluation failures.
# TYPE prometheus_rule_evaluation_failures_total counter
prometheus_rule_evaluation_failures_total{rule_type="alerting"} 0
prometheus_rule_evaluation_failures_total{rule_type="recording"} 0
# HELP prometheus_target_interval_length_seconds Actual intervals between scrapes.
# TYPE prometheus_target_interval_length_seconds summary
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.01"} 59.9982779
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.05"} 59.999698542
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.5"} 60.000020089
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.9"} 60.000227985
prometheus_target_interval_length_seconds{interval="1m0s",quantile="0.99"} 60.001273134
prometheus_target_interval_length_seconds_sum{interval="1m0s"} 20280.013181240007
prometheus_target_interval_length_seconds_count{interval="1m0s"} 338
# HELP prometheus_target_scrape_pool_sync_total Total number of syncs that were executed on a scrape pool.
# TYPE prometheus_target_scrape_pool_sync_total counter
prometheus_target_scrape_pool_sync_total{scrape_job="kubernetes-cluster"} 420
prometheus_target_scrape_pool_sync_total{scrape_job="kubernetes-service-endpoints"} 420
prometheus_target_scrape_pool_sync_total{scrape_job="prometheus"} 1
# HELP prometheus_target_sync_length_seconds Actual interval to sync the scrape pool.
# TYPE prometheus_target_sync_length_seconds summary
prometheus_target_sync_length_seconds{scrape_job="kubernetes-cluster",quantile="0.01"} 7.5178e-05
prometheus_target_sync_length_seconds{scrape_job="kubernetes-cluster",quantile="0.05"} 9.2337e-05
prometheus_target_sync_length_seconds{scrape_job="kubernetes-cluster",quantile="0.5"} 0.000121274
prometheus_target_sync_length_seconds{scrape_job="kubernetes-cluster",quantile="0.9"} 0.000163984
prometheus_target_sync_length_seconds{scrape_job="kubernetes-cluster",quantile="0.99"} 0.000911436
prometheus_target_sync_length_seconds_sum{scrape_job="kubernetes-cluster"} 0.06991762300000004
prometheus_target_sync_length_seconds_count{scrape_job="kubernetes-cluster"} 420
prometheus_target_sync_length_seconds{scrape_job="kubernetes-service-endpoints",quantile="0.01"} 3.6621e-05
prometheus_target_sync_length_seconds{scrape_job="kubernetes-service-endpoints",quantile="0.05"} 4.1419e-05
prometheus_target_sync_length_seconds{scrape_job="kubernetes-service-endpoints",quantile="0.5"} 5.6628e-05
prometheus_target_sync_length_seconds{scrape_job="kubernetes-service-endpoints",quantile="0.9"} 0.000100734
prometheus_target_sync_length_seconds{scrape_job="kubernetes-service-endpoints",quantile="0.99"} 0.000223529
prometheus_target_sync_length_seconds_sum{scrape_job="kubernetes-service-endpoints"} 0.034602930999999996
prometheus_target_sync_length_seconds_count{scrape_job="kubernetes-service-endpoints"} 420
prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.01"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.05"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.5"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.9"} NaN
prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.99"} NaN
prometheus_target_sync_length_seconds_sum{scrape_job="prometheus"} 5.1208e-05
prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 1