Created
August 5, 2020 09:22
-
-
Save yangchuansheng/102595fc50436cf4a2ce18744467718c to your computer and use it in GitHub Desktop.
prometheus-rules-others.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| apiVersion: monitoring.coreos.com/v1 | |
| kind: PrometheusRule | |
| metadata: | |
| labels: | |
| prometheus: others | |
| role: alert-rules | |
| name: prometheus-others-rules | |
| namespace: monitoring | |
| spec: | |
| groups: | |
| - name: node-exporter.rules | |
| rules: | |
| - expr: | | |
| count without (cpu) ( | |
| count without (mode) ( | |
| node_cpu_seconds_total{job="node-exporter"} | |
| ) | |
| ) | |
| record: instance:node_num_cpu:sum | |
| - expr: | | |
| 1 - avg without (cpu, mode) ( | |
| rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[1m]) | |
| ) | |
| record: instance:node_cpu_utilisation:rate1m | |
| - expr: | | |
| ( | |
| node_load1{job="node-exporter"} | |
| / | |
| instance:node_num_cpu:sum{job="node-exporter"} | |
| ) | |
| record: instance:node_load1_per_cpu:ratio | |
| - expr: | | |
| 1 - ( | |
| node_memory_MemAvailable_bytes{job="node-exporter"} | |
| / | |
| node_memory_MemTotal_bytes{job="node-exporter"} | |
| ) | |
| record: instance:node_memory_utilisation:ratio | |
| - expr: | | |
| rate(node_vmstat_pgmajfault{job="node-exporter"}[1m]) | |
| record: instance:node_vmstat_pgmajfault:rate1m | |
| - expr: | | |
| rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) | |
| record: instance_device:node_disk_io_time_seconds:rate1m | |
| - expr: | | |
| rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) | |
| record: instance_device:node_disk_io_time_weighted_seconds:rate1m | |
| - expr: | | |
| sum without (device) ( | |
| rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[1m]) | |
| ) | |
| record: instance:node_network_receive_bytes_excluding_lo:rate1m | |
| - expr: | | |
| sum without (device) ( | |
| rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[1m]) | |
| ) | |
| record: instance:node_network_transmit_bytes_excluding_lo:rate1m | |
| - expr: | | |
| sum without (device) ( | |
| rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[1m]) | |
| ) | |
| record: instance:node_network_receive_drop_excluding_lo:rate1m | |
| - expr: | | |
| sum without (device) ( | |
| rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[1m]) | |
| ) | |
| record: instance:node_network_transmit_drop_excluding_lo:rate1m | |
| - name: k8s.rules | |
| rules: | |
| - expr: | | |
| sum(rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])) by (namespace) | |
| record: namespace:container_cpu_usage_seconds_total:sum_rate | |
| - expr: | | |
| sum by (cluster, namespace, pod, container) ( | |
| rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m]) | |
| ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( | |
| 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) | |
| ) | |
| record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate | |
| - expr: | | |
| container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} | |
| * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, | |
| max by(namespace, pod, node) (kube_pod_info{node!=""}) | |
| ) | |
| record: node_namespace_pod_container:container_memory_working_set_bytes | |
| - expr: | | |
| container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} | |
| * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, | |
| max by(namespace, pod, node) (kube_pod_info{node!=""}) | |
| ) | |
| record: node_namespace_pod_container:container_memory_rss | |
| - expr: | | |
| container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} | |
| * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, | |
| max by(namespace, pod, node) (kube_pod_info{node!=""}) | |
| ) | |
| record: node_namespace_pod_container:container_memory_cache | |
| - expr: | | |
| container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} | |
| * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, | |
| max by(namespace, pod, node) (kube_pod_info{node!=""}) | |
| ) | |
| record: node_namespace_pod_container:container_memory_swap | |
| - expr: | | |
| sum(container_memory_usage_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}) by (namespace) | |
| record: namespace:container_memory_usage_bytes:sum | |
| - expr: | | |
| sum by (namespace) ( | |
| sum by (namespace, pod) ( | |
| max by (namespace, pod, container) ( | |
| kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"} | |
| ) * on(namespace, pod) group_left() max by (namespace, pod) ( | |
| kube_pod_status_phase{phase=~"Pending|Running"} == 1 | |
| ) | |
| ) | |
| ) | |
| record: namespace:kube_pod_container_resource_requests_memory_bytes:sum | |
| - expr: | | |
| sum by (namespace) ( | |
| sum by (namespace, pod) ( | |
| max by (namespace, pod, container) ( | |
| kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} | |
| ) * on(namespace, pod) group_left() max by (namespace, pod) ( | |
| kube_pod_status_phase{phase=~"Pending|Running"} == 1 | |
| ) | |
| ) | |
| ) | |
| record: namespace:kube_pod_container_resource_requests_cpu_cores:sum | |
| - expr: | | |
| max by (cluster, namespace, workload, pod) ( | |
| label_replace( | |
| label_replace( | |
| kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, | |
| "replicaset", "$1", "owner_name", "(.*)" | |
| ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( | |
| 1, max by (replicaset, namespace, owner_name) ( | |
| kube_replicaset_owner{job="kube-state-metrics"} | |
| ) | |
| ), | |
| "workload", "$1", "owner_name", "(.*)" | |
| ) | |
| ) | |
| labels: | |
| workload_type: deployment | |
| record: mixin_pod_workload | |
| - expr: | | |
| max by (cluster, namespace, workload, pod) ( | |
| label_replace( | |
| kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, | |
| "workload", "$1", "owner_name", "(.*)" | |
| ) | |
| ) | |
| labels: | |
| workload_type: daemonset | |
| record: mixin_pod_workload | |
| - expr: | | |
| max by (cluster, namespace, workload, pod) ( | |
| label_replace( | |
| kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, | |
| "workload", "$1", "owner_name", "(.*)" | |
| ) | |
| ) | |
| labels: | |
| workload_type: statefulset | |
| record: mixin_pod_workload | |
| - name: node.rules | |
| rules: | |
| - expr: | | |
| sum(min(kube_pod_info{node!=""}) by (cluster, node)) | |
| record: ':kube_pod_info_node_count:' | |
| - expr: | | |
| topk by(namespace, pod) (1, | |
| max by (node, namespace, pod) ( | |
| label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)") | |
| )) | |
| record: 'node_namespace_pod:kube_pod_info:' | |
| - expr: | | |
| count by (cluster, node) (sum by (node, cpu) ( | |
| node_cpu_seconds_total{job="node-exporter"} | |
| * on (namespace, pod) group_left(node) | |
| node_namespace_pod:kube_pod_info: | |
| )) | |
| record: node:node_num_cpu:sum | |
| - expr: | | |
| sum( | |
| node_memory_MemAvailable_bytes{job="node-exporter"} or | |
| ( | |
| node_memory_Buffers_bytes{job="node-exporter"} + | |
| node_memory_Cached_bytes{job="node-exporter"} + | |
| node_memory_MemFree_bytes{job="node-exporter"} + | |
| node_memory_Slab_bytes{job="node-exporter"} | |
| ) | |
| ) by (cluster) | |
| record: :node_memory_MemAvailable_bytes:sum | |
| - name: kube-prometheus-node-recording.rules | |
| rules: | |
| - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY | |
| (instance) | |
| record: instance:node_cpu:rate:sum | |
| - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) | |
| BY (instance) | |
| record: instance:node_filesystem_usage:sum | |
| - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) | |
| record: instance:node_network_receive_bytes:rate:sum | |
| - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) | |
| record: instance:node_network_transmit_bytes:rate:sum | |
| - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT | |
| (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) | |
| BY (instance, cpu)) BY (instance) | |
| record: instance:node_cpu:ratio | |
| - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) | |
| record: cluster:node_cpu:sum_rate5m | |
| - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) | |
| BY (instance, cpu)) | |
| record: cluster:node_cpu:ratio | |
| - name: kube-prometheus-general.rules | |
| rules: | |
| - expr: count without(instance, pod, node) (up == 1) | |
| record: count:up1 | |
| - expr: count without(instance, pod, node) (up == 0) | |
| record: count:up0 | |
| - name: kube-state-metrics | |
| rules: | |
| - alert: KubeStateMetricsListErrors | |
| annotations: | |
| message: kube-state-metrics is experiencing errors at an elevated rate in | |
| list operations. This is likely causing it to not be able to expose metrics | |
| about Kubernetes objects correctly or at all. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricslisterrors | |
| expr: | | |
| (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) | |
| / | |
| sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m]))) | |
| > 0.01 | |
| for: 15m | |
| labels: | |
| severity: critical | |
| - alert: KubeStateMetricsWatchErrors | |
| annotations: | |
| message: kube-state-metrics is experiencing errors at an elevated rate in | |
| watch operations. This is likely causing it to not be able to expose metrics | |
| about Kubernetes objects correctly or at all. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricswatcherrors | |
| expr: | | |
| (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) | |
| / | |
| sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m]))) | |
| > 0.01 | |
| for: 15m | |
| labels: | |
| severity: critical | |
| - name: node-exporter | |
| rules: | |
| - alert: NodeFilesystemSpaceFillingUp | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available space left and is filling | |
| up. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup | |
| summary: Filesystem is predicted to run out of space within the next 24 hours. | |
| expr: | | |
| ( | |
| node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 40 | |
| and | |
| predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: NodeFilesystemSpaceFillingUp | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available space left and is filling | |
| up fast. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup | |
| summary: Filesystem is predicted to run out of space within the next 4 hours. | |
| expr: | | |
| ( | |
| node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15 | |
| and | |
| predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: critical | |
| - alert: NodeFilesystemAlmostOutOfSpace | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available space left. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace | |
| summary: Filesystem has less than 5% space left. | |
| expr: | | |
| ( | |
| node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 5 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: NodeFilesystemAlmostOutOfSpace | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available space left. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace | |
| summary: Filesystem has less than 3% space left. | |
| expr: | | |
| ( | |
| node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 3 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: critical | |
| - alert: NodeFilesystemFilesFillingUp | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available inodes left and is filling | |
| up. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup | |
| summary: Filesystem is predicted to run out of inodes within the next 24 hours. | |
| expr: | | |
| ( | |
| node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 40 | |
| and | |
| predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: NodeFilesystemFilesFillingUp | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available inodes left and is filling | |
| up fast. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup | |
| summary: Filesystem is predicted to run out of inodes within the next 4 hours. | |
| expr: | | |
| ( | |
| node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 20 | |
| and | |
| predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: critical | |
| - alert: NodeFilesystemAlmostOutOfFiles | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available inodes left. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles | |
| summary: Filesystem has less than 5% inodes left. | |
| expr: | | |
| ( | |
| node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 5 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: NodeFilesystemAlmostOutOfFiles | |
| annotations: | |
| description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} | |
| has only {{ printf "%.2f" $value }}% available inodes left. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles | |
| summary: Filesystem has less than 3% inodes left. | |
| expr: | | |
| ( | |
| node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 3 | |
| and | |
| node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 | |
| ) | |
| for: 1h | |
| labels: | |
| severity: critical | |
| - alert: NodeNetworkReceiveErrs | |
| annotations: | |
| description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered | |
| {{ printf "%.0f" $value }} receive errors in the last two minutes.' | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworkreceiveerrs | |
| summary: Network interface is reporting many receive errors. | |
| expr: | | |
| increase(node_network_receive_errs_total[2m]) > 10 | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: NodeNetworkTransmitErrs | |
| annotations: | |
| description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered | |
| {{ printf "%.0f" $value }} transmit errors in the last two minutes.' | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworktransmiterrs | |
| summary: Network interface is reporting many transmit errors. | |
| expr: | | |
| increase(node_network_transmit_errs_total[2m]) > 10 | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: NodeHighNumberConntrackEntriesUsed | |
| annotations: | |
| description: '{{ $value | humanizePercentage }} of conntrack entries are used.' | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodehighnumberconntrackentriesused | |
| summary: Number of conntrack are getting close to the limit. | |
| expr: | | |
| (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 | |
| labels: | |
| severity: warning | |
| - alert: NodeTextFileCollectorScrapeError | |
| annotations: | |
| description: Node Exporter text file collector failed to scrape. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodetextfilecollectorscrapeerror | |
| summary: Node Exporter text file collector failed to scrape. | |
| expr: | | |
| node_textfile_scrape_error{job="node-exporter"} == 1 | |
| labels: | |
| severity: warning | |
| - alert: NodeClockSkewDetected | |
| annotations: | |
| message: Clock on {{ $labels.instance }} is out of sync by more than 300s. | |
| Ensure NTP is configured correctly on this host. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclockskewdetected | |
| summary: Clock skew detected. | |
| expr: | | |
| ( | |
| node_timex_offset_seconds > 0.05 | |
| and | |
| deriv(node_timex_offset_seconds[5m]) >= 0 | |
| ) | |
| or | |
| ( | |
| node_timex_offset_seconds < -0.05 | |
| and | |
| deriv(node_timex_offset_seconds[5m]) <= 0 | |
| ) | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: NodeClockNotSynchronising | |
| annotations: | |
| message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP | |
| is configured on this host. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclocknotsynchronising | |
| summary: Clock not synchronising. | |
| expr: | | |
| min_over_time(node_timex_sync_status[5m]) == 0 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - name: kubernetes-apps | |
| rules: | |
| - alert: KubePodCrashLooping | |
| annotations: | |
| message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container | |
| }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping | |
| expr: | | |
| rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) * 60 * 5 > 0 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubePodNotReady | |
| annotations: | |
| message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready | |
| state for longer than 15 minutes. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready | |
| expr: | | |
| sum by (namespace, pod) ( | |
| max by(namespace, pod) ( | |
| kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"} | |
| ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) ( | |
| 1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"}) | |
| ) | |
| ) > 0 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeDeploymentGenerationMismatch | |
| annotations: | |
| message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment | |
| }} does not match, this indicates that the Deployment has failed but has | |
| not been rolled back. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch | |
| expr: | | |
| kube_deployment_status_observed_generation{job="kube-state-metrics"} | |
| != | |
| kube_deployment_metadata_generation{job="kube-state-metrics"} | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeDeploymentReplicasMismatch | |
| annotations: | |
| message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not | |
| matched the expected number of replicas for longer than 15 minutes. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch | |
| expr: | | |
| ( | |
| kube_deployment_spec_replicas{job="kube-state-metrics"} | |
| != | |
| kube_deployment_status_replicas_available{job="kube-state-metrics"} | |
| ) and ( | |
| changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[5m]) | |
| == | |
| 0 | |
| ) | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeStatefulSetReplicasMismatch | |
| annotations: | |
| message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has | |
| not matched the expected number of replicas for longer than 15 minutes. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch | |
| expr: | | |
| ( | |
| kube_statefulset_status_replicas_ready{job="kube-state-metrics"} | |
| != | |
| kube_statefulset_status_replicas{job="kube-state-metrics"} | |
| ) and ( | |
| changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m]) | |
| == | |
| 0 | |
| ) | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeStatefulSetGenerationMismatch | |
| annotations: | |
| message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset | |
| }} does not match, this indicates that the StatefulSet has failed but has | |
| not been rolled back. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch | |
| expr: | | |
| kube_statefulset_status_observed_generation{job="kube-state-metrics"} | |
| != | |
| kube_statefulset_metadata_generation{job="kube-state-metrics"} | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeStatefulSetUpdateNotRolledOut | |
| annotations: | |
| message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update | |
| has not been rolled out. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout | |
| expr: | | |
| ( | |
| max without (revision) ( | |
| kube_statefulset_status_current_revision{job="kube-state-metrics"} | |
| unless | |
| kube_statefulset_status_update_revision{job="kube-state-metrics"} | |
| ) | |
| * | |
| ( | |
| kube_statefulset_replicas{job="kube-state-metrics"} | |
| != | |
| kube_statefulset_status_replicas_updated{job="kube-state-metrics"} | |
| ) | |
| ) and ( | |
| changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m]) | |
| == | |
| 0 | |
| ) | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeDaemonSetRolloutStuck | |
| annotations: | |
| message: Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet | |
| {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck | |
| expr: | | |
| kube_daemonset_status_number_ready{job="kube-state-metrics"} | |
| / | |
| kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} < 1.00 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeContainerWaiting | |
| annotations: | |
| message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} | |
| has been in waiting state for longer than 1 hour. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting | |
| expr: | | |
| sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: KubeDaemonSetNotScheduled | |
| annotations: | |
| message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset | |
| }} are not scheduled.' | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled | |
| expr: | | |
| kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} | |
| - | |
| kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: KubeDaemonSetMisScheduled | |
| annotations: | |
| message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset | |
| }} are running where they are not supposed to run.' | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled | |
| expr: | | |
| kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeCronJobRunning | |
| annotations: | |
| message: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more | |
| than 1h to complete. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning | |
| expr: | | |
| time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600 | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: KubeJobCompletion | |
| annotations: | |
| message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more | |
| than one hour to complete. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion | |
| expr: | | |
| kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 | |
| for: 1h | |
| labels: | |
| severity: warning | |
| - alert: KubeJobFailed | |
| annotations: | |
| message: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed | |
| expr: | | |
| kube_job_failed{job="kube-state-metrics"} > 0 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeHpaReplicasMismatch | |
| annotations: | |
| message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the | |
| desired number of replicas for longer than 15 minutes. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch | |
| expr: | | |
| (kube_hpa_status_desired_replicas{job="kube-state-metrics"} | |
| != | |
| kube_hpa_status_current_replicas{job="kube-state-metrics"}) | |
| and | |
| changes(kube_hpa_status_current_replicas[15m]) == 0 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: KubeHpaMaxedOut | |
| annotations: | |
| message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at | |
| max replicas for longer than 15 minutes. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout | |
| expr: | | |
| kube_hpa_status_current_replicas{job="kube-state-metrics"} | |
| == | |
| kube_hpa_spec_max_replicas{job="kube-state-metrics"} | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - name: kubernetes-resources | |
| rules: | |
| - alert: KubeCPUOvercommit | |
| annotations: | |
| message: Cluster has overcommitted CPU resource requests for Pods and cannot | |
| tolerate node failure. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit | |
| expr: | | |
| sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{}) | |
| / | |
| sum(kube_node_status_allocatable_cpu_cores) | |
| > | |
| (count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores) | |
| for: 5m | |
| labels: | |
| severity: warning | |
| - alert: KubeMemoryOvercommit | |
| annotations: | |
| message: Cluster has overcommitted memory resource requests for Pods and cannot | |
| tolerate node failure. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit | |
| expr: | | |
| sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{}) | |
| / | |
| sum(kube_node_status_allocatable_memory_bytes) | |
| > | |
| (count(kube_node_status_allocatable_memory_bytes)-1) | |
| / | |
| count(kube_node_status_allocatable_memory_bytes) | |
| for: 5m | |
| labels: | |
| severity: warning | |
| - alert: KubeCPUQuotaOvercommit | |
| annotations: | |
| message: Cluster has overcommitted CPU resource requests for Namespaces. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit | |
| expr: | | |
| sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) | |
| / | |
| sum(kube_node_status_allocatable_cpu_cores) | |
| > 1.5 | |
| for: 5m | |
| labels: | |
| severity: warning | |
| - alert: KubeMemoryQuotaOvercommit | |
| annotations: | |
| message: Cluster has overcommitted memory resource requests for Namespaces. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit | |
| expr: | | |
| sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) | |
| / | |
| sum(kube_node_status_allocatable_memory_bytes{job="node-exporter"}) | |
| > 1.5 | |
| for: 5m | |
| labels: | |
| severity: warning | |
| - alert: KubeQuotaExceeded | |
| annotations: | |
| message: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage | |
| }} of its {{ $labels.resource }} quota. | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded | |
| expr: | | |
| kube_resourcequota{job="kube-state-metrics", type="used"} | |
| / ignoring(instance, job, type) | |
| (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) | |
| > 0.90 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: CPUThrottlingHigh | |
| annotations: | |
| message: '{{ $value | humanizePercentage }} throttling of CPU in namespace | |
| {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ | |
| $labels.pod }}.' | |
| runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh | |
| expr: | | |
| sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) | |
| / | |
| sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace) | |
| > ( 25 / 100 ) | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - name: prometheus | |
| rules: | |
| - alert: PrometheusBadConfig | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to | |
| reload its configuration. | |
| summary: Failed Prometheus configuration reload. | |
| expr: | | |
| # Without max_over_time, failed scrapes could create false negatives, see | |
| # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. | |
| max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring"}[5m]) == 0 | |
| for: 10m | |
| labels: | |
| severity: critical | |
| - alert: PrometheusNotificationQueueRunningFull | |
| annotations: | |
| description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} | |
| is running full. | |
| summary: Prometheus alert notification queue predicted to run full in less | |
| than 30m. | |
| expr: | | |
| # Without min_over_time, failed scrapes could create false negatives, see | |
| # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. | |
| ( | |
| predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring"}[5m], 60 * 30) | |
| > | |
| min_over_time(prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| ) | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers | |
| annotations: | |
| description: '{{ printf "%.1f" $value }}% errors while sending alerts from | |
| Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.' | |
| summary: Prometheus has encountered more than 1% errors sending alerts to | |
| a specific Alertmanager. | |
| expr: | | |
| ( | |
| rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| / | |
| rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| ) | |
| * 100 | |
| > 1 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusErrorSendingAlertsToAnyAlertmanager | |
| annotations: | |
| description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts | |
| from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.' | |
| summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. | |
| expr: | | |
| min without(alertmanager) ( | |
| rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| / | |
| rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| ) | |
| * 100 | |
| > 3 | |
| for: 15m | |
| labels: | |
| severity: critical | |
| - alert: PrometheusNotConnectedToAlertmanagers | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected | |
| to any Alertmanagers. | |
| summary: Prometheus is not connected to any Alertmanagers. | |
| expr: | | |
| # Without max_over_time, failed scrapes could create false negatives, see | |
| # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. | |
| max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring"}[5m]) < 1 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusTSDBReloadsFailing | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected | |
| {{$value | humanize}} reload failures over the last 3h. | |
| summary: Prometheus has issues reloading blocks from disk. | |
| expr: | | |
| increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0 | |
| for: 4h | |
| labels: | |
| severity: warning | |
| - alert: PrometheusTSDBCompactionsFailing | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected | |
| {{$value | humanize}} compaction failures over the last 3h. | |
| summary: Prometheus has issues compacting blocks. | |
| expr: | | |
| increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0 | |
| for: 4h | |
| labels: | |
| severity: warning | |
| - alert: PrometheusNotIngestingSamples | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting | |
| samples. | |
| summary: Prometheus is not ingesting samples. | |
| expr: | | |
| rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusDuplicateTimestamps | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping | |
| {{ printf "%.4g" $value }} samples/s with different values but duplicated | |
| timestamp. | |
| summary: Prometheus is dropping samples with duplicate timestamps. | |
| expr: | | |
| rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusOutOfOrderTimestamps | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping | |
| {{ printf "%.4g" $value }} samples/s with timestamps arriving out of order. | |
| summary: Prometheus drops samples with out-of-order timestamps. | |
| expr: | | |
| rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusRemoteStorageFailures | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send | |
| {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ | |
| $labels.url }} | |
| summary: Prometheus fails to send samples to remote storage. | |
| expr: | | |
| ( | |
| rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| / | |
| ( | |
| rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| + | |
| rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| ) | |
| ) | |
| * 100 | |
| > 1 | |
| for: 15m | |
| labels: | |
| severity: critical | |
| - alert: PrometheusRemoteWriteBehind | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write | |
| is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url | |
| }}. | |
| summary: Prometheus remote write is behind. | |
| expr: | | |
| # Without max_over_time, failed scrapes could create false negatives, see | |
| # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. | |
| ( | |
| max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| - on(job, instance) group_right | |
| max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| ) | |
| > 120 | |
| for: 15m | |
| labels: | |
| severity: critical | |
| - alert: PrometheusRemoteWriteDesiredShards | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write | |
| desired shards calculation wants to run {{ $value }} shards for queue {{ | |
| $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ | |
| printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-k8s",namespace="monitoring"}` | |
| $labels.instance | query | first | value }}. | |
| summary: Prometheus remote write desired shards calculation wants to run more | |
| than configured max shards. | |
| expr: | | |
| # Without max_over_time, failed scrapes could create false negatives, see | |
| # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. | |
| ( | |
| max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| > | |
| max_over_time(prometheus_remote_storage_shards_max{job="prometheus-k8s",namespace="monitoring"}[5m]) | |
| ) | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusRuleFailures | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to | |
| evaluate {{ printf "%.0f" $value }} rules in the last 5m. | |
| summary: Prometheus is failing rule evaluations. | |
| expr: | | |
| increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 | |
| for: 15m | |
| labels: | |
| severity: critical | |
| - alert: PrometheusMissingRuleEvaluations | |
| annotations: | |
| description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ | |
| printf "%.0f" $value }} rule group evaluations in the last 5m. | |
| summary: Prometheus is missing rule evaluations due to slow rule group evaluation. | |
| expr: | | |
| increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 | |
| for: 15m | |
| labels: | |
| severity: warning | |
| - name: alertmanager.rules | |
| rules: | |
| - alert: AlertmanagerConfigInconsistent | |
| annotations: | |
| message: | | |
| The configuration of the instances of the Alertmanager cluster `{{ $labels.namespace }}/{{ $labels.service }}` are out of sync. | |
| {{ range printf "alertmanager_config_hash{namespace=\"%s\",service=\"%s\"}" $labels.namespace $labels.service | query }} | |
| Configuration hash for pod {{ .Labels.pod }} is "{{ printf "%.f" .Value }}" | |
| {{ end }} | |
| expr: | | |
| count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="monitoring"})) != 1 | |
| for: 5m | |
| labels: | |
| severity: critical | |
| - alert: AlertmanagerFailedReload | |
| annotations: | |
| message: Reloading Alertmanager's configuration has failed for {{ $labels.namespace | |
| }}/{{ $labels.pod}}. | |
| expr: | | |
| alertmanager_config_last_reload_successful{job="alertmanager-main",namespace="monitoring"} == 0 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: AlertmanagerMembersInconsistent | |
| annotations: | |
| message: Alertmanager has not found all other members of the cluster. | |
| expr: | | |
| alertmanager_cluster_members{job="alertmanager-main",namespace="monitoring"} | |
| != on (service) GROUP_LEFT() | |
| count by (service) (alertmanager_cluster_members{job="alertmanager-main",namespace="monitoring"}) | |
| for: 5m | |
| labels: | |
| severity: critical | |
| - name: general.rules | |
| rules: | |
| - alert: TargetDown | |
| annotations: | |
| message: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service | |
| }} targets in {{ $labels.namespace }} namespace are down.' | |
| expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, | |
| namespace, service)) > 10 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: Watchdog | |
| annotations: | |
| message: | | |
| This is an alert meant to ensure that the entire alerting pipeline is functional. | |
| This alert is always firing, therefore it should always be firing in Alertmanager | |
| and always fire against a receiver. There are integrations with various notification | |
| mechanisms that send a notification when this alert is not firing. For example the | |
| "DeadMansSnitch" integration in PagerDuty. | |
| expr: vector(1) | |
| labels: | |
| severity: none | |
| - name: node-network | |
| rules: | |
| - alert: NodeNetworkInterfaceFlapping | |
| annotations: | |
| message: Network interface "{{ $labels.device }}" changing it's up status | |
| often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}" | |
| expr: | | |
| changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 | |
| for: 2m | |
| labels: | |
| severity: warning | |
| - name: prometheus-operator | |
| rules: | |
| - alert: PrometheusOperatorReconcileErrors | |
| annotations: | |
| message: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace | |
| }} Namespace. | |
| expr: | | |
| rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - alert: PrometheusOperatorNodeLookupErrors | |
| annotations: | |
| message: Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace. | |
| expr: | | |
| rate(prometheus_operator_node_address_lookup_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1 | |
| for: 10m | |
| labels: | |
| severity: warning | |
| - name: Host and hardware | |
| rules: | |
| - alert: HostOutOfMemory | |
| expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 | |
| for: 5m | |
| labels: | |
| severity: warning | |
| annotations: | |
| summary: "Host out of memory (instance {{ $labels.instance }})" | |
| description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" | |
| - alert: HostMemoryUnderMemoryPressure | |
| expr: rate(node_vmstat_pgmajfault[1m]) > 1000 | |
| for: 5m | |
| labels: | |
| severity: warning | |
| annotations: | |
| summary: "Host memory under memory pressure (instance {{ $labels.instance }})" | |
| description: "The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment