etcd-alerts.yaml

groups:
- name: etcd
  rules:
  - alert: EtcdLeader
    expr: |
      max(etcd_server_has_leader_gauge{environment="production"}) < 1
    for: 3m
    labels:
      severity: critical
    annotations:
      summary: "etcd Leader Alert"
      description: "Etcd in {{ $labels.environment }} doesnt have leader"
  - alert: EtcdDB
    expr: |
      etcd_mvcc_db_total_size_in_bytes_gauge{environment="production"} >= 1999900000
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "etcd db size"
      description: "Etcd in {{ $labels.environment }} db has almost exceeded 2GB"
  - alert: EtcdProcess
    expr: |
      procstat_lookup_running{exe="etcd",environment="production"} < 1
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "etcd process is down"
      description: "Etcd in {{ $labels.environment }} is down"