alerts.yml

######################################
# Teku Application Alert Rules
######################################
groups:

  - name: Teku-Slot-Metrics
    rules:
      - alert: 'Teku Beacon Slot Increase'
        expr: sum (irate(beacon_slot{job="prod-teku-node"}[40s])) by (instance) == 0
        for: 1m
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} beacon_slot value has not increased for 1 minute."
          description: "{{ $labels.instance }} of job {{ $labels.job }} beacon_slot value has not increased for 1 minute."

      - alert: 'Teku Beacon Head Slot Increase'
        expr: sum (irate(beacon_head_slot{job="prod-teku-node"}[360s])) by (instance) == 0
        for: 80m
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} beacon_head_slot has not increased for 80 minutes."
          description: "{{ $labels.instance }} of job {{ $labels.job }} beacon_head_slot value has not increased for 80 minutes."

      - alert: 'Teku Chain Finalizing'
        expr: >-
          (( beacon_epoch{job="prod-teku-node"} - beacon_finalized_epoch{job="prod-teku-node"} ) > 4 ) and
          ( beacon_slot{job="prod-teku-node"} - 20) < beacon_head_slot{job="prod-teku-node"}
        for: 5m
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} chain finalization is dragged"
          description: "{{ $labels.instance }} of job {{ $labels.job }} chain finalization is dragged."

      - alert: 'Teku Peer Count'
        expr: libp2p_peers{job="prod-teku-node"} < 60
        for: 5m
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} has less than 60 peers"
          description: "{{ $labels.instance }} of job {{ $labels.job }} has less than 60 peers [{{ $value }}]"

      - alert: 'Teku Validators Producing Attestations'
        expr: >-
          validator_local_validator_count{job="prod-teku-node"} > 0 and
          ( beacon_slot{job="prod-teku-node"} - 20 ) < beacon_head_slot{job="prod-teku-node"} and
          irate(validator_beacon_node_published_attestation_total{job="prod-teku-node"}[60s]) == 0
        for: 768s
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} attestations are not produced"
          description: "{{ $labels.instance }} of job {{ $labels.job }} validators have not been producing attestations for 768 seconds"

      - alert: 'Teku Attestations Included'
        expr: >-
          ( validator_performance_included_attestations{job="prod-teku-node"} / validator_performance_expected_attestations{job="prod-teku-node"} ) < 0.6 and
          validator_local_validator_count{job="prod-teku-node"} > 0
        for: 5m
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} attestations are not included"
          description: "{{ $labels.instance }} of job {{ $labels.job }} validators have not been including attestations for 5 minutes"

      - alert: 'Teku Excessive GC'
        expr: rate(jvm_gc_collection_seconds_sum{job="prod-teku-node"}[1m]) > 0.5
        for: 30s
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} excessive garbage collection"
          description: "{{ $labels.instance }} of job {{ $labels.job }} excessive garbage collection for {{ $labels.gc }}"

      - alert: 'Teku Full Execute Queue'
        expr: >-
          (executor_beaconchain_queue_size{job="prod-teku-node"} > 400) or
          (executor_p2p_queue_size{job="prod-teku-node"} > 400) or
          (executor_powchain_queue_size{job="prod-teku-node"} > 400) or
          (executor_validator_queue_size{job="prod-teku-node"} > 400)
        for: 30s
        labels:
          severity: critical
          client: teku
        annotations:
          summary: "Instance {{ $labels.instance }} executor queue is greater than 400"
          description: "{{ $labels.instance }} of job {{ $labels.job }} executor queue is greater than 400"