Skip to content

Instantly share code, notes, and snippets.

@david-martin
Last active April 13, 2022 14:35
Show Gist options
  • Save david-martin/176b82d4a290cd2c005b994d5a02dcfa to your computer and use it in GitHub Desktop.
Save david-martin/176b82d4a290cd2c005b994d5a02dcfa to your computer and use it in GitHub Desktop.
dhall-to-yaml --explain <<< '(./rules.dhall)'
-- dhall-to-yaml --explain <<< '(./rules.dhall)'
let Severity : Type = < critical | warning >
let Labels : Type =
{ severity : Severity
}
let Annotations : Type =
{ summary : Text
, description : Text
, sop_url : Text
}
let Alert =
{ alert : Text
, expr : Text
, for : Text
, labels : Labels
, annotations : Annotations
}
let Rules : Type =
{ name: Text
, rules: List Alert
}
let makeLabels = \(severity : Severity) ->
let labels : Labels = { severity }
in labels
let makeAnnotations = \(summary: Text) -> \(description: Text) -> \(sop_url: Text) ->
let annotations : Annotations =
{ summary
, description = "${description} Refer to ${sop_url} for how to debug this."
, sop_url
}
in annotations
let makeAlert = \(alert: Text) -> \(expr: Text) -> \(for: Text) -> \(severity: Severity) -> \(summary: Text) -> \(description: Text) -> \(sop_url: Text) ->
let alert : Alert = {alert, expr, for, labels = makeLabels severity, annotations = makeAnnotations summary description sop_url}
in alert
-- My backend server alert rules
let backendAlerts : Rules =
{ name = "backend"
, rules =
[ makeAlert
"BackendServiceStuck"
"backend_state != 1"
"5m"
Severity.critical
"The backend service is stuck in a non-ready state."
"Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has been in a non-ready state for 5 minutes."
"https://example.com/backend_service_stuck.asciidoc"
, makeAlert
"BackendServiceSlow"
"backend_latency_percentile > 99"
"5m"
Severity.critical
"The backend service latency is high."
"Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} is showing signs of high latency requests."
"https://example.com/backend_service_latency.asciidoc"
]
}
-- My frontend server alert rules
let frontendAlerts : Rules =
{ name = "frontend"
, rules =
[ makeAlert
"FrontendError"
"frontend_error_rate > 10"
"5m"
Severity.critical
"The frontend is having a lot of errors."
"Frontend service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has a high rate of errors/500s."
"https://example.com/frontend_errors.asciidoc"
]
}
in {
groups =
[ backendAlerts
, frontendAlerts
]
}
groups:
- name: backend
rules:
- alert: BackendServiceStuck
annotations:
description: "Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has been in a non-ready state for 5 minutes. Refer to https://example.com/backend_service_stuck.asciidoc for how to debug this."
sop_url: https://example.com/backend_service_stuck.asciidoc
summary: The backend service is stuck in a non-ready state.
expr: "backend_state != 1"
for: "5m"
labels:
severity: critical
- alert: BackendServiceSlow
annotations:
description: "Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} is showing signs of high latency requests. Refer to https://example.com/backend_service_latency.asciidoc for how to debug this."
sop_url: https://example.com/backend_service_latency.asciidoc
summary: The backend service latency is high.
expr: "backend_latency_percentile > 99"
for: "5m"
labels:
severity: critical
- name: frontend
rules:
- alert: FrontendError
annotations:
description: "Frontend service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has a high rate of errors/500s. Refer to https://example.com/frontend_errors.asciidoc for how to debug this."
sop_url: https://example.com/frontend_errors.asciidoc
summary: The frontend is having a lot of errors.
expr: "frontend_error_rate > 10"
for: "5m"
labels:
severity: critical
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment