Last active
April 13, 2022 14:35
-
-
Save david-martin/176b82d4a290cd2c005b994d5a02dcfa to your computer and use it in GitHub Desktop.
dhall-to-yaml --explain <<< '(./rules.dhall)'
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- dhall-to-yaml --explain <<< '(./rules.dhall)' | |
let Severity : Type = < critical | warning > | |
let Labels : Type = | |
{ severity : Severity | |
} | |
let Annotations : Type = | |
{ summary : Text | |
, description : Text | |
, sop_url : Text | |
} | |
let Alert = | |
{ alert : Text | |
, expr : Text | |
, for : Text | |
, labels : Labels | |
, annotations : Annotations | |
} | |
let Rules : Type = | |
{ name: Text | |
, rules: List Alert | |
} | |
let makeLabels = \(severity : Severity) -> | |
let labels : Labels = { severity } | |
in labels | |
let makeAnnotations = \(summary: Text) -> \(description: Text) -> \(sop_url: Text) -> | |
let annotations : Annotations = | |
{ summary | |
, description = "${description} Refer to ${sop_url} for how to debug this." | |
, sop_url | |
} | |
in annotations | |
let makeAlert = \(alert: Text) -> \(expr: Text) -> \(for: Text) -> \(severity: Severity) -> \(summary: Text) -> \(description: Text) -> \(sop_url: Text) -> | |
let alert : Alert = {alert, expr, for, labels = makeLabels severity, annotations = makeAnnotations summary description sop_url} | |
in alert | |
-- My backend server alert rules | |
let backendAlerts : Rules = | |
{ name = "backend" | |
, rules = | |
[ makeAlert | |
"BackendServiceStuck" | |
"backend_state != 1" | |
"5m" | |
Severity.critical | |
"The backend service is stuck in a non-ready state." | |
"Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has been in a non-ready state for 5 minutes." | |
"https://example.com/backend_service_stuck.asciidoc" | |
, makeAlert | |
"BackendServiceSlow" | |
"backend_latency_percentile > 99" | |
"5m" | |
Severity.critical | |
"The backend service latency is high." | |
"Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} is showing signs of high latency requests." | |
"https://example.com/backend_service_latency.asciidoc" | |
] | |
} | |
-- My frontend server alert rules | |
let frontendAlerts : Rules = | |
{ name = "frontend" | |
, rules = | |
[ makeAlert | |
"FrontendError" | |
"frontend_error_rate > 10" | |
"5m" | |
Severity.critical | |
"The frontend is having a lot of errors." | |
"Frontend service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has a high rate of errors/500s." | |
"https://example.com/frontend_errors.asciidoc" | |
] | |
} | |
in { | |
groups = | |
[ backendAlerts | |
, frontendAlerts | |
] | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
groups: | |
- name: backend | |
rules: | |
- alert: BackendServiceStuck | |
annotations: | |
description: "Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has been in a non-ready state for 5 minutes. Refer to https://example.com/backend_service_stuck.asciidoc for how to debug this." | |
sop_url: https://example.com/backend_service_stuck.asciidoc | |
summary: The backend service is stuck in a non-ready state. | |
expr: "backend_state != 1" | |
for: "5m" | |
labels: | |
severity: critical | |
- alert: BackendServiceSlow | |
annotations: | |
description: "Service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} is showing signs of high latency requests. Refer to https://example.com/backend_service_latency.asciidoc for how to debug this." | |
sop_url: https://example.com/backend_service_latency.asciidoc | |
summary: The backend service latency is high. | |
expr: "backend_latency_percentile > 99" | |
for: "5m" | |
labels: | |
severity: critical | |
- name: frontend | |
rules: | |
- alert: FrontendError | |
annotations: | |
description: "Frontend service {{ $labels.name }} in the {{ $labels.namespace }} namespace, managed by operator {{ $labels.pod }} has a high rate of errors/500s. Refer to https://example.com/frontend_errors.asciidoc for how to debug this." | |
sop_url: https://example.com/frontend_errors.asciidoc | |
summary: The frontend is having a lot of errors. | |
expr: "frontend_error_rate > 10" | |
for: "5m" | |
labels: | |
severity: critical |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment