Monzo's Alertmanager Slack templates

monzo-alertmanager-config.yaml

###################################################
##
## Alertmanager YAML configuration for routing.
##
## Will route alerts with a code_owner label to the slack-code-owners receiver 
## configured above, but will continue processing them to send to both a
## central Slack channel (slack-monitoring) and PagerDuty receivers
## (pd-warning and pd-critical)
##

routes:
  ###################################################
  ## Duplicate code_owner routes to teams
  ## These will send alerts to team channels but continue
  ## processing through the rest of the tree to handled by on-call
  - match_re:
      code_owner: '.+'
    routes:
      - match: {severity: info|warning|critical}
        continue: true
        receiver: slack-code-owners

  ###################################################
  ## Standard on-call routes
  - match_re:
      severity: info|warning|critical
    receiver: slack-monitoring
    continue: true


  ###################################################
  ## Pagerduty routes
  - match: {severity: warning}
    routes:
    - receiver: pd-warning
  - match: {severity: critical}
    routes:
    - receiver: pd-critical

monzo-alertmanager-receiver.yaml

receivers:
###################################################
## Slack Receivers
- name: slack-code-owners
  slack_configs:
  - channel: '#{{- template "slack.monzo.code_owner_channel" . -}}'
    send_resolved: true
    title: '{{ template "slack.monzo.title" . }}'
    icon_emoji: '{{ template "slack.monzo.icon_emoji" . }}'
    color: '{{ template "slack.monzo.color" . }}'
    text: '{{ template "slack.monzo.text" . }}'
    actions:
    - type: button
      text: 'Runbook :green_book:'
      url: '{{ (index .Alerts 0).Annotations.runbook }}'
    - type: button
      text: 'Query :mag:'
      url: '{{ (index .Alerts 0).GeneratorURL }}'
    - type: button
      text: 'Dashboard :grafana:'
      url: '{{ (index .Alerts 0).Annotations.dashboard }}'
    - type: button
      text: 'Silence :no_bell:'
      url: '{{ template "__alert_silence_link" . }}'
    - type: button
      text: '{{ template "slack.monzo.link_button_text" . }}'
      url: '{{ .CommonAnnotations.link_url }}'

monzo-slack-templates.tmpl

# This builds the silence URL.  We exclude the alertname in the range
# to avoid the issue of having trailing comma separator (%2C) at the end
# of the generated URL
{{ define "__alert_silence_link" -}}
    {{ .ExternalURL }}/#/silences/new?filter=%7B
    {{- range .CommonLabels.SortedPairs -}}
        {{- if ne .Name "alertname" -}}
            {{- .Name }}%3D"{{- .Value -}}"%2C%20
        {{- end -}}
    {{- end -}}
    alertname%3D"{{ .CommonLabels.alertname }}"%7D
{{- end }}



{{ define "__alert_severity_prefix" -}}
    {{ if ne .Status "firing" -}}
    :lgtm:
    {{- else if eq .Labels.severity "critical" -}}
    :fire:
    {{- else if eq .Labels.severity "warning" -}}
    :warning:
    {{- else -}}
    :question:
    {{- end }}
{{- end }}

{{ define "__alert_severity_prefix_title" -}}
    {{ if ne .Status "firing" -}}
    :lgtm:
    {{- else if eq .CommonLabels.severity "critical" -}}
    :fire:
    {{- else if eq .CommonLabels.severity "warning" -}}
    :warning:
    {{- else if eq .CommonLabels.severity "info" -}}
    :information_source:
    {{- else -}}
    :question:
    {{- end }}
{{- end }}


{{/* First line of Slack alerts */}}
{{ define "slack.monzo.title" -}}
    [{{ .Status | toUpper -}}
    {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}}
    ] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }}
{{- end }}


{{/* Color of Slack attachment (appears as line next to alert )*/}}
{{ define "slack.monzo.color" -}}
    {{ if eq .Status "firing" -}}
        {{ if eq .CommonLabels.severity "warning" -}}
            warning
        {{- else if eq .CommonLabels.severity "critical" -}}
            danger
        {{- else -}}
            #439FE0
        {{- end -}}
    {{ else -}}
    good
    {{- end }}
{{- end }}


{{/* Emoji to display as user icon (custom emoji supported!) */}}
{{ define "slack.monzo.icon_emoji" }}:prometheus:{{ end }}

{{/* The test to display in the alert */}}
{{ define "slack.monzo.text" -}}
    {{ range .Alerts }}
        {{- if .Annotations.message }}
            {{ .Annotations.message }}
        {{- end }}
        {{- if .Annotations.description }}
            {{ .Annotations.description }}
        {{- end }}
    {{- end }}
{{- end }}



{{- /* If none of the below matches, send to #monitoring-no-owner, and we 
can then assign the expected code_owner to the alert or map the code_owner
to the correct channel */ -}}
{{ define "__get_channel_for_code_owner" -}}
    {{- if eq . "platform-team" -}}
        platform-alerts
    {{- else if eq . "security-team" -}}
        security-alerts
    {{- else -}}
        monitoring-no-owner
    {{- end -}}
{{- end }}

{{- /* Select the channel based on the code_owner. We only expect to get
into this template function if the code_owners label is present on an alert.
This is to defend against us accidentally breaking the routing logic. */ -}}
{{ define "slack.monzo.code_owner_channel" -}}
    {{- if .CommonLabels.code_owner }}
        {{ template "__get_channel_for_code_owner" .CommonLabels.code_owner }}
    {{- else -}}
        monitoring
    {{- end }}
{{- end }}

{{ define "slack.monzo.link_button_text" -}}
    {{- if .CommonAnnotations.link_text -}}
        {{- .CommonAnnotations.link_text -}}
    {{- else -}}
        Link
    {{- end }} :link:
{{- end }}