Burrow configuration with slack and pagerduty notification

burrow.toml

[general]
pidfile="burrow.pid"
stdout-logfile="burrow.out"

[logging]
filename="/opt/burrow/logs/burrow.log"
level="info"
maxsize=100
maxbackups=10
maxage=10
use-localtime=false
use-compression=true

[zookeeper]
servers=["x.x.x.x:2181","y.y.y.y:2181","z.z.z.z.140:2181"]
timeout=6

[client-profile.myclient]
kafka-version="2.2.0"

[httpserver.mylistener]
address=":8080"
timeout=300

[storage.mystorage]
class-name="inmemory"

[evaluator.mystorage]
class-name="caching"

[cluster.my-cluster]
class-name="kafka"
servers=["p.p.p.p:9092","q.q.q.q:9092","r.r.r.r:9092"]
client-profile="myclient"

[consumer.myconsumers]
class-name="kafka"
cluster="my-cluster"
servers=["p.p.p.p:9092","q.q.q.q:9092","r.r.r.r:9092"]

[notifier.pagerduty]
class-name="http"
interval=300
threshold=3
send-close=true
method-open="POST"
method-close="POST"
extras={integration_key='xxxxxx'}
template-open="/opt/burrow/config/default-pg-post.tmpl"
template-close="/opt/burrow/config/default-pg-delete.tmpl"
url-open="https://events.pagerduty.com/v2/enqueue"
url-close="https://events.pagerduty.com/v2/enqueue"

[notifier.slacknotify]
class-name="http"
interval=300
threshold=3
send-close=true
template-open='/opt/burrow/config/default-slack-post.tmpl'
template-close='/opt/burrow/config/default-slack-delete.tmpl'
url-open='https://hooks.slack.com/services/xxxx'
url-close='https://hooks.slack.com/services/xxxx'

default-pg-delete.tmpl

{
 "payload": {
    "summary": "[{{.Result.Cluster}}]A kafka consumer is no longer lagging: {{ .Result.TotalLag}}",
    "timestamp": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
    "source": "xxx-prod-burrow",
    "severity": "info",
    "component": "{{.Result.Cluster}}",
    "group": "{{.Result.Group}}",
    "total_lag": "{{ .Result.TotalLag}}"
  },
  "routing_key": "{{ index .Extras "integration_key"}}",
  "dedup_key": "{{.Result.Group}}",
  "event_action": "resolve",
  "client": "burrow-hostname",
  "client_url": "http://x.x.x.x:8080/v3/kafka/{{.Result.Cluster}}"
}

default-pg-post.tmpl

{
 "payload": {
    "summary": "[{{.Result.Cluster}}]Kafka Consumer Lag Detected: {{len .Result.Partitions}} partitions have problems",
    "timestamp": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
    "source": "xxxx-prod-burrow",
    "severity": "warning",
    "component": "kafka",
    "group": "{{.Result.Group}}",
    "class": "Consumer Lag",
    "custom_details": {
      "cluster": "{{.Result.Cluster}}",
      "consumer_group": "{{.Result.Group}}",
      "total_lag": "{{ .Result.TotalLag}}",
      "status": "{{.Result.Status.String}}",
      "complete": "{{.Result.Complete}}",
      "errors": "{{len .Result.Partitions}} partitions have problems",
      "start": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
      "partitions": "{{range .Result.Partitions}} {{.Status.String}} {{.Topic}}:{{.Partition}} ({{.Start.Timestamp}}, {{.Start.Offset}}, {{.Start.Lag}}) -> ({{.End.Timestamp}}, {{.End.Offset}}, {{.End.Lag}}), \n{{end}}"
    }
  },
  "routing_key": "{{ index .Extras "integration_key"}}",
  "dedup_key": "{{.Result.Group}}",
  "links": [{
    "href": "https://github.com/linkedin/Burrow/wiki/HTTP-Endpoint",
    "text": "Burrow endpoints for troubleshooting"
  }],
  "event_action": "trigger",
  "client": "burrow-hostname",
  "client_url": "http://x.x.x.x:8080/v3/kafka/{{.Result.Cluster}}"
}

default-slack-delete.tmpl

{
	"attachments": [{
		"color": "good",
		"title": "A kafka consumer is no longer lagging",
		"fields": [{
			"title": "Group",
			"value": "{{ .Group }}",
			"short": false
		}, {
			"title": "Cluster",
			"value": "{{ .Cluster }}",
			"short": true
		}, {
			"title": "Total Lag",
			"value": "{{ .Result.TotalLag}}",
			"short": true
		}, {
			"title": "Start",
			"value": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
			"short": true
		}]
	}]
}

default-slack-post.tmpl

{
	"attachments": [{
		"color": "danger",
		"title": "A kafka consumer is lagging behind!",
		"fields": [{
			"title": "Group",
			"value": "{{ .Group }}",
			"short": false
		}, {
			"title": "Cluster",
			"value": "{{ .Cluster }}",
			"short": true
		}, {
			"title": "Total Lag",
			"value": "{{ .Result.TotalLag}}",
			"short": true
		}, {
			"title": "Start",
			"value": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
			"short": true
		}, {
			"title": "Errors",
			"value": "{{len .Result.Partitions}} partitions have problems",
			"short": true
		}, {
			"title": ".....",
			"value": "{{range .Result.Partitions}}{{.Status.String}} {{.Topic}}:{{.Partition}} ({{.Start.Timestamp}}, {{.Start.Offset}}, {{.Start.Lag}}) -> ({{.End.Timestamp}}, {{.End.Offset}}, {{.End.Lag}})\n{{end}}",
			"short": false
		}]
	}]
}