SLOs, SLAs, and Error Budgets: The SRE Approach to Reliability

snippet-2.yaml

# prometheus/recording_rules.yml
groups:
  - name: slo_availability
    interval: 30s
    rules:
      - record: job:http_requests_total:success_rate5m
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[5m])) by (job)
          /
          sum(rate(http_requests_total[5m])) by (job)

      - record: job:http_request_duration_seconds:p99_5m
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (job, le)
          )

      - record: job:http_requests_total:error_rate28d
        expr: |
          1 - (
            sum(increase(http_requests_total{status!~"5.."}[28d])) by (job)
            /
            sum(increase(http_requests_total[28d])) by (job)
          )

snippet-3.go

package budget

import "time"

type ErrorBudget struct {
	SLOTarget       float64       // e.g. 0.999
	WindowDuration  time.Duration // e.g. 28 * 24 * time.Hour
	TotalRequests   int64
	FailedRequests  int64
}

// Remaining returns the fraction of error budget still available (0.0 to 1.0).
func (b *ErrorBudget) Remaining() float64 {
	allowedFailureRate := 1.0 - b.SLOTarget
	allowedFailures := float64(b.TotalRequests) * allowedFailureRate
	consumed := float64(b.FailedRequests)
	if allowedFailures == 0 {
		return 0
	}
	remaining := (allowedFailures - consumed) / allowedFailures
	if remaining < 0 {
		return 0
	}
	return remaining
}

// BurnRate returns how fast the budget is being consumed relative to ideal.
// A burn rate > 1.0 means the budget will be exhausted before the window ends.
func (b *ErrorBudget) BurnRate(windowElapsed time.Duration) float64 {
	expectedConsumed := windowElapsed.Seconds() / b.WindowDuration.Seconds()
	actualConsumed := 1.0 - b.Remaining()
	if expectedConsumed == 0 {
		return 0
	}
	return actualConsumed / expectedConsumed
}

snippet-4.yaml

# alertmanager/slo_alerts.yml
groups:
  - name: slo_burn_rate
    rules:
      - alert: HighBurnRate
        expr: |
          (
            job:http_requests_total:error_rate1h > (14.4 * 0.001)
            and
            job:http_requests_total:error_rate5m > (14.4 * 0.001)
          )
        for: 2m
        labels:
          severity: critical
          slo: availability
        annotations:
          summary: "High error budget burn rate on {{ $labels.job }}"
          description: >
            Burn rate is {{ $value | humanizePercentage }} — at this rate
            the 28-day error budget will be exhausted in ~2 hours.

      - alert: ModerateBurnRate
        expr: |
          (
            job:http_requests_total:error_rate6h > (6 * 0.001)
            and
            job:http_requests_total:error_rate30m > (6 * 0.001)
          )
        for: 15m
        labels:
          severity: warning
          slo: availability
        annotations:
          summary: "Elevated error budget consumption on {{ $labels.job }}"

snippet-5.sql

CREATE TABLE error_budget_snapshots (
    id          BIGSERIAL PRIMARY KEY,
    service     TEXT        NOT NULL,
    window_days INTEGER     NOT NULL DEFAULT 28,
    slo_target  NUMERIC(6,4) NOT NULL,
    recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    total_requests  BIGINT  NOT NULL,
    failed_requests BIGINT  NOT NULL,
    budget_remaining NUMERIC(6,4) GENERATED ALWAYS AS (
        CASE
            WHEN total_requests = 0 THEN 1.0
            ELSE GREATEST(
                0,
                1 - (failed_requests::numeric /
                     (total_requests * (1 - slo_target)))
            )
        END
    ) STORED
);

-- Rolling 28-day budget consumption per service
SELECT
    service,
    AVG(budget_remaining)        AS avg_budget_remaining,
    MIN(budget_remaining)        AS worst_day,
    COUNT(*) FILTER (WHERE budget_remaining < 0.10) AS days_near_exhaustion
FROM error_budget_snapshots
WHERE recorded_at >= NOW() - INTERVAL '90 days'
GROUP BY service
ORDER BY worst_day ASC;

snippet-6.sh

#!/usr/bin/env bash
set -euo pipefail

PROMETHEUS_URL="${PROMETHEUS_URL:-http://prometheus:9090}"
SERVICE="${1:?Usage: deploy-gate.sh <service>}"
BUDGET_THRESHOLD=0.10  # block deploys below 10% remaining

budget_remaining=$(curl -sf \
  "${PROMETHEUS_URL}/api/v1/query" \
  --data-urlencode "query=job:http_requests_total:budget_remaining{job=\"${SERVICE}\"}" \
  | jq -r '.data.result[0].value[1] // "1"')

echo "Error budget remaining for ${SERVICE}: $(echo "$budget_remaining * 100" | bc)%"

if (( $(echo "$budget_remaining < $BUDGET_THRESHOLD" | bc -l) )); then
  echo "ERROR: Error budget below ${BUDGET_THRESHOLD}. Halting deployment."
  echo "To override (incident response only): FORCE_DEPLOY=1 $0 $*"
  [[ "${FORCE_DEPLOY:-0}" == "1" ]] || exit 1
fi

echo "Budget gate passed. Proceeding with deployment..."
exec ./scripts/deploy.sh "${SERVICE}"

snippet.go

package middleware

import (
	"net/http"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
	requestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "http_requests_total",
		Help: "Total HTTP requests partitioned by status and method.",
	}, []string{"method", "path", "status"})

	requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "http_request_duration_seconds",
		Help:    "HTTP request latency distributions.",
		Buckets: []float64{0.05, 0.1, 0.2, 0.5, 1.0, 2.5},
	}, []string{"method", "path"})
)

func SLIMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		rec := &statusRecorder{ResponseWriter: w, status: 200}

		next.ServeHTTP(rec, r)

		duration := time.Since(start).Seconds()
		status := strconv.Itoa(rec.status)

		requestsTotal.WithLabelValues(r.Method, r.URL.Path, status).Inc()
		requestDuration.WithLabelValues(r.Method, r.URL.Path).Observe(duration)
	})
}

type statusRecorder struct {
	http.ResponseWriter
	status int
}

func (r *statusRecorder) WriteHeader(status int) {
	r.status = status
	r.ResponseWriter.WriteHeader(status)
}