Chaos Engineering: Building Confidence Through Controlled Failure

snippet-2.sh

#!/usr/bin/env bash
# inject-latency.sh — add 200ms ± 50ms jitter to egress on eth0
# Requires: iproute2, root or CAP_NET_ADMIN

IFACE="${1:-eth0}"
DELAY_MS="${2:-200}"
JITTER_MS="${3:-50}"
DURATION_S="${4:-60}"

echo "[chaos] Injecting ${DELAY_MS}ms ± ${JITTER_MS}ms on ${IFACE} for ${DURATION_S}s"

tc qdisc add dev "$IFACE" root netem delay "${DELAY_MS}ms" "${JITTER_MS}ms" distribution normal

cleanup() {
  echo "[chaos] Removing netem qdisc"
  tc qdisc del dev "$IFACE" root 2>/dev/null
}
trap cleanup EXIT INT TERM

sleep "$DURATION_S"

snippet-3.go

package circuit

import (
	"errors"
	"sync"
	"time"
)

type State int

const (
	StateClosed State = iota
	StateOpen
	StateHalfOpen
)

var ErrCircuitOpen = errors.New("circuit breaker open")

type Breaker struct {
	mu           sync.Mutex
	state        State
	failures     int
	threshold    int
	resetTimeout time.Duration
	openedAt     time.Time
}

func New(threshold int, resetTimeout time.Duration) *Breaker {
	return &Breaker{threshold: threshold, resetTimeout: resetTimeout}
}

func (b *Breaker) Call(fn func() error) error {
	b.mu.Lock()
	switch b.state {
	case StateOpen:
		if time.Since(b.openedAt) > b.resetTimeout {
			b.state = StateHalfOpen
		} else {
			b.mu.Unlock()
			return ErrCircuitOpen
		}
	}
	b.mu.Unlock()

	err := fn()

	b.mu.Lock()
	defer b.mu.Unlock()

	if err != nil {
		b.failures++
		if b.failures >= b.threshold {
			b.state = StateOpen
			b.openedAt = time.Now()
		}
		return err
	}

	b.failures = 0
	b.state = StateClosed
	return nil
}

snippet-4.yaml

# experiments/payment-latency.yaml
experiment:
  name: payment-service-latency-degradation
  hypothesis: >
    When payment-service p99 latency exceeds 500ms,
    checkout error rate stays below 0.5% and order
    throughput stays above 80% of baseline.
  steady_state:
    - metric: checkout_error_rate
      max: 0.005
    - metric: order_throughput_rps
      min: 80
  method:
    - type: network-latency
      target: payment-service
      latency_ms: 500
      jitter_ms: 100
      duration_s: 120
  rollback:
    - type: network-latency
      target: payment-service
      action: remove
  abort_conditions:
    - metric: checkout_error_rate
      threshold: 0.05
      action: rollback_immediately

snippet-5.sql

WITH chaos_window AS (
  SELECT
    COUNT(*) FILTER (WHERE status = 'failed')::float /
    NULLIF(COUNT(*), 0) AS failure_rate,
    'chaos' AS period
  FROM orders
  WHERE created_at BETWEEN '2026-03-14 14:00:00' AND '2026-03-14 14:02:00'
),
baseline_window AS (
  SELECT
    COUNT(*) FILTER (WHERE status = 'failed')::float /
    NULLIF(COUNT(*), 0) AS failure_rate,
    'baseline' AS period
  FROM orders
  WHERE created_at BETWEEN '2026-03-14 13:00:00' AND '2026-03-14 14:00:00'
)
SELECT period, ROUND((failure_rate * 100)::numeric, 3) AS failure_pct
FROM chaos_window
UNION ALL
SELECT period, ROUND((failure_rate * 100)::numeric, 3)
FROM baseline_window;

snippet-6.dockerfile

FROM alpine:3.19

RUN apk add --no-cache iproute2 bash curl

WORKDIR /chaos

COPY inject-latency.sh ./
COPY healthcheck.sh ./

RUN chmod +x inject-latency.sh healthcheck.sh

HEALTHCHECK --interval=5s --timeout=3s CMD ./healthcheck.sh

ENV IFACE=eth0
ENV DELAY_MS=200
ENV JITTER_MS=50
ENV DURATION_S=60

ENTRYPOINT ["./inject-latency.sh"]
CMD ["$IFACE", "$DELAY_MS", "$JITTER_MS", "$DURATION_S"]

snippet-7.sh

#!/usr/bin/env bash
# ci-chaos.sh — run in staging after integration tests pass
set -euo pipefail

EXPERIMENT="${1:-experiments/payment-latency.yaml}"
BASELINE_DURATION=30
CHAOS_DURATION=120

echo "==> Measuring baseline for ${BASELINE_DURATION}s"
sleep "$BASELINE_DURATION"
BASELINE=$(curl -sf http://localhost:9090/metrics | jq '.checkout_error_rate')

echo "==> Running chaos experiment: $EXPERIMENT"
docker run --rm --net=host \
  --cap-add NET_ADMIN \
  -e DURATION_S="$CHAOS_DURATION" \
  chaos-agent:latest

echo "==> Validating steady state"
ERROR_RATE=$(curl -sf http://localhost:9090/metrics | jq '.checkout_error_rate')
MAX_ALLOWED=0.005

if (( $(echo "$ERROR_RATE > $MAX_ALLOWED" | bc -l) )); then
  echo "FAIL: error_rate $ERROR_RATE exceeded threshold $MAX_ALLOWED"
  exit 1
fi

echo "PASS: system held steady state during chaos window"

snippet.go

package main

import (
	"expvar"
	"net/http"
	"sync/atomic"
	"time"
)

var (
	requestsTotal   = expvar.NewInt("requests_total")
	errorsTotal     = expvar.NewInt("errors_total")
	latencyP99Ms    = expvar.NewFloat("latency_p99_ms")
	activeGoroutines = expvar.NewInt("active_goroutines")
)

type LatencyTracker struct {
	samples []float64
	mu      sync.Mutex
}

func (t *LatencyTracker) Record(d time.Duration) {
	t.mu.Lock()
	defer t.mu.Unlock()
	ms := float64(d.Milliseconds())
	t.samples = append(t.samples, ms)
	if len(t.samples) > 1000 {
		t.samples = t.samples[len(t.samples)-1000:]
	}
	latencyP99Ms.Set(t.p99())
}

func (t *LatencyTracker) p99() float64 {
	if len(t.samples) == 0 {
		return 0
	}
	sorted := make([]float64, len(t.samples))
	copy(sorted, t.samples)
	sort.Float64s(sorted)
	idx := int(float64(len(sorted)) * 0.99)
	return sorted[idx]
}

func main() {
	http.Handle("/metrics", expvar.Handler())
	http.ListenAndServe(":9090", nil)
}