Zero-Downtime Deployments: Blue-Green, Canary, and Rolling Strategies

snippet-2.go

func deepHealthHandler(db *sql.DB, cache *redis.Client) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
        defer cancel()

        type check struct {
            name string
            fn   func() error
        }

        checks := []check{
            {"postgres", func() error { return db.PingContext(ctx) }},
            {"redis", func() error { return cache.Ping(ctx).Err() }},
        }

        results := make(map[string]string, len(checks))
        healthy := true

        for _, c := range checks {
            if err := c.fn(); err != nil {
                results[c.name] = err.Error()
                healthy = false
            } else {
                results[c.name] = "ok"
            }
        }

        status := http.StatusOK
        if !healthy {
            status = http.StatusServiceUnavailable
        }
        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(status)
        json.NewEncoder(w).Encode(results)
    }
}

snippet-3.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: api-canary
  namespace: production
  annotations:
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "5"
    nginx.ingress.kubernetes.io/canary-by-header: "X-Canary"
    nginx.ingress.kubernetes.io/canary-by-header-value: "always"
spec:
  rules:
  - host: api.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: api-v2
            port:
              number: 8080

snippet-4.go

type CanaryAnalysis struct {
    PrometheusURL string
    Threshold     float64 // e.g., 0.01 for 1% error rate
}

func (a *CanaryAnalysis) IsHealthy(ctx context.Context, version string) (bool, error) {
    query := fmt.Sprintf(
        `sum(rate(http_requests_total{version=%q,status=~"5.."}[5m])) / `+
            `sum(rate(http_requests_total{version=%q}[5m]))`,
        version, version,
    )
    resp, err := http.Get(fmt.Sprintf(
        "%s/api/v1/query?query=%s",
        a.PrometheusURL, url.QueryEscape(query),
    ))
    if err != nil {
        return false, fmt.Errorf("prometheus query failed: %w", err)
    }
    defer resp.Body.Close()

    var result struct {
        Data struct {
            Result []struct {
                Value [2]json.RawMessage `json:"value"`
            } `json:"result"`
        } `json:"data"`
    }
    if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
        return false, err
    }
    if len(result.Data.Result) == 0 {
        return true, nil // no data yet, optimistically healthy
    }

    var errorRate float64
    if err := json.Unmarshal(result.Data.Result[0].Value[1], &errorRate); err != nil {
        return false, err
    }
    return errorRate < a.Threshold, nil
}

snippet-5.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: api
  namespace: production
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0       # never reduce capacity below 100%
      maxSurge: 2             # add at most 2 new pods at a time
  minReadySeconds: 30         # pod must be ready 30s before counted as available
  template:
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: api
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 5"]  # drain in-flight requests
        readinessProbe:
          httpGet:
            path: /healthz/deep
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          failureThreshold: 3

snippet-6.sql

-- Deploy 1 (expand): add new column alongside old one
ALTER TABLE users ADD COLUMN display_name TEXT;
UPDATE users SET display_name = username;

-- Deploy 2: application reads display_name, writes both columns
-- (no SQL migration needed — it's a code change)

-- Deploy 3 (contract): drop old column once all instances are on new code
ALTER TABLE users DROP COLUMN username;

snippet.sh

#!/usr/bin/env bash
set -euo pipefail

NAMESPACE="production"
SERVICE="api-gateway"
NEW_COLOR="${1:?Usage: $0 <blue|green>}"
OLD_COLOR=$(kubectl get svc "$SERVICE" -n "$NAMESPACE" \
  -o jsonpath='{.spec.selector.color}')

echo "Shifting traffic from $OLD_COLOR → $NEW_COLOR"

# Wait for new deployment to be fully ready
kubectl rollout status "deploy/api-$NEW_COLOR" \
  -n "$NAMESPACE" --timeout=5m

# Smoke test the new environment via its dedicated service
SMOKE_URL="http://api-${NEW_COLOR}.${NAMESPACE}.svc.cluster.local/healthz/deep"
for i in $(seq 1 10); do
  STATUS=$(curl -sf -o /dev/null -w "%{http_code}" "$SMOKE_URL") || true
  [[ "$STATUS" == "200" ]] && break
  echo "Attempt $i: got $STATUS, retrying in 5s..."
  sleep 5
done
[[ "$STATUS" != "200" ]] && { echo "Smoke test failed. Aborting."; exit 1; }

# Atomic traffic shift
kubectl patch svc "$SERVICE" -n "$NAMESPACE" \
  -p "{\"spec\":{\"selector\":{\"color\":\"$NEW_COLOR\"}}}"

echo "Traffic live on $NEW_COLOR. $OLD_COLOR is warm standby."