Debugging CPU Throttling and CFS Bandwidth Control Issues in High-Throughput Go Containers Running on Kubernetes — code snippets

snippet-1.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: high-throughput-service
  namespace: production
spec:
  replicas: 10
  template:
    metadata:
      labels:
        app: high-throughput-service
    spec:
      containers:
      - name: go-app
        image: release-registry.internal/go-app:v1.4.2
        ports:
        - containerPort: 8080
        # MISCONFIGURATION: No GOMAXPROCS environment variable set, and no automaxprocs library loaded.
        resources:
          requests:
            cpu: "2"
            memory: "4Gi"
          limits:
            cpu: "4" # CFS quota = 400ms per 100ms period
            memory: "8Gi"

snippet-2.go

package main

import (
	"fmt"
	"runtime"
	"time"
)

func main() {
	// Print runtime limits to logs on startup
	fmt.Printf("[Startup] Go Runtime Environment Settings:\n")
	fmt.Printf("[Startup] - NumCPU (Logical Host Cores): %d\n", runtime.NumCPU())
	fmt.Printf("[Startup] - GOMAXPROCS (Active Go Processors): %d\n", runtime.GOMAXPROCS(0))

	// Keep-alive loop
	for {
		time.Sleep(10 * time.Minute)
	}
}

snippet-3.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: container-cpu-throttling-rules
  namespace: monitoring
spec:
  groups:
  - name: container-cpu.rules
    rules:
    - alert: GoContainerSevereThrottling
      expr: |
        sum(rate(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) by (namespace, pod, container)
        /
        sum(rate(container_cpu_cfs_periods_total{container!=""}[5m])) by (namespace, pod, container) * 100 > 10
      for: 5m
      labels:
        severity: critical
        team: platform-reliability
      annotations:
        summary: "Container {{ $labels.container }} in pod {{ $labels.pod }} is severely throttled"
        description: "The container has spent more than 10% of its scheduled CFS periods in a throttled state over the last 5 minutes. This will cause high latency spikes. Current rate: {{ printf \"%.2f\" $value }}%."

snippet-4.go

package main

import (
	"context"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	// Automatically configures GOMAXPROCS to match Linux container CPU quota
	_ "go.uber.org/automaxprocs"
)

func main() {
	log.Printf("[Main] Application starting up...")
	
	// Create a simple, robust HTTP server
	mux := http.NewServeMux()
	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("OK"))
	})
	
	server := &http.Server{
		Addr:         ":8080",
		Handler:      mux,
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
		IdleTimeout:  120 * time.Second,
	}

	go func() {
		log.Printf("[Main] Listening on :8080")
		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Fatalf("[Main] Server failed: %v", err)
		}
	}()

	// Graceful shutdown sequence
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
	<-quit

	log.Printf("[Main] Shutting down server...")
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	if err := server.Shutdown(ctx); err != nil {
		log.Fatalf("[Main] Server forced to shutdown: %v", err)
	}
	log.Printf("[Main] Server exited cleanly.")
}

snippet-5.sh

#!/usr/bin/env bash
# Diagnostic script to check CPU throttling using cgroups (v1 and v2)

set -euo pipefail

echo "========================================="
echo " Container CPU Throttling Diagnostic Tool"
echo "========================================="

# Detect Cgroup Version
if [ -f "/sys/fs/cgroup/cpu.stat" ]; then
    echo "[Cgroup Version] Detected Cgroup v2"
    
    # Read Limits
    if [ -f "/sys/fs/cgroup/cpu.max" ]; then
        read -r quota period < "/sys/fs/cgroup/cpu.max"
        echo "[CFS Limits] Quota: $quota, Period: $period"
        if [ "$quota" = "max" ]; then
            echo "[CFS Limits] No CPU limits are set (Quota is max)"
        else
            CORES=$(awk "BEGIN {print $quota / $period}")
            echo "[CFS Limits] Equivalent Cores Allocated: $CORES"
        fi
    fi
    
    # Read Stat
    echo "[Throttling Stats] Content of /sys/fs/cgroup/cpu.stat:"
    cat "/sys/fs/cgroup/cpu.stat"
    
elif [ -d "/sys/fs/cgroup/cpu" ]; then
    echo "[Cgroup Version] Detected Cgroup v1"
    
    # Read Limits
    if [ -f "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" ] && [ -f "/sys/fs/cgroup/cpu/cpu.cfs_period_us" ]; then
        quota=$(cat "/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
        period=$(cat "/sys/fs/cgroup/cpu/cpu.cfs_period_us")
        echo "[CFS Limits] Quota: $quota, Period: $period"
        if [ "$quota" -eq -1 ]; then
            echo "[CFS Limits] No CPU limits are set"
        else
            CORES=$(awk "BEGIN {print $quota / $period}")
            echo "[CFS Limits] Equivalent Cores Allocated: $CORES"
        fi
    fi
    
    # Read Stat
    echo "[Throttling Stats] Content of /sys/fs/cgroup/cpu/cpu.stat:"
    cat "/sys/fs/cgroup/cpu/cpu.stat"
else
    echo "ERROR: Could not locate cgroup stats directories."
    exit 1
fi

snippet-6.go

package main

import (
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
)

// configureScheduler applies configuration parameters to the Go runtime.
// It prioritizes explicit GOMAXPROCS env overrides, otherwise defaults safely.
func configureScheduler() {
	if envVal := os.Getenv("GOMAXPROCS"); envVal != "" {
		if val, err := strconv.Atoi(envVal); err == nil && val > 0 {
			runtime.GOMAXPROCS(val)
			fmt.Printf("[Config] Manual GOMAXPROCS set from environment: %d\n", val)
			return
		}
	}

	// Fallback logic: check custom limit env if running in non-standard container orchestrators
	if cpuLimitStr := os.Getenv("CONTAINER_CPU_LIMIT"); cpuLimitStr != "" {
		if limit, err := strconv.ParseFloat(cpuLimitStr, 64); err == nil && limit > 0 {
			// Round up the fractional cores to the nearest integer
			procs := int(math.Ceil(limit))
			runtime.GOMAXPROCS(procs)
			fmt.Printf("[Config] Derived GOMAXPROCS from custom CPU limit env: %d\n", procs)
			return
		}
	}

	// Default behavior if no overrides exist
	fmt.Printf("[Config] No overrides found. Using default Go runtime GOMAXPROCS: %d\n", runtime.GOMAXPROCS(0))
}

func init() {
	configureScheduler()
}

func main() {
	// Your application logic here
}