Optimizing TCP Connection Pool Exhaustion Debugging with eBPF and bpftrace — code snippets

snippet-1.go

package main

import (
	"context"
	"io"
	"log"
	"net/http"
	"time"
)

// LeakyClient demonstrates a classic connection leak: forgetting to drain and close the body.
func LeakyClient(ctx context.Context, url string) {
	// A poorly configured client with no transport limits
	client := &http.Client{
		Timeout: 5 * time.Second,
	}

	for {
		select {
		case <-ctx.Done():
			return
		default:
			req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
			if err != nil {
				log.Printf("Failed to create request: %v", err)
				time.Sleep(100 * time.Millisecond)
				continue
			}

			resp, err := client.Do(req)
			if err != nil {
				log.Printf("HTTP request failed: %v", err)
				time.Sleep(100 * time.Millisecond)
				continue
			}

			// BUG: Missing 'defer resp.Body.Close()' and io.Copy(io.Discard, resp.Body)
			// Because the response body is never closed or read to completion,
			// the underlying HTTP transport cannot reuse this TCP connection.
			// It remains allocated, forcing the client to establish a new TCP connection
			// on the next iteration of the loop.
			_ = resp
			
			time.Sleep(50 * time.Millisecond)
		}
	}
}

snippet-2.sh

# Check socket states for port 8080 outbound connections
ss -atp dst :8080

# View summary of all socket states on the host
ss -s

snippet-3.txt

/*
 * tcp_state_tracer.bt
 * Traces TCP connection state changes for IPv4 sockets.
 */

#include <net/sock.h>
#include <linux/socket.h>

BEGIN
{
	printf("%-8s %-16s %-6s %-15s %-6s %-15s %-6s %-15s\n", 
		"TIME", "COMM", "PID", "SADDR", "SPORT", "DADDR", "DPORT", "STATE_CHANGE");
}

tracepoint:sock:inet_sock_set_state
{
	$sk = (struct sock *)args->skaddr;
	$family = $sk->__sk_common.skc_family;

	// Trace only IPv4 connections (AF_INET = 2)
	if ($family == 2) {
		$saddr = ntop($sk->__sk_common.skc_rcv_saddr);
		$daddr = ntop($sk->__sk_common.skc_daddr);
		$sport = $sk->__sk_common.skc_num;
		
		// Parse destination port (big-endian conversion)
		$dport = $sk->__sk_common.skc_dport;
		$dport = ($dport >> 8) | (($dport & 0xff) << 8);

		// Convert state integers to readable strings
		// Reference: include/net/tcp_states.h
		$states = associative_array();
		$states[1]  = "ESTABLISHED";
		$states[2]  = "SYN_SENT";
		$states[3]  = "SYN_RECV";
		$states[4]  = "FIN_WAIT1";
		$states[5]  = "FIN_WAIT2";
		$states[6]  = "TIME_WAIT";
		$states[7]  = "CLOSE";
		$states[8]  = "CLOSE_WAIT";
		$states[9]  = "LAST_ACK";
		$states[10] = "LISTEN";
		$states[11] = "CLOSING";

		$old_str = $states[args->oldstate] ? $states[args->oldstate] : "UNKNOWN";
		$new_str = $states[args->newstate] ? $states[args->newstate] : "UNKNOWN";

		printf("%-8s %-16s %-6d %-15s %-6d %-15s %-6d %s -> %s\n",
			elapsed / 1e9, comm, pid, $saddr, $sport, $daddr, $dport, $old_str, $new_str);
	}
}

snippet-4.txt

/*
 * tcp_connection_duration.bt
 * Measures how long a TCP connection remains open in the ESTABLISHED state.
 */

#include <net/sock.h>

// Associate socket pointers to their ESTABLISHED start time
@start_time[struct sock *] = uint64;

tracepoint:sock:inet_sock_set_state
{
	$sk = (struct sock *)args->skaddr;
	$family = $sk->__sk_common.skc_family;

	if ($family == 2) {
		$oldstate = args->oldstate;
		$newstate = args->newstate;

		// State 1 is TCP_ESTABLISHED
		if ($newstate == 1) {
			@start_time[$sk] = nsecs;
		}

		// When leaving ESTABLISHED state
		if ($oldstate == 1 && @start_time[$sk]) {
			$duration_ms = (nsecs - @start_time[$sk]) / 1000000;
			$saddr = ntop($sk->__sk_common.skc_rcv_saddr);
			$daddr = ntop($sk->__sk_common.skc_daddr);
			$sport = $sk->__sk_common.skc_num;
			$dport = $sk->__sk_common.skc_dport;
			$dport = ($dport >> 8) | (($dport & 0xff) << 8);

			// Alert if a connection lasted longer than 30 seconds
			// In microservices, long-lived connections are fine if reused, 
			// but constant closures of long-duration sockets can indicate leaks.
			if ($duration_ms > 30000) {
				printf("[WARNING] Long-lived Socket Closed | PID: %d (%s) | Target: %s:%d -> %s:%d | Active: %d ms\n",
					pid, comm, $saddr, $sport, $daddr, $dport, $duration_ms);
			} else {
				printf("Socket Closed | PID: %d (%s) | Target: %s:%d -> %s:%d | Active: %d ms\n",
					pid, comm, $saddr, $sport, $daddr, $dport, $duration_ms);
			}

			delete(@start_time[$sk]);
		}
	}
}

END
{
	clear(@start_time);
}

snippet-5.txt

/*
 * tcp_backlog_overflow.bt
 * Traces when incoming TCP connections are dropped due to listening socket queue overflow.
 */

#include <net/sock.h>

kprobe:tcp_v4_conn_request
{
	$sk = (struct sock *)arg0;
	
	// sk_acceptq_is_full check: compare current queue length against max backlog
	$ack_backlog = $sk->sk_ack_backlog;
	$max_ack_backlog = $sk->sk_max_ack_backlog;

	if ($ack_backlog > $max_ack_backlog) {
		$saddr = ntop($sk->__sk_common.skc_rcv_saddr);
		$sport = $sk->__sk_common.skc_num;
		
		printf("[ALERT] SYN Drop due to Backlog Overflow! Local: %s:%d | Queue: %d / Limit: %d\n",
			$saddr, $sport, $ack_backlog, $max_ack_backlog);
	}
}

snippet-6.sh

# Execute the state tracer, filtering output for our 'api-gateway' binary
bpftrace tcp_state_tracer.bt | grep "api-gateway"

snippet-7.go

package main

import (
	"context"
	"net"
	"net/http"
	"time"
)

// NewProductionHTTPClient returns a configured client that prevents leaks.
func NewProductionHTTPClient() *http.Client {
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			// Limit the TCP handshake phase to prevent blocking resources
			Timeout:   3 * time.Second,
			// Enable TCP keep-alive probes to detect dead connections
			KeepAlive: 30 * time.Second,
		}).DialContext,

		// MaxIdleConns limits total idle connections across all hosts
		MaxIdleConns: 200,
		
		// MaxIdleConnsPerHost must match your expected concurrency.
		// Setting this high prevents connection churning under load.
		MaxIdleConnsPerHost: 100,

		// MaxConnsPerHost sets a hard limit on total (active + idle) connections.
		// Prevents runaway connection creation if the downstream server stalls.
		MaxConnsPerHost: 200,

		// Close connections that sit idle for longer than 90 seconds
		IdleConnTimeout: 90 * time.Second,
		
		TLSHandshakeTimeout:   5 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		
		// Ensure HTTP/2 multiplexing is used if supported by the server
		ForceAttemptHTTP2: true,
	}

	return &http.Client{
		Transport: transport,
		// Timeout governs the entire request execution (handshake + request + response body read)
		Timeout: 10 * time.Second,
	}
}

snippet-8.txt

# /etc/sysctl.d/99-network-tuning.conf
# Optimizing TCP stack for high-concurrency microservices

# Increase the ephemeral port range to allow more concurrent connections
net.ipv4.ip_local_port_range = 10240 65535

# Enable safe reuse of TIME_WAIT sockets for outbound connections.
# This is safe in modern cloud environments where TCP Timestamps are active.
net.ipv4.tcp_tw_reuse = 1

# Reduce the time sockets remain in FIN-WAIT-2 state before being reclaimed
net.ipv4.tcp_fin_timeout = 15

# Accelerate dead connection detection via aggressive keep-alive settings
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_keepalive_probes = 5

# Increase the socket listen backlog (somaxconn)
# Controls the maximum size of the queue for sockets in SYN_RECV/established state waiting for accept()
net.core.somaxconn = 4096

# Increase maximum network interface backlog queue
net.core.netdev_max_backlog = 8192