Debugging DNS Resolution Latency Spikes in Kubernetes: Profiling CoreDNS and UDP Packet Drops with eBPF — code snippets

snippet-1.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-service
  namespace: production
spec:
  replicas: 10
  template:
    spec:
      containers:
      - name: app
        image: production-registry.internal/payment-service:v2.4.1
      dnsConfig:
        options:
          - name: ndots
            value: "2"
          - name: single-request-reopen
          - name: timeout
            value: "2"
          - name: attempts
            value: "3"

snippet-2.txt

#include <linux/skbuff.h>
#include <linux/ip.h>
#include <net/sock.h>

tracepoint:skb:kfree_skb
{
    $skb = (struct sk_buff *)args->skbaddr;
    $ip = (struct iphdr *)($skb->head + $skb->network_header);
    
    // Verify protocol is UDP (17)
    if ($ip->protocol == 17) {
        $src = ntop($ip->saddr);
        $dst = ntop($ip->daddr);
        
        // Offset to transport header for UDP ports
        $udp = (struct udphdr *)($skb->head + $skb->transport_header);
        $dport = ($udp->dest >> 8) | (($udp->dest & 0xff) << 8); // ntohs manual swap for safety
        $sport = ($udp->source >> 8) | (($udp->source & 0xff) << 8);
        
        if ($dport == 53 || $sport == 53) {
            printf("[%s] UDP DNS Packet Dropped. Src: %s:%d -> Dst: %s:%d. Reason: %d\n",
                strftime("%H:%M:%S", nsecs), $src, $sport, $dst, $dport, args->reason);
            print(stack);
        }
    }
}

snippet-3.txt

#include <linux/bpf.h>
#include <linux/ptrace.h>
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <linux/udp.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>

struct dns_drop_event {
    __u32 src_ip;
    __u32 dst_ip;
    __u16 src_port;
    __u16 dst_port;
    __u64 timestamp;
};

struct {
    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
    __uint(key_size, sizeof(int));
    __uint(value_size, sizeof(int));
} dns_drops SEC(".maps");

SEC("kprobe/kfree_skb")
int trace_kfree_skb(struct pt_regs *ctx) {
    struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx);
    if (!skb)
        return 0;

    // Read network header offset
    unsigned char *head;
    __u16 network_header;
    __u16 transport_header;

    bpf_probe_read_kernel(&head, sizeof(head), &skb->head);
    bpf_probe_read_kernel(&network_header, sizeof(network_header), &skb->network_header);
    bpf_probe_read_kernel(&transport_header, sizeof(transport_header), &skb->transport_header);

    struct iphdr ip;
    bpf_probe_read_kernel(&ip, sizeof(ip), head + network_header);

    // Verify IPv4 and UDP
    if (ip.version != 4 || ip.protocol != 17) // 17 is IPPROTO_UDP
        return 0;

    struct udphdr udp;
    bpf_probe_read_kernel(&udp, sizeof(udp), head + transport_header);

    __u16 dst_port = __bpf_ntohs(udp.dest);
    __u16 src_port = __bpf_ntohs(udp.source);

    // Track only DNS queries or responses (port 53)
    if (dst_port == 53 || src_port == 53) {
        struct dns_drop_event event = {
            .src_ip = ip.saddr,
            .dst_ip = ip.daddr,
            .src_port = src_port,
            .dst_port = dst_port,
            .timestamp = bpf_ktime_get_ns(),
        };
        bpf_perf_event_output(ctx, &dns_drops, BPF_F_CURRENT_CPU, &event, sizeof(event));
    }

    return 0;
}

char LICENSE[] SEC("license") = "GPL";

snippet-4.go

package main

import (
	"bytes"
	"encoding/binary"
	"log"
	"net"
	"os"
	"os/signal"
	"syscall"

	"github.com/cilium/ebpf"
	"github.com/cilium/ebpf/link"
	"github.com/cilium/ebpf/perf"
	"github.com/cilium/ebpf/rlimit"
)

type DnsDropEvent struct {
	SrcIP     uint32
	DstIP     uint32
	SrcPort   uint16
	DstPort   uint16
	Timestamp uint64
}

func intToIP(val uint32) net.IP {
	ip := make(net.IP, 4)
	binary.LittleEndian.PutUint32(ip, val)
	return ip
}

func main() {
	// Allow current process to lock memory for eBPF resources
	if err := rlimit.RemoveMemlock(); err != nil {
		log.Fatalf("failed to remove memlock: %v", err)
	}

	// Load pre-compiled eBPF assets (normally generated by bpf2go)
	spec, err := loadDnsTracerSpec()
	if err != nil {
		log.Fatalf("failed to load spec: %v", err)
	}

	var objs dnsTracerObjects
	if err := spec.LoadAndAssign(&objs, nil); err != nil {
		log.Fatalf("failed to load objects: %v", err)
	}
	defer objs.Close()

	// Attach kprobe to kfree_skb
	kp, err := link.Kprobe("kfree_skb", objs.TraceKfreeSkb, nil)
	if err != nil {
		log.Fatalf("failed to attach kprobe: %v", err)
	}
	defer kp.Close()

	// Open perf reader
	rd, err := perf.NewReader(objs.DnsDrops, os.Getpagesize()*8)
	if err != nil {
		log.Fatalf("failed to open perf reader: %v", err)
	}
	defer rd.Close()

	log.Println("Monitoring CoreDNS UDP packet drops... Press Ctrl+C to exit.")

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		<-sig
		rd.Close()
	}()

	var event DnsDropEvent
	for {
		record, err := rd.Read()
		if err != nil {
			if perf.IsClosed(err) {
				return
			}
			log.Printf("reading from perf buffer failed: %v", err)
			continue
		}

		if err := binary.Read(bytes.NewBuffer(record.RawSample), binary.LittleEndian, &event); err != nil {
			log.Printf("failed to parse struct: %v", err)
			continue
		}

		log.Printf("[DROP DETECTED] %s:%d -> %s:%d | Timestamp: %d ns",
			intToIP(event.SrcIP), event.SrcPort,
			intToIP(event.DstIP), event.DstPort,
			event.Timestamp,
		)
	}
}

snippet-5.txt

.:53 {
    errors
    health {
       lameduck 5s
    }
    ready
    kubernetes cluster.local in-addr.arpa ip6.arpa {
       pods verified
       fallthrough in-addr.arpa ip6.arpa
       ttl 30
    }
    prometheus :9153
    pprof :8080
    forward . /etc/resolv.conf {
       max_concurrent 1000
       prefer_udp
    }
    cache 30 {
       success 10000
       denial 5000
       prefetch 10 10m 10%
    }
    loop
    reload
    loadbalance
}

snippet-6.sh

# Check current conntrack metrics and limits
sysctl net.netfilter.nf_conntrack_max
sysctl net.netfilter.nf_conntrack_count

# Inspect system-wide conntrack table insertions failures (drops)
conntrack -S

# View dropped packet statistics specifically for UDP/IP
nstat -az | grep -E "Udp(RcvbufErrors|SndbufErrors|NoPorts)|Ip(InDiscards|OutDiscards)"

# Set optimized kernel sysctl limits for high-throughput DNS traffic
sudo sysctl -w net.netfilter.nf_conntrack_max=1048576
sudo sysctl -w net.netfilter.nf_conntrack_tcp_timeout_established=86400
sudo sysctl -w net.core.rmem_max=16777216
sudo sysctl -w net.core.wmem_max=16777216