Tracing CPU Cache Misses and TLB Thrashing in High-Throughput Java Applications using eBPF and perf_events — code snippets

snippet-1.txt

package com.performance.cache;

import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;

/**
 * Demonstrates the cache-friendly layout pattern.
 * Compares standard pointer-chasing node traversal against a contiguous, 
 * cache-prefetcher-friendly off-heap array layout.
 */
public class MemoryLayoutBench {

    // Cache-unfriendly: Heap-allocated objects with random pointer references
    public static class PointerNode {
        long timestamp;
        long id;
        double value;
        PointerNode next; // Reference indirection causing random memory access
    }

    // Cache-friendly: Consecutively laid out structure in native memory
    public static class ContiguousArena {
        private final MemorySegment segment;
        private final long elementSize = 24; // 8 bytes timestamp + 8 bytes id + 8 bytes double
        private final long capacity;

        public ContiguousArena(long capacity) {
            this.capacity = capacity;
            // Allocates a single contiguous block of off-heap memory
            this.segment = Arena.ofShared().allocate(capacity * elementSize);
        }

        public void set(long index, long timestamp, long id, double value) {
            long offset = index * elementSize;
            segment.set(ValueLayout.JAVA_LONG, offset, timestamp);
            segment.set(ValueLayout.JAVA_LONG, offset + 8, id);
            segment.set(ValueLayout.JAVA_DOUBLE, offset + 16, value);
        }

        public double traverseSum() {
            double sum = 0;
            // Sequential access triggers CPU hardware stream prefetchers
            // bringing next cache lines into L1 before they are even requested
            for (long i = 0; i < capacity; i++) {
                long offset = i * elementSize;
                long ts = segment.get(ValueLayout.JAVA_LONG, offset);
                long id = segment.get(ValueLayout.JAVA_LONG, offset + 8);
                double val = segment.get(ValueLayout.JAVA_DOUBLE, offset + 16);
                sum += val;
            }
            return sum;
        }
    }
}

snippet-2.sh

#!/usr/bin/env bash
# Script to dump JIT symbols for a running JVM and record baseline hardware performance counters

set -euo pipefail

# Ensure we run as root to access perf counters
if [ "$EUID" -ne 0 ]; then
    echo "Please run as root or using sudo." >&2
    exit 1
fi

JAVA_PID=$(pgrep -f "high-throughput-app")
if [ -z "$JAVA_PID" ]; then
    echo "Error: Target JVM process not found." >&2
    exit 1
fi

echo "JVM PID detected: $JAVA_PID"

# 1. Trigger async-profiler to dump /tmp/perf-PID.map via JVMTI
# This attaches to the JVM and writes the memory mappings for JIT compiled code
ASYNC_PROFILER_PATH="/opt/async-profiler/asprof"
if [ ! -f "$ASYNC_PROFILER_PATH" ]; then
    echo "Downloading async-profiler..."
    mkdir -p /opt/async-profiler
    curl -L https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz | tar -xz -C /opt/async-profiler --strip-components=1
fi

echo "Dumping JIT symbols..."
"$ASYNC_PROFILER_PATH" -f "/tmp/perf-${JAVA_PID}.map" -d 5 "$JAVA_PID"

# Change ownership so BPF loaders and perf can read the map file
chown root:root "/tmp/perf-${JAVA_PID}.map"
chmod 0644 "/tmp/perf-${JAVA_PID}.map"
echo "Symbol map generated at /tmp/perf-${JAVA_PID}.map"

# 2. Execute a system-wide hardware counter baseline check on the JVM process
echo "Profiling hardware PMU counters for 15 seconds..."
perf stat \
  -e cycles,instructions \
  -e cache-references,cache-misses \
  -e L1-dcache-loads,L1-dcache-load-misses \
  -e LLC-loads,LLC-load-misses \
  -e dTLB-loads,dTLB-load-misses \
  -p "$JAVA_PID" -- sleep 15

snippet-3.txt

#!/usr/bin/env bpftrace
/*
 * Profile CPU cache misses on hardware counter overflow.
 * Usage: sudo bpftrace cache_miss_profile.bt <JVM_PID>
 */

#ifndef BPFTRACE_H
#define BPFTRACE_H
#endif

// Hook into the CPU hardware cache-misses event.
// We trigger a sample every 10,000 LLC (Last Level Cache) misses.
// This prevents overwhelming the system while providing statistical accuracy.
hardware:cache-misses:10000
/pid == $1/
{
    // Record the user-space call stack (ustack) of the target JVM process
    @[ustack] = count();
}

// Print top stack traces every 10 seconds and clear the buffer
interval:s:10
{
    printf("\n=== Top Java/Native Stacks Causing LLC Cache Misses ===\n");
    print(@, 10);
    clear(@);
}

snippet-4.txt

#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>

char LICENSE[] SEC("license") = "Dual BSD/GPL";

// Configure a BPF Hash Map to count occurrences of unique stack IDs
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __type(key, int);       // Stack Map ID returned by bpf_get_stackid
    __type(value, u64);     // Frequency count
    __uint(max_entries, 10240);
} stack_counts SEC(".maps");

// Configure a BPF Stack Trace Map to store the actual frame IPs
struct {
    __uint(type, BPF_MAP_TYPE_STACK_TRACE);
    __uint(key_size, sizeof(u32));
    __uint(value_size, 127 * sizeof(u64)); // Collect up to 127 frames deep
    __uint(max_entries, 5000);
} stack_traces SEC(".maps");

// Read-only global variable set by user-space loader
const volatile u32 target_pid = 0;

SEC("perf_event")
int on_pmu_overflow(struct bpf_perf_event_data *ctx) {
    u64 pid_tgid = bpf_get_current_pid_tgid();
    u32 pid = pid_tgid >> 32;

    // Filter events to track only the target Java process
    if (pid != target_pid) {
        return 0;
    }

    // Capture the user-space call stack. BPF_F_USER_STACK is mandatory
    // because JIT compiled code only exists in user-space memory mappings.
    int stack_id = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK);
    if (stack_id < 0) {
        return 0;
    }

    // Atomically increment the miss count for this stack path
    u64 *count = bpf_map_lookup_elem(&stack_counts, &stack_id);
    if (count) {
        __sync_fetch_and_add(count, 1);
    } else {
        u64 initial_count = 1;
        bpf_map_update_elem(&stack_counts, &stack_id, &initial_count, BPF_ANY);
    }

    return 0;
}

snippet-5.py

#!/usr/bin/env python3
# User-space loader script to parse PMU maps and resolve JVM symbols

import sys
import time
from bcc import BPF

if len(sys.argv) < 3:
    print(f"Usage: {sys.argv[0]} <PID> <PMU_EVENT_NAME>")
    print("Events: 'cache-misses', 'dTLB-load-misses', 'L1-dcache-load-misses'")
    sys.exit(1)

target_pid = int(sys.argv[1])
event_name = sys.argv[2]

# C Kernel program loaded dynamically
bpf_source = """
#include <uapi/linux/ptrace.h>

BPF_HASH(stack_counts, int, u64, 20480);
BPF_STACK_TRACE(stack_traces, 8192);

int on_pmu_overflow(struct pt_regs *ctx) {
    u64 pid_tgid = bpf_get_current_pid_tgid();
    u32 pid = pid_tgid >> 32;

    if (pid != TARGET_PID) {
        return 0;
    }

    // Capture user space call stack
    int stack_id = stack_traces.get_stackid(ctx, BPF_F_USER_STACK);
    if (stack_id < 0) {
        return 0;
    }

    stack_counts.increment(stack_id);
    return 0;
}
"""

bpf_source = bpf_source.replace("TARGET_PID", str(target_pid))

# Compile and load the eBPF program
print(f"Compiling eBPF program for JVM PID {target_pid}...")
b = BPF(text=bpf_source)

# Resolve and attach the requested hardware counter
try:
    b.attach_perf_event(
        ev_type=BPF.PERF_TYPE_HARDWARE,
        ev_config=BPF.perf_event_attr(), # Automap standard PMU type
        name=event_name,
        fn_name="on_pmu_overflow",
        sample_period=50000 # Sample every 50,000 hardware occurrences
    )
except Exception as e:
    print(f"Error: PMU event '{event_name}' could not be registered: {e}")
    print("Verify that your kernel is running on bare-metal or hypervisor with PMU virtualization (vPMU) enabled.")
    sys.exit(1)

print(f"Sampling '{event_name}' on PID {target_pid}. Press Ctrl+C to collect data...")

try:
    while True:
        time.sleep(1)
except KeyboardInterrupt:
    print("\nDetaching and processing hardware stack counts...")

stack_counts = b.get_table("stack_counts")
stack_traces = b.get_table("stack_traces")

print("\n" + "=" * 80)
print(f"REPORT: Hot Java Stacks for PMU Event: {event_name}")
print("=" * 80)

# Retrieve samples, sorted by frequency
sorted_stacks = sorted(stack_counts.items(), key=lambda x: x[1].value, reverse=True)

for stack_id, count in sorted_stacks:
    print(f"\n[Count: {count.value} hits]")
    try:
        addresses = stack_traces.walk(stack_id.value)
    except KeyError:
        print("  -> [Stack Trace Corrupted or Evicted]")
        continue

    for addr in addresses:
        # Resolve JIT dynamic compilation pointers using /tmp/perf-PID.map
        # BCC parses this file automatically for target_pid
        symbol = b.sym(addr, target_pid, show_offset=True).decode('utf-8', 'replace')
        if symbol == "[unknown]":
            print(f"  -> {hex(addr)} (unknown native or unmapped address)")
        else:
            print(f"  -> {symbol}")

snippet-6.sh

#!/usr/bin/env bash
# Tuner script to configure OS-level huge pages and optimized sysctl values for high-throughput JVMs

set -euo pipefail

echo "===> 1. Tuning sysctl for high-frequency PMU events"
# Prevent kernel from throttling perf samples under high load
sysctl -w kernel.perf_event_max_sample_rate=100000
# Permit non-privileged users to collect kernel-level stats if necessary
sysctl -w kernel.perf_event_paranoid=1

echo "===> 2. Configuring OS Page Size for TLB Mitigation"
# Option A: Enable Transparent Huge Pages (THP) for dynamic kernel allocations
echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
echo "always" > /sys/kernel/mm/transparent_hugepage/defrag

# Option B: Configure Static Huge Pages (recommended for dedicated databases and JVM hosts)
# Calculate required 2MB pages: 24GB Heap = 24 * 1024 / 2 = 12288 pages
sysctl -w vm.nr_hugepages=12288

echo "===> 3. Enabling NUMA-aware allocation policy"
# Enforce interleave policy across memory nodes for system utilities, pin JVM later
numactl --interleave=all true

echo "===> Configuration complete. Start your JVM with the following optimized flags:"
echo "--------------------------------------------------------------------------------"
cat << 'EOF'
java \
  -server \
  -Xms24g -Xmx24g \
  -XX:+UnlockDiagnosticVMOptions \
  -XX:+PreserveFramePointer \
  -XX:+UseLargePages \
  -XX:+UseTransparentHugePages \
  -XX:+UseNUMA \
  -XX:ObjectAlignmentInBytes=16 \
  -XX:+AlwaysPreTouch \
  -jar high-throughput-app.jar
EOF