Perf contention paper relevant

example-conf.md

#!/bin/bash
# Periodic CPU Contention Experiment

EXPERIMENT_NAME='test'
TARGET_NODE='node-3'
VICTIM_SERVICES='search'
NOISY_NEIGHBOR_TYPE='cpu'

CONTENTION_BURSTS=$(burst 0 60 1)

# Experiment configuration
ITERATIONS=3
ITERATION_DELAY=10

# Windowed sampling
ENABLE_WINDOWED_SAMPLING=true
WINDOW_INTERVAL_MS=100
PERF_EVENTS='cycles,instructions,LLC-loads,LLC-load-misses,branch-instructions,branch-misses,dTLB-loads,dTLB-load-misses,page-faults'
TIMING_BUFFER_SIZE=16384

# Jaeger tracing
JAEGER_SAMPLE_RATIO=0.01  # 1% sampling, set to 0 to disable

# wrk2 workload
WRK2_TARGET_SERVICE='frontend'
WRK2_TARGET_IP='192.168.200.37'  # CHANGE TO YOUR IP
WRK2_TARGET_PORT=5000
WRK2_SCRIPT='../wrk2/scripts/hotel-reservation/mixed-workload_type_1.lua'
WRK2_RATE=50
WRK2_THREADS=3
WRK2_CONNECTIONS=4
WRK2_DIST='fixed'

ghz-loadgen.md

~/go/bin/grpcurl -plaintext \
    -import-path ../services \
    -proto profile/proto/profile.proto \
    -d '{"hotelIds":["1","2","3","4","5"],"locale":"en"}' \
    10.10.1.4:31081 profile.Profile/GetProfiles



ghz --insecure \
    --proto ../services/profile/proto/profile.proto -i ../services \
    --call profile.Profile/GetProfiles \
    -n 3 --format json \
    -d '{"hotelIds":["1","2","3","4","5"],"locale":"en"}' \
    10.10.1.4:31081 | python3 -m json.tool | grep -A2 statusCodeDistribution
 

perf-contention-cmds.md

build-binary for hotelreserve

cd hotelReservation

# Build basic perf_api library
gcc -c services/perf/perf_api.c -o services/perf/perf_api.o
ar rcs services/perf/libperf_api.a services/perf/perf_api.o

# Build windowed perf_api library
gcc -c services/perf/perf_api_windowed.c -o services/perf/perf_api_windowed.o
ar rcs services/perf/libperf_api_windowed.a services/perf/perf_api_windowed.o

# Build MSR reader library (per-window CPU freq utilization)
gcc -c services/perf/msr_reader.c -o services/perf/msr_reader.o
ar rcs services/perf/libmsr_reader.a services/perf/msr_reader.o

# Verify libraries were created (should now show 3 .a files)
ls -lh services/perf/*.a


# Set CGO environment
export CGO_ENABLED=1
export GOOS=linux

# Build search service
go build -o build/search ./cmd/search/

# Verify binary was created
ls -lh build/search
file build/search

# Confirm the binary contains both static-lib payloads
nm build/search 2>/dev/null | grep -E ' (T|t) (perf_window_init|msr_reader_init)'

# Compile with verbose output (now 3 inputs)
gcc -c -v services/perf/perf_api.c          -o services/perf/perf_api.o          2>&1 | grep error
gcc -c -v services/perf/perf_api_windowed.c -o services/perf/perf_api_windowed.o 2>&1 | grep error
gcc -c -v services/perf/msr_reader.c        -o services/perf/msr_reader.o        2>&1 | grep error

# Build with verbose output
CGO_ENABLED=1 go build -v -x ./cmd/search/ 2>&1 | tee build.log

Debug for instrumentation implementation

# Check the pod is using the windowed image
kubectl get deployment search -o jsonpath='{.spec.template.spec.containers[0].image}'

# Check environment variables
kubectl exec deployment/search -- env | grep -E "ENABLE_WINDOWED|ITERATION_ID|WINDOW_INTERVAL"

# Check if windowed sampling started
kubectl logs deployment/search | grep -i "windowed\|sampling\|iteration"

# After experiment runs for 30s, check for data file
kubectl exec deployment/search -- ls -la /data/

# View the data
kubectl exec deployment/search -- cat /data/run_data_search_iter1.json | jq '.sample_count'


# Look for the initialization message or error
kubectl logs deployment/search | grep -A5 -B5 "Initialized windowed perf\|Failed to"

# Check if perf_window_init returned NULL
kubectl logs deployment/search 2>&1 | grep -i "failed to initialize perf"

Perf setup for node where contention exp happens

sudo apt update
sudo apt install linux-tools-common linux-tools-generic linux-tools-$(uname -r)

sudo apt install cpuid
cpuid -1 | grep "number of counters"

echo 0 | sudo tee /proc/sys/kernel/nmi_watchdog
echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid

perf stat -e cycles,instructions,LLC-loads,LLC-load-misses,branch-instructions,branch-misses,dTLB-loads,dTLB-load-misses,page-faults sleep 1

echo msr | sudo tee /etc/modules-load.d/msr.conf
sudo modprobe msr    
lsmod | grep '^msr '

# verify ssh from controller node
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no node-3 'sudo modprobe msr &&
lsmod | grep msr'

Change num of replicas for services

# Check replicas for all deployments
kubectl get deployments -o custom-columns="NAME:.metadata.name,DESIRED:.spec.replicas,READY:.status.readyReplicas,AVAILABLE:.status.availableReplicas"
# or
kubectl get deployments -o custom-columns="NAME:.metadata.name,REPLICAS:.spec.replicas" | grep -v "nfs\|NAME" | sort

# see which services have more than 1 replica
kubectl get deployments -o custom-columns="NAME:.metadata.name,REPLICAS:.spec.replicas" --no-headers | awk '$2 > 1 {print $1, $2}'
#scale them back to 1 replica
for deploy in $(kubectl get deployments -o custom-columns="NAME:.metadata.name,REPLICAS:.spec.replicas" --no-headers | awk '$2 > 1 {print $1}'); do
  echo "Scaling $deploy to 1 replica"
  kubectl scale deployment $deploy --replicas=1
done

# check for stale consul registry
CONSUL_POD=$(kubectl get pods -l io.kompose.service=consul -o jsonpath='{.items[0].metadata.name}')
kubectl exec $CONSUL_POD -- consul catalog services

Safe Resource Limite and Request update (Update ONE service at a time (not bulk), and then clean up consul registration)

# Define the services to update (exclude infrastructure: consul, jaeger, mongodb, memcached)
SERVICES="frontend geo rate profile recommendation user reservation search"

for svc in $SERVICES; do
  echo "=== Updating $svc ==="
  
  # Update resources
  kubectl set resources deployment $svc --limits=cpu=3 --requests=cpu=1
  
  # Wait for rollout to complete
  kubectl rollout status deployment $svc --timeout=120s
  
  # Wait for Consul registration
  sleep 10
  
  echo "$svc updated successfully"
  echo ""
done

# Update MongoDB deployments one at a time
for db in mongodb-geo mongodb-rate mongodb-recommendation mongodb-reservation mongodb-profile mongodb-user; do
  echo "Updating $db"
  kubectl set resources deployment $db --limits=cpu=2 --requests=cpu=500m
  kubectl rollout status deployment $db --timeout=120s
  sleep 5
done

# Check cur resource configs for all services
for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}'); do	
  echo "=== $pod ==="	
  kubectl get pod $pod -o jsonpath='{.spec.containers[*].resources}' | jq .	
  echo ""	
done


CONSUL_POD=$(kubectl get pods -l io.kompose.service=consul -o jsonpath='{.items[0].metadata.name}')

# Get current valid pod IPs
CURRENT_IPS=$(kubectl get pods -o wide --no-headers | awk '{print $6}')

# List registered services and check for stale
kubectl exec $CONSUL_POD -- curl -s http://localhost:8500/v1/agent/services | python3 -c "
import sys,json
current_ips = '''$CURRENT_IPS'''.split()
services = json.load(sys.stdin)
stale = []
for sid, svc in services.items():
    if svc['Address'] not in current_ips and svc['Service'].startswith('srv-'):
        stale.append(sid)
        print(f\"STALE: {sid} -> {svc['Address']} ({svc['Service']})\")
if not stale:
    print('No stale registrations found')
"

# Deregister stale (only do it when necessary)

# Get stale IDs and deregister each
kubectl exec $CONSUL_POD -- curl -s http://localhost:8500/v1/agent/services | python3 -c "
import sys, json
current_ips = '''$CURRENT_IPS'''.split()
services = json.load(sys.stdin)
for sid, svc in services.items():
    if svc['Address'] not in current_ips and svc['Service'].startswith('srv-'):
        print(sid)
" | xargs -I {} kubectl exec $CONSUL_POD -- consul services deregister -id={}

Check image, cpu pinning etc used by pod/deployment/workload

# Check pods in default namespace
kubectl get pods -n default -o custom-columns='POD:.metadata.name,IMAGE:.spec.containers[*].image'

# Check deployments in default namespace
kubectl get deployments -n default -o custom-columns='DEPLOYMENT:.metadata.name,IMAGE:.spec.template.spec.containers[*].image'

# Check all workload types in default namespace
kubectl get deploy,sts,ds -n default -o custom-columns='TYPE:.kind,NAME:.metadata.name,IMAGE:.spec.template.spec.containers[*].image'


# Get the new pod name
NEW_POD=$(kubectl get pods -l io.kompose.service=recommendation -o jsonpath='{.items[0].metadata.name}')

# Check CPU affinity (should show 12-14, not 0-31)
echo "=== CPU Affinity ==="
kubectl exec $NEW_POD -- taskset -cp 1

# Check the actual running command (should have taskset)
echo "=== Process Command Line ==="
kubectl exec $NEW_POD -- cat /proc/1/cmdline | tr '\0' ' ' && echo

# Verify CPU_SET environment variable is set
echo "=== CPU_SET Environment ==="
kubectl exec $NEW_POD -- printenv CPU_SET

# Check perf initialization (should show proper FD counts)
echo "=== Perf Initialization ==="
kubectl logs $NEW_POD | grep "Initialized windowed perf"

Verify DB status, and drop then restart if necesary

# Profile DB
MONGO_PROFILE=$(kubectl get pods -l io.kompose.service=mongodb-profile -o jsonpath='{.items[0].metadata.name}')
kubectl exec $MONGO_PROFILE -- mongo profile-db --eval "db.hotels.count()"

# Geo DB
MONGO_GEO=$(kubectl get pods -l io.kompose.service=mongodb-geo -o jsonpath='{.items[0].metadata.name}')
kubectl exec $MONGO_GEO -- mongo geo-db --eval "db.geo.count()"

# Recommendation DB
MONGO_REC=$(kubectl get pods -l io.kompose.service=mongodb-recommendation -o jsonpath='{.items[0].metadata.name}')
kubectl exec $MONGO_REC -- mongo recommendation-db --eval "db.recommendation.count()"

# Rate DB
MONGO_RATE=$(kubectl get pods -l io.kompose.service=mongodb-rate -o jsonpath='{.items[0].metadata.name}')
kubectl exec $MONGO_RATE -- mongo rate-db --eval "db.inventory.count()"

# Reservation DB
MONGO_RESERVE=$(kubectl get pods -l io.kompose.service=mongodb-reservation -o jsonpath='{.items[0].metadata.name}')
kubectl exec $MONGO_RESERVE -- mongo reservation-db --eval "db.reservation.count(); db.number.count()"

# User DB
MONGO_USER=$(kubectl get pods -l io.kompose.service=mongodb-user -o jsonpath='{.items[0].metadata.name}')
kubectl exec $MONGO_USER -- mongo user-db --eval "db.user.count()"

# Profile DB
echo "Dropping profile-db.hotels..."
kubectl exec $MONGO_PROFILE -- mongo profile-db --eval "db.hotels.drop()"

# Geo DB
echo "Dropping geo-db.geo..."
kubectl exec $MONGO_GEO -- mongo geo-db --eval "db.geo.drop()"

# Recommendation DB
echo "Dropping recommendation-db.recommendation..."
kubectl exec $MONGO_REC -- mongo recommendation-db --eval "db.recommendation.drop()"

# Rate DB
echo "Dropping rate-db.inventory..."
kubectl exec $MONGO_RATE -- mongo rate-db --eval "db.inventory.drop()"

# Reservation DB (has multiple collections)
echo "Dropping reservation-db collections..."
kubectl exec $MONGO_RESERVE -- mongo reservation-db --eval "db.reservation.drop(); db.number.drop()"

# User DB
echo "Dropping user-db.user..."
kubectl exec $MONGO_USER -- mongo user-db --eval "db.user.drop()"

query-interceptor-data.md

# Find samples with processing_time mean < 100,000 ns (100 microseconds)

jq '.samples | map(select(.timing_window.processing_time.mean_ns > 0 and .timing_window.processing_time.mean_ns < 100000)) | sort_by(.timing_window.processing_time.mean_ns) | .[] | {sample_id, mean_ns: .timing_window.processing_time.mean_ns, min_ns: .timing_window.processing_time.min_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json


# Find samples with processing_time min < 50,000 ns (50 microseconds)
jq '.samples | map(select(.timing_window.processing_time.min_ns > 0 and .timing_window.processing_time.min_ns < 50000)) | sort_by(.timing_window.processing_time.min_ns) | .[] | {sample_id, min_ns: .timing_window.processing_time.min_ns, mean_ns: .timing_window.processing_time.mean_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# Find samples where ANY of (min, mean) is very small (< 100us)
jq '.samples | map(select((.timing_window.processing_time.min_ns > 0 and .timing_window.processing_time.min_ns < 100000) or (.timing_window.processing_time.mean_ns > 0 and .timing_window.processing_time.mean_ns < 100000))) | sort_by(.timing_window.processing_time.mean_ns) | .[] | {sample_id, min_ns: .timing_window.processing_time.min_ns, mean_ns: .timing_window.processing_time.mean_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# Get bottom 20 samples by mean processing time (excluding zeros)
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 0)) | sort_by(.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, mean_ns: .timing_window.processing_time.mean_ns, min_ns: .timing_window.processing_time.min_ns, max_ns: .timing_window.processing_time.max_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# See distribution - count how many samples fall into different time ranges
jq '[.samples[] | select(.timing_window.processing_time.mean_ns > 0) | .timing_window.processing_time.mean_ns] | group_by(. < 100000) | map({range: (if .[0] < 100000 then "< 100us" else ">= 100us" end), count: length})' raw/windowed/profile/run_data_iter1.json

# Find samples with large mean processing time (e.g., > 1ms = 1,000,000 ns)
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 1000000)) | sort_by(-.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, mean_ns: .timing_window.processing_time.mean_ns, min_ns: .timing_window.processing_time.min_ns, max_ns: .timing_window.processing_time.max_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# Find top 20 samples by mean processing time
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 0)) | sort_by(-.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, mean_ns: .timing_window.processing_time.mean_ns, min_ns: .timing_window.processing_time.min_ns, max_ns: .timing_window.processing_time.max_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# Find samples with large max processing time (e.g., > 5ms = 5,000,000 ns)
jq '.samples | map(select(.timing_window.processing_time.max_ns > 5000000)) | sort_by(-.timing_window.processing_time.max_ns) | .[0:20] | .[] | {sample_id, max_ns: .timing_window.processing_time.max_ns, mean_ns: .timing_window.processing_time.mean_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# With all timing fields included
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 0)) | sort_by(-.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, request_count: .timing_window.request_count, processing_time: .timing_window.processing_time, offset_ms}' raw/windowed/profile/run_data_iter1.json

# Find samples with large mean processing time (e.g., > 1ms = 1,000,000 ns)
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 1000000)) | sort_by(-.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, mean_ns: .timing_window.processing_time.mean_ns, min_ns: .timing_window.processing_time.min_ns, max_ns: .timing_window.processing_time.max_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# Find top 20 samples by mean processing time
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 0)) | sort_by(-.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, mean_ns: .timing_window.processing_time.mean_ns, min_ns: .timing_window.processing_time.min_ns, max_ns: .timing_window.processing_time.max_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# Find samples with large max processing time (e.g., > 5ms = 5,000,000 ns)
jq '.samples | map(select(.timing_window.processing_time.max_ns > 5000000)) | sort_by(-.timing_window.processing_time.max_ns) | .[0:20] | .[] | {sample_id, max_ns: .timing_window.processing_time.max_ns, mean_ns: .timing_window.processing_time.mean_ns, request_count: .timing_window.request_count}' raw/windowed/profile/run_data_iter1.json

# With all timing fields included
jq '.samples | map(select(.timing_window.processing_time.mean_ns > 0)) | sort_by(-.timing_window.processing_time.mean_ns) | .[0:20] | .[] | {sample_id, request_count: .timing_window.request_count, processing_time: .timing_window.processing_time, offset_ms}' raw/windowed/profile/run_data_iter1.json