plot_qa.gnuplot

#!/usr/bin/gnuplot -c
set terminal pngcairo transparent enhanced linewidth 2 font 'Helvetica,15' size 2000, 1200 background rgb 'white'
set xtics nomirror rotate by -45
set ytics nomirror
set grid ytics
set grid xtics
set ylabel "Bytes"
set datafile separator ","
set output ARG3
set key autotitle columnheader outside
set termoption enhanced
set tics font ",12"

stats ARG1 nooutput

set multiplot layout 2,1

set lmargin at screen 0.15
set rmargin at screen 0.70

set format y '%.1s%cB'

set title "Slow run"
plot for [i=7:STATS_columns] ARG1 u 1:i with lines, \
     ARG1 u 1:6 with lines lt 1 dt 3 lc rgb 'black'

set xlabel "Packets"
set title "Fast run"
plot for [i=7:STATS_columns] ARG2 u 1:i with lines, \
     ARG2 u 1:6 with lines lt 1 dt 3 lc rgb 'black'

unset multiplot

run_qa.sh

#!/bin/bash

BPID="1111111111"

sysctl net.core.rmem_max=$[32*1024*1024]

## should be default
### sysctl net.core.wmem_max=$[212992]
ethtool -K lo tso off

function finish {
    echo "[-] Cleaning up"
    kill $BPID
    echo
}
trap finish EXIT

~marek/bin/bpftrace -q tcp_qa.bt -o bt-slow.txt &
BPID="$!"
sleep 1
python3 window_qa.py slow
kill $BPID


~marek/bin/bpftrace -q tcp_qa.bt -o bt-fast.txt &
BPID="$!"
sleep 1
python3 window_qa.py fast
kill $BPID

sed '/Lost/d' bt-slow.txt|egrep -v "^@"|sort -n|sponge bt-slow.txt
sed '/Lost/d' bt-fast.txt|egrep -v "^@"|sort -n|sponge bt-fast.txt


/usr/bin/gnuplot -c ./plot_qa.gnuplot \
    bt-slow.txt \
    bt-fast.txt \
    out.png

tcp_qa.bt

BEGIN {
	printf(",not\\\\_tcprcvcoalesce,tcprcvcollapsed,tcprcvqdrop,truesize\\\\_ratio,rcv\\\\_ssthresh,skmem\\\\_r,skmem\\\\_rb,rcv\\\\_win,recv-q\n");
	@idx = 0;
}



kprobe:tcp_rcv_established {
	$sk = (struct sock *) arg0;
	$tp = (struct tcp_sock *) $sk;
	$skb = (struct sk_buff *) arg1;
	$isk = (struct inet_sock *) $sk;
	$p = $isk->inet_sport;
	if ($p != 0xd204) {return;}
	@idx += 1;

	$siocinq = $tp->rcv_nxt - $tp->copied_seq;

	printf("%u, %u, %u, %u, %u, %u, %u, %u, %u, %u\n",
	       @idx,
	       0,
	       0,
	       0,
	       0,
	       $tp->rcv_ssthresh,
	       $sk->sk_rcvbuf,
	       $sk->sk_backlog.rmem_alloc.counter,
	       (uint64)$tp->rcv_wnd - ((uint64)$tp->rcv_nxt - (uint64)$tp->rcv_wup),
	       $siocinq
		);
}

window_qa.py

from socket import SOCK_STREAM, AF_INET, SOL_SOCKET, SO_REUSEADDR, SOL_TCP, TCP_MAXSEG, SO_RCVBUF, SO_SNDBUF, IPPROTO_IP, TCP_NODELAY, TCP_QUICKACK
import ctypes
import os
import socket
import struct
import sys
import time
import threading
import select

fast = sys.argv[1] == 'fast'

sd = socket.socket(AF_INET, SOCK_STREAM, 0)
sd.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sd.setsockopt(SOL_TCP, TCP_MAXSEG, 1024)
sd.setsockopt(SOL_SOCKET, SO_RCVBUF, 16*1024*1024)
sd.bind(('127.0.0.3', 1234))
sd.listen(32)

cd = socket.socket(AF_INET, SOCK_STREAM, 0)
cd.setsockopt(SOL_SOCKET, SO_SNDBUF, 4*1024*1024)

cd.bind(('127.0.0.2', 0))
cd.connect(('127.0.0.3', 1234))

# move to ESTAB state ASAP, since tcp_rcv_established is for ESTAB indeed
ssd, _ = sd.accept()

done = False

waitinsec = 0.05
def thread_fill():
    burst_payload = b'a'* (1*1024*1024)

    cd.setblocking(False)
    while not done:
        try:
            cd.send(burst_payload)
            if fast:
                ssd.setsockopt(SOL_TCP, TCP_QUICKACK, 1)
            select.select([], [cd], [], waitinsec / 1000)
        except BlockingIOError:
            time.sleep(waitinsec)

def thread_drain():
    bytespersec = 12500000

    bytesperiter = int(bytespersec * waitinsec)
    ssd.setblocking(False)
    offset = 0.
    while not done:
        wanted = waitinsec - offset
        t0 = time.time()
        try:
            ssd.recv(bytesperiter)
        except BlockingIOError:
            pass
        time.sleep(wanted)
        td = time.time()-t0
        offset = td - wanted


fill_thrd = threading.Thread(target=thread_fill, args=())
fill_thrd.start()
drain_thrd = threading.Thread(target=thread_drain, args=())
drain_thrd.start()
try:
    time.sleep(1.5)
except KeyboardInterrupt:
    print("Ctrl+C pressed...")
done = True
fill_thrd.join()
drain_thrd.join()
os.system('ss -memoi sport = :1234 or dport = :1234|cat')