AF_XDP/DPDK comparison

build.sh

#!/bin/bash -e

for BASE in fedora ubuntu
do
    docker build --build-arg BASE=$BASE -t krsna1729/afxdp:$BASE .
done

Dockerfile

ARG BASE=fedora

FROM fedora as fedora-build
RUN dnf -y groupinstall "Development Tools" && \
    dnf -y install \
        bc \
        bison \
        clang \
        elfutils-libelf-devel \
        flex \
        kernel-devel \
        llvm-devel \
        numactl-devel \
        openssl-devel \
        wget \
        which \
        xz

FROM ubuntu as ubuntu-build
RUN apt-get update && \
    apt-get -y install \
        bc \
        bison \
        build-essential \
        ca-certificates \
        clang \
        flex \
        git \
        libelf-dev \
        libnuma-dev \
        libssl-dev \
        llvm-dev \
        pkg-config \
        unzip \
        wget

FROM $BASE-build as build

#ARG LIBBPF_COMMIT=master
#RUN wget -qO libbpf.zip https://github.com/libbpf/libbpf/archive/${LIBBPF_COMMIT}.zip
#RUN unzip libbpf.zip && \
#    cd libbpf-${LIBBPF_COMMIT}/src && \
#    make install && \
#    mkdir -p /etc/ld.so.conf.d && \
#    echo -e "/usr/lib64\n/usr/local/lib64" >/etc/ld.so.conf.d/libbpf-x86_64.conf && \
#    ldconfig && \
#    cp -f ../include/uapi/linux/if_xdp.h /usr/include/linux

ARG LINUX_VER='5.1.15'
ENV LINUX_DIR='/linux'
RUN wget -qO linux.tar.xz https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-${LINUX_VER}.tar.xz
RUN mkdir -p $LINUX_DIR && \
    tar -xf linux.tar.xz -C $LINUX_DIR --strip-components 1 && \
    cp -f $LINUX_DIR/include/uapi/linux/if_xdp.h /usr/include/linux && \
    cd $LINUX_DIR/tools/lib/bpf/ && \
    make install_lib && \
    make install_headers && \
    cd $LINUX_DIR && \
    wget -q https://src.fedoraproject.org/rpms/kernel/raw/master/f/kernel-x86_64.config -O .config && \
    yes '' | make oldconfig >/dev/null && \
    make -j $CPUS headers_install && \
    make -j $CPUS samples/bpf/
ENV PATH="$PATH:$LINUX_DIR/samples/bpf/"

RUN mkdir -p /etc/ld.so.conf.d && \
    echo -e "/usr/lib64\n/usr/local/lib64" > /etc/ld.so.conf.d/libbpf-x86_64.conf && \
    ldconfig

ARG DPDK_VER='master'
ENV DPDK_DIR='/dpdk'
ENV RTE_TARGET='x86_64-native-linuxapp-gcc'
RUN git clone -b $DPDK_VER -q --depth 1 http://dpdk.org/git/dpdk $DPDK_DIR 2>&1
RUN cd ${DPDK_DIR} && \
    sed -ri 's,(IGB_UIO=).*,\1n,' config/common_linux* && \
    sed -ri 's,(KNI_KMOD=).*,\1n,' config/common_linux* && \
    sed -ri 's,(LIBRTE_BPF=).*,\1n,' config/common_base && \
    sed -ri 's,(AF_XDP=).*,\1y,' config/common_base && \
    make config T=x86_64-native-linuxapp-gcc && \
    make -j $CPUS
ENV PATH="$PATH:$DPDK_DIR/build/app/"

RX

Promiscous mode:

# Alternative xdpsock_user.c can be modified to have the right destination MAC
sudo ip link set enp175s0f1 promsic on

Flow director:

sudo ethtool -N enp175s0f1 flow-type udp4 action 30
sudo ethtool -u enp175s0f1

Note action 30 and --queue=30

xdpsock

docker run --rm -it --privileged --net=host --cpuset-cpus 31 krsna1729/afxdp:ubuntu \
xdpsock --interface=enp175s0f1 --queue=30 --zero-copy --rxdrop  # --poll

testpmd

docker run --rm -it --privileged --net=host --cpuset-cpus 30-31 krsna1729/afxdp:ubuntu \
testpmd --vdev=net_af_xdp0,iface=enp175s0f1,start_queue=30,queue_count=1,pmd_zero_copy=1 --no-huge -m 2048 -- \
--stats-period=1 --nb-cores=1 --nb-ports=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=rxonly

TX

xdpsock

docker run --rm -it --privileged --net=host --cpuset-cpus 31 krsna1729/afxdp:ubuntu \
xdpsock --interface=enp175s0f1 --queue=30 --zero-copy --txonly  # --poll

testpmd

docker run --rm -it --privileged --net=host --cpuset-cpus 30-31 krsna1729/afxdp:ubuntu \
testpmd --vdev=net_af_xdp0,iface=enp175s0f1,start_queue=30,queue_count=1,pmd_zero_copy=1 --no-huge -m 2048 -- \
--stats-period=1 --nb-cores=1 --nb-ports=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=txonly

Benchmarking

• poll() vs not
• zero-copy vs copy
• same queue, cpu number vs not