Sets up DPDK + Silicom FM10K patches and Open vSwitch withd DPDK support, with inline Switch Managers that supports, between others, Silicom PE3100G2DQiRM-QX4 cards.

dpdk-fm10k-git.sh

#!/bin/bash

# Sets up DPDK + Silicom FM10K patches and Open vSwitch withd DPDK support, with inline Switch Managers that supports, between others, Silicom PE3100G2DQiRM-QX4 cards.
# Make sure to edit /etc/dpdk_fm10k.conf to your tastes after initial run.
# For config reference and other help, look at fm10k_config.c on DPDK source
#
# Remove system's dpdk and openvswitch packages beforehand
# Requires some packages:
# Debian/Ubuntu: apt install -y build-essential libcap-ng0 libcap-ng-dev libssl-dev autoconf automake libtool-bin curl git-core libnuma-dev driverctl meson ninja-build linux-headers
# Proxmox: apt install -y build-essential libcap-ng0 libcap-ng-dev libssl-dev autoconf automake libtool-bin curl git-core libnuma-dev driverctl meson ninja-build pve-headers
#
# Requires normal setup of DPDK prerequisites (IOMMU, hugepages, binding, etc.), plus Open vSwitch + DPDK pages.
# Switch logs can be followed on /var/log/openvswitch/ovs-vswitchd.log
#
# PE3100G2DQiRM-QX4 notes:
# * On Silicom PE3100G2DQiRM-QX4 you need to connect all lanes of the first card at least (2x 8xPCIe), so you need bifurcation support on a 16x slot.
# * In addition, daughter card also seems needed, but you may be able to edit the config / fm10k_switch.h as well.
# * dpdk_fm10k.conf defaults to 100G speeds. You can set it to 100G, 40G, 25G (splits into 4x25G), 10G (splits into 4x10G). Such setting applies to all external ports.
# * After testing, this is how internal ports are mapped: PF 0 is daughter card PCIe #2, PF 1 is daughter card PCIe #1, PF 2 is main card PCIe #1, PF 3 is main card PCIe #2, 
# * dpdk_fm10k.conf has ability to set flows. This has not been tested.
# * Current working config doesn't have 2xPF merged onto one for 100G speeds per port. Further research is needed.
#

# Here follows some example setup of devices, probably not working, just for reference:
#
# #list pcie devices to map
# lshw -c network -businfo
#
# #permanent way
# driverctl set-override 0000:03:00.0 vfio-pci
# driverctl set-override 0000:04:00.0 vfio-pci
# driverctl set-override 0000:82:00.0 vfio-pci
# driverctl set-override 0000:83:00.0 vfio-pci

# #temporary way
# python3 "/usr/src/dpdk/usertools/dpdk-devbind.py" --bind=vfio-pci 0000:03:00.0
# python3 "/usr/src/dpdk/usertools/dpdk-devbind.py" --bind=vfio-pci 0000:04:00.0
# python3 "/usr/src/dpdk/usertools/dpdk-devbind.py" --bind=vfio-pci 0000:82:00.0
# python3 "/usr/src/dpdk/usertools/dpdk-devbind.py" --bind=vfio-pci 0000:83:00.0
# python3 "/usr/src/dpdk/usertools/dpdk-devbind.py" --status
#
# export PATH=$PATH:/usr/share/openvswitch/scripts
# export DB_SOCK=/var/run/openvswitch/db.sock
# ovs-ctl --system-id=random --db-sock="$DB_SOCK" start
# ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# ovs-vsctl --no-wait set Open_vSwitch . other_config:hw-offload=false
# ovs-appctl vlog/set file:dpdk:dbg
# ovs-appctl vlog/set file:netdev_dpdk:dbg
# ovs-appctl vlog/set file:netdev_offload_dpdk:dbg
# ovs-ctl --db-sock="$DB_SOCK" restart
#
# ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
# ovs-vsctl add-port br0 dpdk-p0 tag=1 vlan_mode=native-untagged -- set Interface dpdk-p0 type=dpdk options:dpdk-devargs=0000:04:00.0 mtu_request=9000
# ovs-vsctl add-port br0 dpdk-p1 tag=1 vlan_mode=native-untagged -- set Interface dpdk-p1 type=dpdk options:dpdk-devargs=0000:03:00.0 mtu_request=9000
# # Not used ovs-vsctl add-port br0 dpdk-p2 tag=1 vlan_mode=native-untagged -- set Interface dpdk-p2 type=dpdk options:dpdk-devargs=0000:82:00.0 mtu_request=9000
# # Not used ovs-vsctl add-port br0 dpdk-p3 tag=1 vlan_mode=native-untagged -- set Interface dpdk-p3 type=dpdk options:dpdk-devargs=0000:83:00.0 mtu_request=9000
#
# ovs-vsctl add-port br0 vlan1 tag=1 -- set Interface vlan1 type=internal mtu_request=9000
# ip address add 10.100.100.2/24 dev vlan1
# ip link set vlan1 up

DPDK_TAG="af52427f21ce44dd8a9de20c6188f9ddd32bd0bd"
OVS_TAG="v2.14.0"

pushd /usr/src/ || exit 1

if [ ! -d dpdk ]; then
  git clone https://github.com/WeebDataHoarder/dpdk-fm10k.git dpdk
fi
pushd "dpdk" || exit 1

git fetch --all
git reset --hard "${DPDK_TAG}"

#Enable self management
sed -i 's/CONFIG_RTE_FM10K_SWITCH_MANAGEMENT=.*$/CONFIG_RTE_FM10K_SWITCH_MANAGEMENT=y/g' config/common_base
#Enable vhost support
sed -i 's/CONFIG_RTE_LIBRTE_VHOST=.*$/CONFIG_RTE_LIBRTE_VHOST=y/g' config/common_base
sed -i 's/CONFIG_RTE_LIBRTE_PMD_VHOST=.*$/CONFIG_RTE_LIBRTE_PMD_VHOST=y/g' config/common_base
sed -i 's/CONFIG_RTE_BUILD_SHARED_LIB=.*$/CONFIG_RTE_BUILD_SHARED_LIB=n/g' config/common_base

export DPDK_DIR="$(pwd)"
export DPDK_TARGET="$(arch)-native-linuxapp-gcc"
export DPDK_BUILD=$DPDK_DIR/$DPDK_TARGET

make config T=$DPDK_TARGET MAKE_PAUSE=n

make install -j $(nproc) T=$DPDK_TARGET DESTDIR=/usr MAKE_PAUSE=n EXTRA_CFLAGS="-Ofast -g"

popd || exit 1


if [ ! -d openvswitch ]; then
  git clone https://github.com/openvswitch/ovs.git openvswitch
fi

pushd "openvswitch" || exit 1
git fetch --all
git reset --hard "${OVS_TAG}"
./boot.sh

./configure \
--prefix=/usr --localstatedir=/var --with-rundir=/var/run/openvswitch --libdir=/usr/lib/$(dpkg-architecture -q DEB_HOST_MULTIARCH) --sysconfdir=/etc \
--with-dpdk="${DPDK_BUILD}" --with-linux="/lib/modules/$(uname -r)/build" \
CFLAGS="-msse4.2 -mpopcnt -march=native -mtune=native -Ofast"
make -j $(nproc)
make check TESTSUITEFLAGS="-j$(nproc)"
make install
make modules_install

Z_dpdk_fm10k.example.conf

# Tell how many PF ports are bound in DPDK.
# Driver will check the  number when initializes.
dpdk.bind.pf.number int 4


# Set external port speed, 40 means 40G, 100 means 100G.
extern.port.speed int 40


# Map 1 or 2 PF ports to one DPDK port.
# If mapped PF number is 2, traffic will be
# load balance between the 2 PFs.
# And the DPDK port queue number will be configured
# more than 2(each PF need at least 1 DPDK port queue).
dpdk.port.0.map.pf int 0
#dpdk.port.0.map.pf int 2

dpdk.port.1.map.pf int 1
#dpdk.port.1.map.pf int 3

#dpdk.port.2.map.pf int 2
#dpdk.port.3.map.pf int 3


# Map 1 or 2 PF to one external port. If mapped PF number is 2,
# traffic will be load balance between the 2 PF.
extern.port.1.map.pf int 0
#extern.port.1.map.pf int 2
extern.port.2.map.pf int 1
#extern.port.2.map.pf int 3


# Define flow rule
flowset.start string default
flowset.enable string default
flowset.stop string default

Z_fm10k_config_excerpt.c

// excerpt

/* configuration key strings */
static struct fm10k_cfg_key_item fm10k_config_key_items[] = {
		/* enable debug print */
		{ {"debug", "print", "enable"},
			FM10K_CONFIG_DEBUG_ENABLE},
		{ {"debug", "print", "config"},
			FM10K_CONFIG_DEBUG_CONFIG},
		{ {"debug", "print", "ffu", "init"},
			FM10K_CONFIG_DEBUG_FFU_INIT},
		{ {"debug", "print", "ffu", "register"},
			FM10K_CONFIG_DEBUG_FFU_REG},
		{ {"debug", "print", "ffu", "rule"},
			FM10K_CONFIG_DEBUG_FFU_RULE},
		{ {"debug", "print", "stats", "port"},
			FM10K_CONFIG_DEBUG_STATS_PORT},
		{ {"debug", "print", "stats", "queue"},
			FM10K_CONFIG_DEBUG_STATS_QUEUE},
		{ {"debug", "print", "stats", "rule"},
			FM10K_CONFIG_DEBUG_STATS_FFU},
		{ {"debug", "print", "stats", "detail"},
			FM10K_CONFIG_DEBUG_STATS_MORE},
		{ {"debug", "print", "stats", "interval"},
			FM10K_CONFIG_DEBUG_STATS_INTERVAL},
		/* general configuration */
		{ {"dpdk", "bind", "pf", "number"},
			FM10K_CONFIG_BIND_PF_NUMBER},
		{ {"extern", "port", "speed"},
			FM10K_CONFIG_EXT_PORT_SPEED},
		/* internal redirect configuration */
		{ {"dpdk", "port", "*", "map", "pf"},
			FM10K_CONFIG_DPDK_PORT_MAP_PF},
		{ {"extern", "port", "*", "map", "pf"},
			FM10K_CONFIG_EXT_PORT_MAP_PF},
		/* external redirect configuration */
		{ {"flowset", "start"},
			FM10K_CONFIG_FLOWSET_START},
		{ {"flowset", "stop"},
			FM10K_CONFIG_FLOWSET_STOP},
		{ {"flowset", "enable"},
			FM10K_CONFIG_FLOWSET_ENABLE},
		{ {"flow", "*", "condition", "source", "extern", "port"},
			FM10K_CONFIG_FLOW_COND_SRC_EXT_PORT},
		{ {"flow", "*", "condition", "source", "dpdk", "port"},
			FM10K_CONFIG_FLOW_COND_SRC_DPDK_PORT},
		{ {"flow", "*", "condition", "vlan"},
			FM10K_CONFIG_FLOW_COND_VLAN},
		{ {"flow", "*", "action", "forward", "extern", "port"},
			FM10K_CONFIG_FLOW_ACT_FW_EXT_PORT},
		{ {"flow", "*", "action", "forward", "dpdk", "port"},
			FM10K_CONFIG_FLOW_ACT_FW_DPDK_PORT},
		{ {"flow", "*", "action", "forward", "vlan"},
			FM10K_CONFIG_FLOW_ACT_FW_VALN},
};