How to test VFIO with virtio-net-pci device

Nested_VFIO.md

Prepare the host

Get the image clear-XXXXX-kvm.img

wget https://download.clearlinux.org/releases/30080/clear/clear-30080-kvm.img.xz
unxz clear-30080-kvm.img.xz

Add intel_iommu=on to the kernel boot parameters

mkdir mount_dir
sudo mount -o loop,offset=$((2048 * 512)) /home/sebastien/workloads/clear-30080-kvm.img mount_dir/
# Modify mount_dir/loader/entries/Clear-linux-kvm-XXX.conf to add intel_iommu=on to the kernel command line
sudo umount mount_dir/

Get the EFI firmware OVMF.fd Download from official OVMF repo.

Setup network

sudo ip link add name testbr type bridge
sudo ip link add link testbr name testvlan type macvlan mode bridge
sudo ip link set dev testvlan up
sudo ip link set dev testbr up
sudo ip link add link testbr name testtap type macvtap mode bridge
sudo ip link set testtap up
sudo ip addr add 192.168.222.1/24 dev testvlan
tapdev=/dev/tap"$(cat /sys/class/net/testtap/ifindex)"
mac_addr="$(cat /sys/class/net/testtap/address)"

Start first virtualization layer

Start QEMU

sudo -E qemu-system-x86_64 \
    -machine q35,accel=kvm,kernel_irqchip=split \
    -bios OVMF.fd \
    -smp sockets=1,cpus=4,cores=2 \
    -cpu host \
    -m 1024 \
    -vga none -nographic \
    -drive file=clear-30080-kvm.img,format=raw \
    -netdev user,id=mynet0,hostfwd=tcp::10022-:22,hostfwd=tcp::12375-:2375 \
    -device virtio-net-pci,netdev=mynet0 \
    -device virtio-rng-pci \
    -netdev tap,fd=3,id=mynet1,vhost=on,vhostfd=4 3<>$tapdev 4<>/dev/vhost-net \
    -device virtio-net-pci,netdev=mynet1,disable-legacy=on,iommu_platform=on,ats=on,mac=$mac_addr \
    -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on

root@clr# mkdir -p /etc/ssh
root@clr# echo "PermitRootLogin yes" >> /etc/ssh/sshd_config

Connect to the VM through ssh

ssh -p 10022 [email protected]

From the VM, we need to insert the vfio modules properly

sudo modprobe vfio_iommu_type1 allow_unsafe_interrupts
sudo modprobe vfio_pci

Find out the device PCI address

root@clr# lspci
00:00.0 Host bridge: Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
00:01.0 Ethernet controller: Red Hat, Inc. Virtio network device
00:02.0 Unclassified device [00ff]: Red Hat, Inc. Virtio RNG
00:03.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01)
00:04.0 SCSI storage controller: Red Hat, Inc. Virtio block device
00:1f.0 ISA bridge: Intel Corporation 82801IB (ICH9) LPC Interface Controller (rev 02)
00:1f.2 SATA controller: Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode] (rev 02)
00:1f.3 SMBus: Intel Corporation 82801I (ICH9 Family) SMBus Controller (rev 02)

We want to use the second Ethernet Controller that shows up as 00:03.0.

Let's bind the device to VFIO driver

# Unbind from virtio-net driver
sudo bash -c "echo 0000:00:03.0 > /sys/bus/pci/devices/0000\:00\:03.0/driver/unbind"
# Bind to vfio
sudo bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
# Check what we have under vfio
ls -la /dev/vfio
total 0
drwxr-xr-x  2 root root       80 Jun 26 02:44 .
drwxr-xr-x 15 root root     3040 Jun 26 02:39 ..
crw-------  1 root root 245,   0 Jun 26 02:44 3
crw-rw-rw-  1 root root  10, 196 Jun 26 02:39 vfio

Start second layer of virtualization (with QEMU)

Prepare guest environment

# Get kata-static to get a statically built QEMU
wget https://github.com/kata-containers/runtime/releases/download/1.8.0-alpha2/kata-static-1.8.0-alpha2-x86_64.tar.xz
unxz kata-static-1.8.0-alpha2-x86_64.tar.xz
tar -xf kata-static-1.8.0-alpha2-x86_64.tar
rm kata-static-1.8.0-alpha2-x86_64.tar
# Download ClearLinux cloud image. We cannot use kvm image as it is too large
# Download OVMF.fd
# Get kvmvapic.bin

Start QEMU

sudo -E ./qemu-system-x86_64 \
    -machine q35,accel=kvm,kernel_irqchip=split \
    -bios /root/OVMF.fd \
    -smp sockets=1,cpus=1,cores=1 \
    -cpu host \
    -m 256 \
    -net none \
    -vga none -nographic \
    -drive file=/root/clear.img,format=raw \
    -device vfio-pci,host=00:03.0

Check the device shows up as virtio-net

# Check dmesg for virtio-net
# Check the device is tied to virtio driver
# Try to ping the host

Command to run QEMU with direct kernel boot:

sudo -E ~/build-x86_64/x86_64-softmmu/qemu-system-x86_64 \
    -machine q35,accel=kvm,kernel_irqchip=split \
    -kernel /home/sebastien/git_projects/virtio-pmem/linux/arch/x86/boot/bzImage \
    -append  'root=/dev/vda3 console=ttyS0 console=hvc0 kvm-intel.nested=1 rw intel_iommu=on \ vfio_iommu_type1.allow_unsafe_interrupts=1'
    -smp sockets=1,cpus=4,cores=2 \
    -cpu host \
    -m 2048 \
    -vga none \
    -nographic \
    -device virtio-blk-pci,drive=image \
    -drive if=none,id=image,file=/home/sebastien/workloads/vfio/clear-kvm.img,format=raw \
    -drive file=/home/sebastien/workloads/vfio/clear-kvm-2.img,if=virtio,aio=threads,format=raw \
    -netdev user,id=mynet0,hostfwd=tcp::10022-:22,hostfwd=tcp::12375-:2375 \
    -device virtio-net-pci,netdev=mynet0 \
    -device virtio-rng-pci \
    -monitor telnet:127.0.0.1:55555,server,nowait \
    -netdev tap,fd=3,id=mynet1,vhost=on,vhostfd=4 3<>$tapdev 4<>/dev/vhost-net \
    -device virtio-net-pci,netdev=mynet1,disable-legacy=on,iommu_platform=on,ats=on,mac=$mac_addr \
    -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on \
    -bios /home/sebastien/workloads/OVMF.fd