TRex traffic generator in Vagrant

faq.md

FAQ

How to configure host

Passwordless access to qemu system session

Create /etc/polkit-1/rules.d/50-allow-libvirt.rules with

/* Allow users in kvm group to manage the libvirt
 * daemon without authentication */
polkit.addRule(function(action, subject) {
        if (action.id == "org.libvirt.unix.manage" &&
                subject.isInGroup("yourgroup")) {
                return polkit.Result.YES;
        }
});

where yourgroup is the unix group you belong to.

VM boots significantly slower when physical NIC passed through

This is probably due to Intel Boot Agent (PXE). Boot with graphics and check. In Vagrant, the only way to disable boot ROMs is to flash your NIC's persistent memory. If you brave enough do the following:

1. Download and unpack Intel Preboot utility for Linux
2. Reboot your host with IOMMU disabled
3. Go to APPS/BootUtil/Linux_x64
4. Install and load kernel module using sudo DRIVER/install
5. Run sudo ./bootutil64e and find your NIC numbers
6. You should see a lines like

Port Network Address Location Series  WOL Flash Firmware                Version
==== =============== ======== ======= === ============================= =======
  N   XXXXXXXXXXXX     3:00.0 Gigabit YES UEFI,PXE Enabled              1.5.88

7. Run sudo ./bootutil64e -NIC=N -BOOTENABLE=DISABLED
8. Run sudo ./bootutil64e -NIC=N -FD
9. Reboot to IOMMU again and here it is.

traffgen.yml

---
- hosts: all
  tasks:
    - name: add trex rpm repository
      environment: "{{ proxy_env }}"
      yum_repository:
        name: trex
        includepkgs: trex-core
        enablegroups: false
        skip_if_unavailable: no
        description: "Copr repo fo trex traffic generator"
        baseurl: "https://copr-be.cloud.fedoraproject.org/results/snizovtsev/trex/epel-8-$basearch"
        gpgkey: "https://copr-be.cloud.fedoraproject.org/results/snizovtsev/trex/pubkey.gpg"

    - name: install required packages
      environment: "{{ proxy_env }}"
      notify: system upgraded
      dnf:
        state: latest
        name:
          - kernel # force update for trex
          - trex-core # cisco traffic generator
          - kernel-core # upgrade to match kernel-devel
          - tuned # sysctl and boot options
          - nmap # nping tests
          - wireshark-cli # for debugging
          - driverctl # binding/unbinding drivers
          - dnsmasq # pxe boot

    - name: mkdir /etc/tuned/traffgen
      file: path=/etc/tuned/traffgen state=directory
    - name: install tuned profile
      notify: switch tuned profile
      copy:
        dest: /etc/tuned/traffgen/tuned.conf
        content: |
          [main]
          include=network-latency
          [sysctl]
          vm.nr_hugepages=256
          [bootloader]
          [sysfs]

    - name: trex configuration file
      copy:
        dest: /etc/trex_cfg.yaml
        content: |
          - version: 2
            interfaces: ["03:00.0", "04:00.0"]
            port_limit: 2
            limit_memory: 512
            low_end: true

  handlers:
    - name: switch tuned profile
      shell: tuned-adm profile traffgen
    - name: system upgraded
      reboot:

Vagrantfile

#!/usr/bin/env ruby

ENV["PYTHONUNBUFFERED"] = "1"

Vagrant.configure("2") do |config|

  # Better defaults
  config.vm.provider :libvirt do |libvirt|
    libvirt.qemu_use_session = false
    libvirt.cpu_mode = 'host-passthrough' # Prefer performance to reproducibility here
    libvirt.machine_type = 'q35' # better support device passthrough since ICH9 uses a PCI-E bus
                                 # whereas the I440FX only supports a PCI bus

    # Avoid unpredictable memory latencies for better reproducibility
    libvirt.memorybacking :hugepages
    libvirt.memorybacking :nosharepages
    libvirt.memorybacking :locked # Forbids swapping
    libvirt.memorybacking :source, :type => 'file'
    libvirt.memorybacking :allocation, :mode => 'immediate'
  end

  config.vm.define :traffgen do |node|
    node.vm.box = "centos/8"
    node.vm.hostname = "traffgen"
    node.vm.synced_folder './shared', '/vagrant', type: 'rsync'

    node.vm.provider :libvirt do |qemu|
      qemu.graphics_type = 'none' # use serial
      qemu.cpus = 3 # Recommended value from TRex FAQ
      qemu.cputopology :sockets => 1, :cores => 3, :threads => 1
      qemu.memory = 1024 # Reduced value for debugging
      #qemu.memory = 4096 # Recommended value from TRex FAQ

      # HostNIC-1/p0 (PCI 01:00.0) <-cable-> HostNIC-2/p0 (enp3s0f0)
      qemu.pci :bus => '01', :slot => '00', :function => '0'
      # HostNIC-1/p1 (PCI 01:00.1) <-cable-> HostNIC-2/p2 (enp3s0f2)
      qemu.pci :bus => '01', :slot => '00', :function => '1'
    end

    node.vm.provision :ansible do |ansible|
      ansible.playbook = "./traffgen.yml"
      ansible.become = true
      ansible.compatibility_mode = "2.0"
      ansible.verbose = "vv"
    end
  end

  ########################################################################
  # System-under-test
  #
  # Should be connected using physical cable to traffic generator VM.
  # We use macvtap because currently KOS dosn't handle passed-through NICs
  # properly (bug #3980834).
  ########################################################################
  config.vm.define :sut do |node|
    node.vm.provider :libvirt do |qemu|
      qemu.graphics_type = 'none' # use serial
      qemu.kernel = "#{__dir__}/kernel"
      qemu.machine_type = 'pc'
      qemu.cpus = 1
      qemu.memory = 1024
      qemu.mgmt_attach = false # useless on KOS

      # Use OVMF UEFI instead of SeaBIOS
      #qemu.loader = "/usr/share/OVMF/OVMF_CODE.fd"
      #qemu.storage :file, :device => :cdrom, :bus => "sata", :path => "#{__dir__}/uboot-efi.img"

      # IOMMU
      # qemu.features = ['acpi', 'apic', 'pae', 'ioapic'] # 3 vagrant default + ioapic
      # qemu.qemuargs :value => "-device"
      # qemu.qemuargs :value => "intel-iommu"
    end

    node.vm.network :public_network,
                    :dev => "enp3s0f0",
                    :mode => "private",
                    :model_type => "e1000",
                    :trust_guest_rx_filters => true

    node.vm.network :public_network,
                    :dev => "enp3s0f2",
                    :mode => "private",
                    :model_type => "e1000",
                    :trust_guest_rx_filters => true
  end

  ########################################################################
  # NetBSD reference system
  ########################################################################
  config.vm.define :netbsd do |node|
    node.vm.box = "generic/netbsd8"

    node.vm.provider :libvirt do |qemu|
      # qemu.graphics_type should be non-none or netbsd won't boot
      qemu.cpus = 1
      qemu.memory = 1024
      qemu.machine_type = 'pc'
    end

    node.vm.network :public_network,
                    :dev => "enp3s0f0",
                    :mode => "private",
                    :model_type => "rtl8139",
                    :trust_guest_rx_filters => true

    node.vm.network :public_network,
                    :dev => "enp3s0f2",
                    :mode => "private",
                    :model_type => "rtl8139",
                    :trust_guest_rx_filters => true
  end

end