proxmox-vfio

etc-modules

pci_stub
vfio
vfio_iommu_type1
vfio_pci
kvm
kvm_intel

get videocard ROM

# http://vfio.blogspot.md/2014/08/does-my-graphics-card-rom-support-efi.html

# cd /sys/bus/pci/devices/0000:01:00.0/
# echo 1 > rom
# cat rom > /tmp/image.rom
# echo 0 > rom

$ git clone https://github.com/awilliam/rom-parser
$ cd rom-parser
$ make

$ ./rom-parser GT635.rom 
Valid ROM signature found @0h, PCIR offset 190h
 PCIR: type 0, vendor: 10de, device: 1280, class: 030000
 PCIR: revision 0, vendor revision: 1
Valid ROM signature found @f400h, PCIR offset 1ch
 PCIR: type 3, vendor: 10de, device: 1280, class: 030000
 PCIR: revision 3, vendor revision: 0
  EFI: Signature Valid
 Last image

kernel boot args

# https://www.pugetsystems.com/labs/articles/Multiheaded-NVIDIA-Gaming-using-Ubuntu-14-04-KVM-585/
# http://vfio.blogspot.md/2014/08/iommu-groups-inside-and-out.html
# http://vfio.blogspot.md/2015/05/vfio-gpu-how-to-series-part-3-host.html

# if boot from USB HDD
rootdelay=5

# activate iommu
intel_iommu=on

# sets the IOMMU into passthrough mode for host devices This reduces the overhead of the IOMMU for host owned devices, 
# but also removes any protection the IOMMU may have provided again errant DMA from devices.  
# If you weren't using the IOMMU before, there's nothing lost.  
# Regardless of passthrough mode, the IOMMU will provide the same degree of isolation for assigned devices.
iommu=pt 

# disable vgaarb ???
nomodeset

# assign driver for GPU
pci-stub.ids=10de:13c2,10de:0fbb
$ dmesg | grep pci-stub

# Enable interrupt remapping. Depending on your motherboard, this may or may not be necessary.
vfio_iommu_type1.allow_unsafe_interrupts=1

other

# check if motherboard support IOMMU/DMAR
dmesg | grep -e DMAR -e IOMMU

# check GPU device ID
lspci | grep VGA
lspci -n
lspci -tv

run script

#!/bin/bash
##################################################
# CONFIG PART
##################################################
# https://pve.proxmox.com/wiki/USB_physical_port_mapping#Proxmox_2.x_and_later
BIN="qemu-system-x86_64"
NAME="xen1"
CFGDIR="/kvm"
CMD="-name ${NAME}"
CMD+=" -machine q35,accel=kvm -cpu host,kvm=off -smp 8,cores=8 -m 4000"
CMD+=" -boot order=dc,menu=on"
CMD+=" -vnc :0"
CMD+=" -net bridge,br=vmbr0"
CMD+=" -net nic,model=virtio,macaddr=00:16:3e:12:f8:65"
CMD+=" -drive file=/var/lib/vz/windows7.qcow2,if=virtio,media=disk"
#CMD+=" -drive file=/var/lib/vz/template/iso/W7AIO.x86-x64.120520.RU-CtrlSoft.iso,media=cdrom"
#CMD+=" -drive file=/var/lib/vz/template/iso/virtio-win-0.1.102.iso,media=cdrom"
CMD+=" -drive file=/var/lib/vz/template/iso/W7AIO.x86-x64.120520.RU-CtrlSoft.iso,id=isocd.0 -device ide-cd,bus=ide.0,drive=isocd.0"
CMD+=" -drive file=/var/lib/vz/template/iso/virtio-win-0.1.102.iso,id=isocd.1 -device ide-cd,bus=ide.1,drive=isocd.1"
CMD+=" -device ioh3420,bus=pcie.0,addr=1c.0,multifunction=on,port=1,chassis=1,id=root.1"
CMD+=" -device vfio-pci,host=01:00.0,bus=root.1,addr=00.0,multifunction=on,x-vga=on,romfile=/root/Pitcairn.rom -device vfio-pci,host=01:00.1 -vga none"
#CMD+=" -device vfio-pci,host=00:1b.0"
#CMD+=" -usb -usbdevice host:046d:c05b -usbdevice host:04d9:1702"
CMD+=" -usb -device usb-host,hostbus=1,hostport=3.4.3"
CMD+=" -usb -device usb-host,hostbus=1,hostport=3.4.4"
#CMD+=" -usbdevice tablet"
CMD+=" -daemonize"

[ ! -f /dev/vfio/1 ] && /root/vfio-bind 0000:01:00.0 0000:01:00.1
[ ! -d $CFGDIR ] && mkdir $CFGDIR;
##################################################
# FUNCTIONS PART
##################################################
start() {
	$BIN ${CMD}
	echo "[$0] Executing $BIN ${CMD}"
	ps -ef | grep kvm | grep ${NAME} | grep -v grep | grep -v tmux | awk '{print $2}' > ${CFGDIR}/${NAME}.pid
	if [ -n "`cat ${CFGDIR}/${NAME}.pid`" ]; then
		echo "[$0] status: started"
	else
		echo "[$0] status: cant start"
	fi
}
stop() {
	PID="`ps -ef | grep qemu | grep ${NAME} | grep -v grep | grep -v tmux | awk '{print $2}'`"
	PIDFILE="`cat ${CFGDIR}/${NAME}.pid 2>/dev/null`"
	if [ -n "$PID" ] && [ "$PID" = "$PIDFILE" ]; then
		kill -9 $PID
		rm -v ${CFGDIR}/${NAME}.pid
		echo "[$0] Send 'kill -9 $PID'"
	fi
}
status() {
	PID="`ps -ef | grep qemu | grep ${NAME} | grep -v grep | grep -v tmux | awk '{print $2}'`"
	PIDFILE="`cat ${CFGDIR}/${NAME}.pid 2>/dev/null`"
	if [ -n "$PID" ] && [ "$PID" = "$PIDFILE" ]; then
		echo "[$0] status: running"
	else
		echo "[$0] status: stopped"
	fi
}
##################################################
# MAIN PART
##################################################
case "$1" in
 "start")
	start
	;;
 "stop")
	stop
	;;
 "restart")
	stop
	start
	;;
 "status")
	status
	;;
 *)
	echo "Usage: $0 start|stop|restart|status"
	;;
esac

vfio-bind

#!/bin/bash
# /root/vfio-bind

modprobe vfio-pci

for dev in "$@"; do
  vendor=$(cat /sys/bus/pci/devices/$dev/vendor)
  device=$(cat /sys/bus/pci/devices/$dev/device)
  if [ -e /sys/bus/pci/devices/$dev/driver ]; then
    echo $dev > /sys/bus/pci/devices/$dev/driver/unbind
  fi
  echo $vendor $device > /sys/bus/pci/drivers/vfio-pci/new_id
done

vfio-pci-override-vga.sh

#!/bin/sh
# /sbin/vfio-pci-override-vga.sh

for i in $(find /sys/devices/pci* -name boot_vga); do
        if [ $(cat $i) -eq 0 ]; then
                GPU=$(dirname $i)
                AUDIO=$(echo $GPU | sed -e "s/0$/1/")
                echo "vfio-pci" > $GPU/driver_override
                if [ -d $AUDIO ]; then
                        echo "vfio-pci" > $AUDIO/driver_override
                fi
        fi
done

modprobe -i vfio-pci