Skip to content

Instantly share code, notes, and snippets.

@muink

muink/PCI_HW_passthrough.sh

Last active March 10, 2025 09:09
Show Gist options
  • Save muink/5de439284486ef0dd1ba777da052b04f to your computer and use it in GitHub Desktop.
Save muink/5de439284486ef0dd1ba777da052b04f to your computer and use it in GitHub Desktop.
Download ZIP
Proxmox VE install tools
Raw
init_base.sh
#!/bin/sh
# debian sources
source /etc/os-release
[ "$VERSION_CODENAME" = "bookworm" ] && NONFREE="non-free non-free-firmware" || NONFREE="non-free"
sed -Ei "/^deb https?.+ contrib$/{s|$| $NONFREE|g}" /etc/apt/sources.list
sed -i 's|^deb http://ftp.debian.org|deb https://mirrors.ustc.edu.cn|g' /etc/apt/sources.list
sed -i 's|^deb http://security.debian.org|deb https://mirrors.ustc.edu.cn/debian-security|g' /etc/apt/sources.list
# enterprise sources
source /etc/os-release
echo "deb https://mirrors.ustc.edu.cn/proxmox/debian/pve $VERSION_CODENAME pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list
echo "deb http://download.proxmox.com/debian/pve $VERSION_CODENAME pve-no-subscription" >> /etc/apt/sources.list.d/pve-no-subscription.list
sed -i 's|deb|#deb|' /etc/apt/sources.list.d/pve-enterprise.list
# ceph sources
if [ -f /etc/apt/sources.list.d/ceph.list ]; then CEPH_CODENAME=`ceph -v | grep ceph | awk '{print $(NF-1)}'`; source /etc/os-release; echo "deb https://mirrors.ustc.edu.cn/proxmox/debian/ceph-$CEPH_CODENAME $VERSION_CODENAME no-subscription" > /etc/apt/sources.list.d/ceph.list; fi
# end
apt update
# web port 443-->8006
iptables -t nat -I PREROUTING [-i <ifname>|-d <iface_ip>] -p tcp --dport 443 -j DNAT --to-destination <service_ip>:8006
# OR
iptables -t nat -A PREROUTING -p tcp --dport 443 -j REDIRECT --to-ports 8006
# Persistence
echo -e '#!/bin/sh\niptables -t nat -A PREROUTING -p tcp --dport 443 -j REDIRECT --to-ports 8006' > /etc/network/if-pre-up.d/webport
chmod +x /etc/network/if-pre-up.d/webport
# hostname
sed -i 's/pve/<newname>/g' /etc/hostname
sed -i 's/pve/<newname>/g' /etc/hosts
sed -i 's/pve/<newname>/g' /etc/postfix/main.cf
# dns
sed -i '$a\nameserver 64.6.65.6\nnameserver 223.5.5.5' /etc/resolv.conf
# install
apt update && apt install -y apt-transport-https ca-certificates curl tar unzip git jq neofetch iperf3 cpufrequtils
# Proxmox_VE_Status*
cd ~/Proxmox_VE_Status 2>/dev/null && {
git fetch
git reset --hard master
git pull --rebase
} || {
git clone https://github.com/KoolCore/Proxmox_VE_Status.git ~/Proxmox_VE_Status
cd ~/Proxmox_VE_Status
}
bash ./Proxmox_VE_Status_en.sh
# rm 'No valid subscription'*
sed -zi.bak "s/res === null || res === undefined || \!res || res\s\+\.data\.status\.toLowerCase() !== 'active'/false/g" /usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js
# CT Templates sources*
cp /usr/share/perl5/PVE/APLInfo.pm /usr/share/perl5/PVE/APLInfo.pm_back
sed -i 's|http://download.proxmox.com|https://mirrors.ustc.edu.cn/proxmox|g' /usr/share/perl5/PVE/APLInfo.pm
systemctl restart pvedaemon.service
# remove old kernel
apt-cache search proxmox-
uname -r # current kernel
dpkg --list | grep proxmox-
apt remove <kernel> [headers]
dpkg --get-selections | grep kernel
dpkg --purge <kernel>
[ -z "$(efibootmgr -v | grep systemd-boot)" ] && update-grub || pve-efiboot-tool refresh
Raw
ipv6.txt
# address configure via SLAAC and request prefix via DHCPv6
iface enp1s0 inet6 auto
dhcp 1
request_prefix 1
# providers will send addresses via DHCPv6, but the default route via SLAAC
iface enp1s0 inet6 dhcp
accept_ra 2
request_prefix 1
# manual unique local address
iface eth0 inet6 static
address fdc3:cafe::3/64
# use SLAAC to get global IPv6 address from the router
# we may not enable ipv6 forwarding, otherwise SLAAC gets disabled
autoconf 1
accept_ra 2
iface vmbr0 inet6 static
address fd00:aaaa:bbbb::123/64
gateway fd00:aaaa:bbbb::1
autoconf 1
accept_ra 2
ifreload -a
Raw
PCI_HW_passthrough.sh
#!/bin/bash
# Constants
DEFGRUB="/etc/default/grub"
KLCMDL="/etc/kernel/cmdline"
MODULES_CONF="/etc/modules"
I915_SRIOV_DIR="$HOME/i915-sriov-dkms"
pause() {
read -s -n1 -p "Press any key to continue ... "
return ${1:-0};
}
checkIommu() {
if ! dmesg | grep -e DMAR -e IOMMU -e AMD-Vi &>/dev/null; then
echo -e "The device does not support IOMMU.\nThis may be because the BIOS does not enable Virtualization support.\n"
pause 1
fi
}
getCPUplatform() {
case $(lscpu | grep '^Model name' | grep -E 'Intel|AMD') in
*Intel*) cpu_platform="intel" ;;
*AMD*) cpu_platform="amd" ;;
*) echo -e "Unsupported platform.\n"; pause 1 ;;
esac
}
getIommu() {
pvever=$(/usr/bin/pveversion)
getCPUplatform || return 1
case $cpu_platform in
intel) iommu="intel_iommu=on" ;;
amd) iommu="amd_iommu=on" ;;
esac
iommu="$iommu iommu=pt pcie_acs_override=downstream,multifunction"
}
getPstate() {
getCPUplatform || return 1
unset pstatedriver
case $cpu_platform in
intel) pstate="intel_pstate=passive" ;;
amd) pstate="amd_pstate=passive" ;;
esac
}
getBootloader() {
if ! efibootmgr -v | grep systemd-boot &>/dev/null; then
BL='grub'
bootconf="$DEFGRUB"
bootrefresh='update-grub'
else
BL='systemd-boot'
bootconf="$KLCMDL"
bootrefresh='pve-efiboot-tool refresh'
fi
}
getCPUgen() {
# https://www.intel.com/content/www/us/en/support/articles/000093216/graphics/processor-graphics.html
local model=$(lscpu | grep "^Model name" | sed -En "s|.*\b(i[3579]-.+)\b.*|\1|p")
model=$(echo "$model" | sed -En "s|.+-([0-9]+)[a-zA-Z]*|\1|p")
if [ -z "$cpugen" ]; then
echo -ne "Unable to identify cpu generation.\nIf your CPU is Rocket Lake series, please enter 10.\nPlease type manually: [ ]\b\b"
read cpugen
else
cpugen="${model:0:-3}"
[ "$cpugen" = 11 ] && { echo -ne "Need to reconfirm.\nIf your CPU is Tiger Lake series, please enter 11.\nIf your CPU is Rocket Lake series, please enter 10.\nPlease type manually: [ ]\b\b"; read cpugen; }
fi
}
getGPUbusID() {
gpu_id=$(lspci | grep -E 'VGA|3D|Display' | awk -F'.' '{print $1}' | sort -u)
if [ -z "$gpu_id" ]; then
echo -ne "No available GPU device found.\n"
pause 1
fi
}
getGPUtotalvfs() {
getGPUbusID || return $?
gpu_totalvfs=$(cat /sys/bus/pci/devices/0000:${gpu_id}.0/sriov_totalvfs 2>/dev/null)
if [ -z "$gpu_totalvfs" ]; then
echo -ne "GPU device not support SR-IOV.\n"
pause 1
fi
}
enable_iommu() {
clear
checkIommu || return $?
getIommu || return $?
getBootloader
if ! grep -q "$iommu" "$bootconf" 2>/dev/null; then
[ "$BL" = "grub" ] \
&& sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT/{s|quiet|quiet $iommu|}" "$bootconf" \
|| sed -i "/root=/{s|$| $iommu|}" "$bootconf"
sleep 1; $bootrefresh
if ! grep -q "^vfio" "$MODULES_CONF" 2>/dev/null;then
cat <<-EOF >> "$MODULES_CONF"
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd #not needed if on kernel 6.2 or newer
EOF
fi
sleep 1; update-initramfs -u -k all
echo -e "Success. Please restart the device manually later.\n"
pause 0
else
echo -e "Has been enabled, no need to enabled again.\n"
pause 0
fi
}
disable_iommu() {
clear
checkIommu || return $?
getIommu || return $?
getBootloader
if ! grep -q "$iommu" "$bootconf" 2>/dev/null; then
echo -e "Not enabled, no need to disable.\n"
pause 0
else
[ "$BL" = "grub" ] \
&& sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT/{s| $iommu||}" "$bootconf" \
|| sed -i "/root=/{s| $iommu||}" "$bootconf"
sleep 1; $bootrefresh
sed -i '/^vfio/d' "$MODULES_CONF"
sleep 1; update-initramfs -u -k all
echo -e "Success. Please restart the device manually later.\n"
pause 0
fi
}
# Sub functions for SR-IOV and GVT-g
# enable_sriov [kernel-release]
enable_sriov() {
local krelease=$1
getGPUtotalvfs || return $?
apt update && apt install -y git pve-headers mokutil # install proxmox-headers-$(uname -r)
cd "$I915_SRIOV_DIR" 2>/dev/null && {
git fetch
git reset --hard master
#git pull --rebase
} || {
git clone https://github.com/strongtz/i915-sriov-dkms.git "$I915_SRIOV_DIR"
cd "$I915_SRIOV_DIR"
}
#cp -fa "$I915_SRIOV_DIR/dkms.conf"{,.bak}
#sed -i 's/"@_PKGBASE@"/"i915-sriov-dkms"/g' dkms.conf
#sed -i 's/"@PKGVER@"/"'"$KERNEL"'"/g' dkms.conf
#sed -i 's|^\(AUTOINSTALL\)=.*|\1=no|' dkms.conf
cat dkms.conf
eval "$(sed -n '/^PACKAGE_NAME=/p' dkms.conf)"
eval "$(sed -n '/^PACKAGE_VERSION=/p' dkms.conf)"
rm -rf /var/lib/dkms/$PACKAGE_NAME
rm -rf /usr/src/${PACKAGE_NAME}-*
apt install -y dkms
dkms add . # Creating symlink /var/lib/dkms/$PACKAGE_NAME/$PACKAGE_VERSION/source -> /usr/src/$PACKAGE_NAME-$PACKAGE_VERSION
cd /usr/src/$PACKAGE_NAME-$PACKAGE_VERSION
dkms status # $PACKAGE_NAME/$PACKAGE_VERSION: added
dkms install -m $PACKAGE_NAME -v $PACKAGE_VERSION -k ${krelease:-$(uname -r)} --force -j $(nproc)
#dkms install -m $PACKAGE_NAME -v $PACKAGE_VERSION -k 6.5.13-5-pve --force -j $(nproc)
#dkms install -m $PACKAGE_NAME -v $PACKAGE_VERSION -k 6.8.4-2-pve --force -j $(nproc)
# Sign command: /lib/modules/$(uname -r)/build/scripts/sign-file
# Binary /lib/modules/$(uname -r)/build/scripts/sign-file not found, modules won't be signed
# Error! Your kernel headers for kernel $(uname -r) cannot be found at /lib/modules/$(uname -r)/build or /lib/modules/$(uname -r)/source.
# Please install the proxmox-headers-$(uname -r) package or use the --kernelsourcedir option to tell DKMS where it's located.
# ====================================================================================
# Sign command: /lib/modules/$(uname -r)/build/scripts/sign-file
# Signing key: /var/lib/dkms/mok.key
# Public certificate (MOK): /var/lib/dkms/mok.pub
#
# Building module:
# Cleaning build area...
# make -j8 KERNELRELEASE=$(uname -r) -C /lib/modules/$(uname -r)/build M=/var/lib/dkms/$PACKAGE_NAME/$PACKAGE_VERSION/build....................
# Signing module /var/lib/dkms/$PACKAGE_NAME/$PACKAGE_VERSION/build/i915.ko
# Cleaning build area...
#
# i915.ko:
# Running module version sanity check.
# - Original module
# - Installation
# - Installing to /lib/modules/$(uname -r)/updates/dkms/
# depmod...
dkms status # $PACKAGE_NAME/$PACKAGE_VERSION, $(uname -r), x86_64: installed
#mokutil --import /var/lib/dkms/mok.pub # Please restart the system later and register mok in the bootloader screen.
cd $HOME
# rm source to prevent automatic building on kernel updates
rm -rf /var/lib/dkms/$PACKAGE_NAME
rm -rf /usr/src/${PACKAGE_NAME}-*
if ! grep -q 'i915.enable_guc=3' "$bootconf" 2>/dev/null; then
[ "$BL" = "grub" ] \
&& sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT/{s|\"$| i915.enable_guc=3 i915.max_vfs=${gpu_totalvfs}\"|}" "$bootconf" \
|| sed -i "/root=/{s|$| i915.enable_guc=3 i915.max_vfs=${gpu_totalvfs}|}" "$bootconf"
sleep 1; $bootrefresh
apt install -y sysfsutils
echo "devices/pci0000:00/0000:${gpu_id}.0/sriov_numvfs = ${gpu_totalvfs}" > /etc/sysfs.conf
#echo 7 > /sys/devices/pci0000:00/0000:${gpu_id}.0/sriov_numvfs # manual method
fi
sleep 1; update-initramfs -u -k all
echo -e "Success. Please restart the device manually later.\n"
pause 0
}
disable_sriov() {
getGPUtotalvfs || return $?
cd "$I915_SRIOV_DIR" 2>/dev/null && {
eval "$(sed -n '/^PACKAGE_NAME=/p' dkms.conf)"
eval "$(sed -n '/^PACKAGE_VERSION=/p' dkms.conf)"
} || {
PACKAGE_NAME=i915-sriov-dkms
PACKAGE_VERSION=$(uname -r | sed 's|-pve$||')
}
rm -rf /var/lib/dkms/$PACKAGE_NAME
rm -rf /usr/src/${PACKAGE_NAME}-*
dkms status
# rmall /lib/modules/$(uname -r)/updates/dkms/i915.ko
dkms remove -m $PACKAGE_NAME -v $PACKAGE_VERSION --all \
|| rm -f /lib/modules/*/updates/dkms/i915.ko
dkms status
if ! grep -q 'i915.enable_guc=3' "$bootconf" 2>/dev/null; then
echo -e "Not enabled, no need to disable.\n"
pause 0
else
[ "$BL" = "grub" ] \
&& sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT/{s| i915.enable_guc=3 i915.max_vfs=${gpu_totalvfs}||}" "$bootconf" \
|| sed -i "/root=/{s| i915.enable_guc=3 i915.max_vfs=${gpu_totalvfs}||}" "$bootconf"
sleep 1; $bootrefresh
sleep 1; update-initramfs -u -k all
rm -f /etc/sysfs.conf
echo -e "Success. Please restart the device manually later.\n"
pause 0
fi
}
enable_gvt() {
if ! grep -q 'i915.enable_gvt=1' "$bootconf" 2>/dev/null; then
[ "$BL" = "grub" ] \
&& sed -i '/^GRUB_CMDLINE_LINUX_DEFAULT/{s|"$| i915.enable_gvt=1"|}' "$bootconf" \
|| sed -i "/root=/{s|$| i915.enable_gvt=1|}" "$bootconf"
sleep 1; $bootrefresh
if ! grep -q "^kvmgt" "$MODULES_CONF" 2>/dev/null; then
echo "kvmgt" >> "$MODULES_CONF"
fi
sleep 1; update-initramfs -u -k all
echo -e "Success. Please restart the device manually later.\n"
pause 0
else
echo -e "GVT-g has already been enabled.\n"
pause 0
fi
}
disable_gvt() {
if ! grep -q 'i915.enable_gvt=1' "$bootconf" 2>/dev/null; then
echo -e "Not enabled, no need to disable.\n"
pause 0
else
[ "$BL" = "grub" ] \
&& sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT/{s| i915.enable_gvt=1||}" "$bootconf" \
|| sed -i "/root=/{s| i915.enable_gvt=1||}" "$bootconf"
sleep 1; $bootrefresh
sed -i '/^kvmgt/d' "$MODULES_CONF"
sleep 1; update-initramfs -u -k all
echo -e "Success. Please restart the device manually later.\n"
pause 0
fi
}
# enable_vg [kernel-release]
enable_vg() {
clear
checkIommu || return $?
getBootloader
getCPUgen
getGPUbusID || return $?
if [ "$cpugen" -ge 11 ]; then
# SR-IOV
enable_sriov "$1" || return $?
elif [ "$cpugen" -ge 6 -a "$cpugen" -le 10 ]; then
# GVT-g
enable_gvt || return $?
else
echo -e "Unsupported generation.\n"
pause 0
fi
}
enable_vg_latest() {
local krelease=$(grep -Eo "vmlinuz-\S+" /boot/grub/grub.cfg | head -1 | sed 's|^vmlinuz-||')
[ -n "$krelease" ] && enable_vg "$krelease"
}
disable_vg() {
clear
checkIommu || return $?
getBootloader
getCPUgen
getGPUbusID || return $?
if [ "$cpugen" -ge 11 ]; then
# SR-IOV
disable_sriov || return $?
elif [ "$cpugen" -ge 6 -a "$cpugen" -le 10 ]; then
# GVT-g
disable_gvt || return $?
else
echo -e "Unsupported generation.\n"
pause 0
fi
}
set_cpufreq() {
clear
getBootloader
getPstate || return 1
#cat /sys/devices/system/cpu/cpufreq/policy*/scaling_driver 2>/dev/null | sort -u
[ -n "$pstate" ] && {
if ! grep -q "$pstate" "$bootconf" 2>/dev/null; then
apt update && apt install -y cpufrequtils
[ "$BL" = "grub" ] \
&& sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT/{s|\"$| $pstate\"|}" "$bootconf" \
|| sed -i "/root=/{s|$| $pstate|}" "$bootconf"
sleep 1; $bootrefresh
#echo "passive" > /sys/devices/system/cpu/intel_pstate/status # manual method
fi
}
local governors="$(cat /sys/devices/system/cpu/cpufreq/policy*/scaling_available_governors 2>/dev/null | sort -u | sed 's|\s|\n|g' | sed '/^$/d' | nl)"
local governors_count=$(echo "$governors" | wc -l)
local governorid=
echo "Available governors:"
echo "$governors"
echo -ne "Please select: [ ]\b\b"
read -t 60 governorid
governorid=${governorid:-1}
[ "$governorid" -ge 1 -a "$governorid" -le $governors_count ] || governorid=1
governor=$(echo "$governors" | sed -n "${governorid}p" | sed -En 's|^\s+[0-9]+\s*||p')
cat <<-EOF > /etc/default/cpufrequtils
GOVERNOR="$governor"
EOF
systemctl restart cpufrequtils
echo -e "Success. Please restart the device manually later.\n"
pause 0
}
#verify_iommu
dmesg | grep 'remapping' | grep -q -e 'AMD-Vi: Interrupt remapping enabled' -e 'DMAR-IR: Enabled IRQ remapping in x2apic mode'
[ $? = 0 ] && isiommu="-. IOMMU is enabled" || isiommu="-. IOMMU is not enabled"
#verify_iommu_group
checkIommu || isiommug="-. IOMMU is not supported"
if find /sys/kernel/iommu_groups/ -type l >/dev/null 2>&1; then
isiommug="-. IOMMU Groups is enabled"
else
isiommug="-. IOMMU Groups is not enabled."
fi
#verify_vg
getGPUbusID || { isgsriov="-. No GPU found."; isgvt="-. No GPU found."; }
if [ -n "${gpu_id}" ]; then
#sriov
gpu_vfs=$(cat /sys/bus/pci/devices/0000:${gpu_id}.0/sriov_numvfs 2>/dev/null)
#gpu_vfs=$(( $(lspci | grep -E 'VGA|3D|Display' | wc -l) - 1))
if [ "$gpu_vfs" -gt 0 ]; then
isgsriov="-. GPU SR-IOV is enabled. ${gpu_vfs} VFs."
else
isgsriov="-. GPU SR-IOV is not enabled."
fi
#gvt
if ls /sys/bus/pci/devices/0000:${gpu_id}.0/mdev_supported_types >/dev/null 2>&1; then
isgvt="-. GPU GVT-g is enabled."
else
isgvt="-. GPU GVT-g is not enabled."
fi
fi
#verify_numa
lscpu | grep -q -i numa 2>/dev/null && isnuma="-. CPU NUMA is supported" || isnuma="-. CPU NUMA is not supported"
#cpu_governors
cpugovernor="-. CPU governor: $(cat /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 2>/dev/null | sort -u)"
# Main menu
while :; do
clear
echo "Proxmox VE HW Tool"
cat <<-EOF
1. Enable IOMMU
2. Disable IOMMU
3. Enable Intel GVT-g/SR-IOV (current kernel)
4. Enable Intel GVT-g/SR-IOV (latest kernel)
5. Disable Intel GVT-g/SR-IOV
6. Set CPU Power governors
--------------------------------
$isiommu
$isiommug
$isgsriov
$isgvt
$isnuma
$cpugovernor
x. Exit
EOF
echo -ne "Please select: [ ]\b\b"
read -t 60 menuid
menuid=${menuid:-x}
case ${menuid} in
1) enable_iommu ;;
2) disable_iommu ;;
3) enable_vg ;;
4) enable_vg_latest ;;
5) disable_vg ;;
6) set_cpufreq ;;
x) exit 0 ;;
*) ;;
esac
done
Raw
qcow2_convert_lvm_thin.txt
VMFile:
srcvf=/var/lib/vz/template/iso/<openwrt.qcow2>.iso # upload $srcvf
qm importdisk <vmid> $srcvf <proxmox-storage-name> --format qcow2
cp -f $srcvf /var/lib/vz/images/<vmid>/<vm-<vmid>-disk-?>.qcow2
LVM-Thin:
srcvf=/var/lib/vz/template/iso/<openwrt.qcow2>.iso # upload $srcvf
qm importdisk <vmid> $srcvf <proxmox-storage-name> --format qcow2
#OR qemu-img convert -nf qcow2 -O raw -t none $srcvf /dev/pve/<vm-<vmid>-disk-?>
VMCONF:
/etc/pve/nodes/<pve_node_name>/qemu-server/100.conf
Raw
view_iommu_groups.sh
#!/bin/bash
shopt -s nullglob
for g in $(find /sys/kernel/iommu_groups/* -maxdepth 0 -type d | sort -V); do
echo "IOMMU Group ${g##*/}:"
for d in $g/devices/*; do
echo -e "\t$(lspci -nns ${d##*/})"
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment