Install the latest NVIDIA drivers on a Proxmox host and LXC container with a GPU configured for passthrough.

proxmox_lxc_install_latest_nvidia_drivers.sh

#!/bin/bash
#
# This script installs the latest NVIDIA drivers on a Proxmox host
# and LXC container guests (Debian based) with a card passed through to them.
#
# It essentially automates the repeatable steps from this guide for re-running after kernel upgrades
# https://forums.plex.tv/t/plex-hw-acceleration-in-lxc-container-anyone-with-success/219289/35
# It automatically grabs the latest version of the driver from: https://github.com/keylase/nvidia-patch
#
# HOST: Place this script in /opt/nvidia/ and execute
# LXC Guest: Bind-mount the host's /opt/nvidia/ directory to /opt/pve-host, e.g. with:
#   mp2: /opt/nvidia,mp=/opt/pve-host,mountoptions=noatime,ro=1,replicate=0,shared=1
#
# For the guests, it assumes a configuration similar to the following for passing through the GPU:
# lxc.cgroup2.devices.allow: c 195:* rwm
# lxc.cgroup2.devices.allow: c 235:* rwm
# lxc.cgroup2.devices.allow: c 511:* rwm
# lxc.mount.entry: /dev/nvidia0 dev/nvidia0 none bind,optional,create=file
# lxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file
# lxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file
# lxc.mount.entry: /dev/nvidia-modeset dev/nvidia-modeset none bind,optional,create=file
# lxc.mount.entry: /dev/nvidia-uvm-tools dev/nvidia-uvm-tools none bind,optional,create=file
#
# Verified to work on Proxmox 6/7/8, with nVidia Quadro P1000/P2000 cards. Tested with Debian and Ubuntu LXC guests.
# Should work (and patch) for RTX/gaming cards as well.

# Bail on any errors
set -e

#
# functions
#

function die() {
	echo "[FATAL] $@" >&2
	exit 1
}

function ensure_prereqs() {
	required_pkgs=("build-essential" "proxmox-headers-$(uname -r | cut -d . -f 1,2)" "git")
	pkgs_to_install=()

	# Build list of missing packages
	for pkg in "${required_pkgs[@]}"; do
		if ! dpkg -s "$pkg" &>/dev/null; then
			pkgs_to_install+=("$pkg")
		fi
	done

	# Install any missing packages
	if [ "${#pkgs_to_install[@]}" -gt 0 ]; then

		# Set non-interactive frontend for apt
		export DEBIAN_FRONTEND=noninteractive
		export DEBCONF_NONINTERACTIVE_SEEN=true

		apt-get update -q

		for pkg in "${pkgs_to_install[@]}"; do
			apt-get install -y --no-install-recommends "$pkg"
		done
	fi
}

function fetch_driver_if_not_exists() {
	cd "${nv_base_dir}"

	if [[ ! -e "NVIDIA-Linux-x86_64-${driver_version}.run" ]]; then
		if ! wget "https://international.download.nvidia.com/XFree86/Linux-x86_64/${driver_version}/NVIDIA-Linux-x86_64-${driver_version}.run" ; then
			echo "Failed to download current NVIDIA Driver Installer (${driver_version})"
		exit 1
		fi
	else
		echo "Current driver installer present (${nv_installer})"
	fi
}

function install_host() {
	if [[ "$nvidia_module_uptodate" == "true" ]]; then
		die "[Host] nVidia module(s) already up to date and loaded - aborting"
	else
		echo "[Host] nVidia module(s) not up to date or currently loaded - proceeding to install"
		# Install nVidia binary driver
		cd "${nv_base_dir}"
		chmod 755 "$nv_installer"
		if ! $nv_installer --no-questions --ui=none --disable-nouveau ; then
			die "[Host] Failed to run $nv_installer"
		fi
		if ! update-initramfs -u ; then
			die "[Host] Failed to run update-initramfs"
		fi

		# Install systemd service
		if ! cd $(dirname "$nv_persistenced_installer"); then
			echo "[Host] nvidia-persistenced repository not present, cloning..."
			cd "${nv_base_dir}"
			git clone --depth 1 https://github.com/NVIDIA/nvidia-persistenced.git
			cd $(dirname "$nv_persistenced_installer")
		else
			echo "[Host] nvidia-persistenced repository present, pulling..."
			git pull
		fi
		if ! $nv_persistenced_installer ; then
			die "[Host] Failed to run $nv_persistenced_installer"
		fi

		if ! systemctl enable nvidia-persistenced.service ; then
			die '[Host] Failed to enable nvidia-persistenced.service'
		fi

		if ! systemctl start nvidia-persistenced.service ; then
			die '[Host] Failed to start nvidia-persistenced.service'
		fi

		# Patch driver to remove concurrent stream limits
		if ! cd $(dirname "$nv_patch_installer"); then
			echo "[Host] nvidia-patch repository not present, cloning..."
			cd "${nv_base_dir}"
			git clone --depth 1 https://github.com/keylase/nvidia-patch.git
			cd $(dirname "$nv_patch_installer")
		else
			echo "[Host] nvidia-patch repository present, pulling..."
			git pull
		fi
		if ! $nv_patch_installer ; then
			die "[Host] Failed to run $nv_patch_installer"
		fi

		# Run nvidia-smi to verify that everything is working
		if ! nvidia-smi ; then
			die "[LXC] Failed to run nvidia-smi"
		fi
	fi
}

function install_lxc() {
	# Install nVidia binary driver
	cd $(dirname "$nv_installer")
	if ! $nv_installer --no-questions --ui=none --no-kernel-module ; then
		die "[LXC] Failed to run $nv_installer"
	fi

	if ! nvidia-smi ; then
		die "[LXC] Failed to run nvidia-smi"
	fi
}

# Bail early if this isn't being ran as root

[[ "$EUID" -eq 0 ]] || die "$0 must be ran as root - aborting."

#
# Global variables
#

driver_arch=$(uname -m)

if lspci | grep -qE "(Quadro P1000)"; then
	# Legacy device - capped at driver 580.x
	driver_version_match="580\."
	echo "Legacy device detected - filtering available driver version strings with \"${driver_version_match}\"" >&2
else
	driver_version_match=".*"
fi

driver_version=$(curl -s https://raw.githubusercontent.com/keylase/nvidia-patch/master/README.md | grep -E "NVIDIA-Linux-${driver_arch}-${driver_version_match}" | sed -e 's/[()]/^/g' | awk -F '^' '/ \| YES \| YES \| \[Driver link\]\^/ {print $2}' | tail -n1 | cut -d'/' -f6)

if grep -q '^lxc$' /run/systemd/container 2>/dev/null ; then
	is_container="LXC"
	nv_base_dir="/opt/pve-host"
else
	is_container="Host"
	nv_base_dir="/opt/nvidia"
fi

# https://raw.githubusercontent.com/keylase/nvidia-patch/master/README.md
nv_installer="${nv_base_dir}/NVIDIA-Linux-${driver_arch}-${driver_version}.run"
# https://github.com/NVIDIA/nvidia-persistenced
nv_persistenced_installer="${nv_base_dir}/nvidia-persistenced/init/install.sh"
# https://github.com/keylase/nvidia-patch
nv_patch_installer="${nv_base_dir}/nvidia-patch/patch.sh"

cur_driver_ver=$(modinfo -F version nvidia 2>/dev/null) || true

# Empty = not installed, therefore not up to date
if [[ -z "$cur_driver_ver" ]]; then
    nvidia_module_uptodate="false"
else
    # sort -V: lowest version sorts first; if cur >= target, cur appears last (or equal)
    lowest=$(printf '%s\n%s' "$cur_driver_ver" "$driver_version" | sort -V | head -1)
    if [[ "$lowest" == "$driver_version" ]]; then
        nvidia_module_uptodate="true"   # cur_driver_ver >= driver_version
    else
        nvidia_module_uptodate="false"
    fi
fi


#
# main
#

echo "Current system context: $is_container"

fetch_driver_if_not_exists

if [[ "$is_container" == "LXC" ]]; then
	install_lxc
elif [[ "$is_container" == "Host" ]]; then
	# Ensure pre-reqs are present
	ensure_prereqs

	install_host
else
	die "Can not determine if we are a container or host"
fi