what i do after i install pve on a new node

pve-post_install.sh

#!/bin/bash


## init pve-node

# rsa is less secure
ssh-keygen -t ed25519 

# add my apt cache
echo 'Acquire::http::Proxy "http://192.168.10.14:3142";' > /etc/apt/apt.conf.d/00aptproxy 

# etckeeper
apt update ; echo "pve/" > /etc/.gitignore ; apt install etckeeper -y

# proxmox post-install script ; y to everything except update and reboot
bash -c "$(wget -qLO - https://github.com/community-scripts/ProxmoxVE/raw/main/misc/post-pve-install.sh)"

# fix sources , intel-microcode
sed -i 's/contrib/contrib non-free non-free-firmware/g' /etc/apt/sources.list
apt install -y intel-microcode

# my config comforts
apt install -y tmux fish powerline
mkdir ~/dl ; cd ~/dl ; git clone https://github.com/wommy/dotfiles.git ; cd ; ln -s dl/dotfiles/.config .
# req for tmux/powerline
sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
echo 'LANG=en_US.UTF-8' > /etc/default/locale && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG=en_US.UTF-8

# do it yourself so you can see logs
apt upgrade -y ; reboot


## zram
# mostly from https://github.com/garyexplains/examples/blob/master/how_to_enable_ZRAM_Raspberry_Pi.md
# but also a lil https://wiki.archlinux.org/title/Zram
apt install -y systemd-zram-generator

lvcreate -L 20G pve -n writeback # figure out perfect size

cat << 'EOF' >> /etc/systemd/zram-generator.conf
zram-size = ram * 2
compression-algorithm = lz4 zstd(level=9)
writeback-device = /dev/pve/writeback
swap-priority = 100
EOF

# 99-vm-zram-parameters.conf
cat << 'EOF' > /etc/sysctl.d/99-vm-zram-parameters.conf
vm.vfs_cache_pressure=500
vm.swappiness=100
vm.dirty_background_ratio=1
vm.dirty_ratio=50
vm.watermark_boost_factor = 0
vm.watermark_scale_factor = 125
vm.page-cluster = 0
EOF

systemctl daemon-reload ; systemctl start [email protected]


## zfs

lvcreate -L 200G pve -n spcl

zpool create -o ashift=12 -o autotrim=on \
	-O acltype=posixacl -O xattr=sa -O dnodesize=auto \
	-O normalization=formD -O relatime=on -O compression=zstd \
	-O sync=disabled -O recordsize=1M -O special_small_blocks=64k \
	rpool /dev/disk/by-id/ata-ST500LM000-1EJ162_W761KD37 \
	special /dev/disk/by-id/dm-name-pve-spcl

zfs create rpool/data


## nvidia

# best guide is currently https://digitalspaceport.com/proxmox-lxc-gpu-passthru-setup-guide/
apt install pve-headers build-essential software-properties-common make nvtop htop -y

mkdir ~/dl/nvidia ; pushd ~/dl/nvidia 
# easier to get drivers here https://github.com/keylase/nvidia-patch?tab=readme-ov-file#version-table
wget http://international.download.nvidia.com/XFree86/Linux-x86_64/570.86.16/NVIDIA-Linux-x86_64-570.86.16.run
NVIDIA_DRIVER=NVIDIA-Linux-x86_64-570.86.16.run
chmod +x NVIDIA_DRIVER && ./NVIDIA_DRIVER --dkms
# LoRes-DIY GPU LXC passthru - https://www.youtube.com/watch?v=-Us8KPOhOCY
# based on theOrangeOne's writeup -  https://theorangeone.net/posts/lxc-nvidia-gpu-passthrough/
cat << 'EOF' > /etc/modules-load.d/nvidia.conf
nvidia
nvidia_uvm
EOF
cat << 'EOF' > /etc/udev/rules.d/70-nvidia.rules
KERNEL=="nvidia", RUN+="/bin/bash -c '/usr/bin/nvidia-smi -L && /bin/chmod 666 /dev/nvidia*'"
KERNEL=="nvidia_uvm", RUN+="/bin/bash -c '/usr/bin/nvidia-modprobe -c0 -u && /bin/chmod 666 /dev/nvidia-uvm*'"
EOF
# kernel params / grub
sed -i 's/quiet/iommu=pt mitigations=off nvidia-drm.modeset=1/g' /etc/default/grub
update-grub ; update-initramfs -u -k all ; reboot


## wip

# lxc docker helper-script
bash -c "$(wget -qLO - https://github.com/community-scripts/ProxmoxVE/raw/main/ct/docker.sh)"

ls -al /dev/nvidia*

for i in 195 234 240
do
  lxc.cgroup2.devices.allow: c $i:* rwm
done

for i in 0 ctl -modeset -uvm -uvm-tools -cap1 -cap2
do
  echo "lxc.mount.entry: /dev/nvidia$i dev/nvidia$i none bind,optional,create=file" >> /etc/pve/lxc/$CT_NUM.conf 
done

echo "lxc.mount.entry: /dev/nvidia-uvm-tools dev/nvidia-caps/nvidia-cap{1,2} none bind,optional,create=file" >> /etc/pve/lxc/$CT_NUM.conf 

pct push $CT_NUM NVIDIA_DRIVER /root/NVIDIA_DRIVER
lxc-attach $CT_NUM
./$NVIDIA_DRIVER --no-kernel-modules

# nvidia container toolkit
apt install gpg curl
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && \
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
apt update ; apt install nvidia-container-toolkit

sed -i 's/#no-cgroups = false/no-cgroups = true/g' /etc/nvidia-container-runtime/config.toml
nvidia-ctk runtime configure --runtime=docker