K3s on Ubuntu 20.04 with root on encrypted ZFS

01-pre-install.sh

#!/bin/sh
# Run as user `ubuntu` from the Ubuntu Desktop installer's live environment.

sudo apt-add-repository universe
sudo apt update
passwd ubuntu
sudo apt install --yes openssh-server screen vim
ip addr show scope global | grep inet

02-install.sh

#!/bin/sh
# Run as user `root` from the Ubuntu Desktop installer's live environment, e.g., via ssh/sudo.

# Change these as appropriate.
read HOSTNAME
read FQDN
IFACE=$(ip link show scope global | grep -v '\(NO-CARRIER\|LOOPBACK\|link/\)' | awk '{print $2}' | sed s/://)

apt install --yes debootstrap gdisk zfs-initramfs
systemctl stop zed

# These IDs correspond to the LU, not the underlying physical disk!
disks=$(ls -l /dev/disk/by-id/scsi* \
            | fgrep sd \
            | grep -v part \
            | sed -e 's/^.*\/dev/\/dev/' \
            | awk '{print $3 " " $1}' \
            | sort \
            | awk '{print $2}')
for disk in $disks; do ls -l $disk; done
echo disks=$((for disk in $disks; do ls -l $disk; done) | wc -l)

# Explicitly force Linux to re-read the partition table as there seems to be a short delay between sgdisk exiting and the entries in /dev updating.
efi_parts=""
bios_parts=""
swap_parts=""
bpool_parts=""
rpool_parts=""
for DISK in $disks; do
    sgdisk --zap-all                 $DISK
    partprobe
    # EFI boot
    sgdisk     -n1:1M:+512M   -t1:EF00 $DISK
    dd if=/dev/zero of=${DISK}-part1 bs=4K count=131072
    efi_parts="${efi_parts} ${DISK}-part1"
    # legacy BIOS boot
    sgdisk -a1 -n5:24K:+1000K -t5:EF02 $DISK
    dd if=/dev/zero of=${DISK}-part5 bs=4K count=250
    bios_parts="${bios_parts} ${DISK}-part5"
    # mirrored swap (can't use zvol b/c it deadlocks)
    sgdisk     -n2:0:+4G      -t2:FD00 $DISK
    dd if=/dev/zero of=${DISK}-part2 bs=4K count=1048576
    swap_parts="${swap_parts} ${DISK}-part2"
    # boot pool
    sgdisk     -n3:0:+2G      -t3:BE00 $DISK
    bpool_parts="${bpool_parts} ${DISK}-part3"
    # root pool (ZFS encryption)
    sgdisk     -n4:0:0        -t4:BF00 $DISK
    rpool_parts="${rpool_parts} ${DISK}-part4"
done
partprobe
sleep 60
for part_list in efi_parts bios_parts swap_parts bpool_parts rpool_parts; do
    ctr=0
    for part in ${!part_list}; do
        ls -l "${part}"
        ctr=$(expr ${ctr} + 1)
    done
    echo "${part_list}=${ctr}"
done

# Create a separate boot pool for /boot with the features limited to only those that GRUB supports, allowing the root pool to use any/all features.
zpool create -f \
    -o ashift=12 -d \
    -o feature@async_destroy=enabled \
    -o feature@bookmarks=enabled \
    -o feature@embedded_data=enabled \
    -o feature@empty_bpobj=enabled \
    -o feature@enabled_txg=enabled \
    -o feature@extensible_dataset=enabled \
    -o feature@filesystem_limits=enabled \
    -o feature@hole_birth=enabled \
    -o feature@large_blocks=enabled \
    -o feature@lz4_compress=enabled \
    -o feature@spacemap_histogram=enabled \
    -o feature@zpool_checkpoint=enabled \
    -O acltype=posixacl -O canmount=off -O compression=lz4 \
    -O devices=off -O normalization=formD -O relatime=on -O xattr=sa \
    -O mountpoint=/boot -R /mnt \
    bpool mirror ${bpool_parts}

# Create the root pool with ZFS encryption.
zpool create -f \
    -o ashift=12 \
    -O encryption=aes-256-gcm \
    -O keylocation=prompt -O keyformat=passphrase \
    -O acltype=posixacl -O canmount=off -O compression=lz4 \
    -O dnodesize=auto -O normalization=formD -O relatime=on \
    -O xattr=sa -O mountpoint=/ -R /mnt \
    rpool raidz2 ${rpool_parts}

zfs create -o canmount=off -o mountpoint=none rpool/ROOT
zfs create -o canmount=off -o mountpoint=none bpool/BOOT

UUID=$(dd if=/dev/urandom of=/dev/stdout bs=1 count=100 2>/dev/null | \
    tr -dc 'a-z0-9' | cut -c-6)

zfs create -o canmount=noauto -o mountpoint=/ \
    -o com.ubuntu.zsys:bootfs=yes \
    -o com.ubuntu.zsys:last-used=$(date +%s) rpool/ROOT/ubuntu_$UUID
zfs mount rpool/ROOT/ubuntu_$UUID

zfs create -o canmount=noauto -o mountpoint=/boot \
    bpool/BOOT/ubuntu_$UUID
zfs mount bpool/BOOT/ubuntu_$UUID

zfs create -o com.ubuntu.zsys:bootfs=no \
    rpool/ROOT/ubuntu_$UUID/srv
zfs create -o com.ubuntu.zsys:bootfs=no -o canmount=off \
    rpool/ROOT/ubuntu_$UUID/usr
zfs create rpool/ROOT/ubuntu_$UUID/usr/local
zfs create -o com.ubuntu.zsys:bootfs=no -o canmount=off \
    rpool/ROOT/ubuntu_$UUID/var
zfs create rpool/ROOT/ubuntu_$UUID/var/games
zfs create rpool/ROOT/ubuntu_$UUID/var/lib
zfs create rpool/ROOT/ubuntu_$UUID/var/lib/AccountsService
zfs create rpool/ROOT/ubuntu_$UUID/var/lib/apt
zfs create rpool/ROOT/ubuntu_$UUID/var/lib/dpkg
zfs create rpool/ROOT/ubuntu_$UUID/var/lib/NetworkManager
zfs create rpool/ROOT/ubuntu_$UUID/var/log
zfs create rpool/ROOT/ubuntu_$UUID/var/mail
zfs create rpool/ROOT/ubuntu_$UUID/var/snap
zfs create rpool/ROOT/ubuntu_$UUID/var/spool
zfs create rpool/ROOT/ubuntu_$UUID/var/www

zfs create -o canmount=off -o mountpoint=/ \
    rpool/USERDATA
zfs create -o com.ubuntu.zsys:bootfs-datasets=rpool/ROOT/ubuntu_$UUID \
    -o canmount=on -o mountpoint=/root \
    rpool/USERDATA/root_$UUID
    
zfs create bpool/BOOT/ubuntu_$UUID/grub

zfs create -o com.ubuntu.zsys:bootfs=no \
    rpool/ROOT/ubuntu_$UUID/tmp
chmod 1777 /mnt/tmp

# Install the Ubuntu base environment.
debootstrap focal /mnt

echo $HOSTNAME > /mnt/etc/hostname
tee -a /mnt/etc/hosts <<EOF
127.0.1.1       ${FQDN} ${HOSTNAME}
EOF

# Change the network configuration as appropriate.
tee /mnt/etc/netplan/01-netcfg.yaml <<EOF
network:
  version: 2
  ethernets:
    ${IFACE}:
      dhcp4: true
EOF

tee /mnt/etc/apt/sources.list <<EOF
deb http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse
deb http://security.ubuntu.com/ubuntu focal-security main restricted universe multiverse
EOF

mount --rbind /dev  /mnt/dev
mount --rbind /proc /mnt/proc
mount --rbind /sys  /mnt/sys
chroot /mnt /usr/bin/env \
    HOSTNAME="${HOSTNAME}" \
    FQDN="${FQDN}" \
    disks="${disks}" \
    efi_parts="${efi_parts}" \
    bios_parts="${bios_parts}" \
    swap_parts="${swap_parts}" \
    bpool_parts="${bpool_parts}" \
    rpool_parts="${rpool_parts}" \
    UUID="${UUID}" \
    bash --login

03-post-install.sh

#!/bin/sh
# Run as user `root` within the chroot environment created by the previous step.

apt update

dpkg-reconfigure locales
dpkg-reconfigure tzdata

apt install --yes vim

apt install --yes dosfstools

# This equires zeroing partitions, else blkid returns the wrong values.
# https://www.reddit.com/r/zfs/comments/alo3vm/ubuntu_zfs_mirrored_boot_with_grub/
efi_part=$(echo ${efi_parts} | cut --delimiter " " --fields 1)
mkdosfs -F 32 -s 1 -n EFI $efi_part
mkdir /boot/efi
echo UUID=$(blkid -s UUID -o value ${efi_part}) \
    /boot/efi vfat umask=0022,fmask=0022,dmask=0022 0 1 >> /etc/fstab
mount /boot/efi

remaining_efi_parts=$(echo ${efi_parts} | cut --delimiter " " --fields 2-)
ctr=2
for efi_part in ${remaining_efi_parts}; do
    mkdosfs -F 32 -s 1 -n EFI $efi_part
    mkdir /boot/efi${ctr}
    echo UUID=$(blkid -s UUID -o value ${efi_part}) \
        /boot/efi${ctr} vfat umask=0022,fmask=0022,dmask=0022 0 1 >> /etc/fstab
    mount /boot/efi${ctr}
    ctr=$(expr ${ctr} + 1)
done

apt install --yes grub-pc linux-image-generic zfs-initramfs zsys
dpkg --purge os-prober

# Set the root password.
passwd

apt install --yes cryptsetup mdadm

# Configure encrypted swap.  Adjust the level (ZFS raidz = MD raid5, raidz2 = raid6) and raid-devices if necessary and specify the actual devices.
mdadm --create /dev/md0 --metadata=1.2 --level=raid6 \
    --raid-devices=$(tr -dc ' ' <<< "${swap_parts}" | wc -c) \
    ${swap_parts}
echo swap /dev/md0 /dev/urandom \
      swap,cipher=aes-xts-plain64:sha256,size=512 >> /etc/crypttab
echo /dev/mapper/swap none swap defaults 0 0 >> /etc/fstab

# Skip tmpfs creation.

addgroup --system lpadmin
addgroup --system lxd
addgroup --system sambashare

sudo apt install --yes curl patch

curl https://launchpadlibrarian.net/478315221/2150-fix-systemd-dependency-loops.patch | \
    sed "s|/etc|/lib|;s|\.in$||" | (cd / ; patch -p1)

grub-probe /boot

update-initramfs -c -k all

vi /etc/default/grub
# Add init_on_alloc=0 to: GRUB_CMDLINE_LINUX_DEFAULT
# Comment out: GRUB_TIMEOUT_STYLE=hidden
# Set: GRUB_TIMEOUT=5
# Below GRUB_TIMEOUT, add: GRUB_RECORDFAIL_TIMEOUT=5
# Remove quiet and splash from: GRUB_CMDLINE_LINUX_DEFAULT
# Uncomment: GRUB_TERMINAL=console
# Save and quit.

update-grub

for DISK in ${disks}; do
    grub-install ${DISK}
done

systemctl mask grub-initrd-fallback.service

mkdir /etc/zfs/zfs-list.cache
touch /etc/zfs/zfs-list.cache/bpool
touch /etc/zfs/zfs-list.cache/rpool
ln -s /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh /etc/zfs/zed.d
zed -F &
zfs set canmount=noauto bpool/BOOT/ubuntu_$UUID
zfs set canmount=noauto rpool/ROOT/ubuntu_$UUID
kill %1

apt install --yes openssh-server

vi /etc/ssh/sshd_config
# Set: PermitRootLogin yes

04-cleanup.sh

#!/bin/bash
# Run as user `root` from the Ubuntu Desktop installer's live environment after exiting the chroot environment.

mount | grep -v zfs | tac | awk '/\/mnt/ {print $3}' | \
    xargs -i{} umount -lf {}
zpool export -a

reboot

05-first-boot.sh

#!/bin/bash
# Run as user `root` from the server run-time environment.

# Change this.
read USERNAME

ROOT_DS=$(zfs list -o name | awk '/ROOT\/ubuntu_/{print $1;exit}')
UUID=$(echo ${ROOT_DS} | sed -e 's/.*_//')

zfs create -o com.ubuntu.zsys:bootfs-datasets=$ROOT_DS \
    -o canmount=on -o mountpoint=/home/${USERNAME} \
    rpool/USERDATA/${USERNAME}_${UUID}
adduser ${USERNAME}

cp -a /etc/skel/. /home/${USERNAME}
chown -R ${USERNAME}:${USERNAME} /home/${USERNAME}
usermod -a -G adm,cdrom,dip,lpadmin,lxd,plugdev,sambashare,sudo ${USERNAME}

# Complete the Ubuntu installation.
apt dist-upgrade --yes
apt install --yes ubuntu-standard

for file in /etc/logrotate.d/* ; do
    if grep -Eq "(^|[^#y])compress" "$file" ; then
        sed -i -r "s/(^|[^#y])(compress)/\1#\2/" "$file"
    fi
done

reboot

06-finalization.sh

#!/bin/bash
# Run as user `root` from the server run-time environment.

usermod -p '*' root

vi /etc/ssh/sshd_config
# Remove: PermitRootLogin yes
systemctl restart ssh

07-k3s.sh

#!/bin/bash
# Run as user `root` from the server run-time environment.

# Change this.
read SECRET

ROOT_DS=$(zfs list -o name | awk '/ROOT\/ubuntu_/{print $1;exit}')
UUID=$(echo ${ROOT_DS} | sed -e 's/.*_//')

zfs create -o canmount=off -o mountpoint=/ \
    rpool/K3S
for mnt in containerd docker dockershim kubelet rancher; do
  zfs create -o com.ubuntu.zsys:bootfs-datasets=${ROOT_DS} \
    -o canmount=on -o mountpoint=/var/lib/${mnt} \
    rpool/K3S/${mnt}_${UUID}
done

curl https://releases.rancher.com/install-docker/19.03.sh | sh
curl -sfL https://get.k3s.io | K3S_TOKEN=${SECRET} sh -s - \
  --cluster-init \
  --docker \
  --secrets-encryption \
  --tls-san cloud.irtnog.net

k3s kubectl get pods --all-namespaces
# This should print something like:
# NAMESPACE     NAME                                     READY   STATUS      RESTARTS   AGE
# kube-system   coredns-66c464876b-8kf69                 1/1     Running     0          111s
# kube-system   local-path-provisioner-7ff9579c6-cgmsx   1/1     Running     0          111s
# kube-system   metrics-server-7b4f8b595-nxx26           1/1     Running     0          111s
# kube-system   helm-install-traefik-4prm2               0/1     Completed   0          111s
# kube-system   svclb-traefik-9qt8p                      2/2     Running     0          86s
# kube-system   traefik-5dd496474-6qg77                  1/1     Running     0          86s

docker ps
# This should print something like:
# CONTAINER ID        IMAGE                            COMMAND                  CREATED             STATUS              PORTS               NAMES
# e2055d0d3ee0        897ce3c5fc8f                     "entry"                  2 minutes ago       Up 2 minutes                            k8s_lb-port-443_svclb-traefik-9qt8p_kube-system_200a9f98-32e5-4679-949b-10b4dc6166d4_0
# 3d7428f5ec20        rancher/klipper-lb               "entry"                  2 minutes ago       Up 2 minutes                            k8s_lb-port-80_svclb-traefik-9qt8p_kube-system_200a9f98-32e5-4679-949b-10b4dc6166d4_0
# 68b2321dd1ad        rancher/library-traefik          "/traefik --configfi…"   2 minutes ago       Up 2 minutes                            k8s_traefik_traefik-5dd496474-6qg77_kube-system_7a8116fb-6ae7-4ef7-8031-d4bd6d3f5d3d_0
# 26c973c7a10e        rancher/pause:3.1                "/pause"                 2 minutes ago       Up 2 minutes                            k8s_POD_traefik-5dd496474-6qg77_kube-system_7a8116fb-6ae7-4ef7-8031-d4bd6d3f5d3d_0
# 1685f5ea28be        rancher/pause:3.1                "/pause"                 2 minutes ago       Up 2 minutes                            k8s_POD_svclb-traefik-9qt8p_kube-system_200a9f98-32e5-4679-949b-10b4dc6166d4_0
# 1bc228a36ad4        rancher/metrics-server           "/metrics-server"        2 minutes ago       Up 2 minutes                            k8s_metrics-server_metrics-server-7b4f8b595-nxx26_kube-system_cc777e4c-3df3-46f6-939c-660d8f7a3b1a_0
# 40a4b83e524b        rancher/local-path-provisioner   "local-path-provisio…"   2 minutes ago       Up 2 minutes                            k8s_local-path-provisioner_local-path-provisioner-7ff9579c6-cgmsx_kube-system_05a99cd3-adf3-4bb9-bdfe-5f902e41a482_0
# 1aa63ab1ba1f        rancher/coredns-coredns          "/coredns -conf /etc…"   2 minutes ago       Up 2 minutes                            k8s_coredns_coredns-66c464876b-8kf69_kube-system_68348ea3-443a-4e59-b93f-e6207589c3ad_0
# 20055605654c        rancher/pause:3.1                "/pause"                 2 minutes ago       Up 2 minutes                            k8s_POD_metrics-server-7b4f8b595-nxx26_kube-system_cc777e4c-3df3-46f6-939c-660d8f7a3b1a_0
# bcc6fae176ad        rancher/pause:3.1                "/pause"                 2 minutes ago       Up 2 minutes                            k8s_POD_coredns-66c464876b-8kf69_kube-system_68348ea3-443a-4e59-b93f-e6207589c3ad_0
# d12dd1e4d517        rancher/pause:3.1                "/pause"                 2 minutes ago       Up 2 minutes                            k8s_POD_local-path-provisioner-7ff9579c6-cgmsx_kube-system_05a99cd3-adf3-4bb9-bdfe-5f902e41a482_0

# Additional reading:
# https://rancher.com/docs/k3s/latest/en/installation/private-registry/
# https://rancher.com/blog/2020/implementing-gitops-kubernetes-k3s-rancher-vault-argocd
# https://rancher.com/blog/2020/take-guesswork-out-of-secure-kubernetes-deployment-rancher-stackrox

# https://rancher.com/docs/k3s/latest/en/helm/
curl https://baltocdn.com/helm/signing.asc | apt-key add -
apt install --yes apt-transport-https
echo "deb https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list
apt update
apt install --yes helm
helm repo add stable https://charts.helm.sh/stable
# Creates files in $HOME; invoke as follows:
# env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm ...

# cf. https://rancher.com/docs/k3s/latest/en/installation/kube-dashboard/
GITHUB_URL=https://github.com/kubernetes/dashboard/releases
VERSION_KUBE_DASHBOARD=$(curl -w '%{url_effective}' -I -L -s -S ${GITHUB_URL}/latest -o /dev/null | sed -e 's|.*/||')
k3s kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/${VERSION_KUBE_DASHBOARD}/aio/deploy/recommended.yaml
cat > dashboard.admin-user.yml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
EOF
cat > dashboard.admin-user-role.yml <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF
k3s kubectl create -f dashboard.admin-user.yml -f dashboard.admin-user-role.yml
k3s kubectl -n kubernetes-dashboard describe secret admin-user-token | grep ^token
k3s kubectl -n kubernetes-dashboard get service kubernetes-dashboard | awk 'FNR == 1 {next} {print $3}'
# TODO: publish to LAN via Traefik?

08-openfaas.sh

#!/bin/sh
# Run as user `root` from the server run-time environment.

# https://github.com/openfaas/faas-netes/tree/master/chart/openfaas#deploy-openfaas
git clone https://github.com/openfaas/faas-netes.git
cd faas-entes
helm template \
  openfaas chart/openfaas/ \
    --namespace openfaas \
    --set basic_auth=true \
    --set operator.create=true \
    --set ingress.enabled=true \
    --set functionNamespace=openfaas-fn > openfaas.yaml
k3s kubectl apply -f namespaces.yml,openfaas.yaml