Information on my system for troubleshooting ZFS on root

boot-grub-grub.cfg

#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub-mkconfig using templates
# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
insmod part_gpt
insmod part_msdos
if [ -s $prefix/grubenv ]; then
  load_env
fi
if [ "${next_entry}" ] ; then
   set default="${next_entry}"
   set next_entry=
   save_env next_entry
   set boot_once=true
else
   set default="0"
fi

if [ x"${feature_menuentry_id}" = xy ]; then
  menuentry_id_option="--id"
else
  menuentry_id_option=""
fi

export menuentry_id_option

if [ "${prev_saved_entry}" ]; then
  set saved_entry="${prev_saved_entry}"
  save_env saved_entry
  set prev_saved_entry=
  save_env prev_saved_entry
  set boot_once=true
fi

function savedefault {
  if [ -z "${boot_once}" ]; then
    saved_entry="${chosen}"
    save_env saved_entry
  fi
}

function load_video {
  if [ x$feature_all_video_module = xy ]; then
    insmod all_video
  else
    insmod efi_gop
    insmod efi_uga
    insmod ieee1275_fb
    insmod vbe
    insmod vga
    insmod video_bochs
    insmod video_cirrus
  fi
}

if loadfont unicode ; then
  set gfxmode=auto
  load_video
  insmod gfxterm
  set locale_dir=$prefix/locale
  set lang=en_US
  insmod gettext
fi
terminal_input console
terminal_output gfxterm
if [ x$feature_timeout_style = xy ] ; then
  set timeout_style=menu
  set timeout=5
# Fallback normal timeout code in case the timeout_style feature is
# unavailable.
else
  set timeout=5
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/10_linux ###
menuentry 'Arch Linux' --class arch --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-/dev/sda1_/dev/nvme0n1p2' {
	load_video
	set gfxpayload=keep
	insmod gzio
	insmod part_gpt
	insmod part_gpt
	insmod zfs
	set root='hd0,gpt2'
	if [ x$feature_platform_search_hint = xy ]; then
	  search --no-floppy --fs-uuid --set=root  --hint-ieee1275='ieee1275//disk@0,gpt2' --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2  9b7c9e0017403ea5
	else
	  search --no-floppy --fs-uuid --set=root 9b7c9e0017403ea5
	fi
	echo	'Loading Linux linux ...'
	linux	/boot/arch@/vmlinuz-linux root=ZFS=/root/arch rw  loglevel=3 nvme_load=YES
	echo	'Loading initial ramdisk ...'
	initrd	/boot/arch@/intel-ucode.img /boot/arch@/initramfs-linux.img
}
submenu 'Advanced options for Arch Linux' $menuentry_id_option 'gnulinux-advanced-/dev/sda1_/dev/nvme0n1p2' {
	menuentry 'Arch Linux, with Linux linux' --class arch --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-linux-advanced-/dev/sda1_/dev/nvme0n1p2' {
		load_video
		set gfxpayload=keep
		insmod gzio
		insmod part_gpt
		insmod part_gpt
		insmod zfs
		set root='hd0,gpt2'
		if [ x$feature_platform_search_hint = xy ]; then
		  search --no-floppy --fs-uuid --set=root  --hint-ieee1275='ieee1275//disk@0,gpt2' --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2  9b7c9e0017403ea5
		else
		  search --no-floppy --fs-uuid --set=root 9b7c9e0017403ea5
		fi
		echo	'Loading Linux linux ...'
		linux	/boot/arch@/vmlinuz-linux root=ZFS=/root/arch rw  loglevel=3 nvme_load=YES
		echo	'Loading initial ramdisk ...'
		initrd	/boot/arch@/intel-ucode.img /boot/arch@/initramfs-linux.img
	}
	menuentry 'Arch Linux, with Linux linux (fallback initramfs)' --class arch --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-linux-fallback-/dev/sda1_/dev/nvme0n1p2' {
		load_video
		set gfxpayload=keep
		insmod gzio
		insmod part_gpt
		insmod part_gpt
		insmod zfs
		set root='hd0,gpt2'
		if [ x$feature_platform_search_hint = xy ]; then
		  search --no-floppy --fs-uuid --set=root  --hint-ieee1275='ieee1275//disk@0,gpt2' --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2  9b7c9e0017403ea5
		else
		  search --no-floppy --fs-uuid --set=root 9b7c9e0017403ea5
		fi
		echo	'Loading Linux linux ...'
		linux	/boot/arch@/vmlinuz-linux root=ZFS=/root/arch rw  loglevel=3 nvme_load=YES
		echo	'Loading initial ramdisk ...'
		initrd	/boot/arch@/initramfs-linux-fallback.img
	}
}

### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###
### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries.  Simply type the
# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f  ${config_directory}/custom.cfg ]; then
  source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f  $prefix/custom.cfg ]; then
  source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###

df-h

Filesystem             Size  Used Avail Use% Mounted on
nand/root/arch         193G  1.4G  192G   1% /
bpool/boot/arch        350M   61M  289M  18% /boot
rust/home              3.5T  384K  3.5T   1% /home
rust/home/bpotter      3.5T  256K  3.5T   1% /home/bpotter
rust/var/lib/docker    3.5T  256K  3.5T   1% /var/lib/docker
rust/var/lib/libvirt   3.5T  256K  3.5T   1% /var/lib/libvirt
rust/var/lib/machines  3.5T  256K  3.5T   1% /var/lib/machines
rust/var/cache         3.5T  372M  3.5T   1% /var/cache
udev                    31G     0   31G   0% /dev
shm                     31G     0   31G   0% /dev/shm
run                     31G   94M   31G   1% /run
tmp                     31G     0   31G   0% /tmp
airootfs               256M  348K  256M   1% /etc/resolv.conf
/dev/sdh               238M  112M  127M  47% /run/mount/usb

etc-default-grub

# GRUB boot loader configuration

GRUB_DEFAULT=0
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="Arch"
GRUB_CMDLINE_LINUX_DEFAULT="loglevel=3 nvme_load=YES"
GRUB_CMDLINE_LINUX=""

# Preload both GPT and MBR modules so that they are not missed
GRUB_PRELOAD_MODULES="part_gpt part_msdos"

# Uncomment to enable booting from LUKS encrypted devices
#GRUB_ENABLE_CRYPTODISK=y

# Set to 'countdown' or 'hidden' to change timeout behavior,
# press ESC key to display menu.
GRUB_TIMEOUT_STYLE=menu

# Uncomment to use basic console
GRUB_TERMINAL_INPUT=console

# Uncomment to disable graphical terminal
#GRUB_TERMINAL_OUTPUT=console

# The resolution used on graphical terminal
# note that you can use only modes which your graphic card supports via VBE
# you can see them in real GRUB with the command `vbeinfo'
GRUB_GFXMODE=auto

# Uncomment to allow the kernel use the same resolution used by grub
GRUB_GFXPAYLOAD_LINUX=keep

# Uncomment if you want GRUB to pass to the Linux kernel the old parameter
# format "root=/dev/xxx" instead of "root=/dev/disk/by-uuid/xxx"
#GRUB_DISABLE_LINUX_UUID=true

# Uncomment to disable generation of recovery mode menu entries
GRUB_DISABLE_RECOVERY=true

# Uncomment and set to the desired menu colors.  Used by normal and wallpaper
# modes only.  Entries specified as foreground/background.
#GRUB_COLOR_NORMAL="light-blue/black"
#GRUB_COLOR_HIGHLIGHT="light-cyan/blue"

# Uncomment one of them for the gfx desired, a image background or a gfxtheme
#GRUB_BACKGROUND="/path/to/wallpaper"
#GRUB_THEME="/path/to/gfxtheme"

# Uncomment to get a beep at GRUB start
#GRUB_INIT_TUNE="480 440 1"

# Uncomment to make GRUB remember the last selection. This requires
# setting 'GRUB_DEFAULT=saved' above.
#GRUB_SAVEDEFAULT="true"


#menuentry "Arch Linux" {
#  search -u UUID
#  linux /bpool/boot/arch/@/vmlinuz-linux zfs=nand/root/arch rw
#  initrd /bpool/boot/arch/@/initramfs-linux.img
#}

etc-default-zfs

# ZoL userland configuration.

# To enable a boolean setting, set it to yes, on, true, or 1.
# Anything else will be interpreted as unset.

# Run `zfs mount -a` during system start?
ZFS_MOUNT='yes'

# Run `zfs unmount -a` during system stop?
ZFS_UNMOUNT='yes'

# Run `zfs share -a` during system start?
# nb: The shareiscsi, sharenfs, and sharesmb dataset properties.
ZFS_SHARE='yes'

# Run `zfs unshare -a` during system stop?
ZFS_UNSHARE='yes'

# By default, a verbatim import of all pools is performed at boot based on the
# contents of the default zpool cache file.  The contents of the cache are
# managed automatically by the 'zpool import' and 'zpool export' commands.
#
# By setting this to 'yes', the system will instead search all devices for
# pools and attempt to import them all at boot, even those that have been
# exported.  Under this mode, the search path can be controlled by the
# ZPOOL_IMPORT_PATH variable and a list of pools that should not be imported
# can be listed in the ZFS_POOL_EXCEPTIONS variable.
#
# Note that importing all visible pools may include pools that you don't
# expect, such as those on removable devices and SANs, and those pools may
# proceed to mount themselves in places you do not want them to.  The results
# can be unpredictable and possibly dangerous.  Only enable this option if you
# understand this risk and have complete physical control over your system and
# SAN to prevent the insertion of malicious pools.
ZPOOL_IMPORT_ALL_VISIBLE='yes'

# Specify specific path(s) to look for device nodes and/or links for the
# pool import(s). See zpool(8) for more information about this variable.
# It supersedes the old USE_DISK_BY_ID which indicated that it would only
# try '/dev/disk/by-id'.
# The old variable will still work in the code, but is deprecated.
#ZPOOL_IMPORT_PATH="/dev/disk/by-vdev:/dev/disk/by-id"

# List of pools that should NOT be imported at boot
# when ZPOOL_IMPORT_ALL_VISIBLE is 'yes'.
# This is a space separated list.
#ZFS_POOL_EXCEPTIONS="test2"

# List of pools that SHOULD be imported at boot by the initramfs
# instead of trying to import all available pools.  If this is set
# then ZFS_POOL_EXCEPTIONS is ignored.
# Only applicable for Debian GNU/Linux {dkms,initramfs}.
# This is a semi-colon separated list.
#ZFS_POOL_IMPORT="pool1;pool2"

# Should the datasets be mounted verbosely?
# A mount counter will be used when mounting if set to 'yes'.
VERBOSE_MOUNT='no'

# Should we allow overlay mounts?
# This is standard in Linux, but not ZFS which comes from Solaris where this
# is not allowed).
DO_OVERLAY_MOUNTS='no'

# Any additional option to the 'zfs import' commandline?
# Include '-o' for each option wanted.
# You don't need to put '-f' in here, unless you want it ALL the time.
# Using the option 'zfsforce=1' on the grub/kernel command line will
# do the same, but on a case-to-case basis.
ZPOOL_IMPORT_OPTS="-l"

# Full path to the ZFS cache file?
# See "cachefile" in zpool(8).
# The default is "/etc/zfs/zpool.cache".
#ZPOOL_CACHE="/etc/zfs/zpool.cache"
#
# Setting ZPOOL_CACHE to an empty string ('') AND setting ZPOOL_IMPORT_OPTS to
# "-c /etc/zfs/zpool.cache" will _enforce_ the use of a cache file.
# This is needed in some cases (extreme amounts of VDEVs, multipath etc).
# Generally, the use of a cache file is usually not recommended on Linux
# because it sometimes is more trouble than it's worth (laptops with external
# devices or when/if device nodes changes names).
#ZPOOL_IMPORT_OPTS="-c /etc/zfs/zpool.cache"
#ZPOOL_CACHE=""

# Any additional option to the 'zfs mount' command line?
# Include '-o' for each option wanted.
MOUNT_EXTRA_OPTIONS=""

# Build kernel modules with the --enable-debug switch?
# Only applicable for Debian GNU/Linux {dkms,initramfs}.
ZFS_DKMS_ENABLE_DEBUG='no'

# Keep debugging symbols in kernel modules?
# Only applicable for Debian GNU/Linux {dkms,initramfs}.
ZFS_DKMS_DISABLE_STRIP='no'

# Wait for this many seconds in the initrd pre_mountroot?
# This delays startup and should be '0' on most systems.
# Only applicable for Debian GNU/Linux {dkms,initramfs}.
ZFS_INITRD_PRE_MOUNTROOT_SLEEP='0'

# Wait for this many seconds in the initrd mountroot?
# This delays startup and should be '0' on most systems. This might help on
# systems which have their ZFS root on a USB disk that takes just a little
# longer to be available
# Only applicable for Debian GNU/Linux {dkms,initramfs}.
ZFS_INITRD_POST_MODPROBE_SLEEP='0'

# List of additional datasets to mount after the root dataset is mounted?
#
# The init script will use the mountpoint specified in the 'mountpoint'
# property value in the dataset to determine where it should be mounted.
#
# This is a space separated list, and will be mounted in the order specified,
# so if one filesystem depends on a previous mountpoint, make sure to put
# them in the right order.
#
# It is not necessary to add filesystems below the root fs here. It is
# taken care of by the initrd script automatically. These are only for
# additional filesystems needed. Such as /opt, /usr/local which is not
# located under the root fs.
# Example: If root FS is 'rpool/ROOT/rootfs', this would make sense.
#ZFS_INITRD_ADDITIONAL_DATASETS="rpool/ROOT/usr rpool/ROOT/var"

# Optional arguments for the ZFS Event Daemon (ZED).
# See zed(8) for more information on available options.
#ZED_ARGS="-M"

etc-mkinitcpio.conf

# vim:set ft=sh
# MODULES
# The following modules are loaded before any boot hooks are
# run.  Advanced users may wish to specify all system modules
# in this array.  For instance:
#     MODULES=(piix ide_disk reiserfs)
MODULES=(zfs nvme_core)

# BINARIES
# This setting includes any additional binaries a given user may
# wish into the CPIO image.  This is run last, so it may be used to
# override the actual binaries included by a given hook
# BINARIES are dependency parsed, so you may safely ignore libraries
BINARIES=()

# FILES
# This setting is similar to BINARIES above, however, files are added
# as-is and are not parsed in any way.  This is useful for config files.
FILES=()

# HOOKS
# This is the most important setting in this file.  The HOOKS control the
# modules and scripts added to the image, and what happens at boot time.
# Order is important, and it is recommended that you do not change the
# order in which HOOKS are added.  Run 'mkinitcpio -H <hook name>' for
# help on a given hook.
# 'base' is _required_ unless you know precisely what you are doing.
# 'udev' is _required_ in order to automatically load modules
# 'filesystems' is _required_ unless you specify your fs modules in MODULES
# Examples:
##   This setup specifies all modules in the MODULES setting above.
##   No raid, lvm2, or encrypted root is needed.
#    HOOKS=(base)
#
##   This setup will autodetect all modules for your system and should
##   work as a sane default
#    HOOKS=(base udev autodetect block filesystems)
#
##   This setup will generate a 'full' image which supports most systems.
##   No autodetection is done.
#    HOOKS=(base udev block filesystems)
#
##   This setup assembles a pata mdadm array with an encrypted root FS.
##   Note: See 'mkinitcpio -H mdadm' for more information on raid devices.
#    HOOKS=(base udev block mdadm encrypt filesystems)
#
##   This setup loads an lvm2 volume group on a usb device.
#    HOOKS=(base udev block lvm2 filesystems)
#
##   NOTE: If you have /usr on a separate partition, you MUST include the
#    usr, fsck and shutdown hooks.
HOOKS=(base udev autodetect modconf block keyboard zfs filesystems)

# COMPRESSION
# Use this to compress the initramfs image. By default, gzip compression
# is used. Use 'cat' to create an uncompressed image.
#COMPRESSION="gzip"
#COMPRESSION="bzip2"
#COMPRESSION="lzma"
#COMPRESSION="xz"
#COMPRESSION="lzop"
#COMPRESSION="lz4"

# COMPRESSION_OPTIONS
# Additional options for the compressor
#COMPRESSION_OPTIONS=()

fdisk-list

Disk /dev/sda: 223.58 GiB, 240057409536 bytes, 468862128 sectors
Disk model: INTEL SSDSC2CW24
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: gpt
Disk identifier: 6DAB9E38-305B-4673-81B5-D15E16E289BD

Device         Start       End   Sectors  Size Type
/dev/sda1       2048 419432447 419430400  200G Solaris /usr & Apple ZFS
/dev/sda2  419432448 420456447   1024000  500M Solaris /usr & Apple ZFS


Disk /dev/sdb: 1.84 TiB, 2000398934016 bytes, 3907029168 sectors
Disk model: ST2000DM001-1ER1
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 4096 bytes / 4096 bytes
Disklabel type: gpt
Disk identifier: 86066D5F-FCED-2F47-8970-8F5E5FA9FCBC

Device          Start        End    Sectors  Size Type
/dev/sdb1        2048 3907012607 3907010560  1.8T Solaris /usr & Apple ZFS
/dev/sdb9  3907012608 3907028991      16384    8M Solaris reserved 1


Disk /dev/sdc: 1.84 TiB, 2000398934016 bytes, 3907029168 sectors
Disk model: ST2000DM001-1ER1
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 4096 bytes / 4096 bytes
Disklabel type: gpt
Disk identifier: D6F7F232-A5E5-584C-8E66-A4CBC62E0DD0

Device          Start        End    Sectors  Size Type
/dev/sdc1        2048 3907012607 3907010560  1.8T Solaris /usr & Apple ZFS
/dev/sdc9  3907012608 3907028991      16384    8M Solaris reserved 1




Disk /dev/sde: 1.84 TiB, 2000398934016 bytes, 3907029168 sectors
Disk model: ST2000DM001-1ER1
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 4096 bytes / 4096 bytes
Disklabel type: gpt
Disk identifier: BA51D0DF-6B5F-6D45-8FEF-DEE040314825

Device          Start        End    Sectors  Size Type
/dev/sde1        2048 3907012607 3907010560  1.8T Solaris /usr & Apple ZFS
/dev/sde9  3907012608 3907028991      16384    8M Solaris reserved 1


Disk /dev/sdd: 1.84 TiB, 2000398934016 bytes, 3907029168 sectors
Disk model: ST2000DM001-1ER1
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 4096 bytes / 4096 bytes
Disklabel type: gpt
Disk identifier: E57A962B-5658-A346-8D51-F5E02934835D

Device          Start        End    Sectors  Size Type
/dev/sdd1        2048 3907012607 3907010560  1.8T Solaris /usr & Apple ZFS
/dev/sdd9  3907012608 3907028991      16384    8M Solaris reserved 1


Disk /dev/sdf: 14.66 GiB, 15728640000 bytes, 30720000 sectors
Disk model: USB DISK        
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x18730557

Device     Boot Start     End Sectors  Size Id Type
/dev/sdf1  *        0 1820671 1820672  889M  0 Empty
/dev/sdf2         164  131235  131072   64M ef EFI (FAT-12/16/32)


Disk /dev/loop0: 775.29 MiB, 812945408 bytes, 1587784 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes


Disk /dev/nvme0n1: 953.89 GiB, 1024209543168 bytes, 2000409264 sectors
Disk model: SSDPEKKF010T8 NVMe INTEL 1024GB         
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: gpt
Disk identifier: 5AC90A49-F29E-4A85-8002-59C1A8C5F8DC

Device              Start        End   Sectors  Size Type
/dev/nvme0n1p1       2048    1026047   1024000  500M EFI System
/dev/nvme0n1p2    1026048  420456447 419430400  200G Solaris /usr & Apple ZFS
/dev/nvme0n1p3  420456448  424650751   4194304    2G Solaris /usr & Apple ZFS
/dev/nvme0n1p4  424650752 1001367551 576716800  275G Solaris /usr & Apple ZFS
/dev/nvme0n1p5 1001367552 1525655551 524288000  250G Linux filesystem
/dev/nvme0n1p6 1525655552 1526679551   1024000  500M EFI System


Disk /dev/sdh: 238 MiB, 249561088 bytes, 487424 sectors
Disk model: USB DISK 12X    
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x00000000

zfs-list

NAME                    USED  AVAIL     REFER  MOUNTPOINT
bpool                  62.5M   289M      192K  none
bpool/boot             60.8M   289M      192K  none
bpool/boot/arch        60.6M   289M     60.6M  legacy
nand                   1.36G   191G      320K  none
nand/root              1.36G   191G      320K  none
nand/root/arch         1.36G   191G     1.36G  legacy
rust                    376M  3.41T      256K  none
rust/home               523K  3.41T      267K  legacy
rust/home/bpotter       256K  3.41T      256K  legacy
rust/var                373M  3.41T      256K  none
rust/var/cache          372M  3.41T      372M  legacy
rust/var/lib           1023K  3.41T      256K  none
rust/var/lib/docker     256K  3.41T      256K  legacy
rust/var/lib/libvirt    256K  3.41T      256K  legacy
rust/var/lib/machines   256K  3.41T      256K  legacy

zpool-list

NAME    SIZE  ALLOC   FREE  CKPOINT  EXPANDSZ   FRAG    CAP  DEDUP    HEALTH  ALTROOT
bpool   480M    63M   417M        -         -     5%    13%  1.00x    ONLINE  -
nand    199G  1.36G   198G        -         -     0%     0%  1.00x    ONLINE  -
rust   7.27T   777M  7.26T        -         -     0%     0%  1.00x    ONLINE  -