HOWTO: Hetzner + FreeBSD 10.3 + mirrored root ZFS mountpoint optimized for 4K drives

zfs

[root@rescue ~]# kldload zfs
[root@rescue ~]# sysctl kern.geom.label.gptid.enable=0
kern.geom.label.gptid.enable: 1 -> 0
[root@rescue ~]# gpart destroy -F nvd0
nvd0 destroyed
[root@rescue ~]# gpart destroy -F nvd1
nvd1 destroyed
# Getting "gpart: arg0 'nvdN': Invalid argument" is alright as it means that there was no partition table on the disk anyway.

[root@rescue ~]# gpart create -s gpt nvd0
nvd0 created
[root@rescue ~]# gpart create -s gpt nvd1
nvd1 created

# Set minimum sector size. Default value is 9 which represents 512-bytes sector
# (2^9 = 512) while 12 represents 4096-bytes sector (2^12 = 4096).
# Before setting anything, make sure your disk can use 4096-bytes sectors and announces that to the OS. In order to get information about your disks you can use either diskinfo or camcontrol:
[root@rescue ~]# diskinfo -v nvd0 | grep "sectorsize"
	512         	# sectorsize
[root@rescue ~]# diskinfo -v nvd1 | grep "sectorsize"
	512         	# sectorsize
[root@rescue ~]# camcontrol identify ada0 | grep "sector size"
sector size           logical 512, physical 512, offset 0
[root@rescue ~]# camcontrol identify ada1 | grep "sector size"
sector size           logical 512, physical 512, offset 0

# If physical shows 4096, you are good to go with next lines. Otherwise, go to
# next part as there is no point in forcing gpart and ZFS to set 4k
# as you don’t benefit anything anyway.

# NOTE: SSD disks are 4k from the beginning so if you use SSD disks, you better force it.

# If your disks do use 4K blocks, these lines are for you:
[root@rescue ~]# sysctl vfs.zfs.min_auto_ashift=12
vfs.zfs.min_auto_ashift: 9 -> 12

# Boot partition for ZFS on UEFI.
[root@rescue ~]# gpart add -a 4k -s 800K -t efi nvd0
nvd0p1 added
[root@rescue ~]# gpart add -a 4k -s 800K -t efi nvd1
nvd1p1 added

# Boot partition for ZFS on legacy. Start from 4096 block of the disk.
[root@rescue ~]# gpart add -a 4k -b 4096 -s 512K -t freebsd-boot nvd0
nvd0p2 added
[root@rescue ~]# gpart add -a 4k -b 4096 -s 512K -t freebsd-boot nvd1
nvd1p2 added

# FreeBSD boot partition. Start from 8192 block of the disk.
[root@rescue ~]# gpart add -a 4k -b 8192 -s 4G -t freebsd-zfs -l zboot0 nvd0
nvd0p3 added
[root@rescue ~]# gpart add -a 4k -b 8192 -s 4G -t freebsd-zfs -l zboot1 nvd1
nvd1p3 added

# Last partition for everything else.
[root@rescue ~]# gpart add -a 4k -t freebsd-zfs -l zroot0 nvd0
nvd0p4 added
[root@rescue ~]# gpart add -a 4k -t freebsd-zfs -l zroot1 nvd1
nvd1p4 added

# If your disks don’t use 4K blocks, these lines are for you:
[root@rescue ~]# gpart add -s 800K -t efi nvd0
nvd0p1 added
[root@rescue ~]# gpart add -s 800K -t efi nvd1
nvd1p1 added
[root@rescue ~]# gpart add -b 2048 -s 512K -t freebsd-boot nvd0
nvd0p2 added
[root@rescue ~]# gpart add -b 2048 -s 512K -t freebsd-boot nvd1
nvd1p2 added
[root@rescue ~]# gpart add -b 4096 -s 4G -t freebsd-zfs -l zboot0 nvd0
nvd0p3 added
[root@rescue ~]# gpart add -b 4096 -s 4G -t freebsd-zfs -l zboot1 nvd1
nvd1p3 added
[root@rescue ~]# gpart add -t freebsd-zfs -l zroot0 nvd0
nvd0p4 added
[root@rescue ~]# gpart add -t freebsd-zfs -l zroot1 nvd1
nvd1p4 added

# If you want to check how disk was partitioned, you can check that with gpart show:
[root@rescue ~]# gpart show nvd0 nvd1
=>        34  1000215149  nvd0  GPT  (477G)
          34           6        - free -  (3.0K)
          40        1600     1  efi  (800K)
        1640        2456        - free -  (1.2M)
        4096        1024     2  freebsd-boot  (512K)
        5120        3072        - free -  (1.5M)
        8192     8388608     3  freebsd-zfs  (4.0G)
     8396800   991818376     4  freebsd-zfs  (473G)
  1000215176           7        - free -  (3.5K)

=>        34  1000215149  nvd1  GPT  (477G)
          34           6        - free -  (3.0K)
          40        1600     1  efi  (800K)
        1640        2456        - free -  (1.2M)
        4096        1024     2  freebsd-boot  (512K)
        5120        3072        - free -  (1.5M)
        8192     8388608     3  freebsd-zfs  (4.0G)
     8396800   991818376     4  freebsd-zfs  (473G)
  1000215176           7        - free -  (3.5K)

# Next steps are for whenever you set 4K, or not.

# For ZFS on UEFI.
[root@rescue ~]# dd if=/boot/boot1.efifat of=/dev/nvd0p1 
1600+0 records in
1600+0 records out
819200 bytes transferred in 0.029730 secs (27554583 bytes/sec)
[root@rescue ~]# dd if=/boot/boot1.efifat of=/dev/nvd1p1 
1600+0 records in
1600+0 records out
819200 bytes transferred in 0.029721 secs (27562983 bytes/sec)

# For ZFS on legacy.
[root@rescue ~]# gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 2 nvd0
bootcode written to nvd0
[root@rescue ~]# gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 2 nvd1
bootcode written to nvd1

[root@rescue ~]# mdconfig -a -t malloc -s 128m -u 2
[root@rescue ~]# newfs -O2 /dev/md2
/dev/md2: 128.0MB (262144 sectors) block size 32768, fragment size 4096
	using 4 cylinder groups of 32.03MB, 1025 blks, 4224 inodes.
super-block backups (for fsck_ffs -b #) at:
 192, 65792, 131392, 196992
[root@rescue ~]# mount /dev/md2 /boot/zfs
[root@rescue ~]# zpool create -m none zboot mirror gpt/zboot0 gpt/zboot1
[root@rescue ~]# zpool create -m none zroot mirror gpt/zroot0 gpt/zroot1

# If you want to check if it is 4K (2^12=4096) and not 512 (2^9=512) just run:
[root@rescue ~]# zdb | grep ashift
		  ashift: 12
		  ashift: 12

# Disable mounting zroot and zboot pool themselves. We are going to create and use zroot/root and zboot/boot as dedicated datesets.
[root@rescue ~]# zfs set canmount=off zroot
[root@rescue ~]# zfs set canmount=off zboot

# Set checksum mechanism to use fletcher4. If you are paranoid, you can change it to sha256. I don't like being paranoid.
[root@rescue ~]# zfs set checksum=fletcher4 zroot
[root@rescue ~]# zfs set checksum=fletcher4 zboot

# Turn off writing a metadata change every time a file is accessed to increase performance.
[root@rescue ~]# zfs set atime=off zroot
[root@rescue ~]# zfs set atime=off zboot

[root@rescue ~]# zfs create                                               -o mountpoint=/boot/zfs/zroot zroot/root
[root@rescue ~]# zfs create                                               -o mountpoint=/boot/zfs/zroot/zboot zboot/boot
[root@rescue ~]# zfs create -o compression=on   -o exec=on  -o setuid=off -o mountpoint=/boot/zfs/zroot/tmp zroot/tmp
[root@rescue ~]# zfs create                                               -o mountpoint=/boot/zfs/zroot/home zroot/home
[root@rescue ~]# zfs create                                               -o mountpoint=/boot/zfs/zroot/usr zroot/usr
[root@rescue ~]# zfs create                                               -o mountpoint=/boot/zfs/zroot/usr/local zroot/usr/local
[root@rescue ~]# zfs create -o compression=lz4              -o setuid=off -o mountpoint=/boot/zfs/zroot/usr/ports zroot/usr/ports
[root@rescue ~]# zfs create -o compression=off  -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/usr/ports/distfiles zroot/usr/ports/distfiles
[root@rescue ~]# zfs create -o compression=off  -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/usr/ports/packages zroot/usr/ports/packages
[root@rescue ~]# zfs create -o compression=lz4  -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/usr/src zroot/usr/src
[root@rescue ~]# zfs create -o reservation=10G                            -o mountpoint=/boot/zfs/zroot/var zroot/var
[root@rescue ~]# zfs create -o compression=lz4  -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/var/crash zroot/var/crash
[root@rescue ~]# zfs create                     -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/var/db zroot/var/db
[root@rescue ~]# zfs create -o compression=lz4  -o exec=on  -o setuid=off -o mountpoint=/boot/zfs/zroot/var/db/pkg zroot/var/db/pkg
[root@rescue ~]# zfs create                     -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/var/empty zroot/var/empty
[root@rescue ~]# zfs create -o compression=lz4  -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/var/log zroot/var/log
[root@rescue ~]# zfs create -o compression=gzip -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/var/mail zroot/var/mail
[root@rescue ~]# zfs create                     -o exec=off -o setuid=off -o mountpoint=/boot/zfs/zroot/var/run zroot/var/run
[root@rescue ~]# zfs create -o compression=lz4  -o exec=on  -o setuid=off -o mountpoint=/boot/zfs/zroot/var/tmp zroot/var/tmp

[root@rescue ~]# chmod 1777 /boot/zfs/zroot/tmp
[root@rescue ~]# chmod 1777 /boot/zfs/zroot/var/tmp

# Install FreeBSD.
[root@rescue ~]# cd /boot/zfs/zroot/
[root@rescue /boot/zfs/zroot]# fetch ftp://ftp.freebsd.org/pub/FreeBSD/releases/ISO-IMAGES/10.3/FreeBSD-10.3-RELEASE-amd64-disc1.iso
FreeBSD-10.3-RELEASE-amd64-disc1.iso          100% of  696 MB   20 MBps 00m32s
[root@rescue /boot/zfs/zroot]# mount -t cd9660 /dev/`mdconfig -f FreeBSD-10.3-RELEASE-amd64-disc1.iso` /mnt
[root@rescue /boot/zfs/zroot]# unxz -c /mnt/usr/freebsd-dist/base.txz   | tar xpf -
[root@rescue /boot/zfs/zroot]# unxz -c /mnt/usr/freebsd-dist/kernel.txz | tar xpf -
[root@rescue /boot/zfs/zroot]# unxz -c /mnt/usr/freebsd-dist/lib32.txz  | tar xpf -
[root@rescue /boot/zfs/zroot]# unxz -c /mnt/usr/freebsd-dist/ports.txz  | tar xpf -
[root@rescue /boot/zfs/zroot]# unxz -c /mnt/usr/freebsd-dist/src.txz    | tar xpf -
[root@rescue /boot/zfs/zroot]# umount /mnt
[root@rescue /boot/zfs/zroot]# rm FreeBSD-10.3-RELEASE-amd64-disc1.iso
[root@rescue /boot/zfs/zroot]# zfs set readonly=on zroot/var/empty
[root@rescue /boot/zfs/zroot]# chroot /boot/zfs/zroot /bin/csh
root@rescue:/ # mv boot zboot/
root@rescue:/ # ln -fs zboot/boot
root@rescue:/ # echo 'hostname="thrall.heimdall.pl"' > /etc/rc.conf
root@rescue:/ # echo 'ifconfig_em0="DHCP"' >> /etc/rc.conf
root@rescue:/ # echo 'ifconfig_em0_ipv6="inet6 accept_rtadv"' >> /etc/rc.conf
root@rescue:/ # echo 'zfs_enable="YES"' >> /etc/rc.conf
root@rescue:/ # echo 'sshd_enable="YES"' >> /etc/rc.conf
root@rescue:/ # echo 'dumpdev="AUTO"' >> /etc/rc.conf
root@rescue:/ # touch /etc/fstab
root@rescue:/ # echo 'vfs.zfs.prefetch_disable="1"' > /boot/loader.conf
root@rescue:/ # echo 'vfs.root.mountfrom="zfs:zroot/root"' >> /boot/loader.conf
root@rescue:/ # echo 'zfs_load="YES"' >> /boot/loader.conf

# Turn off ACPI errors.
root@rescue:/ # echo 'debug.acpi.disabled="thermal"' >> /boot/loader.conf

root@rescue:/ # cd /etc/mail
root@rescue:/etc/mail # make aliases
/usr/sbin/sendmail -bi -OAliasFile=/etc/mail/aliases
/etc/mail/aliases: 29 aliases, longest 10 bytes, 297 bytes total
chmod 0640 /etc/mail/aliases.db
root@rescue:/etc/mail # tzsetup
root@rescue:/etc/mail # kbdmap
root@rescue:/etc/mail # passwd root
root@rescue:/etc/mail # adduser
root@rescue:/etc/mail # exit
exit
[root@rescue /boot/zfs/zroot]# cd

# Prepare ZFS before rebooting the system.
[root@rescue ~]# cp /boot/zfs/zpool.cache /boot/zfs/zroot/boot/zfs/zpool.cache
[root@rescue ~]# zfs unmount -af
[root@rescue ~]# zfs set mountpoint=/ zroot/root
[root@rescue ~]# zfs set mountpoint=/zboot zboot/boot
[root@rescue ~]# zfs set mountpoint=/tmp zroot/tmp
[root@rescue ~]# zfs set mountpoint=/home zroot/home
[root@rescue ~]# zfs set mountpoint=/usr zroot/usr
[root@rescue ~]# zfs set mountpoint=/usr/local zroot/usr/local
[root@rescue ~]# zfs set mountpoint=/usr/ports zroot/usr/ports
[root@rescue ~]# zfs set mountpoint=/usr/ports/distfiles zroot/usr/ports/distfiles
[root@rescue ~]# zfs set mountpoint=/usr/ports/packages zroot/usr/ports/packages
[root@rescue ~]# zfs set mountpoint=/usr/src zroot/usr/src
[root@rescue ~]# zfs set mountpoint=/var zroot/var
[root@rescue ~]# zfs set mountpoint=/var/crash zroot/var/crash
[root@rescue ~]# zfs set mountpoint=/var/db zroot/var/db
[root@rescue ~]# zfs set mountpoint=/var/db/pkg zroot/var/db/pkg
[root@rescue ~]# zfs set mountpoint=/var/empty zroot/var/empty
[root@rescue ~]# zfs set mountpoint=/var/log zroot/var/log
[root@rescue ~]# zfs set mountpoint=/var/mail zroot/var/mail
[root@rescue ~]# zfs set mountpoint=/var/run zroot/var/run
[root@rescue ~]# zfs set mountpoint=/var/tmp zroot/var/tmp

# Set booting from zboot/boot dataset.
[root@rescue ~]# zpool set bootfs=zboot/boot zboot

[root@rescue ~]# sync
[root@rescue ~]# shutdown -r now