see https://git.sr.ht/~dch/diy-jails/ for some background

jr.fish

# Defined in /home/dch/.config/fish/functions/jr.fish @ line 1
function jr
	set local
	set JAIL $argv[1]
	set INSTANCE (jls -dj $JAIL path)
	pkill -TERM -j $JAIL
	sleep 3
	doas jail -rv $JAIL

	set MOUNTS (mount -p | grep $INSTANCE/ | cut -wf 2)
	echo removing $MOUNTS...

	set MOUNTS (mount -p | grep $INSTANCE/ | cut -wf 2 | egrep dev/fd\$)
	for node in $MOUNTS
		doas umount -f $node
	end
	set MOUNTS (mount -p | grep $INSTANCE/ | cut -wf 2 | egrep dev\$)
	for node in $MOUNTS
		doas umount -f $node
	end
	set MOUNTS (mount -p | grep $INSTANCE/ | cut -wf 2 | egrep tmp\$)
	for node in $MOUNTS
		doas umount -f $node
	end
end

jz.fish

# Defined in /home/dch/.config/fish/functions/jz.fish @ line 1
function jz
    # see https://git.io/JfHOu
    set local
    set MACH (sysctl -n hw.machine)
    set ARCH (sysctl -n hw.machine_arch)
    set DOMAIN skunkwerks.at
    set RELEASE 15.0-RELEASE
    set RELDATE 1500068
    set FLAVOUR $RELEASE-$MACH-$ARCH
    # grab the name
    set JAIL $argv[1]
    # the container
    set TEMPLATE /jails/templates/$FLAVOUR
    set INSTANCE /jails/instances/$FLAVOUR/$JAIL

    # find the zpool for jails and jailed datasets
    set ZPOOL (zfs list -Ho name /jails | cut -f 1 -d /)

    # create a /jailed dataset to contain optional per-jail jailable datasets
    # this can be located on an alternative zpool but must not be mountable
    zfs list -Ho name -d1 |grep /jailed\$ > /dev/null 2>&1 \
        || doas zfs create -o mountpoint=none -o canmount=off $ZPOOL/jailed
    # re-fetch after possible creation
    set JAILED (zfs list -Ho name -d1 | grep /jailed\$)

    # create mountpoint if missing
    zfs list -Ho name $ZPOOL/jails > /dev/null 2>&1 \
        || doas zfs create -o mountpoint=/jails -p $ZPOOL/jails

    # clone a template if missing
    test -d $INSTANCE \
        || doas zfs clone $ZPOOL$TEMPLATE@ready $ZPOOL$INSTANCE
    test -d $INSTANCE/.zfs/snapshot/ready \
        || doas zfs snapshot $ZPOOL$INSTANCE@ready

    # add a softlink if missing
    ln -sf $INSTANCE /jails/tags/

    # derive a hopefully non-clashing IP address
    set IPA (echo -n $JAIL | sha512 | cut -c 1,2)
    set IPB (echo -n $JAIL | sha512 | cut -c 3,4)
    set IP6 $IPA$IPB
    set IP (printf "%d.%d" 0x$IPA 0x$IPB)
    # get the zt 6plane
    set IFACE (ifconfig | sort -r | /usr/bin/egrep -o '^zt[a-z0-9]+' | head -1)
    set SUBNET (ifconfig $IFACE inet6 | rg -Po 'fc\S+::' | uniq)
    set ALIAS "$IFACE|$SUBNET$IP6"
    # display config and try to start it
    echo name=$JAIL ip=$IP ip6=$IP6 iface=$IFACE subnet=$SUBNET alias=$ALIAS
    jls -j $JAIL > /dev/null 2>&1
    if test $status = 0
        doas jexec $JAIL tmux -u new-session -DAs default
    else
        doas jail -vcmr \
        name=$JAIL \
        path=$INSTANCE \
        exec.created="zfs jail $JAIL $JAILED/$JAIL || true" \
        exec.created="test -c $INSTANCE/dev/null || mount -t devfs devfs $INSTANCE/dev" \
        exec.created="test -c $INSTANCE/tmp || mount -t tmpfs tmpfs $INSTANCE/tmp" \
        exec.start="/bin/sh /etc/rc" \
        exec.poststop="umount $INSTANCE/tmp" \
        exec.poststop="umount $INSTANCE/dev" \
        exec.poststop="logger -p info -t jz removed $JAIL" \
        host.domainname=$DOMAIN \
        host.hostid=0 \
        host.hostname=$JAIL.$DOMAIN \
        host.hostuuid=(uuidgen) \
        ip4.addr="lo1|100.64.$IP" \
        ip6.addr="$ALIAS" \
        allow.chflags \
        allow.mount \
        allow.mount.devfs \
        allow.mount.fdescfs \
        allow.mount.nolinprocfs \
        allow.mount.nullfs \
        allow.mount.procfs \
        allow.mount.tmpfs \
        allow.mount.zfs \
        allow.nomlock \
        allow.noquotas \
        allow.noread_msgbuf \
        allow.noset_hostname \
        allow.nosocket_af \
        allow.nosysvipc \
        allow.raw_sockets \
        allow.reserved_ports \
        allow.sysvipc=1 \
        children.max=0 \
        devfs_ruleset=4 \
        enforce_statfs=1 \
        osreldate=$RELDATE \
        osrelease=$RELEASE \
        persist \
        securelevel=2 \
        sysvmsg=disable \
        sysvsem=disable \
         command=/bin/sh -c "env -i TERM=$TERM tmux -u new-session -DAs default"
    end
    # command=/bin/sh
    # allow.nounprivileged_proc_debug \
    # allow.novmm \
end

Ideas for improved FreeBSD kernel jail support

• how to pass env vars through via jexec? we need an allowlist
• single user id in jail, no exec of root at all aallowed
• way to "map" a uuid from a zfs dataset to a randomly assigned one in a jail
• uuids for jails please, or at least names, and not specify jail id
• create jails by non-privileged user (means we inherit pf rules, zfs delegation, predefined)
• need to mount both devfs & tmpfs in prior to starting jail app
• attach random info to each jail, optionally to be made available inside the jail
• a way to extract current state of all jails (or a subset thereof, more efficiently) to use for administration purposes
• when jails get wedged during upgrades and restarts, most operators end up rebooting to cure it. This shouldn't need manual recovery each time it happens. Whether this is implementing a statemachine representation for jails, or providing a way to resume wedged operations after fixing this, something needs to be improved. Usually when this happens, its not a good time to get diagnostic information, or wait for a response on a mailing list to elucidate the breakage. Perhaps a coredump would be sufficient.

Quick intros

• discuss some current issues with jails from each other
• discuss next steps

My 2c

Infra today

• private dns services & some FreeBSD CI
• anycast + BGP ECMP to haproxy
• plan in 2023 to commercialise (ie finish) both of these
• jails are not dynamic enough for my usage

Wish list

• looking to hook haproxy routing across multiple jail hosts and have dynamic state of active jails available to haproxy for routing decisions

• need metadata to attach to jails

• jail lifecycle mgmt support (jail post scripts don’t get run in some circumstances)

• treat jails as simple packages (with optional zfs support)

• jail JSON API for integration via unix socket

• user space jails please

• kernel enforced single uid jails (non root)

• kernel-enforced UUIDs like flake

• inbuilt lua scripting for state changes

post jails call

• libxo support needed
• improved docs around jail / jail.conf needed
• lot of shared logic (or even code) out there, albeit not intended
• jail tools but not in shell
• what could we add to base (or userland)
• /etc/start_if_interfacename for simplifying config
• rootless jails
• mac framework
• user jails
• attach misc config data via https://man.freebsd.org/osd
• ucl config + live json state