readonly root via overlayfs

etc_initramfs-tools_hooks_overlay

#!/bin/sh
# goes in /etc/initramfs-tools/hooks/overlay

# Ensure that overlayfs is loaded.

PREREQ=""
prereqs()
{
    echo "$PREREQ"
}

case $1 in
    prereqs)
        prereqs
        exit 0
        ;;
esac

. /usr/share/initramfs-tools/hook-functions

manual_add_modules overlay

etc_initramfs-tools_scripts_init-bottom_overlay

#!/bin/sh
# goes in /etc/initramfs-tools/scripts/init-bottom/overlay

PREREQ=""
prereqs()
{
    echo "$PREREQ"
}

case $1 in
    prereqs)
        prereqs
        exit 0
        ;;
esac

. /scripts/functions

RW_SIZE=256m

# Should we run at all?

OVERLAY="no"
for arg in $(cat /proc/cmdline)
do
    case "$arg" in
        overlay=*)
            OVERLAY=${arg#overlay=}
            ;;
        *)
            ;;
    esac
done

if [ "$OVERLAY" != "yes" ]
then
    log_warning_msg "overlay not enabled, leaving the root fs alone"
    exit 0
fi

log_begin_msg "Remounting root fs in overlay mode"

if ! modprobe -qb overlay
then
    log_failure_msg "can't load overlay module"
    exit 0
fi

# initially we have:

#  /           initramfs
#  ${rootmnt}  readonly-mounted real root FS  (mounted from device ${ROOT})
#
# We want:
#
#  /              initramfs
#  ${rootmnt}     overlayfs using ${rootmnt}/ro and ${rootmnt}/rw
#  ${rootmnt}/ro  readonly-mounted real root FS  (mounted from device ${ROOT})
#  ${rootmnt}/rw  tmpfs that will be used for changes
#  ${rootmnt}/rw/upper  overlayfs upper dir
#  ${rootmnt}/rw/work   overlayfs work dir
#

# create and mount /ro and /rw on the initramfs, we will move them later
for dir in /ro /rw
do
  [ -d "${dir}" ] || mkdir -p "${dir}"
  if [ $? -ne 0 ]
  then
    log_failure_msg "can't create ${dir}"
    exit 0
  fi
done

# set up /rw
if ! mount -t tmpfs -o "size=${RW_SIZE}" overlay-rw /rw
then
    log_failure_msg "can't mount tmpfs on /rw"
    exit 0
fi

for dir in /rw/upper /rw/work
do
  if ! mkdir -p ${dir}
  then
      log_failure_msg "overlay: can't create ${dir}"
      exit 0
  fi
done

# set up /ro
if ! mount -o move "${rootmnt}" /ro
then
    log_failure_msg "can't move root fs to /ro"
    exit 0
fi

# set up an overlayfs on rootmnt
if ! mount -t overlay -o lowerdir=/ro,upperdir=/rw/upper,workdir=/rw/work overlay-root "${rootmnt}"
then
    log_failure_msg "can't move root fs to /ro"

    # try to recover
    if ! mount -o move /ro "${rootmnt}"
    then
        panic "recovering the old root fs failed, panicking"
        exit 0
    fi

    log_warning_msg "overlay: moved regular rootfs back into place and continuing"
    exit 0
fi

for dir in "${rootmnt}/ro" "${rootmnt}/rw"
do
  [ -d "${dir}" ] || mkdir -p "${dir}"
  if [ $? -ne 0 ]
  then
    log_failure_msg "can't create ${dir}"
    exit 0
  fi
done

# move /ro and /rw into place so we can access them later
if ! mount -o move /ro "${rootmnt}/ro"
then
    log_failure_msg "can't move /ro to ${rootmnt}/ro"
    exit 0
fi

if ! mount -o move /rw "${rootmnt}/rw"
then
    log_failure_msg "can't move /rw to ${rootmnt}/rw"
    exit 0
fi

# populate fstab on the new root

{
    cat <<EOF
# overlayfs mount is in use; changes to the root FS are
# written to tmpfs and will be discarded on reboot.
#
# to disable, pass "overlay=no" on the kernel command line
# (in /boot/cmdline.txt)
#
# the underlying root FS is mounted readonly on /ro
# temporary changes are written to /rw

EOF

    {
        while read dev dir fstype fsopts freq pass
        do
            if [ "${dir}" = "${rootmnt}" ]
            then
                echo "${dev} / ${fstype} ${fsopts} ${freq} ${pass}"
            fi
        done
    } </proc/mounts

    echo "# original fstab follows"

    {
        while read dev dir fstype fsopts freq pass
        do
            if [ "${dir}" != "/" ]
            then
                echo "${dev} ${dir} ${fstype} ${fsopts} ${freq} ${pass}"
            fi
        done
    } <"${rootmnt}/ro/etc/fstab"
} >"${rootmnt}/etc/fstab"

log_end_msg
exit 0

OVERLAYFS

- install the two shellscripts into the appropriate places under /etc/initramfs-tools
- run update-initramfs
- put "overlay=yes" on the kernel command line
- reboot

With the overlay in place, the real root is mounted readonly on /ro.
Only the root fs is changed, other filesystems are mounted normally.

Remove "overlay=yes" (or change it to something other than yes) and reboot to go back to readwrite.
(This probably means that you want the commandline config to live somewhere other than on the root fs, e.g. under /boot)

@mutability I tried this on Debian 11 and it didn't seem to work. The steps were as follows:

1. Add the above files to their respective locations and ensure they are executable
2. Run: update-initramfs -u to update initramfs
3. Edit the following in /etc/default/grub: GRUB_CMDLINE_LINUX="overlay=yes"
4. Run update-grub to update the grub bootloader

After rebooting, I try creating a file and then reboot again to see if it is still there and it is (i.e. it's not working). The following is what mounts are listed:

root@ipcast:~# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=965616k,nr_inodes=241404,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,noexec,relatime,size=196488k,mode=755)
/dev/sda1 on / type ext4 (rw,relatime,errors=remount-ro)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
cgroup2 on /sys/fs/cgroup type cgroup2 (rw,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
none on /sys/fs/bpf type bpf (rw,nosuid,nodev,noexec,relatime,mode=700)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=30,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=10736)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,pagesize=2M)
mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,nosuid,nodev,noexec,relatime)
tracefs on /sys/kernel/tracing type tracefs (rw,nosuid,nodev,noexec,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,nosuid,nodev,noexec,relatime)
configfs on /sys/kernel/config type configfs (rw,nosuid,nodev,noexec,relatime)
/dev/sda3 on /mnt/un type ext4 (rw,relatime)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=196484k,nr_inodes=49121,mode=700)

@matthewknill this was a one-off that I haven't touched since -- unfortunately you'll need to diagnose this one yourself.

The solution is to not have the /overlay subfolder and ensure that both the hook and script have 755 permissions set.