Repro: Incus btrfs loop-backed pool backing file is created without nodatacow (+C), unlike block volumes inside the pool. Measures physical write amplification at the block-device level.

incus-loop-nocow-repro.sh

#!/bin/bash
# incus-loop-nocow-repro.sh
#
# Measures physical write amplification of a loop-backed Incus btrfs storage
# pool whose backing file (/var/lib/incus/disks/<pool>.img) lives on a btrfs
# host filesystem, with and without the nodatacow (+C) attribute on the
# backing file, and with and without host-side snapshots of the subvolume
# holding it (simulating snapper snapshotting the root subvolume).
#
# Also records whether Incus sets +C on:
#   - the pool's own backing file          (/var/lib/incus/disks/<pool>.img)
#   - a block volume created inside a pool (custom/<vol>/root.img)
# for both a loop-backed pool and a pool on a real block device.
#
# Two workloads, each run in five configurations (A-E below):
#   bulk    - 1GiB of 4k random overwrites, fdatasync every 64 writes.
#             Sustained-throughput amplification.
#   trickle - 120 x (one 4k random write + fdatasync + 0.5s pause) into a
#             64MiB file; 480KiB logical total. Models a mostly-idle host
#             (logging, databases) where per-commit fixed costs dominate.
#
# Configurations:
#   A - baseline: write directly to the host btrfs, no Incus in the path
#   B - loop-backed pool, .img exactly as Incus creates it (no +C)
#   C - loop-backed pool, .img with +C (nodatacow)
#   D - as B, plus a read-only snapshot of the subvolume holding the .img
#       every 10s during the workload (simulates snapper)
#   E - C and D combined
#
# REQUIREMENTS / WARNINGS
#   - Run as root on a THROWAWAY machine (e.g. a fresh Debian 13 VM).
#   - incus, btrfs-progs and fio installed; incus daemon running but with
#     NO storage pools defined yet (fresh install, "incus admin init" not run).
#   - TESTDEV is WIPED and becomes the simulated "host filesystem";
#     /var/lib/incus is moved onto a subvolume of it.
#   - BLKDEV (optional) is WIPED and used only for the real-block-device
#     pool attribute check. Pass "" to skip.
#
# Usage: ./incus-loop-nocow-repro.sh [TESTDEV] [BLKDEV]
#        e.g. ./incus-loop-nocow-repro.sh /dev/sdb /dev/sdc

set -eu

TESTDEV=${1:-/dev/sdb}     # wiped; the btrfs we measure writes against
BLKDEV=${2:-/dev/sdc}      # wiped; real-device pool check ("" to skip)
MNT=/mnt/hostfs            # top-level (subvolid=5) mount of TESTDEV
POOL=wearpool              # recreated fresh for each nested case
POOL_SIZE=8GiB
SNAP_INTERVAL=10           # seconds between snapshots in cases D and E

FIO_COMMON="--ioengine=psync --randseed=4242 --randrepeat=1 --end_fsync=1"
FIO_BULK="--rw=randwrite --bs=4k --size=1g --io_size=1g --fdatasync=64 $FIO_COMMON"
FIO_TRICKLE="--rw=randwrite --bs=4k --size=64m --io_size=480k --fdatasync=1 \
  --thinktime=500000 $FIO_COMMON"
PREP_MB_bulk=1024          # file is pre-created before measuring, so only
PREP_MB_trickle=64         # the overwrite phase is counted

DEVNAME=$(basename "$TESTDEV")
RESULTS=/tmp/nocow-results.txt
: > "$RESULTS"

log() { echo "==> $*"; }

# Physical bytes written to TESTDEV: /sys/block/<dev>/stat field 7 is
# "sectors written" (always 512-byte sectors).
bytes_written() { awk '{print $7 * 512}' "/sys/block/$DEVNAME/stat"; }

# Flush everything (guest btrfs -> loop -> host btrfs -> disk) and let the
# counters settle before reading them.
settle() { sync; sleep 5; sync; sleep 5; }

# measure <workload> <config> <target-dir>:
# pre-create the target file (sequential, fsync'd) BEFORE the "before"
# counter is read so only the overwrite phase is measured, then run fio and
# record the device-level write delta.
measure() {
    local wl="$1" cfg="$2" dir="$3" prep_mb fio_args before after delta
    if [ "$wl" = bulk ]; then prep_mb=$PREP_MB_bulk; fio_args=$FIO_BULK
    else prep_mb=$PREP_MB_trickle; fio_args=$FIO_TRICKLE; fi
    dd if=/dev/zero of="$dir/wear.0.0" bs=1M count=$prep_mb conv=fsync status=none
    settle
    before=$(bytes_written)
    fio --name=wear --directory="$dir" $fio_args \
        --output="/tmp/fio_${wl}_${cfg}.log" > /dev/null
    settle
    after=$(bytes_written)
    delta=$((after - before))
    echo "$wl $cfg $delta" >> "$RESULTS"
    log "$wl/$cfg: $delta bytes written to $TESTDEV"
}

make_pool() {
    incus storage create "$POOL" btrfs size=$POOL_SIZE
    log "lsattr on pool backing file:"
    lsattr "/var/lib/incus/disks/$POOL.img"
    incus storage volume create "$POOL" scratch size=4GiB
    # Loop-backed btrfs pools stay mounted after use; the custom volume is a
    # plain subvolume we can write into directly.
    findmnt "/var/lib/incus/storage-pools/$POOL" > /dev/null
}
destroy_pool() {
    incus storage volume delete "$POOL" scratch
    incus storage delete "$POOL"
}

snapshots_start() {
    mkdir -p "$MNT/snaps"
    touch /tmp/snapping
    ( i=0
      while [ -f /tmp/snapping ]; do
          btrfs subvolume snapshot -r "$MNT/@incus" "$MNT/snaps/snap_$i" >/dev/null
          i=$((i+1))
          sleep "$SNAP_INTERVAL"
      done ) &
    SNAP_PID=$!
}
snapshots_stop() {
    rm -f /tmp/snapping
    wait "$SNAP_PID" 2>/dev/null || true
    SNAP_COUNT=$(ls -d "$MNT"/snaps/snap_* | wc -l)
    log "snapshots taken during workload: $SNAP_COUNT"
    for s in "$MNT"/snaps/snap_*; do
        btrfs subvolume delete "$s" > /dev/null
    done
}

# run_config <workload> <config>: full lifecycle for one matrix cell.
run_config() {
    local wl="$1" cfg="$2" dir
    log "$wl/$cfg starting"
    case "$cfg" in
        A)
            mkdir -p "$MNT/baseline"
            measure "$wl" A "$MNT/baseline"
            rm -rf "$MNT/baseline"
            return ;;
        B|D) chattr -C /var/lib/incus/disks 2>/dev/null || true ;;
        C|E) chattr +C /var/lib/incus/disks ;;
    esac
    make_pool
    if [ "$cfg" = C ] || [ "$cfg" = E ]; then
        lsattr "/var/lib/incus/disks/$POOL.img" | grep -q '^-*C-* ' \
            || { echo "ERROR: +C did not take on $POOL.img"; exit 1; }
    fi
    dir="/var/lib/incus/storage-pools/$POOL/custom/default_scratch"
    if [ "$cfg" = D ] || [ "$cfg" = E ]; then
        snapshots_start
        measure "$wl" "$cfg" "$dir"
        snapshots_stop
        echo "snapcount $wl $cfg $SNAP_COUNT" >> "$RESULTS"
    else
        measure "$wl" "$cfg" "$dir"
    fi
    destroy_pool
}

# ---------------------------------------------------------------- sanity
[ "$(id -u)" = 0 ] || { echo "run as root"; exit 1; }
[ -b "$TESTDEV" ] || { echo "$TESTDEV is not a block device"; exit 1; }
if [ "$(incus storage list -f csv | wc -l)" != 0 ]; then
    echo "incus already has storage pools; use a fresh install"; exit 1
fi

log "versions"
uname -r
incus version
btrfs --version | head -1
fio --version
. /etc/os-release && echo "$PRETTY_NAME"

# --------------------------------------------- host filesystem simulation
# TESTDEV becomes the "host btrfs". /var/lib/incus moves onto its own
# subvolume (@incus) so it can be snapshotted, mirroring snapper
# snapshotting a root subvolume that contains /var/lib/incus.
log "setting up host btrfs on $TESTDEV, /var/lib/incus on subvolume @incus"
systemctl stop incus.service incus.socket incus-user.socket 2>/dev/null || true
wipefs -a "$TESTDEV" > /dev/null
mkfs.btrfs -f "$TESTDEV" > /dev/null
mkdir -p "$MNT"
mount -o subvolid=5 "$TESTDEV" "$MNT"
btrfs subvolume create "$MNT/@incus" > /dev/null
cp -a /var/lib/incus/. "$MNT/@incus/"
mount -o subvol=@incus "$TESTDEV" /var/lib/incus
systemctl start incus.socket incus.service
mkdir -p /var/lib/incus/disks

# ------------------------------------------------- attribute-only checks
# Default loop-backed pool: does the backing .img get +C? Does a block
# volume inside the pool get +C?
log "attribute check: loop-backed pool, Incus defaults"
chattr -C /var/lib/incus/disks 2>/dev/null || true
incus storage create attrcheck btrfs size=1GiB
log "lsattr on pool backing file:"
lsattr /var/lib/incus/disks/attrcheck.img
incus storage volume create attrcheck blkvol --type=block size=64MiB
log "lsattr on block volume inside loop-backed pool:"
lsattr /var/lib/incus/storage-pools/attrcheck/custom/default_blkvol/root.img
incus storage volume delete attrcheck blkvol
incus storage delete attrcheck

if [ -n "$BLKDEV" ] && [ -b "$BLKDEV" ]; then
    log "attribute check: pool on real block device $BLKDEV"
    wipefs -a "$BLKDEV" > /dev/null
    incus storage create realdev btrfs source="$BLKDEV"
    incus storage volume create realdev blkvol --type=block size=64MiB
    log "lsattr on block volume inside real-device pool:"
    lsattr /var/lib/incus/storage-pools/realdev/custom/default_blkvol/root.img
    incus storage volume delete realdev blkvol
    incus storage delete realdev
fi

# ------------------------------------------------------------ the matrix
for wl in bulk trickle; do
    for cfg in A B C D E; do
        run_config "$wl" "$cfg"
    done
done
chattr -C /var/lib/incus/disks

# ------------------------------------------------------------ results
echo
echo "== RESULTS: physical bytes written to $TESTDEV =="
echo "   (bulk = 1GiB logical; trickle = 480KiB logical)"
for wl in bulk trickle; do
    base=$(awk -v w="$wl" '$1==w && $2=="A" {print $3}' "$RESULTS")
    awk -v w="$wl" -v a="$base" '$1==w && $2!="snapcount" {
        printf "  %-7s %s: %14d bytes  (%8.3f GiB)  %8.2fx of baseline\n", \
            $1, $2, $3, $3/2^30, $3/a }' "$RESULTS"
done
awk '$1=="snapcount" {printf "  snapshots during %s/%s: %d\n", $2, $3, $4}' "$RESULTS"
echo "  fio logs: /tmp/fio_<workload>_<config>.log"