Mover script for MergerFS with Bidirectional Sync and Tiered Percentage keep ...

mover

#!/usr/bin/env bash
# mover: bidirectional sync + retention-mode move/trim.
# - Sync: --sync [--exclude PATTERN ...] [--authoritative-delete src|dst] [--authoritative-grace-sec SEC]
# - Retain mode: --move --move-path P [--retain-pct N] [--retain-min-bytes BYTES]
#                [--ensure-dst-superset] [--max-trim-bytes BYTES]
# Common safety: --min-age-sec SEC, --skip-open (auto: fuser→lsof→/proc), MergerFS branch awareness.
# Locking: per pair/direction/global via --lock-scope, lock files in --lock-dir (default /var/lock/mover)

set -euo pipefail
export LC_ALL=C

# ---------- defaults ----------
DRY_RUN=0
VERBOSE=0
LOG_FILE=""

# lock
LOCK_DIR="/var/lock/mover"
LOCK_SCOPE="pair"   # pair | direction | global

DO_SYNC=0
DO_MOVE=0

# sync
EXCLUDES=()
AUTHORITATIVE_DELETE=""
AUTHORITATIVE_GRACE_SEC=0

# retention mode (required for --move)
MOVE_PATH=""
RETAIN_PCT=-1            # <0 = not specified; 0..100 valid
RETAIN_MIN_BYTES=0
ENSURE_DST_SUPERSET=0
MAX_TRIM_BYTES=0         # 0 = unlimited per run
ORDER_BY="mtime"         # mtime|size

# safety
MIN_AGE_SEC=0
SKIP_OPEN=0

# rsync
RSYNC_BASE=(-aHAXx --numeric-ids --human-readable --info=stats2,progress2)
RSYNC_IOPS=(--inplace)

# ---------- helpers ----------
log(){ local ts; ts="$(date '+%Y-%m-%d %H:%M:%S')"; [[ -n "$LOG_FILE" ]] && printf '[%s] %s\n' "$ts" "$*" >>"$LOG_FILE"; ((VERBOSE)) && printf '[%s] %s\n' "$ts" "$*" >&2; }
die(){ log "ERROR: $*"; echo "ERROR: $*" >&2; exit 1; }
human_bytes(){ local b=$1 u=(B KiB MiB GiB TiB PiB) i=0; while (( b>=1024 && i<${#u[@]}-1 )); do b=$((b/1024)); ((i++)); done; echo "$b ${u[$i]}"; }
ensure_dirs(){ for d in "$@"; do [[ -d "$d" ]] || die "dir not found: $d"; done; }
have_cmd(){ command -v "$1" >/dev/null 2>&1; }
rsync_run(){ if ((DRY_RUN)); then rsync -n "${RSYNC_BASE[@]}" "${RSYNC_IOPS[@]}" "$@"; else rsync "${RSYNC_BASE[@]}" "${RSYNC_IOPS[@]}" "$@"; fi; }

# ---------- MergerFS awareness ----------
findmnt_line(){ findmnt -no FSTYPE,TARGET,SOURCE,OPTIONS -- "$1" 2>/dev/null || true; }
mergerfs_branches(){
  local line; line="$(findmnt_line "$1")" || true
  [[ -z "$line" ]] && return 1
  local fstype target source opts
  fstype="${line%% *}"; line="${line#* }"
  target="${line%% *}"; line="${line#* }"
  source="${line%% *}"; opts="${line#* }"
  [[ "$fstype" == "fuse.mergerfs" || "$fstype" == "mergerfs" ]] || return 1
  local branches_opt; branches_opt="$(sed -n 's/.*\bbranches=\([^,]*\).*/\1/p' <<<"$opts")"
  [[ -z "$branches_opt" ]] && return 1
  local IFS=':' out=()
  for entry in $branches_opt; do out+=( "${entry%%=*}" ); done
  printf '%s\0' "$target"; printf '%s\0' "${out[@]}"
}
candidate_paths_for(){
  local p="$1"; printf '%s\0' "$p"
  local mnb; mnb="$(mergerfs_branches "$p")" || return 0
  local IFS= read -r -d '' mnt <<<"$mnb" || true
  local rest="${mnb#*$'\0'}"
  [[ -n "$mnt" && "$p" == "$mnt"* ]] || return 0
  local rel="${p#$mnt/}"
  while IFS= read -r -d '' br; do
    [[ -d "$br" ]] || continue
    local cand="$br/$rel"
    [[ -e "$cand" ]] && printf '%s\0' "$cand"
  done <<<"$rest"
}

# ---------- open-file detection ----------
OPEN_MODE="auto"
resolve_open_mode(){
  if (( SKIP_OPEN==0 )); then OPEN_MODE="none"; return; fi
  if have_cmd fuser; then OPEN_MODE="fuser"; return; fi
  if have_cmd lsof;  then OPEN_MODE="lsof";  return; fi
  OPEN_MODE="proc"
}
is_open_fuser_one(){ fuser -s -- "$1" 2>/dev/null; }                 # 0=open
is_open_lsof_one(){ lsof -Fn -- "$1" 2>/dev/null | grep -q '^n'; }  # 0=open
is_open_proc_one(){
  local target; target="$(readlink -f -- "$1" 2>/dev/null || true)"; [[ -z "$target" ]] && return 1
  local fdpath real
  for fdpath in /proc/*/fd/*; do
    [[ -e "$fdpath" ]] || continue
    real="$(readlink -f -- "$fdpath" 2>/dev/null || true)"
    [[ -n "$real" && "$real" == "$target" ]] && return 0
  done
  return 1
}
file_is_open(){
  (( SKIP_OPEN==0 )) && return 1
  local p="$1" cand
  while IFS= read -r -d '' cand; do
    case "$OPEN_MODE" in
      fuser) is_open_fuser_one "$cand" && return 0 ;;
      lsof)  is_open_lsof_one  "$cand" && return 0 ;;
      proc)  is_open_proc_one  "$cand" && return 0 ;;
    esac
  done < <(candidate_paths_for "$p")
  return 1
}
old_enough(){ (( MIN_AGE_SEC==0 )) && return 0; local now mt; now=$(date +%s); mt=$(stat -c %Y -- "$1") || return 1; (( now - mt >= MIN_AGE_SEC )); }

# ---------- sync ----------
sync_bidirectional(){
  local SRC=$1 DST=$2
  local excl=(); for e in "${EXCLUDES[@]}"; do excl+=(--exclude "$e"); done
  log "Sync SRC->DST"; rsync_run --update "${excl[@]}" "$SRC"/ "$DST"/
  log "Sync DST->SRC"; rsync_run --update "${excl[@]}" "$DST"/ "$SRC"/
  if [[ -n "$AUTHORITATIVE_DELETE" ]]; then
    local AUTH_PATH OTHER_PATH
    case "$AUTHORITATIVE_DELETE" in
      src) AUTH_PATH="$SRC"; OTHER_PATH="$DST" ;;
      dst) AUTH_PATH="$DST"; OTHER_PATH="$SRC" ;;
      *) die "invalid --authoritative-delete";;
    esac
    local ts_now cutoff; ts_now=$(date +%s); cutoff=$((ts_now - AUTHORITATIVE_GRACE_SEC))
    local prune_args=(); for e in "${EXCLUDES[@]}"; do [[ "$e" == /*/*** ]] && prune_args+=( -path "${OTHER_PATH}${e%%/***}" -prune -o ); done
    while IFS= read -r -d '' f; do
      local rel="${f#$OTHER_PATH/}"; [[ -e "$AUTH_PATH/$rel" ]] && continue
      local mt; mt=$(stat -c %Y -- "$f") || continue
      if (( mt < cutoff )); then log "Delete $f"; (( DRY_RUN )) || rm -f -- "$f"; fi
    done < <(find "$OTHER_PATH" -xdev "${prune_args[@]}" -type f -print0)
  fi
}

# ---------- retention mode helpers ----------
bytes_in_dir(){ local d=$1; [[ -d "$d" ]] || { echo 0; return 0; }; du -sb --apparent-size -- "$d" 2>/dev/null | awk '{print $1}'; }

copy_missing_to_dst(){
  local SRC_SUB=$1 DST_SUB=$2
  local TMP_LIST; TMP_LIST="$(mktemp -t mover-copylist.XXXXXX)"
  trap '[[ -n "${TMP_LIST:-}" ]] && rm -f -- "$TMP_LIST"' RETURN

  if (( MIN_AGE_SEC > 0 )); then
    find "$SRC_SUB" -xdev -type f -not -newermt "-${MIN_AGE_SEC} seconds" -print0 > "$TMP_LIST"
  else
    find "$SRC_SUB" -xdev -type f -print0 > "$TMP_LIST"
  fi

  local cand; cand="$(tr -cd '\0' < "$TMP_LIST" | wc -c)"
  (( cand == 0 )) && { log "copy pass: nothing to copy (no eligible candidates)"; trap - RETURN; rm -f -- "$TMP_LIST" || true; return 0; }

  log "copy pass: scanning $cand candidate file(s); letting rsync copy only missing/size-different files"

  local REL_LIST; REL_LIST="$(mktemp -t mover-copyrel.XXXXXX)"
  {
    local abs rel
    while IFS= read -r -d '' abs; do
      rel="${abs#$SRC_SUB/}"
      printf '%s\0' "$rel"
    done < "$TMP_LIST"
  } > "$REL_LIST"

  local RSYNC_OPTS=(-aHAXx --numeric-ids --info=progress2,stats2 --human-readable
                    --files-from="$REL_LIST" --from0 --relative --size-only)

  local rsync_rc=0
  if (( DRY_RUN )); then
    rsync -n "${RSYNC_OPTS[@]}" "$SRC_SUB"/ "$DST_SUB"/ || rsync_rc=$?
  else
    ionice -c3 nice -n 19 rsync "${RSYNC_OPTS[@]}" "$SRC_SUB"/ "$DST_SUB"/ || rsync_rc=$?
  fi

  trap - RETURN
  rm -f -- "$TMP_LIST" "$REL_LIST" || true
  return "$rsync_rc"
}

trim_src_to_retention(){
  local SRC_SUB=$1 DST_SUB=$2
  local total_bytes retain_by_pct trim_target
  total_bytes="$(bytes_in_dir "$SRC_SUB")"
  retain_by_pct=0
  if (( RETAIN_PCT >= 0 )); then retain_by_pct=$(( (total_bytes * RETAIN_PCT) / 100 )); fi
  (( retain_by_pct < RETAIN_MIN_BYTES )) && retain_by_pct="$RETAIN_MIN_BYTES"
  if (( total_bytes <= retain_by_pct )); then
    log "trim pass: at/under target (total=$(human_bytes "$total_bytes"), retain=$(human_bytes "$retain_by_pct"))"
    return 0
  fi
  trim_target=$(( total_bytes - retain_by_pct ))
  if (( MAX_TRIM_BYTES > 0 && trim_target > MAX_TRIM_BYTES )); then trim_target="$MAX_TRIM_BYTES"; fi
  log "trim pass: plan to delete ~$(human_bytes "$trim_target") from SRC subtree"

  local cum=0 selected=0 skipped_open=0 skipped_young=0 skipped_missingdst=0
  local find_fmt find_sort
  case "$ORDER_BY" in
    mtime) find_fmt='%T@ %p\0'; find_sort='sort -z -n' ;; # oldest first
    size)  find_fmt='%s %p\0';  find_sort='sort -z -n' ;; # smallest first
    *) die "invalid --order-by";;
  esac

  local rel="" srcf="" dstf="" ssz=0 dsz=0

  # --- temporarily disable errexit for the loop + generator ---
  set +e
  while IFS= read -r -d '' srcf; do
    if ! old_enough "$srcf"; then ((skipped_young++)); continue; fi
    if file_is_open "$srcf"; then ((skipped_open++)); continue; fi

    rel="${srcf#$SRC_SUB/}"
    dstf="$DST_SUB/$rel"

    if [[ ! -e "$dstf" ]]; then ((skipped_missingdst++)); continue; fi
    ssz=$(stat -c %s -- "$srcf") || { ((skipped_missingdst++)); continue; }
    dsz=$(stat -c %s -- "$dstf") || { ((skipped_missingdst++)); continue; }
    [[ "$ssz" -eq "$dsz" ]] || { ((skipped_missingdst++)); continue; }

    (( cum + ssz > trim_target )) && break

    if (( DRY_RUN )); then
      log "trim [dry]: delete ${rel:-<unset>} (size=$(human_bytes "$ssz"))"
    else
      rm -f -- "$srcf"
    fi

    cum=$((cum + ssz)); ((selected++))
    (( cum >= trim_target )) && break
  done < <(
    ( find "$SRC_SUB" -xdev -type f -printf "$find_fmt" \
      | $find_sort \
      | awk -v RS='\0' -v ORS='\0' '{ sub(/^[^ ]+ /, "", $0); print }'
    ) || :
  )
  loop_rc=$?
  set -e
  # --- end errexit mask ---

  log "trim pass: deleted=$(human_bytes "$cum") files=$selected skipped_open=$skipped_open skipped_young=$skipped_young skipped_missingdst=$skipped_missingdst"
  (( ! DRY_RUN )) && find "$SRC_SUB" -type d -empty -delete || true

  return 0
}

move_retain_mode(){
  local SRC=$1 DST=$2
  [[ -n "$MOVE_PATH" ]] || die "--move requires --move-path"
  local SRC_SUB="$SRC/$MOVE_PATH" DST_SUB="$DST/$MOVE_PATH"
  ensure_dirs "$SRC_SUB" "$DST_SUB"

  resolve_open_mode
  log "Open-check: $OPEN_MODE"

  if (( ENSURE_DST_SUPERSET )); then
    local rc=0
    copy_missing_to_dst "$SRC_SUB" "$DST_SUB" || rc=$?
    if (( rc != 0 )); then
      log "copy pass failed (rsync rc=$rc); skipping trim to avoid data loss"
      exit "$rc"
    fi
  fi
  trim_src_to_retention "$SRC_SUB" "$DST_SUB"
}

# ---------- locking ----------
realpath_can(){ readlink -f -- "$1"; }
ordered_pair_key(){ local a="$1" b="$2"; if [[ "$a" < "$b" ]]; then printf '%s|%s' "$a" "$b"; else printf '%s|%s' "$b" "$a"; fi; }

setup_lock(){
  mkdir -p "$LOCK_DIR"
  local SRC_CAN DST_CAN LOCK_KEY LOCK_HASH
  SRC_CAN="$(realpath_can "$SRC")"
  DST_CAN="$(realpath_can "$DST")"
  case "$LOCK_SCOPE" in
    global)    LOCK_KEY="GLOBAL" ;;
    pair)
      if (( DO_MOVE )) && [[ -n "$MOVE_PATH" ]]; then
        LOCK_KEY="$(ordered_pair_key "$SRC_CAN" "$DST_CAN")|mp=$MOVE_PATH"
      else
        LOCK_KEY="$(ordered_pair_key "$SRC_CAN" "$DST_CAN")"
      fi
      ;;
    direction)
      if (( DO_MOVE )) && [[ -n "$MOVE_PATH" ]]; then
        LOCK_KEY="$SRC_CAN|$DST_CAN|mp=$MOVE_PATH"
      else
        LOCK_KEY="$SRC_CAN|$DST_CAN"
      fi
      ;;
    *) echo "Invalid --lock-scope: $LOCK_SCOPE" >&2; exit 2 ;;
  esac
  LOCK_HASH="$(printf '%s' "$LOCK_KEY" | sha256sum | awk '{print $1}')"
  LOCK_PATH="$LOCK_DIR/$LOCK_HASH.lock"
  # shellcheck disable=SC3028
  exec {LOCK_FD}>"$LOCK_PATH" || { echo "cannot open lock $LOCK_PATH" >&2; exit 1; }
  if ! flock -n "$LOCK_FD"; then
    echo "another mover is running for scope '$LOCK_SCOPE'
SRC=$(realpath_can "$SRC")
DST=$(realpath_can "$DST")
MOVE_PATH=${MOVE_PATH:-<none>}
lock=$LOCK_PATH
key=$LOCK_KEY" >&2
    exit 1
  fi
}

# ---------- usage ----------
usage(){
  cat <<'USAGE'
Usage:
  mover --src <path> --dst <path>
        [--sync [--exclude PATTERN ...] [--authoritative-delete src|dst] [--authoritative-grace-sec SEC]]
        [--move --move-path RELPATH
            [--retain-pct N] [--retain-min-bytes BYTES] [--ensure-dst-superset]
            [--max-trim-bytes BYTES]
            [--order-by mtime|size] [--min-age-sec SEC] [--skip-open]]
        [--dry-run] [-v] [--log-file FILE]
        [--lock-scope pair|direction|global] [--lock-dir DIR]
USAGE
}

# ---------- argparse ----------
SRC="" ; DST=""
while (( $# )); do
  case "$1" in
    --src) SRC="$2"; shift 2 ;;
    --dst) DST="$2"; shift 2 ;;
    --sync) DO_SYNC=1; shift ;;
    --exclude) EXCLUDES+=("$2"); shift 2 ;;
    --authoritative-delete) AUTHORITATIVE_DELETE="$2"; shift 2 ;;
    --authoritative-grace-sec) AUTHORITATIVE_GRACE_SEC="$2"; shift 2 ;;
    --move) DO_MOVE=1; shift ;;
    --move-path) MOVE_PATH="$2"; shift 2 ;;
    --retain-pct) RETAIN_PCT="$2"; shift 2 ;;
    --retain-min-bytes) RETAIN_MIN_BYTES="$2"; shift 2 ;;
    --ensure-dst-superset) ENSURE_DST_SUPERSET=1; shift ;;
    --max-trim-bytes) MAX_TRIM_BYTES="$2"; shift 2 ;;
    --order-by) ORDER_BY="$2"; shift 2 ;;
    --min-age-sec) MIN_AGE_SEC="$2"; shift 2 ;;
    --skip-open) SKIP_OPEN=1; shift ;;
    --dry-run) DRY_RUN=1; shift ;;
    -v|--verbose) VERBOSE=1; shift ;;
    --log-file) LOG_FILE="$2"; shift 2 ;;
    --lock-scope) LOCK_SCOPE="$2"; shift 2 ;;
    --lock-dir) LOCK_DIR="$2"; shift 2 ;;
    -h|--help) usage; exit 0 ;;
    *) echo "unknown arg: $1" >&2; usage; exit 2 ;;
  esac
done

[[ -n "$SRC" && -n "$DST" ]] || { usage; exit 2; }
ensure_dirs "$SRC" "$DST"

# locking
setup_lock

log "==== mover start src='$SRC' dst='$DST' dry=$DRY_RUN ===="
(( DO_SYNC==0 && DO_MOVE==0 )) && die "nothing to do: pass --sync and/or --move"

(( DO_SYNC )) && sync_bidirectional "$SRC" "$DST"
(( DO_MOVE )) && move_retain_mode "$SRC" "$DST"

log "==== mover done ===="
exit 0

Usage examples ...

Sync with Excludes - Both directions:

# dry-run first (no changes)
mover --src /zfs/ssdpool/restic --dst /zfs/coldstore/restic \
  --sync \
  --exclude '/data/***' \
  --exclude '/locks/***' \
  -v --dry-run

Results in output like this:

[2025-10-20 21:14:16] ==== mover start src='/zfs/ssdpool/restic' dst='/zfs/coldstore/restic' dry=1 ====
[2025-10-20 21:14:16] Sync SRC->DST
              0   0%    0.00kB/s    0:00:00 (xfr#0, to-chk=0/5065)   

Number of files: 5,065 (reg: 5,061, dir: 4)
Number of created files: 0
Number of deleted files: 0
Number of regular files transferred: 0
Total file size: 171.61M bytes
Total transferred file size: 0 bytes
Literal data: 0 bytes
Matched data: 0 bytes
File list size: 327.53K
File list generation time: 0.001 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 413.33K
Total bytes received: 23

sent 413.33K bytes  received 23 bytes  275.57K bytes/sec
total size is 171.61M  speedup is 415.17 (DRY RUN)
[2025-10-20 21:14:17] Sync DST->SRC
              0   0%    0.00kB/s    0:00:00 (xfr#0, to-chk=0/5065)   

Number of files: 5,065 (reg: 5,061, dir: 4)
Number of created files: 0
Number of deleted files: 0
Number of regular files transferred: 0
Total file size: 171.61M bytes
Total transferred file size: 0 bytes
Literal data: 0 bytes
Matched data: 0 bytes
File list size: 327.46K
File list generation time: 0.001 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 413.36K
Total bytes received: 23

sent 413.36K bytes  received 23 bytes  275.59K bytes/sec
total size is 171.61M  speedup is 415.14 (DRY RUN)
[2025-10-20 21:14:18] ==== mover done ====

---

Move with Keep Percentage - Uni-Directional (ensures destination always has the full copy before deletion on source):

mover --src /zfs/ssdpool/restic --dst /zfs/coldstore/restic \
      --move --move-path data \
      --retain-pct 20 \
      --ensure-dst-superset \
      --min-age-sec 600 --skip-open -v --dry-run

Results in output like this (inverted - because my current SSD is actually empty ...):

[2025-10-20 21:13:23] ==== mover start src='/zfs/coldstore/restic' dst='/zfs/ssdpool/restic' dry=1 ====
[2025-10-20 21:13:23] Open-check: fuser
[2025-10-20 21:13:24] copy pass: scanning 31215 candidate file(s); letting rsync copy only missing/size-different files
          1.39T 100% 1296655.66GB/s    0:00:00 (xfr#31215, to-chk=0/31471)

Number of files: 31,471 (reg: 31,215, dir: 256)
Number of created files: 31,471 (reg: 31,215, dir: 256)
Number of deleted files: 0
Number of regular files transferred: 31,215
Total file size: 1.39T bytes
Total transferred file size: 1.39T bytes
Literal data: 0 bytes
Matched data: 0 bytes
File list size: 2.49M
File list generation time: 21.917 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 2.61M
Total bytes received: 95.06K

sent 2.61M bytes  received 95.06K bytes  120.33K bytes/sec
total size is 1.39T  speedup is 514,228.03 (DRY RUN)
[2025-10-20 21:13:54] trim pass: plan to delete ~1 TiB from SRC subtree

Test-Script to verify correct behavior:

fill-test-move.sh:

#!/usr/bin/env bash
set -euo pipefail

# -------- config (matches your mover flags) --------
SRC="/zfs/ssdpool/test"
DST="/zfs/coldstore/test"
RETAIN_PCT=${RETAIN_PCT:-20}
RETAIN_MIN_BYTES=${RETAIN_MIN_BYTES:-0}
MIN_AGE_SEC=${MIN_AGE_SEC:-600}         # 10 minutes
TEST_BAND_PCT=${TEST_BAND_PCT:-10}      # ± band for tests (independent from mover)
PACK_COUNT=${PACK_COUNT:-100}           # baseline pack files to create
PACK_MIN_MB=${PACK_MIN_MB:-1}
PACK_MAX_MB=${PACK_MAX_MB:-5}

# -------- helpers --------
xi() { numfmt --to=iec --suffix=B "$1"; }
now() { date '+%Y-%m-%d %H:%M:%S'; }

# best-effort xattr helpers
setxattr() {
  local f=$1 k=$2 v=$3
  if command -v setfattr >/dev/null 2>&1; then
    setfattr -n "$k" -v "$v" -- "$f" || true
  elif command -v attr >/dev/null 2>&1; then
    attr -s "$k" -V "$v" -- "$f" || true
  fi
}

# deterministic random size (1..5 MiB by default)
rand_mib_for() {
  local n=$1
  local range=$(( PACK_MAX_MB - PACK_MIN_MB + 1 ))
  local v=$(( (n % range) + PACK_MIN_MB ))
  echo "$v"
}

# Make a file of N MiB quickly (zero data to keep test fast/stable)
mk_mib_file() {
  local path=$1 mib=$2
  dd if=/dev/zero of="$path" bs=1M count="$mib" status=none conv=fsync
}

snapshot_bytes() {
  local d=$1
  if [[ -d "$d" ]]; then
    du -sb --apparent-size -- "$d" 2>/dev/null | awk '{print $1}'
  else
    echo 0
  fi
}

count_files() {
  local d=$1
  [[ -d "$d" ]] || { echo 0; return 0; }
  find "$d" -xdev -type f | wc -l
}

eligible_list() {
  local d=$1
  if (( MIN_AGE_SEC > 0 )); then
    # strictly older than MIN_AGE_SEC
    find "$d" -xdev -type f -not -newermt "-${MIN_AGE_SEC} seconds" -print0
  else
    find "$d" -xdev -type f -print0
  fi
}

largest_eligible_file_size() {
  local d=$1
  local tmp; tmp="$(mktemp)"; trap 'rm -f "$tmp"' RETURN
  eligible_list "$d" | xargs -0 -I{} stat -c %s -- "{}" > "$tmp" || true
  if [[ -s "$tmp" ]]; then
    awk 'BEGIN{m=0} {if($1>m)m=$1} END{print m}' "$tmp"
  else
    echo 0
  fi
  trap - RETURN
}

# -------- data prep --------
echo
echo "===== Prepare clean test trees ====="
rm -rf -- "$SRC" "$DST"
mkdir -p "$SRC/data" "$DST/data"

echo
echo "===== Generate baseline packs on SRC ====="
for i in $(seq 1 "$PACK_COUNT"); do
  size_mib=$(rand_mib_for "$i")
  mk_mib_file "$SRC/data/pack_${i}.bin" "$size_mib"
done

# hardlink to a pack to mimic dedup/hardlink cases
ln "$SRC/data/pack_1.bin" "$SRC/data/pack_1_hardlink.bin"

# symlink pointing to an existing pack
ln -s "pack_2.bin" "$SRC/data/pack_2_symlink.bin"

# zero-length file and sparse file
: > "$SRC/data/zero.bin"
truncate -s 64M "$SRC/data/sparse.bin"
# (sparse is sparse only if underlying FS supports it; it’s ok either way)

# xattr on one file (best effort)
setxattr "$SRC/data/pack_3.bin" "user.testmeta" "hello"

# one permission-denied file (not eligible due to min-age anyway)
mk_mib_file "$SRC/data/protected.bin" 2
chmod 000 "$SRC/data/protected.bin" || true

# create "young" files which must not be moved/trimmed
for j in $(seq 1 5); do
  mk_mib_file "$SRC/data/young_${j}.bin" 1
done

# create an actually open file (kept open in background)
OPEN_HOLDER="$SRC/data/open_hold.bin"
mk_mib_file "$OPEN_HOLDER" 1
# open FD 9 write-locked by sleep
exec 9>"$OPEN_HOLDER"
# shellcheck disable=SC3037
flock -n 9 || true
( sleep 120 ) & OPEN_HOLDER_PID=$!

# ensure mtime for packs is old enough; touch young/open to be recent
find "$SRC/data" -type f -name 'pack_*.bin' -exec touch -d "90 minutes ago" {} +
find "$SRC/data" -type f -name 'zero.bin' -exec touch -d "90 minutes ago" {} +
find "$SRC/data" -type f -name 'sparse.bin' -exec touch -d "90 minutes ago" {} +
# make the young/open files recent
touch "$OPEN_HOLDER"
for j in $(seq 1 5); do touch "$SRC/data/young_${j}.bin"; done

echo
echo "===== Seed DST with subset and size-mismatches ====="
# copy half of the packs older than min-age
for i in $(seq 1 "$PACK_COUNT"); do
  if (( i % 2 == 0 )); then
    cp -a --reflink=auto "$SRC/data/pack_${i}.bin" "$DST/data/pack_${i}.bin" || cp -a "$SRC/data/pack_${i}.bin" "$DST/data/pack_${i}.bin"
  fi
done
# create a bunch of mismatches on DST (different sizes than SRC)
for i in 10 20 30 40 50 60 70; do
  mk_mib_file "$DST/data/pack_${i}.bin" 1
done

# keep the symlink only on SRC (makes DST need to copy a symlink)
# seed zero/sparse only in SRC (DST must copy)
# leave xattr case for mover to propagate if rsync preserves (we pass -X)

echo
echo "===== Add metadata files for sync test (outside /data) ====="
mkdir -p "$SRC/config" "$DST/config"
echo "src-meta $(now)" > "$SRC/README.txt"
echo "dst-meta $(now)" > "$DST/NOTES.txt"
mk_mib_file "$SRC/config/info.bin" 2
mk_mib_file "$DST/config/info.bin" 2   # same file; sync should be no-op

# -------- snapshot before --------
SRC_BYTES_BEFORE=$(snapshot_bytes "$SRC/data")
DST_BYTES_BEFORE=$(snapshot_bytes "$DST/data")
SRC_FILES_BEFORE=$(count_files "$SRC/data")
ELIGIBLE_COUNT=$(eligible_list "$SRC/data" | tr -cd '\0' | wc -c)

echo
echo "===== Snapshot baseline sizes ====="
echo "SRC before: $(xi "$SRC_BYTES_BEFORE")  (files=${SRC_FILES_BEFORE}, eligible=${ELIGIBLE_COUNT})"
echo "DST before: $(xi "$DST_BYTES_BEFORE")"

# -------- dry-run mover --------
echo
echo "===== Dry-run mover (superset+retain) ====="
set +e
mover --src "$SRC" --dst "$DST" \
  --move --move-path data \
  --retain-pct "$RETAIN_PCT" \
  --ensure-dst-superset \
  --min-age-sec "$MIN_AGE_SEC" --skip-open -v --dry-run
DRY_RC=$?
echo "Dry-run mover rc=$DRY_RC"

# extract brief rsync summary for visibility (best effort)
DRY_SUMMARY=$(rsync --version >/dev/null 2>&1; echo "created=?, transferred=?, total_size=?, tx_size=?")
echo "Dry-run summary: $DRY_SUMMARY"
set -e

# -------- real mover --------
echo
echo "===== Real mover (superset+retain) ====="
set +e
mover --src "$SRC" --dst "$DST" \
  --move --move-path data \
  --retain-pct "$RETAIN_PCT" \
  --ensure-dst-superset \
  --min-age-sec "$MIN_AGE_SEC" --skip-open -v
REAL_RC=$?
set -e
if (( REAL_RC != 0 )); then
  echo "Real mover rc=$REAL_RC"
  echo "FAIL: mover returned non-zero ($REAL_RC); aborting."
  # cleanup open holder process before exit
  if [[ -n "${OPEN_HOLDER_PID:-}" ]]; then kill "$OPEN_HOLDER_PID" 2>/dev/null || true; fi
  exec 9>&- || true
  exit 1
fi

# -------- post-run stats --------
echo
echo "===== Post-run stats ====="
SRC_BYTES_AFTER=$(snapshot_bytes "$SRC/data")
DST_BYTES_AFTER=$(snapshot_bytes "$DST/data")
SRC_FILES_AFTER=$(count_files "$SRC/data")
DST_FILES_AFTER=$(count_files "$DST/data")
DELETED_BYTES=$(( SRC_BYTES_BEFORE - SRC_BYTES_AFTER ))
RETAIN_TARGET_BYTES=$(( (SRC_BYTES_BEFORE * RETAIN_PCT) / 100 ))
if (( RETAIN_TARGET_BYTES < RETAIN_MIN_BYTES )); then
  RETAIN_TARGET_BYTES=$RETAIN_MIN_BYTES
fi

echo "SRC bytes before:  $(xi "$SRC_BYTES_BEFORE")"
echo "SRC bytes after:   $(xi "$SRC_BYTES_AFTER")"
echo "Deleted from SRC:  $(xi "$DELETED_BYTES")"
echo "Retain target:     $(xi "$RETAIN_TARGET_BYTES") (${RETAIN_PCT}%)"
LOWER_BYTES=$(( RETAIN_TARGET_BYTES - (RETAIN_TARGET_BYTES * TEST_BAND_PCT) / 100 ))
(( LOWER_BYTES < 0 )) && LOWER_BYTES=0
UPPER_BY_PCT=$(( RETAIN_TARGET_BYTES + (RETAIN_TARGET_BYTES * TEST_BAND_PCT) / 100 ))
ELIG_MAX_SIZE=$(largest_eligible_file_size "$SRC/data")
UPPER_BY_FILE=$(( RETAIN_TARGET_BYTES + ELIG_MAX_SIZE ))
UPPER_BYTES=$(( UPPER_BY_PCT > UPPER_BY_FILE ? UPPER_BY_PCT : UPPER_BY_FILE ))
(( UPPER_BYTES > SRC_BYTES_BEFORE )) && UPPER_BYTES=$SRC_BYTES_BEFORE

echo "Retain band:       $(xi "$LOWER_BYTES") .. $(xi "$UPPER_BYTES") (±${TEST_BAND_PCT}% or +largest eligible)"
KEPT_PCT=$(awk -v a="$SRC_BYTES_AFTER" -v b="$SRC_BYTES_BEFORE" 'BEGIN{ if(b==0){print 0}else{printf "%.2f", (a*100.0)/b} }')
echo "Actual kept:       ${KEPT_PCT}% of pre-trim"
echo "SRC files:         before=${SRC_FILES_BEFORE} after=${SRC_FILES_AFTER}"
echo "DST bytes after:   $(xi "$DST_BYTES_AFTER")"
echo "DST files after:   ${DST_FILES_AFTER}"

# -------- verifications --------

echo
echo "===== Verify: DST is a superset for eligible SRC files (older than ${MIN_AGE_SEC}s) ====="
fail=0
while IFS= read -r -d '' f; do
  rel="${f#$SRC/data/}"
  if [[ ! -e "$DST/data/$rel" ]]; then
    echo "Missing on DST: $rel"; fail=1; break
  fi
  ssz=$(stat -c %s -- "$f")
  dsz=$(stat -c %s -- "$DST/data/$rel")
  if [[ "$ssz" -ne "$dsz" ]]; then
    echo "Size mismatch after run: $rel (src=$ssz dst=$dsz)"; fail=1; break
  fi
done < <(eligible_list "$SRC/data")
if (( fail == 0 )); then
  echo "OK: DST contains all eligible SRC files."
else
  echo "FAIL: DST is not a superset of eligible SRC files."
  # cleanup and exit
  if [[ -n "${OPEN_HOLDER_PID:-}" ]]; then kill "$OPEN_HOLDER_PID" 2>/dev/null || true; fi
  exec 9>&- || true
  exit 1
fi

echo
echo "===== Verify: min-age & open-file preserved on SRC ====="
ok=1
for f in "$OPEN_HOLDER" "$SRC/data/young_1.bin" "$SRC/data/young_2.bin" "$SRC/data/young_3.bin" "$SRC/data/young_4.bin" "$SRC/data/young_5.bin"; do
  if [[ -e "$f" ]]; then
    echo "OK: $(basename "$f") present on SRC"
  else
    echo "FAIL: $(basename "$f") missing on SRC"
    ok=0
  fi
done
(( ok == 1 )) || { echo "FAIL: min-age/open preservation failed."; exit 1; }

echo
printf "===== Verify: retain ~%d%% on SRC with granularity tolerance =====\n" "$RETAIN_PCT"
echo "Eligible largest file:  $(xi "$ELIG_MAX_SIZE")"
echo "Target keep:            $(xi "$RETAIN_TARGET_BYTES")"
echo "Accepted band:          $(xi "$LOWER_BYTES") .. $(xi "$UPPER_BYTES")"
echo "Actual kept:            $(xi "$SRC_BYTES_AFTER")"
if (( SRC_BYTES_AFTER < LOWER_BYTES || SRC_BYTES_AFTER > UPPER_BYTES )); then
  echo "FAIL: retention outside allowed band (granularity-aware)."
  exit 1
else
  echo "OK: retention within allowed band."
fi

echo
echo "===== Verify: previously mismatched DST files now match sizes ====="
mismatch_ok=1
for i in 10 20 30 40 50 60 70; do
  rel="pack_${i}.bin"
  if [[ ! -f "$DST/data/$rel" || ! -f "$SRC/data/$rel" ]]; then
    echo "WARN: $rel missing in one side; skipping strict mismatch check"
    continue
  fi
  ssz=$(stat -c %s -- "$SRC/data/$rel")
  dsz=$(stat -c %s -- "$DST/data/$rel")
  if [[ "$ssz" -ne "$dsz" ]]; then
    echo "FAIL: size still mismatched for $rel (src=$ssz dst=$dsz)"
    mismatch_ok=0
  fi
done
(( mismatch_ok == 1 )) && echo "OK: all intended mismatches resolved."

echo
echo "===== Verify: idempotency (run mover again; should be no-op) ====="
set +e
mover --src "$SRC" --dst "$DST" \
  --move --move-path data \
  --retain-pct "$RETAIN_PCT" \
  --ensure-dst-superset \
  --min-age-sec "$MIN_AGE_SEC" --skip-open -v
IDEMP_RC=$?
set -e
if (( IDEMP_RC != 0 )); then
  echo "FAIL: idempotency run returned rc=$IDEMP_RC"
  exit 1
else
  echo "OK: idempotency run returned rc=0"
fi

echo
echo "===== Verify: reverse-direction (sanity; should do minimal/no work) ====="
set +e
mover --src "$DST" --dst "$SRC" \
  --move --move-path data \
  --retain-pct "$RETAIN_PCT" \
  --ensure-dst-superset \
  --min-age-sec "$MIN_AGE_SEC" --skip-open -v --dry-run
REV_RC=$?
set -e
if (( REV_RC == 0 )); then
  echo "OK: reverse-direction dry-run rc=0 (expected: no data loss)."
else
  echo "WARN: reverse-direction dry-run rc=$REV_RC (not fatal for this test)."
fi

# -------- cleanup open file --------
if [[ -n "${OPEN_HOLDER_PID:-}" ]]; then
  kill "$OPEN_HOLDER_PID" 2>/dev/null || true
fi
exec 9>&- || true

echo
echo "===== RESULT ====="
echo "ALL CHECKS PASSED ✅"

This is mainly a WIP using rsync to get going because I have the need right now for something that I can use already. In the future I maybe want this to be a standalone binary compiled with Rust with proper Unit-Tests as a separate project ... (yea, like never)