rclone - reconciliation process for a DVR with Docker compose

compose.yaml

# Periodic reconciliation sweep: copies any source footage MISSING on the backup,
# over SFTP, on an interval. Backstop for the per-upload push when the backup was
# unavailable for a while. COPY ONLY (never sync) — it never deletes from the
# backup, so the backup's own (longer) retention stays in sole control of trimming.
#
# Add this service to your existing docker-compose.yml on the PRIMARY host.
#
# Uses the official rclone image with an SFTP remote configured via environment
# variables (RCLONE_CONFIG_<REMOTE>_*), so there's no config file to manage.
# The remote here is named "backup".
#
# IMPORTANT: rclone requires the SFTP password to be OBSCURED (not plaintext)
# when supplied via env/config. To keep this ergonomic, put your PLAINTEXT
# password in BACKUP_PASS_PLAINTEXT below; the container obscures it at startup
# with `rclone obscure` and exports RCLONE_CONFIG_BACKUP_PASS itself. You never
# run `rclone obscure` by hand.
#
# rclone's SFTP backend does NOT verify host keys unless a known_hosts file is
# given. We omit that, so it accepts whatever key the server presents. The path
# is Tailscale-only and transport is still encrypted; we just don't pin the key.

services:
  footage-sync:
    image: rclone/rclone:latest
    restart: unless-stopped
    # If rclone reads the source footage from a LOCAL directory on the primary,
    # mount it read-only here and point SRC_PATH (below) at the container path.
    volumes:
      - /dvr:/data/source:ro
    environment:
      # ── Backup SFTP remote (named "backup") ──
      RCLONE_CONFIG_BACKUP_TYPE: "sftp"
      RCLONE_CONFIG_BACKUP_HOST: "REPLACE_ME"
      RCLONE_CONFIG_BACKUP_PORT: "REPLACE_ME"
      RCLONE_CONFIG_BACKUP_USER: "REPLACE_ME"
      # Plaintext here; the entrypoint obscures it and sets RCLONE_CONFIG_BACKUP_PASS.
      BACKUP_PASS_PLAINTEXT: "CHANGE_ME_PASSWORD"
      # Sweep interval in seconds (default 15 min). Override here if desired.
      SYNC_INTERVAL: "900"
      # Only copy files untouched for at least this long, so the sweep never
      # grabs a segment the DVR is still writing, nor races SFTPGo's in-flight
      # per-upload push. Set LONGER than your DVR segment length if the DVR
      # keeps a segment open (advancing its mtime) while recording.
      MIN_AGE: "5m"
    entrypoint: ["/bin/sh","-c"]
    command:
      - |
        set -eu
        # Obscure the plaintext password for rclone (required for SFTP).
        export RCLONE_CONFIG_BACKUP_PASS="$$(rclone obscure "$$BACKUP_PASS_PLAINTEXT")"
        unset BACKUP_PASS_PLAINTEXT

        SRC_PATH="/data/source"            # container path to source footage
        DST_PATH="backup:/"                # backup user's root
        INTERVAL="$${SYNC_INTERVAL:-900}"
        echo "footage-sync starting; interval=$${INTERVAL}s"
        while true; do
          echo "[$$(date -u +%FT%TZ)] sweep start"
          rclone copy "$$SRC_PATH" "$$DST_PATH" \
            --min-age "$${MIN_AGE:-5m}" \
            --transfers 4 --checkers 8 \
            --log-level INFO \
            || echo "[$$(date -u +%FT%TZ)] sweep returned non-zero (will retry next interval)"
          echo "[$$(date -u +%FT%TZ)] sweep done; sleeping $${INTERVAL}s"
          sleep "$$INTERVAL"
        done