Rclone copy via cache folder (with tmux): HDD > SSD (cache) > Target

rclone-cached-tmux.sh

#!/bin/bash

set -euo pipefail

if [[ "$1" == "--to-cache" ]]; then
  if [[ -n "$(rclone lsf "$CACHE" --include-from="transfer_files.log" --files-only --recursive --min-size=0)" ]]; then
    echo "Files present to transfer."
    touch copied
    exit 0
  fi

  echo "Copying file(s) to cache..."
  if ! rclone -P copy "$SOURCE" "$CACHE" --include-from="transfer_files.log" --min-size=0 --max-transfer="$CACHE_SIZE" --transfers="$MAX_TRANSFERS" --buffer-size=16M; then
      echo "Some files were copied."
  fi

  touch copied
  exit 0
fi

if [[ "$1" == "--to-target" ]]; then
  while true; do
    echo "Moving from cache to target..."
    if ! rclone -P move "$CACHE" "$TARGET" --include-from="transfer_files.log" --min-size=0 --transfers="$MAX_OUT_TRANSFERS" --buffer-size=16M; then
        echo "ERROR: rclone move failed. Exiting."
        exit 1
    fi
    sleep 5s

    [[ -e copied ]] && break
  done
  exit 0
fi

if [[ $# -lt 3 ]] || [[ $# -gt 6 ]]; then
    echo "usage: $0 <source> <cache> <target> [cache-size] [max-transfers] [max-out-transfers]"
    exit 1
fi

export SOURCE="$1"
export CACHE="$2"
export TARGET="$3"
export CACHE_SIZE="${4:-10G}"
export MAX_TRANSFERS="${5:-1}"
export MAX_OUT_TRANSFERS="${6:-5}"

# Step 2: Pull list of files already on the target
SESSION="rclone_sync_session"
WINDOW="sync"

for (( round=1; ; round++ )); do
    echo "=== Round $round ==="
    echo "Fetching list of files to upload..."
    if rclone check "$SOURCE" "$TARGET" --match="transferred_files.log" --missing-on-dst="transfer_files.log" --one-way &>/dev/null; then
      echo "All files uploaded."
      break
    fi

    echo "Removing transferred files..."
    rclone delete "$CACHE" --files-from="transferred_files.log" --rmdirs

    rm -f copied

    tmux kill-session -t "$SESSION" 2>/dev/null || true
    tmux new-session -d -s "$SESSION" -n "$WINDOW" "$0 --to-cache"
    tmux split-window -h -t "$SESSION:$WINDOW" "$0 --to-target"
    tmux select-layout -t "$SESSION:$WINDOW" even-horizontal
    tmux attach -t "$SESSION"

    echo
done

# echo "Final check."
# rclone -P copy "$SOURCE" "$TARGET"

echo "Finished file(s) transfer."

Use:

./rclone-cached /slow/hdd /fast/ssd/cache s3-deep-archive:

Do a concurrent copy to cache folder from hdd to then send this over slow internet to the target from fast SSD, allowing HDD to spin-down in between. Uses concurrent copy and move via tmux panes.