Created
June 20, 2025 16:50
-
-
Save ayufan/63ee38c676e6f2596c0f1ecf3e834f4a to your computer and use it in GitHub Desktop.
Rclone copy via cache folder: HDD > SSD (cache) > Target
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
set -euo pipefail | |
if [[ "$1" == "--to-cache" ]]; then | |
echo "Copying file(s) to cache..." | |
if ! rclone -P copy "$SOURCE" "$CACHE" --max-transfer="$CACHE_SIZE" --exclude-from="transferred_files.log" --transfers="$MAX_TRANSFERS" --buffer-size=16M; then | |
echo "Some files were copied." | |
fi | |
touch copied | |
exit 0 | |
fi | |
if [[ "$1" == "--to-target" ]]; then | |
while [[ ! -e copied ]]; do | |
echo "Moving from cache to target..." | |
if ! rclone -P move "$CACHE" "$TARGET" --min-size=0 --exclude="*.partial" --transfers="$MAX_OUT_TRANSFERS" --buffer-size=16M; then | |
echo "ERROR: rclone move failed. Exiting." | |
exit 1 | |
fi | |
sleep 5s | |
done | |
exit 0 | |
fi | |
if [[ $# -lt 3 ]] || [[ $# -gt 6 ]]; then | |
echo "usage: $0 <source> <cache> <target> [cache-size] [max-transfers] [max-out-transfers]" | |
exit 1 | |
fi | |
export SOURCE="$1" | |
export CACHE="$2" | |
export TARGET="$3" | |
export CACHE_SIZE="${4:-10G}" | |
export MAX_TRANSFERS="${5:-1}" | |
export MAX_OUT_TRANSFERS="${6:-5}" | |
SESSION="rclone_sync_session" | |
WINDOW="sync" | |
for (( round=1; ; round++ )); do | |
echo "=== Round $round ===" | |
echo "Fetching list of files already on target..." | |
rclone lsf "$TARGET" --files-only --recursive > "transferred_files.log" | |
echo "Checking files to transfer..." | |
if [[ -z "$(rclone lsf "$SOURCE" --files-only --recursive --min-size=0 --exclude="*.partial" --exclude-from="transferred_files.log")" ]]; then | |
echo "No more files to copy." | |
break | |
fi | |
rm -f copied | |
tmux kill-session -t "$SESSION" 2>/dev/null || true | |
tmux new-session -d -s "$SESSION" -n "$WINDOW" "$0 --to-cache" | |
tmux split-window -h -t "$SESSION:$WINDOW" "$0 --to-target" | |
tmux select-layout -t "$SESSION:$WINDOW" even-horizontal | |
tmux attach -t "$SESSION" | |
echo | |
done | |
echo "Final check." | |
rclone -P copy "$SOURCE" "$TARGET" --transfers="$MAX_OUT_TRANSFERS" --buffer-size=16M | |
echo "Finished file(s) transfer." | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Use:
Do a concurrent copy to cache folder from hdd to then send this over slow internet to the target from fast SSD, allowing HDD to spin-down in between.