Sync script for package mirrors that I use

sync.sh

#!/usr/bin/env bash

set -e

# Name of the mirror used for local directory names
mirror="${1}"

# Directory where the repo is stored locally.
target="/srv/ftp/${mirror}"

# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
tmp="/srv/ftp/tmp/${mirror}"

# Lockfile path
# It's recommended to put the lockfile on a tmpfs (such as /tmp on most systems),
# since after a new boot, the sync is aborted.
# If a lockfile was to be persistent in this scenario, it would require manual intervention.
lock="/tmp/.ftp-syncrepo-${mirror}.lck"

# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0

# The source domain+path of the mirror you want to sync from.
source_url="${2}"

# Path of lastsync file
lastsync_path="${3:-/}"

if [ "$lastsync_path" = "none" ]; then
	lastupdate_url="${4:-https://${source_url}/}"
	lastupdate_path='/'
else
	lastupdate_url="${4:-https://${source_url}${lastsync_path}}"
	lastupdate_path="${lastsync_path}"
fi

[ ! -d "${target}" ] && mkdir -p "${target}"
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"

exec 9>"${lock}"
flock -n 9 || exit

rsync_cmd() {
	local -a cmd=(rsync -rlptH --safe-links --delete-after --delay-updates
		"--timeout=600" "--contimeout=60" --no-motd "--temp-dir=${tmp}")

	if stty &>/dev/null; then
		cmd+=(-h -v --progress --stats)
	else
		cmd+=(--quiet)
	fi

	if ((bwlimit>0)); then
		cmd+=("--bwlimit=$bwlimit")
	fi

	"${cmd[@]}" "$@"
}

if [ "$lastupdate_url" != "none" ]; then
	# if we are called without a tty (cronjob) only run when there are changes
	if ! tty -s && [[ -f "${target}${lastupdate_path}lastupdate" ]] && diff -b <(curl -Ls "${lastupdate_url}lastupdate") "${target}${lastupdate_path}lastupdate" >/dev/null; then
		# keep lastsync file in sync for statistics generated by the Arch Linux website
		if [ "$lastsync_path" != "none" ]; then
			rsync_cmd "rsync://${source_url}${lastsync_path}lastsync" "${target}${lastsync_path}lastsync"
			# Uncomment below if you want to add an info file
			#echo "Last sync was $(date -d @"$(cat ${target}${lastsync_path}lastsync)" --rfc-3339=seconds -u)" > "${target}${lastsync_path}info"
			#[ -f "${target}${lastupdate_path}lastupdate" ] && echo "Last update was $(date -d @"$(cat ${target}${lastupdate_path}lastupdate)" --rfc-3339=seconds -u)" >> "${target}${lastsync_path}info"
		fi
		exit 0
	fi
fi

if [ "$lastsync_path" = "none" ]; then
	rsync_exclude+=("--exclude=/lastsync")
fi

if [ -n "$EXCLUDE_PATHS" ]; then
	EXCLUDE_PATHS=$(echo "$EXCLUDE_PATHS" | tr ":" "\n")

	for path in $EXCLUDE_PATHS; do
		rsync_exclude+=("--exclude=/$path")
	done
fi

rsync_cmd \
	"${rsync_exclude[@]}" \
	"rsync://${source_url}" \
	"${target}"

if [ "$lastsync_path" = "none" ]; then
	date -u "+%s" > "${target}/lastsync"

	# Uncomment below if you want to add an info file
	#echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
	#[ -f "${target}${lastupdate_path}lastupdate" ] && echo "Last update was $(date -d @"$(cat ${target}${lastupdate_path}lastupdate)" --rfc-3339=seconds -u)" >> "${target}info"
fi