A mirror script for the arch4edu repositories. Assume MIT License.

01-syncarch4edu.sh

#!/usr/bin/env bash

set -e

# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.

# Directory where the repo is stored locally. Example: /srv/repo
target="/srv/mirror/arch4edu"

# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
# Example: /var/tmp/repo-tmp
tmp="/var/tmp/arch4edu-tmp"

# Lockfile path
# It's recommended to put the lockfile on a tmpfs (such as /tmp on most systems),
# since after a new boot, the sync is aborted.
# If a lockfile was to be persistent in this scenario, it would require manual intervention.
lock="/tmp/.arch4edu-sync-lock"

# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0

# The source URL of the mirror you want to sync from.
# If you are a tier 1 mirror use:
# rsync://mirrors.tuna.tsinghua.edu.cn/arch4edu/
# Otherwise, check if any of the mirrors at
# https://github.com/arch4edu/mirrorlist/blob/master/mirrorlist.arch4edu
# support an rsync connection and use that instead.
source_url='rsync://mirrors.tuna.tsinghua.edu.cn/arch4edu/'

# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
# If you are a tier 1 mirror use: https://mirrors.tuna.tsinghua.edu.cn/arch4edu/lastupdate
# Otherwise use the HTTP(S) URL from your chosen mirror.
lastupdate_url='https://mirrors.tuna.tsinghua.edu.cn/arch4edu/lastupdate'

#### END CONFIG

[ ! -d "${target}" ] && mkdir -p "${target}"
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"

exec 9>"${lock}"
flock -n 9 || exit

rsync_cmd() {
	local -a cmd=(rsync -rtliH --delete-after --delay-updates --safe-links --max-delete=1000 \
	"--timeout=600" "--contimeout=60" -p --no-motd "--temp-dir=${tmp}")

	if stty &>/dev/null; then
		cmd+=(-h -v --progress)
	else
		cmd+=(--quiet)
	fi

	if ((bwlimit>0)); then
		cmd+=("--bwlimit=$bwlimit")
	fi

	"${cmd[@]}" "$@"
}

# Only run when there are changes
if [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
	# If your chosen mirror DOES have a lastsync file, uncomment the next three lines.
	# rsync_cmd "$source_url/lastsync" "$target/lastsync"
	# echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
	# echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"
	exit 0
fi

rsync_exclude=("--exclude=/info")
# Uncomment below if your chosen mirror DOES NOT have a lastsync file
rsync_exclude+=("--exclude=/lastsync")
rsync_cmd \
	"${rsync_exclude[@]}" \
	"${source_url}" \
	"${target}"

# Uncomment below if your chosen mirror DOES NOT have a lastsync file
date -u "+%s" > "$target/lastsync"

# Uncomment below if you've set up everything above correctly.
echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"

02-syncarch4edu-kescher.sh

#!/usr/bin/env bash

set -e

# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.

# Directory where the repo is stored locally. Example: /srv/repo
target="/srv/mirror/arch4edu"

# Directory where files are downloaded to before being moved in place.
# This should be on the same filesystem as $target, but not a subdirectory of $target.
# Example: /var/tmp/repo-tmp
tmp="/var/tmp/arch4edu-tmp"

# Lockfile path
# It's recommended to put the lockfile on a tmpfs (such as /tmp on most systems),
# since after a new boot, the sync is aborted.
# If a lockfile was to be persistent in this scenario, it would require manual intervention.
lock="/tmp/.arch4edu-sync-lock"

# If you want to limit the bandwidth used by rsync set this.
# Use 0 to disable the limit.
# The default unit is KiB (see man rsync /--bwlimit for more)
bwlimit=0

# The source URL of the mirror you want to sync from.
# If you are a tier 1 mirror use:
# rsync://mirrors.tuna.tsinghua.edu.cn/arch4edu/
# Otherwise, check if any of the mirrors at
# https://github.com/arch4edu/mirrorlist/blob/master/mirrorlist.arch4edu
# support an rsync connection and use that instead.
source_url='rsync://at.arch4edu.mirror.kescher.at/mirror/arch4edu/'

# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
# If you are a tier 1 mirror use: https://mirrors.tuna.tsinghua.edu.cn/arch4edu/lastupdate
# Otherwise use the HTTP(S) URL from your chosen mirror.
lastupdate_url='https://at.arch4edu.mirror.kescher.at/lastupdate'

#### END CONFIG

[ ! -d "${target}" ] && mkdir -p "${target}"
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"

exec 9>"${lock}"
flock -n 9 || exit

rsync_cmd() {
	local -a cmd=(rsync -rtliH --delete-after --delay-updates --safe-links --max-delete=1000 \
	"--timeout=600" "--contimeout=60" -p --no-motd "--temp-dir=${tmp}")

	if stty &>/dev/null; then
		cmd+=(-h -v --progress)
	else
		cmd+=(--quiet)
	fi

	if ((bwlimit>0)); then
		cmd+=("--bwlimit=$bwlimit")
	fi

	"${cmd[@]}" "$@"
}

# Only run when there are changes
if [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
	# If your chosen mirror DOES have a lastsync file, uncomment the next three lines.
	rsync_cmd "$source_url/lastsync" "$target/lastsync"
	echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
	echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"
	exit 0
fi

rsync_exclude=("--exclude=/info")
# Uncomment below if your chosen mirror DOES NOT have a lastsync file
# rsync_exclude+=("--exclude=/lastsync")
rsync_cmd \
	"${rsync_exclude[@]}" \
	"${source_url}" \
	"${target}"

# Uncomment below if your chosen mirror DOES NOT have a lastsync file
# date -u "+%s" > "$target/lastsync"

# Uncomment below if you've set up everything above correctly.
echo "Last sync was $(date -d @"$(cat ${target}/lastsync)" --rfc-3339=seconds -u)" > "${target}/info"
echo "Last update was $(date -d @"$(cat ${target}/lastupdate)" --rfc-3339=seconds -u)" >> "${target}/info"