Script for periodical synchronization of data (e.g. mirrors) using rsync

README.adoc

Files

• /etc/periodic/15min/sync-mirror-alpine → /usr/local/bin/sync-mirror (symlink)

• /etc/sync-mirror.d/alpine.conf

• /usr/local/bin/sync-mirror

• /var/www/mirrors/alpine

alpine.conf

# sync-mirror configuration for Alpine aports

src_uri="rsync://rsync.alpinelinux.org/alpine/"
dest_dir="/var/www/mirrors/alpine/"
excludes="/v2.4 /v2.5 /v2.6 /v2.7 /v3.3/latest-stable /snapshots"

sync-mirror

#!/bin/sh
set -eu

CONF_DIR='/etc/sync-mirror.d'
DEFAULT_OPTS='--quiet --recursive --links --hard-links --times --delay-updates --delete-after'
DATE_FORMAT='%Y-%m-%d %H:%M:%S'


# Infer config name from the script name, if not provided as first argument.
if [ $# -eq 0 ]; then
	script_name="$(basename "$0")"
	case "${script_name%%.sh}" in
		sync-mirror-*) name="${script_name#sync-mirror-}";;
		sync-*) name="${script_name#sync-}";;
	esac
else
	name="$1"
fi

if [ -z "$name" ]; then
	echo "Usage: $0 NAME" >&2; exit 1
fi

config="$CONF_DIR/$name.conf"
if [ ! -r "$config" ]; then
	echo "ERROR: Config file $config does not exist or not readable" >&2; exit 1
fi

# Make sure we never run 2 rsync at the same time.
lockfile="/var/lock/sync-mirror-$name.lock"
if [ -z "${FLOCK:-}" ] ; then
	exec env FLOCK=1 flock -n "$lockfile" $0 "$@"
fi

# Kill all subshells at the end.
trap "kill 0" SIGINT SIGTERM


. "$config" || {
	echo "Failed to source $config" >&2; exit 1
}

: ${excludes:=}
: ${logfile:="/var/log/sync-mirror.log"}
: ${run_as:="rsync"}
: ${timeout:=300}  # seconds

rsync_opts="$DEFAULT_OPTS ${rsync_opts:-}"
if [ "$timeout" -gt 0 ]; then
	rsync_opts="$rsync_opts --timeout=$timeout"
fi
for path in $excludes; do
	rsync_opts="$rsync_opts --exclude='$path'"
done
rsync_opts="$rsync_opts $src_uri $dest_dir"

# An awk program to add prefix to all logged lines.
log_prefix="{ print strftime(\"$DATE_FORMAT\"), \"[$name]:\", \$0; fflush(); }"

{
	echo 'starting sync'
	su "$run_as" -c "/usr/bin/rsync $rsync_opts" || {
		echo 'sync failed'; exit 2
	}
	echo 'sync completed'

} | awk "$log_prefix" | tee -a "$logfile" 2>&1