Sync a source directory to a target directory

rsync_parallel.sh

#!/bin/bash

##############################################################################
# This script recursively syncs a source directory to a target directory.
# This script can run prallel with multiple processes for speed.
# It will:
#   - Skip existing files
#   - Copy file and its metadata
#   - Delete empty directories in target directory
#
# Author: Ismet Handzic (02/29/2024)
##############################################################################
set -u

# Define source and target directories (absolute and no trailing "/")
# The contents of the source directory will be the contents of the target directory
SOURCE_DIR="/path/to/source"
TARGET_DIR="/path/to/target"

# Ensure trailing slashes for rsync behavior consistency
SOURCE_DIR="${SOURCE_DIR%/}/"
TARGET_DIR="${TARGET_DIR%/}/"
EXCLUDE_DIRS=()                       # Space separated ("one" "two")
EXCLUDE_EXTENSIONS=("*.foo")          # Space separated ("*.json" "*.foo")
LOGGER_TAG="RSYNC-SYNC"

# Parallel Options
PARALLEL_JOB=true
PARALLEL_JOBS=10
##############################################################################

# Function to log to syslog with a tag and print to stdout
log_message() {
	# USAGE: log_message "message" [level] [message_tag]
	# Levels: debug, info, notice, warning, err, crit, alert, emerg
	local level=${2:-info}
	local message_tag="${3:-$LOGGER_TAG}"
	logger -p local0."$level" -t "$message_tag" "$1" && echo "$1"
}

# Function to log an error message, send a UI notification, and exit
error_handling() {
	local exit_code=$?
	if [ "$exit_code" -ne 0 ]; then
		local error_message="Error occurred during rsync. Exit code: $exit_code"
		log_message "$error_message" "err"
		# Send a notification to the OS UI
		notify-send "Failed rsync" "$error_message" -u critical
	fi
	exit "$exit_code"
}
# Trap ERR to catch rsync errors and any other unexpected errors
trap 'error_handling' ERR

log_message "Syncing $SOURCE_DIR to $TARGET_DIR ..."
log_message "Excluding directories: ${EXCLUDE_DIRS[*]}"
log_message "Excluding file extensions: ${EXCLUDE_EXTENSIONS[*]}"

# Prepare the rsync exclude parameters
EXCLUDE_PARAMS=()
for dir in "${EXCLUDE_DIRS[@]}"; do
	EXCLUDE_PARAMS+=(--exclude="$dir")
done

# Prepare the rsync exclude extension parameters
for ext in "${EXCLUDE_EXTENSIONS[@]}"; do
	EXCLUDE_PARAMS+=(--exclude="$ext")
done

# Rsync options
# NOTE: Add --no-compress for local runs
# NOTE: Add --dry-run for testing
RSYNC_OPTIONS=(
	--archive
	--human-readable
	--info=NAME
	--delete
	--force
	--ignore-existing
	--no-compress
	"${EXCLUDE_PARAMS[@]}"
)

# Log rsync options being used
log_message "Rsync options: ${RSYNC_OPTIONS[*]}"

# Perform the sync using rsync directly for directory structure, including empty directories
log_message "Creating directory structure in target..."
rsync "${RSYNC_OPTIONS[@]}" --include='*/' --exclude='*' "$SOURCE_DIR" "$TARGET_DIR"

# Use find and rsync in parallel for files if parallel jobs are enabled
if [ "$PARALLEL_JOB" = true ]; then
	log_message "Parallel number of jobs running: ${PARALLEL_JOBS}"
	find "$SOURCE_DIR" -type f -print0 | xargs -0 -I {} -P "$PARALLEL_JOBS" bash -c \
		'src="{}"; dest="'"$TARGET_DIR"'${src#'"$SOURCE_DIR"'}"; mkdir -p "$(dirname "$dest")"; rsync '"${RSYNC_OPTIONS[*]}"' "$src" "$dest"'
else
	rsync "${RSYNC_OPTIONS[@]}" "$SOURCE_DIR" "$TARGET_DIR"
fi

# Remove empty directories in the target directory that might have been left from deletions
find "$TARGET_DIR" -type d -empty -delete
echo
log_message "Successfully synced directories!"