copy files in parallel to a destination using rsync and gnu parallel

gistfile1.txt

#!/usr/bin/env bash
# archive_parallel.sh
# Two-pass archival using rsync and GNU Parallel:
#   Pass 1 (sequential): Recreate full directory structure safely.
#   Pass 2 (parallel): Copy files in parallel using rsync.

set -euo pipefail

if [ "$#" -ne 2 ]; then
    echo "Usage: $0 <src_dir> <dst_dir>" >&2
    exit 1
fi

src_dir=$(realpath "$1")
dst_dir=$(realpath "$2")

if [ ! -d "$src_dir" ]; then
    echo "Error: Source directory '$src_dir' does not exist." >&2
    exit 1
fi

mkdir -p "$dst_dir"

echo "=== Pass 1: Recreating directory structure (sequential) ==="
# Create all subdirectories sequentially using mkdir -p (safe & ordered by find)
while IFS= read -r -d '' dir; do
    rel_path=$(realpath --relative-to="$src_dir" "$dir")
    mkdir -p "$dst_dir/$rel_path"
done < <(find "$src_dir" -type d -print0)

echo "=== Pass 2: Parallel rsync of files ==="
export src_dir dst_dir
find "$src_dir" -type f -print0 \
  | parallel -0 -j+0 '
      rel_path=$(realpath --relative-to="$src_dir" {})
      rsync -a "$src_dir/$rel_path" "$dst_dir/$rel_path"
  '

echo "✅ Archival complete: $src_dir → $dst_dir"