Created
October 31, 2014 21:36
-
-
Save simonlynen/82f5f5c421cdc2558bda to your computer and use it in GitHub Desktop.
Parallel rsync Source: http://www.krazyworks.com/making-rsync-faster/
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Define source, target, maxdepth and cd to source | |
source="/tmp/source" | |
target="/tmp/target" | |
depth=1 | |
cd "${source}" | |
# Set the maximum number of concurrent rsync threads | |
maxthreads=30 | |
# How long to wait before checking the number of rsync threads again | |
sleeptime=5 | |
# Find all folders in the source directory within the maxdepth level | |
find . -maxdepth ${depth} -type d | while read dir | |
do | |
# Make sure to ignore the parent folder | |
if [ `echo "${dir}" | awk -F'/' '{print NF}'` -gt ${depth} ] | |
then | |
# Strip leading dot slash | |
subfolder=$(echo "${dir}" | sed 's@^./@@g') | |
if [ ! -d "${target}/${subfolder}" ] | |
then | |
# Create destination folder and set ownership and permissions to match source | |
mkdir -p "${target}/${subfolder}" | |
chown --reference="${source}/${subfolder}" "${target}/${subfolder}" | |
chmod --reference="${source}/${subfolder}" "${target}/${subfolder}" | |
fi | |
# Make sure the number of rsync threads running is below the threshold | |
while [ `ps -ef | grep -c [r]sync` -gt ${maxthreads} ] | |
do | |
echo "Sleeping ${sleeptime} seconds" | |
sleep ${sleeptime} | |
done | |
# Run rsync in background for the current subfolder and move one to the next one | |
nohup rsync -a "${source}/${subfolder}/" "${target}/${subfolder}/" </dev/null >/dev/null 2>&1 & | |
fi | |
done | |
# Find all files above the maxdepth level and rsync them as well | |
find . -maxdepth ${depth} -type f -print0 | rsync -a --files-from=- --from0 ./ "${target}/" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment