rsync examples (two remotes)

rsync_examples.sh

# rsync with sudo
rsync -av \
    --rsync-path="sudo -u www-data rsync" \
    -e 'ssh -o StrictHostKeyChecking=no' \
    path_to_local_data/ [email protected]:/var/www/


# If you wanted to build an exclusion list from a list of files in a directory
# This was useful when trying to sync a list of files but exclude a list of default installed
# so that the destination only contained the differences of what was added that wasn't a part
# of the default list of installed files.
RSYNC_EXCLUSIONS=$(ls /path_containing_files_to_exclude/ | perl -ne 'chomp; print "--exclude $_ "')
rsync -av ${RSYNC_EXCLUSIONS} \
  /source_path/ /destination_path/


# rsync media between two remotes that can not talk to each other
# this creates a port (50000) on the origin server that establishes a tunnel between the origin and destination through
# your computer, as long as your computer can access both servers you can rsync the files between the two
# You will need to either:
# 1. Put public key from origin server (origin.net) into authorized_keys of destination server (destination.com)
# 2. Add a `-A` flag to the ssh command so that your computer's private key gets forwarded for authentication purposes
ssh -R 50000:destination.com:22 origin.net 'rsync -e "ssh -o StrictHostKeyChecking=no -p 50000" -av \
    ~/origin.net/htdocs/media/ root@localhost:/var/www/htdocs/media/'

# push/send from origin to send to destination
# authorize origin server to login to destination server
ssh -R 50000:${DESTINATION_HOST}:22 ${ORIGIN_USER}@${ORIGIN_HOST} "rsync -e 'ssh -o StrictHostKeyChecking=no -p 50000' -av \
    ${ORIGIN_SHARED_PATH}/pub/media/ ${DESTINATION_USER}@localhost:${DESTINATION_SHARED_PATH}/pub/media/"

# pull/get from destination to get from origin
# authorize destination server to login to origin server
ssh -R 50000:${ORIGIN_HOST}:22 ${DESTINATION_USER}@${DESTINATION_HOST} "rsync -e 'ssh -o StrictHostKeyChecking=no -p 50000' -av \
    ${ORIGIN_USER}@localhost:${ORIGIN_SHARED_PATH}/pub/media/ ${DESTINATION_SHARED_PATH}/pub/media/"



# Three way file transfer of backup file adding file timestamp when sending to destination
#
# An SSH connection is established with the destination and key forwarding (use of -A flag 
# on ssh connection) is used in order to establish the ssh connection from the destination 
# to the origin and initiate the rsync from the destination. A progress indicator (use of 
# --progress flag on rsync command) exists to monitor the file transfer progress.
#
# The rsync command escalates the permissions to root with sudo to pull a backup file
# that was generated and only accessible by the root user. The destination file will
# be owned by the destination user.
declare ORIGIN_HOST="my_origin_hostname"
declare ORIGIN_USER="my_origin_username"
declare DEST_HOST="my_destination_hostname"
declare DEST_USER="my_destination_username"
declare BACKUP_FILE_NAME="backup_filename.sql.gz"
declare BACKUP_FILE_PATH="/the/path/to/the/file/you/want/to/transfer/"
declare DEST_PATH="/the/path/you/want/the/destination/file/to/reside/at/"
declare BACKUP_DATE=$(\
    ssh -q ${ORIGIN_USER}@${ORIGIN_HOST} "\
        date -d @\$(sudo stat -c %Y ${BACKUP_FILE_PATH}${BACKUP_FILE_NAME}) +%Y%m%d \
        "\
    )
declare TUNNEL_HOST="localhost"
declare TUNNEL_PORT="50000"
ssh -A -R ${TUNNEL_PORT}:${ORIGIN_HOST}:22 ${DEST_USER}@${DEST_HOST} "rsync \
    -e 'ssh -l ${ORIGIN_USER} -o StrictHostKeyChecking=no -p ${TUNNEL_PORT}' -av --progress --rsync-path=\"sudo rsync\" \
    ${TUNNEL_HOST}:${BACKUP_FILE_PATH}${BACKUP_FILE_NAME} ${DEST_PATH}${BACKUP_DATE}_${BACKUP_FILE_NAME}"



# Rsync file through bastion/jumphost
declare ORIGIN_FILEPATH="/the/path/to/the/file/you/want/to/transfer/"
declare DEST_BAST_USER="my_username"
declare DEST_BAST_HOST="my_bastion_hostname" # Public address of bastion host
declare DEST_SSH_HOST="10.0.0.10" # IP/hostname on local network relative to bastion host
declare DEST_SSH_USER="my_username"
declare DEST_FILEUSER="www-data" # User that owns the file on the destination
declare DEST_FILEPATH="/the/path/you/want/the/destination/file/to/reside/at/"
rsync -av --progress --rsync-path="sudo -u ${DEST_FILEUSER} rsync" \
    -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ProxyCommand=\"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR ${DEST_BAST_USER}@${DEST_BAST_HOST} -W %h:%p\"" \
    ${ORIGIN_FILEPATH} ${DEST_SSH_USER}@${DEST_SSH_HOST}:${DEST_FILEPATH}