Last active
March 6, 2023 03:39
-
-
Save davenicoll/d135deaa28849739f892dbb39c86186f to your computer and use it in GitHub Desktop.
Download and remove files from ftp using lftp and pget chunks to maximise transfers and download speed
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
if [[ -z "$1" ]]; then | |
echo "ERROR: no label provided (i.e. tv)" | |
exit 1 | |
fi | |
login='' | |
pass='' | |
host='' | |
port='21' | |
remote_dir="/files/_finished/$1/" | |
local_dir="/volume1/share/Video/Incoming/$1" | |
nfile='2' | |
nsegment='3' | |
minchunk='1' | |
base_name="$(basename "$0")" | |
lock_file="/tmp/${base_name}.lock" | |
echo "${0} Starting at $(date)" | |
trap "rm -f ${lock_file}" SIGINT SIGTERM | |
find /tmp -name "${base_name}.lock" -mmin +240 -delete > /dev/null | |
if [ -e "${lock_file}" ] | |
then | |
echo "${base_name} is running already." | |
exit | |
else | |
touch "${lock_file}" | |
lftp -p "${port}" -u "${login},${pass}" "${host}" << EOF | |
set ssl:verify-certificate false | |
set ftp:list-options -a | |
set sftp:auto-confirm yes | |
set net:timeout 5 | |
set net:max-retries 2 | |
set net:reconnect-interval-base 5 | |
set ftp:ssl-force yes | |
set ftp:ssl-protect-data | |
#set pget:min-chunk-size ${minchunk} | |
#set pget:default-n ${nsegment} | |
set ftp:sync-mode no | |
#set mirror:use-pget-n ${nsegment} | |
set mirror:parallel-transfer-count ${nfile} | |
set mirror:parallel-directories yes | |
set xfer:use-temp-file yes | |
set xfer:temp-file-name *.lftp | |
mirror -c -v --loop --Remove-source-dirs "${remote_dir}" "${local_dir}" | |
quit | |
EOF | |
chmod 776 -R "${local_dir}" | |
rm -f "${lock_file}" | |
trap - SIGINT SIGTERM | |
echo "${0} Finished at $(date)" | |
exit | |
fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment