Skip to content

Instantly share code, notes, and snippets.

@davenicoll
Last active March 6, 2023 03:39
Show Gist options
  • Save davenicoll/d135deaa28849739f892dbb39c86186f to your computer and use it in GitHub Desktop.
Save davenicoll/d135deaa28849739f892dbb39c86186f to your computer and use it in GitHub Desktop.
Download and remove files from ftp using lftp and pget chunks to maximise transfers and download speed
#!/bin/bash
if [[ -z "$1" ]]; then
echo "ERROR: no label provided (i.e. tv)"
exit 1
fi
login=''
pass=''
host=''
port='21'
remote_dir="/files/_finished/$1/"
local_dir="/volume1/share/Video/Incoming/$1"
nfile='2'
nsegment='3'
minchunk='1'
base_name="$(basename "$0")"
lock_file="/tmp/${base_name}.lock"
echo "${0} Starting at $(date)"
trap "rm -f ${lock_file}" SIGINT SIGTERM
find /tmp -name "${base_name}.lock" -mmin +240 -delete > /dev/null
if [ -e "${lock_file}" ]
then
echo "${base_name} is running already."
exit
else
touch "${lock_file}"
lftp -p "${port}" -u "${login},${pass}" "${host}" << EOF
set ssl:verify-certificate false
set ftp:list-options -a
set sftp:auto-confirm yes
set net:timeout 5
set net:max-retries 2
set net:reconnect-interval-base 5
set ftp:ssl-force yes
set ftp:ssl-protect-data
#set pget:min-chunk-size ${minchunk}
#set pget:default-n ${nsegment}
set ftp:sync-mode no
#set mirror:use-pget-n ${nsegment}
set mirror:parallel-transfer-count ${nfile}
set mirror:parallel-directories yes
set xfer:use-temp-file yes
set xfer:temp-file-name *.lftp
mirror -c -v --loop --Remove-source-dirs "${remote_dir}" "${local_dir}"
quit
EOF
chmod 776 -R "${local_dir}"
rm -f "${lock_file}"
trap - SIGINT SIGTERM
echo "${0} Finished at $(date)"
exit
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment