Skip to content

Instantly share code, notes, and snippets.

@dbiesecke

dbiesecke/rclone-lba.sh

Last active August 21, 2024 13:58
Show Gist options
  • Save dbiesecke/16221c3a5ee28d728fde6489710f840b to your computer and use it in GitHub Desktop.
Save dbiesecke/16221c3a5ee28d728fde6489710f840b to your computer and use it in GitHub Desktop.
Download ZIP
Raw
rclone-lba.sh
#!/bin/bash
#######################################
### NZBGET POST-PROCESSING SCRIPT ###
# Moves downloads to a configured location using rclone and load balances over
# multiple remotes
#
#
# NOTE: Requires sh, coreutils or busybox, screen and rclone. Remotes must be setup in rclone.conf.
########################################
### OPTIONS
# Options to add to the end of the rclone command
#
#
# NOTE: Specify the location of rclone.conf here if it is not in the default
# location for the user running nzbget.
# INFO: Increasing drive-chunk-size will
# increase speed but also increase memory usage.
#
### NZBGET POST-PROCESSING SCRIPT ###
#######################################
export PATH="/shared/bin:$PATH"
RCLONE_DRIVE_CLIENT_ID=456XXXXXXX46v34m6.apps.googleusercontent.com
RCLONE_DRIVE_CLIENT_SECRET=XXXXX
RCLONE_LBA_METHOD=move
OPTIONS="--transfers 2 --drive-chunk-size 64M -P"
CONFIG=/shared/rclone/conf
PROJECT_ID=5dd
NAME=pre
WKEY=
AGENT=`uname -io -sr `
#exit 93
if [ -d "$1" ]; then
NZBPP_FINALDIR=$1
cd "$NZBPP_FINALDIR/../"
PWD=$(pwd)
# NZBPP_CATEGORY=$(basename $PWD)
NZBPP_CATEGORY=$2
echo "$NZBPP_FINALDIR $NZBPP_CATEGORY\n"
FILESIZE=$(du . -c -s | tail -1 | awk '{print $1}')
rclone="rclone"
POSTPROCESS_SUCCESS=93
POSTPROCESS_ERROR=94
index_curr=0
while $(test -d "$NZBPP_FINALDIR")
do
index_curr=$((index_curr + 1))
if ((index_curr >= 5)); then
exit $POSTPROCESS_ERROR
else
index_curr=$((index_curr + 1))
echo "next loop"
fi
echo "[INFO] Remove samples,sfv and other crap "
find "$NZBPP_FINALDIR" -type d -iname "Sample" -exec rm -fR "{}" \; 2>/dev/null
find "$NZBPP_FINALDIR" -type f -iname "*.sample.*" -exec rm "{}" \; 2>/dev/null
find "$NZBPP_FINALDIR" -type f -iname "*.url" -exec rm "{}" \; 2>/dev/null
find "$NZBPP_FINALDIR" -type f -iname "*.nzb" -exec rm "{}" \; 2>/dev/null
find "$NZBPP_FINALDIR" -type f -iname "*.sfv" -exec rm "{}" \; 2>/dev/null
find "$NZBPP_FINALDIR" -type f -iname "*.exe" -exec rm "{}" \; 2>/dev/null
echo "[INFO] Flatten dir: $NZBPP_FINALDIR"
find "$NZBPP_FINALDIR" -type f -exec mv "{}" "$NZBPP_FINALDIR" \; 2>/dev/null
echo "[INFO] Unpack rar's if found..."
for i in $(find . -type f -iname "*.rar"); do unrar x -y "$i" && echo "[X] Unpack OK, remove file" && rm "$i" ;done
echo "[INFO] remove empty dirs: $NZBPP_FINALDIR"
find "$NZBPP_FINALDIR" -type d -exec rmdir "{}" \; 2>/dev/null
REMOTES=$($rclone listremotes --config $CONFIG| xargs | sed -r 's/ /,/g' | sed -r 's/://g')
REMOTE=$(echo $REMOTES | tr , '\n' | shuf | sed -n '1p')
MOVEPATH=""$REMOTE":"${NZBPP_CATEGORY^^}-iNCOMiNG"/"
MSIZE=$((15 * 1024 * 1024))
USPACE=$($rclone --config "$CONFIG" size "$REMOTE:" --json | jq '.bytes')
FREESPACE=$(($MSIZE - $USPACE))
echo "[INFO] REMOTE: "$REMOTE" Free Space: $(($FREESPACE/1024))MB UploadSize: $(($FILESIZE/1024/1024))GB $MOVEPATH"
if (($FREESPACE > $FILESIZE)); then
echo "[INFO] REMOTE:YES"
else
index_curr=$((index_curr + 1))
echo "next loop"
fi
BASE=$(basename "$NZBPP_FINALDIR")
FILE=$(find "$NZBPP_FINALDIR" -type f -printf '%TY-%Tm-%Td %TH:%TM: %p\n'| sort -n | tail -n1 | awk -F' ' '{print $3}' | sed -r 's/^\.//')
#CAT=${NZBPP_CATEGORY^^}
#NZBPP_CATEGORY=$CAT
#ls -lah $NZBPP_FINALDIR
#MOVEPATH=""$REMOTE":"${NZBPP_CATEGORY}"/"$(basename "$NZBPP_FINALDIR")
# CLOUD="/storage/rclone/"${NZBPP_CATEGORY}"/$FILEBASE"
COMMAND="$rclone --config $CONFIG $RCLONE_LBA_METHOD \""$NZBPP_FINALDIR"\" \""$MOVEPATH/"\" "$OPTIONS""
echo "[INFO] RCLONE: "$COMMAND""
eval "$COMMAND"
#msg=$($COMMAND 2>&1)
exit_code=$?
if [ $exit_code -ne 0 ] ; then
echo "ERROR $msg"
# index_curr=$((index_curr + 1))
# echo "next loop"
else
#if no errors occured just log the message
echo "[X] Upload $msg"
# find "$NZBPP_FINALDIR" -type d -exec rmdir "{}" \; 2>/dev/null
# rmdir "$NZBPP_FINALDIR" >/dev/null 2>/dev/null
test -d "$NZBPP_FINALDIR" && echo "[X] Upload ... $(basename $NZBPP_FINALDIR) SUCCESS"
fi
#eval "$COMMAND"
# export NZBPP_FINALDIR="$CLOUD"
done
Raw
rclone_lba.sh
#!/bin/zsh
rclone_lba(){
rclone=rclone
sadir=/etc/rclone/svcacc
#safile="/etc/rclone/svcacc/000003.json"
safile=$(find "$sadir" -type f | shuf -n 1)
MYFILE=$(realpath "$1")
MYFILE_SINGLE=$(basename "$MYFILE")
FILESIZE=$(du "$MYFILE" -c -s | tail -1 | awk '{print $1}')
REMOTE=foilo
BASE=$(basename "$MYFILE")
SUBDIR=$(echo "$(realpath "$MYFILE")" | sed -r "s/\/S[0-9]+.+//g")
BASEDIR=$(basename "$SUBDIR")
#BASEDIR=$(basename "$(dirname "${MYFILE}")")
#SUBDIR=$(echo "$(realpath "$MYFILE")" | sed -r 's/\/S[0-9]+.+//g')
#SUBDIR="$MYFILE"
#path=${path#/*/}
#SECDIR=$(basename "$(realpath "$BASEDIR"./")")
MAINDIR=$(basename $(dirname "$(realpath "${BASEDIR}")"))
MOVE_REMOTE=wcc-sa
MSIZE=$((15 * 1024 * 1024 ))
USPACE=$(rclone about "$REMOTE:" --drive-service-account-file "$safile" --json | jq '.free')
#USPACE=$($rclone about "$REMOTE:" --drive-service-account-file "$safile" --json )
FREESPACE=$(($USPACE - $(($FILESIZE*1024)) ))
#echo "$SUBDIR"
#return 127
#MOVEPATH=""$MOVE_REMOTE":"$MAINDIR"/"$SUBDIR"/"
MOVEPATH=""$MOVE_REMOTE":"$MAINDIR
if [[ "$MYFILE" == *"Movie"* ]]; then
login="8f1c11a97ea3161b48e2"
key="eo40z34pB1hOLK"
folder="D7nAEmXYFTo"
#export MAINDIR="Movies"
if [[ "$MYFILE" == *"Doku"* ]]; then
export MAINDIR="Movies-Doku"
login="1b32e1aff6ab5360e2c3"
key="3rogR2xgaJcXoJ"
folder="MngAp3ZkTTY"
fi
fi
if [[ "$MYFILE" == *"Anime"* ]]; then
export MAINDIR="Anime"
login="f87bdffcdbaf91a17f5b"
key="Y13wpyKA2XTvPrg"
folder="ak-leNIhh3o"
if [[ "$MYFILE" == *"Movie"* ]]; then
MAINDIR="Movies-Anime"
login="e034ff43040429291c9c"
key="Gb48yYrqBrh1wam"
folder="fqEsWVfvom0"
fi
if [[ "$MYFILE" == *"TV"* ]]; then
MAINDIR="TV-Anime"
login="f87bdffcdbaf91a17f5b"
key="Y13wpyKA2XTvPrg"
folder="ak-leNIhh3o"
fi
else
if [[ "$MYFILE" == *"TV-Doku"* || "$MYFILE" == *"History"* ]]; then
#MOVEPATH=""$MOVE_REMOTE":"$MAINDIR"/"$BASEDIR"/"
export MAINDIR="TV-Doku"
login="af93cff344b69403a941"
key="YexAmajgPJse0A"
folder="CLT4lpDVSEk"
if [[ "$MYFILE" == *"Movie"*"DOKU"* ]]; then
MAINDIR="Movies-Doku"
login="1b32e1aff6ab5360e2c3"
key="3rogR2xgaJcXoJ"
folder="MngAp3ZkTTY"
fi
else
if [[ "$MYFILE" == *"Movie"* ]]; then
MAINDIR="Movies"
login="1b32e1aff6ab5360e2c3"
key="3rogR2xgaJcXoJ"
folder="MngAp3ZkTTY"
else
if [[ "$MYFILE" == *"TV"* ]]; then
export MAINDIR="TV"
if [[ "$MYFILE" == *"History"* || "$MYFILE" == *"Doku"* || "$MYFILE" == *"NatGeo"* ]]; then
login="af93cff344b69403a941"
key="YexAmajgPJse0A"
folder="CLT4lpDVSEk"
export MAINDIR="TV-Doku"
fi
login="afd6057c943f5d447d9a"
key="W5O5r6Q1h34"
folder="W5O5r6Q1h34ö"
fi
fi
fi
fi
#MOVEPATH=""$MOVE_REMOTE":"$MAINDIR"/"$BASEDIR"/"
if [[ "$MYFILE" == *"S0"* ]]; then
MOVEPATH=""$MOVE_REMOTE":"$MAINDIR"/"$SUBDIR"/"
else
MOVEPATH=""$MOVE_REMOTE":"$MAINDIR
fi
if [ -f "/mnt/strm/$MAINDIR/$BASEDIR/$BASE.strm" ]; then
echo "File allready exist \t /mnt/strm/$MAINDIR/$BASEDIR/$BASE.strm"
return 127
fi
#if [[ "$MYFILE" == *"TV-Doku"* ]]; then
#export MAINDIR="TV-Doku"
#login="af93cff344b69403a941"
#key="YexAmajgPJse0A"
#folder="CLT4lpDVSEk"
# if [[ "$MYFILE" == *"Movie"* ]]; then
# MAINDIR="Movies-Doku"
# login="1b32e1aff6ab5360e2c3"
# key="3rogR2xgaJcXoJ"
# folder="MngAp3ZkTTY"
# fi
#else
#if [[ "$MYFILE" == *"TV"* ]]; then
#export MAINDIR="TV"
#login="afd6057c943f5d447d9a"
#key="W5O5r6Q1h34"
#folder="W5O5r6Q1h34ö"
#fi
#
#fi
#MOVEPATH=""$MOVE_REMOTE":"$MAINDIR"/"$BASEDIR"/"
MYSTRM="https://onepoint.bcgd1.workers.dev/gdrive/$MAINDIR/$BASEDIR/$MYFILE_SINGLE"
echo "$SUBDIR\t $MAINDIR\t$MYSTRM\t$MOVEPATH\n"
#return 127
echo "[INFO] REMOTE: "$REMOTE" Free Space: $(($FREESPACE/1024))MB UploadSize: $(($FILESIZE/1024/1024))GB $MOVEPATH"
#echo "[INFO] \t Free: $USPACE Free:$(($FREESPACE/1024/1024)) Size:$(($FILESIZE/1024))"
if (( $(($FILESIZE/1024)) > 14958 )); then
echo "[INFO] Filesize to big! $MYFILE $(($FILESIZE/1024))MB"
return -1
fi
if (($(($FREESPACE/1024/1024)) > $(($FILESIZE/1024)) )); then
echo "[INFO] REMOTE:YES \t Free:$(($FREESPACE/1024/1024)) Size:$(($FILESIZE/1024))"
COMMAND='rclone copy "'$MYFILE'" "'$MOVEPATH'" --drive-service-account-file "'$safile'" --ignore-existing -P --onedrive-chunk-size 124M'
eval "$COMMAND"
#msg=$($COMMAND 2>&1)
exit_code=$?
if [ $exit_code -ne 0 ] ; then
echo "ERROR $msg"
#rclone_lba "$@"
else
COMMAND='rclone link "'$MOVEPATH$MYFILE_SINGLE'" --drive-service-account-file "'$safile'"'
echo "$COMMAND"
eval "$COMMAND"
#exit_code=$?
#if [ $exit_code -ne 0 ] ; then
# echo "Error on upload $MYFILE"
# return -1
#fi
echo "$REMOTE:$BASE"
test -d "/mnt/strm/$MAINDIR/$BASEDIR" || mkdir -p "/mnt/strm/$MAINDIR/$BASEDIR"
echo "${MYSTRM// /%20}" | tee "/mnt/strm/$MAINDIR/$BASEDIR/$BASE.strm"
echo "writen: /mnt/strm/$MAINDIR/$BASEDIR/$BASE.strm"
#rclone backend shortcut 'foilo:' 'Gundress_-_(ger.sub)(Xvid).avi' -o 'target=wcc1:' 'Gundress_-_(ger.sub)(Xvid).avi' --drive-service-account-file "/etc/rclone/svcacc/000003.json"
rclone mkdir "$MOVEPATH"
# COMMAND='rclone backend shortcut "'$MOVE_REMOTE':'$BASEDIR'" "'$BASE'" -o "target='$MOVEPATH'" "'$MYFILE_SINGLE'" -v --drive-service-account-file "'$safile'" --onedrive-chunk-size 124M'
COMMAND='rclone backend shortcut "'$MOVEPATH'" "'$BASE'" -o "target='$MOVEPATH'" "'$MYFILE_SINGLE'" -v --drive-service-account-file "'$safile'" --onedrive-chunk-size 124M >/dev/null'
echo "$COMMAND"
eval $COMMAND
# exit_code=$?
# if [ $exit_code -ne 0 ] ; then
# return "Error on link"
# return -1
# fi
tap_upload_generic_remote "${MYSTRM// /%20}" "$login" "$key" "$folder"
test -f "$MYFILE" && rm "$MYFILE"
return 1
fi
fi
return 127
}
rclone_lba "$1"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment