Last active
November 30, 2023 23:25
-
-
Save tyrm/6709892f16290bc66958145d566beb17 to your computer and use it in GitHub Desktop.
i spent too much time on this
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
FILES="users.yaml users2.yaml users3.yaml" | |
OUTPUT_FOLDER=./files/ | |
. venv/bin/activate | |
# The log file where the output will be stored | |
LOG_FILE=./logs/users_$(date +%s).log | |
touch ${LOG_FILE} | |
log() { | |
# Echo the input and use tee to append to the log file | |
# "$@" expands to all the arguments passed to the function | |
timestamp="[$(date "+%Y-%m-%d %H:%M:%S")]" | |
echo "${timestamp}$@" | tee -a "${LOG_FILE}" | |
} | |
# Do the thing | |
for FILE in ${FILES} | |
do | |
# The command to be attempted | |
command_to_run="bdfr download ${OUTPUT_FOLDER} --no-dupes --opts ${FILE} --search-existing --submitted" | |
# Maximum number of attempts | |
max_attempts=3 | |
# Current attempt number | |
attempt=1 | |
while [ $attempt -le $max_attempts ]; do | |
log "(${FILE}) Attempt ${attempt}: Running restarting vpn." | |
# reset vpn | |
nordvpn d | |
sleep 10 | |
nordvpn c United_States | |
log "(${FILE}) Attempt ${attempt}: Running running BDFR." | |
# do scrape | |
$command_to_run | |
# Check if the command succeeded | |
if [ $? -eq 0 ]; then | |
log "(${FILE}) Attempt ${attempt}: Running BDFR succeeded." | |
break | |
fi | |
# Increment the attempt counter | |
((attempt++)) | |
# be nice | |
log "(${FILE}) Attempt ${attempt}: Running BDFR filed." | |
log "(${FILE}) Sleeping to be nice to nord." | |
sleep 60 | |
done | |
# be nice | |
log "Sleeping between files to be nice to nord." | |
sleep 60 | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment