Massive download queues, simplified

wgetter.sh

#!/bin/bash

VERSION=0.3.0
echo "wgetter v$VERSION"

function usage() {
	echo "Usage: $0 textFile [outputDir]"
	echo "Text file contains list of URLs, separated by line"
	exit 
}

OUTDIR=.

if [ $# -lt 1 ]; then
	usage
elif [ $# -eq 2 ]; then
	OUTDIR=$2
	if [ ! -e "$OUTDIR" ]; then
		echo "Directory '$OUTDIR' does not exist, creating..."
		mkdir -p "$OUTDIR"

		if [ "$?" != "0" ]; then
			echo "Unable to create directory '$OUTDIR'. Exiting"
			exit
		fi
	fi
fi

if [ ! -e "$1" ]; then
	echo "File '$1' does not exist. Exiting"
	exit
fi

echo "Attempting to wget all files in $1..."

N=0
while IFS='' read -r line || [[ -n "$line" ]]; do

	# get dat filename
	IFS='/' read -ra ADDR <<< "$line"
	lastIdx=$((${#ADDR[@]}-1))

	if [ $lastIdx -lt 1 ]; then
		continue
	fi

	N=$((N + 1))
	fname="${ADDR[$lastIdx]}"

	if [ -e "$fname" ]; then
		echo "File '$fname' already exists, skipping."
	fi

	echo
	echo "$N. $fname"
	echo "----------------------------------------------------"

	wget "$line" -O "$OUTDIR/$fname"

	if [ "$?" = "0" ]; then
		echo "Successful fetch."
	else
		echo "Could not fetch '$fname'"
		continue
	fi

done < "$1"