dl-000

dl-000.md

List of downloaders

dl-bandcamp

#!/bin/sh

# =========================================================================
#
# mp3 downloader
#
# =========================================================================

[ -t 0 ] || {
	xargs -r -IX "$0" "$@" X
	exit
}

# =========================================================================

for f in ~/.dl-rc ./dl-rc
do
	# shellcheck source=/dev/null
	[ -f "$f" ] \
	&& . "$f"
done

# =========================================================================

re_base="https://.+\.bandcamp\.com"
re_list="$re_base(/(music)?)?"
re_album="$re_base/album/.+"

# =========================================================================

if echo "$1" | grep -E -q "^$re_list\$"
then
	dl_list=1
elif echo "$1" | grep -E -q "^$re_album\$"
then
	dl_list=""
elif [ "$1" = "-f" ]
then
	shift

	dl_list=1

	case "$1" in
	-a ) shift ; dl_list="" ;;
	-* ) echo "Bad option: $1">&2 ; exit 1 ;;
	esac

	echo "$1" | grep -E -q '^https?://' || {
		echo "URL required">&2
		exit 1
	}
else
	me="$( basename "$0" )"
	cat - <<USAGE >&2
Usage:
	$me $re_list
	$me $re_album
	$me -f [-a] self-hosted-bandcamp-url
USAGE
	exit
fi

# =========================================================================

dl() {
	# shellcheck disable=SC2086
	dl_$DL "$@"
}

dl_wget() {
	url="$1"
	name="${2:--}"
	wget ${DL_USERAGENT:+--user-agent="$DL_USERAGENT"} \
		--no-check-certificate -c \
		-O "$name" "$url" \
		2>>"$DL_LOGFILE"
}

dl_curl() {
	url="$1"
	name="${2:--}"
	curl ${DL_USERAGENT:+--user-agent "$DL_USERAGENT"} \
		--insecure --continue-at - \
		-L -o "$name" "$url" \
		2>>"$DL_LOGFILE"
}

# =========================================================================

DL=""

for n in wget curl
do
	command -v "$n" >/dev/null && {
		DL="$n"
		break
	}
done

[ -n "$DL" ] || {
	echo "No downloader found">&2
	exit 1
}

case "$DL_LOG" in
''	) DL_LOGFILE=/dev/null ;;
*	) DL_LOGFILE="$( echo "$1" | sed 's|.*://||; s|/.*||' ).log" ;;
esac

# =========================================================================

if [ -n "$dl_list" ]
then
	base_url="$( echo "$1" | sed 's/\(https\?:\/\/[^\/]*\).*/\1/' )"

	dl "$1" \
	| grep -E '<a href="/(album|track)/[^"]+">' \
	| sed 's/.*href="//; s/".*//' \
	| awk -v base_url="$base_url" '{ print base_url$0 }'

	exit
fi

# =========================================================================

dl "$1" \
| perl -MEncode -MJSON::PP -MDate::Parse -lan0e '
BEGIN {
	%ent = (
		"amp"	=> "&",
		"nbsp"	=> " ",
		"quot"	=> chr 34,
		"apos"	=> chr 39,
		"lt"	=> "<",
		"gt"	=> ">",
	);

	sub safe_path {
		$s = shift;
		$s =~ s/
			(
				&
				(?:
					\# (\d+)		# 2
					|
					\#[xX] ([0-9a-fA-F]+)	# 3
					|
					(\w+)			# 4
				)
				;
			)
		/
			$c = "";
			defined $2 and $c = $2;
			defined $3 and $c = hex $3;
			$c ? Encode::encode("UTF-8", chr $c) : $ent{$4} || $1;
		/egx;
		$s =~ s#[\\/:*?"<>|]#-#g;
		Encode::is_utf8($s, 1) ? Encode::encode("UTF-8", $s) : $s;
	}
}

m{<a class="popupImage" href="([^\"]*)};
$cover = $1;

m{data-tralbum="([^"]*)"};
$_ = $1;

s/&quot;/"/g;

$data = decode_json $_;

exit unless defined $data->{current}->{title} && defined $data->{trackinfo};

#if ( $data->{freeDownloadPage} ) {
#	printf "<FREE-DOWNLOAD-PAGE>\t%s\n", $data->{freeDownloadPage};
#	exit;
#}

#if ( $data->{current}->{minimum_price} == 0
#&& !! $data->{current}->{require_email} ) {
#	printf "<FREE-BY-EMAIL>\t%s\n", $data->{url};
#	exit;
#}

$release_date = $data->{current}->{release_date};
$year = 1900 + ( gmtime str2time $release_date )[5];

printf "<ALBUM-DIR>\t%s\n", safe_path "$year - $data->{current}->{title}";

printf "<ALBUM-IMG>\t%s\n", $cover;

for $track ( @{ $data->{trackinfo} } ) {
	printf "%s\t%02d. %s\n",
		!! $track->{unreleased_track} ? "<NO-RELEASE>"
		: $track->{file}->{"mp3-128"} || "<NO-URL>",
		$track->{track_num},
		safe_path $track->{title};
}
' \
| while IFS="$( printf '\t' )" read -r u n
do
	[ "$u" = "<ALBUM-DIR>" ] && {
		album_dir="$n"
		echo "$album_dir"
		mkdir -p "$album_dir"
		continue
	}

	[ "$u" = "<ALBUM-IMG>" ] && {
		dl "$n" "$album_dir/cover-front.jpg"
		continue
	}

#	[ "$u" = "<FREE-DOWNLOAD-PAGE>" ] && {
#		# perl -MJSON::PP
#		# m{data-blob="([^"]*)"};
#		# !! $data->{has_downloads} == true
#		# !! $data->{multidownload} == false
#		# $data->{download_items}->[0]->{downloads}->{"mp3-320"}->{url}
#		# $data->{download_items}->[0]->{title}
#		# continue
#	}

	echo "$u" | grep -E -q '^<.*>$' && {
		echo "  $u: $n"
		continue
	}

	echo "  $n"
	dl "$u" "$album_dir/$n.mp3"
done

# =========================================================================

# EOF

dl-cbr

#!/bin/sh

# =========================================================================

cbr_daily_url="http://www.cbr.ru/scripts/XML_daily_eng.asp"
cbr_names_url="http://www.cbr.ru/scripts/XML_valFull.asp"

# =========================================================================

print_usage() {
	me="$( basename "$0" )"
	echo "\
Usage:
	$me [-h | -l | currency_list]

-h		Print this help
-l		Print the list of currrencies
currency_list	Print currencies for today (defaults to EUR USD)
"
}

# =========================================================================

main() {
	while getopts "hl" opt
	do
		case "$opt" in
		h ) print_usage ; exit ;;
		l ) check_prereq ; print_names ; exit ;;
		* ) print_usage ; exit 1 ;;
		esac
	done

	check_prereq
	print_daily "$@"
}


# =========================================================================

check_prereq() {
	DL="$( lookup_command wget curl )"

	[ -n "$DL" ] || {
		die "No downloader found"
	}
}

lookup_command() {
	for n in "$@"
	do
		command -v "$n" >/dev/null && {
			echo "$n"
			break
		}
	done
}

# =========================================================================

print_names() {
	dl "$cbr_names_url" \
	| sed 's/></>\n</g' \
	| parse_names
}

parse_names() {
	LANG=C
	awk -F '[<>]' '
	/^<Item/		{ split($0, a, "\""); id = a[2] }
	/^<EngName/		{ engname    = $3 }
	/^<ParentCode/		{ parentcode = $3 }
	/^<ISO_Char_Code/	{ charcode   = $3 }
	/^<ISO_Num_Code/	{ numcode    = $3 }
	/^<\/Item/		{
		if ( parentcode ~ id ) { parentcode = "" }
		printf "%s\t%5s\t%-10s\t%-10s\t%s\n",
			charcode, numcode, id, parentcode, engname
	}
	'
}

# =========================================================================

print_daily() {
	dl "$cbr_daily_url" \
	| sed 's/></>\n</g' \
	| parse_daily "$@"
}

parse_daily() {
	LANG=C
	awk -v re="$*" -F '[<>]' '
	BEGIN	{
		if ( ! re ) { re = "USD EUR" }
		gsub(/^ +| $/, "", re)
		gsub(/[ ,]+/, "|", re)
	}

	/^<ValCurs/	{ split($0, a, "\""); print a[2] }
	/^<Valute/	{ split($0, a, "\""); id = a[2] }
	/^<CharCode/	{ charcode = $3 }
	/^<Value/	{ value    = $3 }
	/^<\/Valute/ && ( id ~ re || charcode ~ re ) {
		printf "%s\t%8s\n", charcode, value
	}
	'
}

# =========================================================================

dl() {
	# shellcheck disable=SC2086
	dl_$DL "$@"
}

dl_wget() {
	wget -q -O - "$@"
}

dl_curl() {
	curl -s "$@"
}

# =========================================================================

die() {
	warn "$*"
	exit 1
}

warn() {
	echo "$*" >&2
}

# =========================================================================

main "$@"

# =========================================================================

# EOF

dl-musify

#!/bin/sh

# =========================================================================
#
# mp3 downloader
#
# =========================================================================

[ -t 0 ] || {
	xargs -r -IX "$0" "$@" X
	exit
}

# =========================================================================

for f in ~/.dl-rc ./dl-rc
do
	# shellcheck source=/dev/null
	[ -f "$f" ] \
	&& . "$f"
done

# =========================================================================

#base_name="myzcloud.me"
#base_url="https://$base_name"
#
#re_list="$base_url/artist/[^/]+/[^/]+/albums"
#re_album="$base_url/album/[^/]+/[^/]+"

#base_name="w1.musify.club"
base_name="musify.club"
base_url="https://$base_name"

re_list="$base_url/artist/[^/]+/releases"
re_album="$base_url/release/[^/]+"

release_types="$( cat - <<-TYPES
	all
	other
	studio
	ep
	single
	bootleg
	live
	compilation
	mixtape
	demo
	dj-mix
	artist-compilation
	split
	unofficial
	soundtrack
TYPES
)"
release_type_default="studio"

# =========================================================================

if echo "$1" | grep -E -q "^$re_list\$"
then
	dl_list=1
elif echo "$1" | grep -E -q "^$re_album\$"
then
	dl_list=""
else
	me="$( basename "$0" )"
	cat - <<USAGE >&2
Usage:
	$me $re_list [release_type]
	$me $re_album

Release types (defaults to "$release_type_default"):
USAGE
	echo "$release_types" | sort | sed 's/^/    /' >&2
	exit 1
fi

# =========================================================================

dl() {
	# shellcheck disable=SC2086
	dl_$DL "$@"
}

dl_wget() {
	url="$1"
	name="${2:--}"
	wget ${DL_USERAGENT:+--user-agent="$DL_USERAGENT"} \
		--no-check-certificate -c \
		-O "$name" "$url" \
		2>>"$DL_LOGFILE"
}

dl_curl() {
	url="$1"
	name="${2:--}"
	curl ${DL_USERAGENT:+--user-agent "$DL_USERAGENT"} \
		--insecure --continue-at - \
		-L -o "$name" "$url" \
		2>>"$DL_LOGFILE"
}

# =========================================================================

DL=""

for n in wget curl
do
	command -v "$n" >/dev/null && {
		DL="$n"
		break
	}
done

[ -n "$DL" ] || {
	echo "No downloader found">&2
	exit 1
}

case "$DL_LOG" in
''	) DL_LOGFILE=/dev/null ;;
*	) DL_LOGFILE="$( echo "$1" | sed 's|.*://||; s|/.*||' ).log" ;;
esac

# =========================================================================

if [ -n "$dl_list" ]
then
	type="$(
		echo "$release_types" \
		| awk -v x="${2:-$release_type_default}" '$0 == x { print NR-1 }'
	)"

	if [ -z "$type" ]
	then
		echo "Bad release type: $2">&2
		exit 1
	fi

	if [ "$type" = "0" ]
	then
		type="[0-9]+"
	fi

	dl "$1" \
	| grep -E '<div data-type="'"$type"'" class="card[^"]*">' -A 1 \
	| grep '<a ' \
	| sed 's/.*href="//; s/".*//' \
	| awk -v base_url="$base_url" '{ print base_url$0 }'

	exit
fi

# =========================================================================

dl "$1" \
| sed '/<!--/,/-->/d' \
| perl -MEncode -lane '
BEGIN {
	%ent = (
		"amp"	=> "&",
		"nbsp"	=> " ",
		"quot"	=> chr 34,
		"apos"	=> chr 39,
		"lt"	=> "<",
		"gt"	=> ">",
	);

	sub safe_path {
		$s = shift;
		$s =~ s/
			(
				&
				(?:
					\# (\d+)		# 2
					|
					\#[xX] ([0-9a-fA-F]+)	# 3
					|
					(\w+)			# 4
				)
				;
			)
		/
			$c = "";
			defined $2 and $c = $2;
			defined $3 and $c = hex $3;
			$c ? Encode::encode("UTF-8", chr $c) : $ent{$4} || $1;
		/egx;
		$s =~ s#[\\/:*?"<>|]#-#g;
		Encode::is_utf8($s, 1) ? Encode::encode("UTF-8", $s) : $s;
	}
}

m{
	<h1>
	(.*)
	</h1>
}x and printf "<ALBUM-DIR>\t%s\n", safe_path $1;

m{
	<img \s+ .* \s+
	(?:data-)?src="([^"]+)"
	.*? \s+ class="[^"]*album-img[^"]*" [^>]* >
}x and printf "<ALBUM-IMG>\t%s\n", $1;

m{
	<a \s+ [Cc]lass="strong"
	\s+ href="([^"]+)" [^>]* > ([^<>]+?) </a>
	|
	<span \s+ [Cc]lass="strong"> ([^<>]+?) </span>
}x and printf "%s\t%02d. %s\n",
	$1 || "<NO-URL>",
	++$n,
	safe_path($2 || $3);
' \
| while IFS="$( printf '\t' )" read -r u n
do
	[ "$u" = "<ALBUM-DIR>" ] && {
		album_dir="$n"
		echo "$album_dir"
		mkdir -p "$album_dir"
		continue
	}

	[ "$u" = "<ALBUM-IMG>" ] && {
		dl "$n" "$album_dir/folder.jpg"
		continue
	}

	echo "$u" | grep -E -q '^<.*>$' && {
		echo "  $u: $n"
		continue
	}

	dl "$base_url$u" \
	| perl -lane '
		m{
			<a \b .* \b itemprop="audio" .*
			href="([^"]+)"
			[^>]* >
		}x and print $1;
	' | {
		read -r d
		[ -n "$d" ] || exit

		echo "  $n"
		dl "$base_url$d" "$album_dir/$n.mp3"
	}
done

# =========================================================================

# EOF

dl-progarchives

#!/bin/sh

# =========================================================================

main_url="www.progarchives.com/artist.asp?id="

# =========================================================================

[ $# -eq 1 ] || {
	echo "Usage: $0 [$main_url]NUMBER" >&2
	exit 1
}

# =========================================================================

for f in ~/.dl-rc ./dl-rc
do
	# shellcheck source=/dev/null
	[ -f "$f" ] \
	&& . "$f"
done

# =========================================================================

dl() {
	# shellcheck disable=SC2086
	dl_$DL "$@"
}

dl_wget() {
	url="$1"
	name="${2:--}"
	wget ${DL_USERAGENT:+--user-agent="$DL_USERAGENT"} \
		--no-check-certificate -c \
		-O "$name" "$url" \
		2>>"$DL_LOGFILE"
}

dl_curl() {
	url="$1"
	name="${2:--}"
	curl ${DL_USERAGENT:+--user-agent "$DL_USERAGENT"} \
		--insecure --continue-at - \
		-L -o "$name" "$url" \
		2>>"$DL_LOGFILE"
}

# =========================================================================

DL=""

for n in wget curl
do
	command -v "$n" >/dev/null && {
		DL="$n"
		break
	}
done

[ -n "$DL" ] || {
	echo "No downloader found">&2
	exit 1
}

case "$DL_LOG" in
''	) DL_LOGFILE=/dev/null ;;
*	) DL_LOGFILE="$( echo "$1" | sed 's|.*://||; s|/.*||' ).log" ;;
esac

# =========================================================================

artist_page="$( echo "$1" | grep -F "$main_url" || echo "$main_url$1" )"

# =========================================================================

echo "$artist_page"

dl "$artist_page" \
| dos2unix \
| perl -ne '
# site
if ( m|<a.*href="([^"]*)".*>Official website</a>| ) {
	$site = $1;
}

# genres
if ( m|<h2[^>]*>(.*)</h2>| && ! $genres ) {
	$_ = $1;

	s/<[^<>]*>//g;
	s/(.* &bull; )/$1Region /;

	s/ &bull; /; /g;

	s/United Kingdom/UK/;
	s/United States/USA/;

	$genres = $_;
}

# bio
if ( m|<div style="padding-left:10px;margin:0;width:690px;float:right; border-left: solid 5px #f0f0f0;">| ... m|</div>| ) {
	s/<br[^<>]*>/\n/g;
	s|<strong>read more</strong>||g;
	s/<[^<>]*>//g;

	$bio .= $_;
}

END {
	$bio =~ s/^\s*//;
	$bio =~ s/\s*$//;
	
	print <<TEXT;
$site

$genres

$bio
TEXT
}
'
# =========================================================================

# EOF

dl-rc

# =========================================================================

# Stored as ~/.dl-rc (in the user's home directory) this file defines
# global settings available everywhere. Stored as ./dl-rc this file
# defines local settings available in the current directory only.

# Both files are searched and sourced in this order: ~/.dl-rc first,
# then ./dl-rc. Thus the local settings overwrite the global ones. It
# provides possibility to define local and global settings separately.

# =========================================================================

# Specify the User-Agent string to send to the HTTP server.
#DL_USERAGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"

# Any non-empty value enables logging to a log file.
#DL_LOG=1

# =========================================================================

# EOF