Archive Magic: The Gathering wallpapers

archive_magic_wallpaper.sh

#!/usr/bin/env bash

set -euo pipefail

main() {
    which pup > /dev/null || (echo "Error: missing 'pup'; see https://github.com/ericchiang/pup" ; exit 1)
    which jq > /dev/null || (echo "Error: missing 'jq'; install with: apt install jq" ; exit 1)
    
    mkdir -p "wizard_wallpapers" && pushd "wizard_wallpapers" >/dev/null || exit

    local start_page=0
    get_images $start_page

    jq -s 'add' wizard_page_*.json > ../all_wizard_wallpapers.json
    echo "Wrote file: \"all_wizard_wallpapers.json\" with all wallpapers. Use 'jq' to do further parsing."
}

get_images() {
    local page_num=$1
    local response=$(curl --silent --fail --show-error "https://magic.wizards.com/en/see-more-wallpaper?page=$page_num&filter_by=DESC&artist=-1&expansion=&title=")

    is_error=$(echo "$response" | jq -r '.status!=0')
    if [ "$is_error" == "true" ]; then
        echo "Status error!"
        exit 1
    fi

    # do some magic and save the response for
    echo "$response" | parse_html_response | to_json_objects | aggregate_result >> "wizard_page_$page_num.json" && echo "saved wizard_page_$page_num.json"

    local has_next_page=$(echo "$response" | jq '.displaySeeMore==1')
    if [ "$has_next_page" == "true" ]; then
        local next_page=$(($page_num + 1))
        get_images "$next_page"
    else
        echo "Finished grabbing all pages. In total: $page_num"
    fi
}

parse_html_response() {
    jq -r ".data"
}

to_json_objects() {
    pup "div.wrap json{}"
}

aggregate_result() {
    jq '[.[] | {title:.children[0].text, subtitle:.children[1].children[0].text, author:.children[2].text, date:.children[1].text, urls:[.children[3].children[] | .children[0] | { res:.text, url:.download }]}]'
}

# call main function
main

download_magic_wallpaper.sh

#!/usr/bin/env bash
# requirements:
# perl
# jq
# curl 7.73+

# sanity check
CURLVER=$(curl --version | head -n 1 | grep -Po '\d+\.\d+.\d+' | head -n 1 )
if [[ ! ${CURLVER//\./} -gt 7730 ]]; then
    echo "minimum Curl version 7.73 is needed. Your version: $CURLVER"
    exit 1
fi


main() {
    local FILE="all_wizard_wallpapers.json"
    local NUM_ITEMS=$(jq 'length' $FILE)
    echo "Total items: $NUM_ITEMS"
    jq -c '.[]' $FILE | xargs -d'\n' -I '{}' -P 8 bash -c "get_image '{}' ."
}

function sanitize_file_name {
    perl -pe 's/^\s+|\s+$//g;' -pe 's/[\?\[\]\/\\=<>:;,''"&\$#*()|~`!{}%+]//g;' -pe 's/[\r\n\t -]+/-/g;' < /dev/stdin
}
export -f sanitize_file_name

get_image()
{
    local image=$1
    local safe_filename=$(echo -n "$image" | jq -rc '.title' | sanitize_file_name)

    # download all images
    while IFS= read -r url; do
        curl --silent --show-error --create-dirs --output-dir "$safe_filename" -O -J -L "$url"
    done < <(echo "$image" | jq -rc '.urls[].url')

    # write info.json
    echo "$image" | jq -r '.' > "$safe_filename/$safe_filename.info.json"
}
export -f get_image

main