humble-downloader.py

from __future__ import print_function

import json
import sys
import re
import hashlib
from os import makedirs, path, remove
from getpass import getpass
from urlparse import urlparse

import requests
from pyquery import PyQuery as pq
from unidecode import unidecode
from clint.textui import prompt


def echo(message, *args, **kwargs):
    print(message, *args, **kwargs)
    sys.stdout.flush()


class Downloader:
    def __init__(self, product, download, struct):
        self.url = struct["url"]["web"]
        self.md5 = struct["md5"]

        self.path = None
        self.filename = None

        self.prepare(product, download, struct)

    #      If you're less lazy than I am, maybe
    # TODO work on the path these three arguments get passed
    def prepare(self, product, download, struct):
        identifier = unidecode(product["human_name"]).lower()
        identifier = re.sub(r"[^a-z 0-9]+", "", identifier)
        identifier = re.sub(r"[ ]+", " ", identifier)

        web_url = struct["url"]["web"]
        parsed_url = urlparse(web_url)

        self.path = path.join("humble", download["platform"], identifier)
        target_filename = path.basename(parsed_url.path)

        self.filename = path.join(self.path, target_filename)

    def get_md5(self):
        if not path.exists(self.filename):
            return None

        block_size = 16 * 1024 * 1024
        hasher = hashlib.md5()

        with open(self.filename, "rb") as file:
            buf = file.read(block_size)
            while len(buf) > 0:
                hasher.update(buf)
                buf = file.read(block_size)

        return hasher.hexdigest()

    def should_download(self):
        if path.exists(self.filename):
            if self.md5 != self.get_md5():
                # Checksum mismatch
                return True

            return False

        return True

    def download(self):
        echo("      %s" % self.filename, end="\r")
        sys.stdout.flush()

        if not self.should_download():
            echo("  >  ")
            return

        if not path.exists(self.path):
            makedirs(self.path)

        r = requests.get(self.url, stream=True)
        if r.status_code == 200:
            expected_size = 0
            if "content-length" in r.headers:
                expected_size = float(r.headers["content-length"])

            loaded_size = 0.0
            with open(self.filename, "wb") as f:
                echo("[   ]", end="\r")

                for chunk in r.iter_content(1024 * 32):
                    if expected_size > 0:
                        percent = 100.0 * (loaded_size / expected_size)
                        echo("[%2d%%]" % int(percent), end="\r")

                    f.write(chunk)
                    loaded_size += len(chunk)

            echo("[ # ]", end="\r")

            if self.md5 == self.get_md5():
                echo("[ v ]")
            else:
                echo("[ X ]")

    def remove(self):
        remove(self.filename)
        echo("")

    def __gt__(self, other):
        return self.filename > other.filename


def parse_download(download, product):
    for struct in download["download_struct"]:
        if "url" in struct:
            downloader = Downloader(product, download, struct)

            if downloader.filename not in seen_downloaders:
                downloaders.append(downloader)
                seen_downloaders.append(downloader.filename)


def parse_product(product):
    if "downloads" in product:
        for download in product["downloads"]:
            parse_download(download, product)


downloaders = []
seen_downloaders = []
gamekey_finder = re.compile(ur"var\s+gamekeys\s*=\s*(.+);")

pre_login_url = "https://www.humblebundle.com/"
login_url = "https://www.humblebundle.com/processlogin"
library_url = "https://www.humblebundle.com/home/library"
api_url = "https://www.humblebundle.com/api/v1/order/%s"
guard_url = "https://www.humblebundle.com/user/humbleguard"

session = requests.Session()

response = session.get(pre_login_url)
document = pq(response.text)
login_document = pq(document("#account-login").html())

login_data = {
    "_le_csrf_token": pq(login_document(".csrftoken")).val(),
    "goto": "#",
    "qs": "",
    "script-wrapper": "login_callback",

    "username": None,
    "password": None,

    "authy-token": "",
    "submit-data": "",
}

if path.exists("login.json"):
    with open("login.json", "r") as login_file:
        login = json.load(login_file)
        for key, value in login.items():
            login_data[key] = value

if not login_data["username"]:
    login_data["username"] = prompt.query("Username: ")

if not login_data["password"]:
    login_data["password"] = getpass("Password: ")

session.post(login_url, login_data)
response = session.get(library_url)

if "Account Protection" in response.text:
    echo("Humble protection wants to verify you. Wait for the e-mail.")

    code = prompt.query("Protection code: ")
    response = session.post(guard_url, {
        "code": code,
    })

    echo(response.text)
    response = session.get(library_url)

game_keys = json.loads(re.search(gamekey_finder, response.text).group(1))

index = 0
for game_key in game_keys:
    index += 1
    echo("Loading key %d of %d" % (index, len(game_keys)))

    response = session.get(api_url % game_key)
    data = response.json()

    if not path.exists("humble/data"):
        makedirs("humble/data")

    with open("humble/data/%s.json" % game_key, "w") as data_file:
        json.dump(data, data_file)

    parse_product(data["product"])

    for product in data["subproducts"]:
        parse_product(product)

echo("")
downloaders.sort()

echo("Downloads have been prepared.")
echo("")

for downloader in downloaders:
    try:
        downloader.download()
    except KeyboardInterrupt as interrupt:
        downloader.remove()
        exit()

login.json

{
  "username": "[email protected]",
  "password": "dontstealmygamespls"
}

requirements.txt

clint==0.4.1
requests==2.7.0
pyquery==1.2.9
Unidecode==0.04.18