Download a file from the web using requests and a pretty progress bar

download.py

# -*- coding: utf-8 -*-
#
# Download a file from the web using requests with a nice progress bar.

from __future__ import print_function

from tqdm import tqdm

import requests
import warnings
import argparse
import math


def download(url, target, headers, block_size):
    # Stream so we can iterate the response.
    r = requests.get(url, stream=True, headers=headers)

    # Total size in bytes.
    total_size = int(r.headers.get('content-length', 0))
    bytes_written = 0

    with open(target, 'wb') as f:
        for data in tqdm(r.iter_content(block_size),
                         total=math.ceil(total_size // block_size),
                         unit='KB', unit_scale=True):
            bytes_written += len(data)
            f.write(data)

    if total_size != 0 and bytes_written != total_size:
        warnings.warn("Only read %i/%i bytes!" % (bytes_written, total_size))
    elif total_size == 0:
        warnings.warn("Downloaded 0 bytes!")



if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Download a file over HTTP "
                                                 "with a nice progress bar.")

    parser.add_argument("--url", dest="url", type=str,
                        help="The URL from which to download.")

    parser.add_argument("--output", dest="output", type=str,
                        help="The absolute path to the file to which to save "
                             "the web content.")

    parser.add_argument("--headers", dest="headers", type=str, default=None,
                        help="Optional headers.")

    parser.add_argument("--block-size", dest="block_size", type=int,
                        default=1024, help="The (optional) block size.")

    parser.set_defaults(url=None, output=None, headers=None, block_size=1024)
    args = parser.parse_args()

    # validate args
    if not all(arg for arg in (args.url, args.output)):
        raise ValueError("URL and output required!")

    download(url=args.url, target=args.output, headers=args.headers,
             block_size=args.block_size)