asyncio_download_queue.py

import asyncio
import requests
import os
import re
import time

progress = {}
chunk_size = 4096
workers = 2
running = 0


@asyncio.coroutine
def download_url(url):
    global running
    running += 1
    r = requests.get(url, stream=True)

    filename = re.findall('/([^\/]*\.jpg)', url)[0]

    progress[filename] = {
        'total': int(r.headers['content-length']),
        'progress': 0,
    }

    # yield chunks of data
    with open(filename, 'wb') as fp:
        for chunk in r.iter_content(chunk_size):
            time.sleep(0.03)  # throttle for debug
            progress[filename]['progress'] += len(chunk)
            fp.write(chunk)
            yield

    running -= 1

    # add a new worker
    if running < workers and len(downloads) > 0:
        add_next_download()


@asyncio.coroutine
def print_progress():
    while True:
        all_done = True
        os.system('clear')
        for filename, data in progress.items():
            percent = data['progress'] / data['total']
            if percent < 1:
                all_done = False
            percent_str = '{:.2%}'.format(percent)
            print('{:>7} {}'.format(percent_str, filename))

        if all_done is True:
            exit()

        yield from asyncio.sleep(1)


def add_next_download():
    asyncio.Task(download_url(downloads.pop()))


downloads = [url for url in '''
http://images2.fanpop.com/image/photos/10500000/Cute-Wallpapers-cute-kittens-10501757-1600-1200.jpg
http://images6.fanpop.com/image/photos/34900000/Kitties-3-cats-34909578-1280-800.jpg

http://www.ihdwal.com/wp-content/uploads/2014/01/cute_kitties_pictures_cute_little_kitten_-_cute_kittens_wallpaper_16288222_-_fanpop.jpg

http://www.mrwallpaper.com/wallpapers/cute-kitties-1600x900.jpg

http://th08.deviantart.net/fs71/PRE/f/2013/353/5/c/lisa_and_tina_as_sexy_kitties_by_arai1986-d6yjmti.jpg
'''.strip().split('\n') if url != '']

for _ in range(workers):  # start the first n workers
    add_next_download()

asyncio.Task(print_progress())
asyncio.get_event_loop().run_forever()