concurrent http requests with aiohttp

aiohttp-example.py

# author: @Daniel_Abeles
# date:   18/12/2017

import asyncio
from aiohttp import ClientSession
from timeit import default_timer
import async_timeout


async def fetch_all(urls: list):
    """ Fetch all URLs """
    tasks = []
    custom_headers = {
        'User-Agent': 'aiohttp client 0.17'
    }

    async with ClientSession(headers=custom_headers) as session:
        for url in urls:
            task = asyncio.ensure_future(fetch(url, session))
            tasks.append(task)

        _ = await asyncio.gather(*tasks)


async def fetch(url: str, session: object):
    """ Fetch a single URL """
    with async_timeout.timeout(10):
        async with session.get(url) as response:
            before_request = default_timer()
            resp = await response.read()
            elapsed = default_timer() - before_request

            print(f'{url:30}{elapsed:5.2f}')

            return {
                'resp': resp,
                'url': url,
                'elapsed': elapsed
            }


def main():
    """ Main Function """
    start_time = default_timer()
    urls = [
        'https://twitter.com',
        'https://9to5mac.com',
        'https://amazon.com'
    ]

    loop = asyncio.get_event_loop()
    total_future = asyncio.ensure_future(fetch_all(urls))

    loop.run_until_complete(total_future)

    total_elapsed = default_timer() - start_time
    print(f'{" total time of".rjust(28, "-")}: {total_elapsed:5.2f}')


if __name__ == "__main__":
    main()