[Python Tornado Yield Multiple Async Requests] #python #tornado #yield #splat #async #asynchttpclient #locals #vars #variables

1. Simpler (no web-server) variation.py

"""
this uses tornado's own IOLoop (i.e. not asyncio's) and it also uses tornado's own
concurrency functions such as `tornado.gen.coroutine` instead of `async/await` as
those are native Python3 functions and not compatible with tornado.
"""

from tornado.gen import coroutine
from tornado.httpclient import AsyncHTTPClient
from tornado.ioloop import IOLoop

AsyncHTTPClient.configure(None, defaults=dict(user_agent="IntegralistTesting"))
http_client = AsyncHTTPClient()


def get_urls():
    host = "www.buzzfeed.com"

    paths = [
        "/news",
        "/quizzes",
        "/tvandmovies",
        "/shopping",
        "/celebrity",
        "/videos",
        "/animals",
        "/newsletters",
        "/trending",
        "/valezabakolli/more-bargain-buys-under-6-that-you-might-actua-kmldud9yf",  # noqa
    ]

    responses = []

    for path in paths:
        responses.append(http_client.fetch(f"https://{host}/{path}"))

    return responses


@coroutine
def process_urls():
    a, b, c, d, e, f, g, h, i, j = yield get_urls()

    for k, v in locals().items():
        print(f"var: {k}, {v.effective_url}: {v.code}")


io_loop = IOLoop.current()
io_loop.run_sync(process_urls)

2. Python Tornado Yield Multiple Async Requests.py

import time
import tornado.ioloop
import tornado.web
import tornado.httpclient

class MainHandler(tornado.web.RequestHandler):
    @tornado.gen.coroutine
    def get(self):
        first_then = time.time()
        response = yield single_async_http_request()
        first_now = time.time()

        print("Single HTTP Request: %d" % round(first_now - first_then))
        print(response.body)

        second_then = time.time()
        future_a, future_b = yield [*multiple_async_http_requests()]
        second_now = time.time()

        print("Multiple HTTP Requests: %d" % round(second_now - second_then))
        print(future_a.body, future_b.body)
        print("Overall time: %d" % round(second_now - first_then))

        self.write('...')
        self.finish()

def multiple_async_http_requests():
    http = tornado.httpclient.AsyncHTTPClient()
    a = http.fetch("http://some-slow-responding-server")
    b = http.fetch("http://some-slow-responding-server")
    return a, b

@tornado.gen.coroutine
def single_async_http_request():
    http = tornado.httpclient.AsyncHTTPClient()
    response = yield http.fetch("http://some-slow-responding-server")
    return response

if __name__ == "__main__":
    app = tornado.web.Application([
        (r"/foo/?", MainHandler),
    ])
    app.listen(8888)

    tornado.ioloop.IOLoop.current().start()