Inception

gistfile1.py

#!/usr/bin/env python

import tornado
import tornado.escape
import tornado.httpclient
import tornado.ioloop
import tornado.web

urls = [
    "http://friendfeed-api.com/v2/feed/bret",
    "http://friendfeed-api.com/v2/feed/paul",
    "http://friendfeed-api.com/v2/feed/paulg",
    "http://friendfeed-api.com/v2/feed/bret",
    "http://friendfeed-api.com/v2/feed/paul",
    "http://friendfeed-api.com/v2/feed/paulg",
    "http://friendfeed-api.com/v2/feed/bret",
    "http://friendfeed-api.com/v2/feed/paul",
    "http://friendfeed-api.com/v2/feed/paulg",
]


class DLMixin:
    """The purpose of this mixin is to provide a *simple* interface to parallel
    downloads with Tornado.
    """
    def dl_urls(self, urls, callback):
        """This function is an entry point for collecting multiple URLs. It receives
        a list of URLs and a function to return the list of results from calling
        Tornado's http.fetch on each URL.

        Each call to http.fetch requires a callback, which is addressed in the form of
        an aggregator function to collect the responses until each URL is finished.
        This list of URL responses is then passed back to the caller of this function
        by calling `callback`.
        """
        call_dl_aggregate = lambda response: self.dl_aggregate(response, callback)
        
        ### Prepare async client
        http = tornado.httpclient.AsyncHTTPClient()

        ### Queue up page fetches
        for url in urls:
            http.fetch(url, callback=self.async_callback(call_dl_aggregate))
        
    def dl_aggregate(self, response, callback):
        """Callback for `dl_urls()` handler. It collects all the responses and writes
        to the client after each page fetch has completed.
        """
        if response.error:
            self._dl_responses.append(tornado.web.HTTPError(500))
        else:
            json = tornado.escape.json_decode(response.body)
            self._dl_responses.append(json)
        if len(self._dl_responses) == len(urls):
            callback(self._dl_responses)


class MainHandler(tornado.web.RequestHandler, DLMixin):
    @tornado.web.asynchronous
    def get(self):
        """Simple get handler that fetches the links listed above.
        """
        print 'Starting data fetch'
        
        self._dl_responses = list()
        self.dl_urls(urls, self.handle_dl)

    def handle_dl(self, responses):
        """Callback for get handler. It collects all the responses and writes
        to the client once every page fetch is complete.
        """
        self.write("Fetched %d entries from the FriendFeed API<br>" %
                   len(responses))

        if len(responses) == len(urls):
            self.finish()


application = tornado.web.Application([
    (r"/", MainHandler),
])

if __name__ == "__main__":
    application.listen(8000)
    tornado.ioloop.IOLoop.instance().start()