Last active
December 21, 2015 04:08
-
-
Save nicky-zs/6246807 to your computer and use it in GitHub Desktop.
Run costly tasks in the background in tornado, along with tornado.gen.engine.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# vim: fileencoding=utf-8 | |
import sys, time | |
from multiprocessing.pool import ThreadPool | |
from tornado import web, gen, stack_context, httpserver, ioloop, util, options | |
_workers = ThreadPool() | |
def costly_task(n): # a costly task like processing images | |
time.sleep(n) | |
return n | |
class BaseHandler(web.RequestHandler): | |
def run_background(self, func, args=(), kws={}, callback=None): | |
exc_info = [None] | |
def _func(*fargs, **fkws): | |
try: | |
return func(*fargs, **fkws) | |
except: | |
exc_info[0] = sys.exc_info() | |
return | |
def _callback(result): | |
exc = exc_info[0] | |
ioloop.IOLoop.instance().add_callback(lambda: callback(result) | |
if exc is None else util.raise_exc_info(exc)) | |
_workers.apply_async(_func, args, kws, stack_context.wrap(_callback)) | |
@web.asynchronous | |
@gen.engine | |
def get(self): | |
count = int(self.get_argument('count', 1)) | |
result = yield gen.Task(self.run_background, costly_task, (count,)) | |
self.finish('the operation costs %d seconds\n' % result) | |
if __name__ == '__main__': | |
options.parse_command_line() # open log | |
httpserver.HTTPServer(web.Application([('/test', BaseHandler)])).listen(8888) | |
ioloop.IOLoop.instance().start() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Note that GIL in python will prevent multiple native threads from executing python bytecodes at once.
So ProcessPool should be used instead of ThreadPool if the background operation will happen inside GIL, suffering from the potential problems of serialization of the arguments and return values.
However, according to the official wiki, https://wiki.python.org/moin/GlobalInterpreterLock, potentially blocking or long-running operations, such as I/O, image processing, and NumPy number crunching, happen outside the GIL.