Python Framework Benchmark

bench.py

import time
import sys
import StringIO

class Bench(object):
    def __init__(self, flags=None):
       ''' Flags:
           * cgi: Build a new app every time.
           * dny: Return the :name part of a /hello/:name url.
       '''
       self.flags = flags or []
       self.url = '/'
       self.body = 'Hello World!'
       if 'urlargs' in self.flags:
           self.url = '/hello/bench'
           self.body = 'bench'

    def start_response(self, status, header):
        if status != '200 OK':
            raise AssertionError('%r %s %r' % (self, status, header))
        pass

    def make_environ(self, run_once=False):
       env = {
         'REQUEST_METHOD': 'GET',
         'SCRIPT_NAME': '',
         'PATH_INFO': self.url,
         'QUERY_STRING': '',
         'SERVER_NAME': 'localhost',
         'SERVER_PORT': '80',
         'SERVER_PROTOCOL': 'HTTP/1.0',
         'wsgi.version': (1,0),
         'wsgi.url_scheme': 'http',
         'wsgi.input': StringIO.StringIO(),
         'wsgi.errors': sys.stdout,
         'wsgi.multithread': False,
         'wsgi.multiprocess': False,
         'wsgi.run_once': run_once,
       }
       return env

    def test_cold(self, mode, n=1000):
        start = time.time()
        for i in xrange(n):
            app = self.make_app()
            body = ''.join(app(self.make_environ(run_once=True), self.start_response))
        if body != self.body: raise AssertionError('%r: %r != %r' % (self, body, self.body))
        return n / (time.time() - start)

    def test_warm(self, mode, n=1000):
        app = self.make_app()
        ''.join(app(self.make_environ(), self.start_response))
        start = time.time()
        for i in xrange(n):
            body = ''.join(app(self.make_environ(), self.start_response))
        if body != self.body: raise AssertionError('%r: %r != %r' % (self, body, self.body))
        return n / (time.time() - start)
       

class WSGIBench(Bench):
    name = 'WSGI'
    def make_app(self):
        def app(environ, start_response):
            start_response('200 OK',  [('content-type','text/html')])
            path = environ['PATH_INFO']
            if path.startswith('/hello/'):
                return [path[7:]]
            return ['Hello World!']
        return app

class BottleBench(Bench):
    name = 'Bottle'
    def make_app(self):
        import bottle
        app = bottle.Bottle()
        @app.route('/')
        @app.route('/hello/:name')
        def hello(name='Hello World!'):
            return name
        return app

class PyramidBench(Bench):
    name = 'Pyramid'
    def make_app(self):
        from pyramid.config import Configurator
        from pyramid.response import Response
        def hello_world(request):
            return Response('Hello World!')
        def hello_name(request):
            return Response(request.matchdict['name'])
        config = Configurator()
        config.add_view(hello_world)
        config.add_route('foo', '/hello/{name}', view=hello_name)
        return config.make_wsgi_app()

class FlaskBench(Bench):
    name = 'Flask'
    def make_app(self):
        import flask
        app = flask.Flask(__name__)
        @app.route('/')
        @app.route('/hello/<name>')
        def hello(name='Hello World!'):
            return name
        return app

suites = [obj for obj in globals().values() if isinstance(obj, type) and issubclass(obj, Bench)]
suites.remove(Bench)
suites.sort(key=lambda c: c.__name__)

if __name__ == '__main__':

    flags = sys.argv[1:]
    n, r = 100, 10

    print
    print 'Python Framework Benchmark'
    print
    print 'All results are shown as requests per second (high is good).'
    print 'In "Normal" mode, setup and first request are not considered (warm start).'
    print 'In "CGI" mode, a new app is created for each request (cold start).'
    print
    print '% 20s  % 10s % 10s' % ('', 'Normal', 'CGI')

    for suite in suites:
        print '% 20s ' % suite.name,
        sys.stdout.flush()
    
        rps = min(suite(flags).test_warm(n) for i in xrange(r))
        print '% 10d' % rps,
        sys.stdout.flush()
    
        rps = min(suite(flags).test_cold(n) for i in xrange(r))
        print '% 10d' % rps
        sys.stdout.flush()