urllib2_gevent_VERSUS_grequests.py

import sys

import gevent
from gevent import monkey

monkey.patch_all()

import grequests
import urllib2

urls = ['http://www.yahoo.com'] * 5


def call_back(resp):
    content = resp.content
    title = content.split('<title>')[1].split('</title>')[0].strip()
    print title
    return title


def worker(url, use_urllib2=False):
    if use_urllib2:
        content = urllib2.urlopen(url).read().lower()
        title = content.split('<title>')[1].split('</title>')[0].strip()
        print title
    else:
        rs = [grequests.get(u) for u in url]
        resps = grequests.map(rs)
        for resp in resps:
            call_back(resp)


def by_grequests():
    """
    Using grequests since use_urllib2 is not given and defaults to false
    """
    worker(urls)


def by_urllib2():
    """
    Using urllib2 and gevent
    """
    jobs = [gevent.spawn(worker, url, True) for url in urls]
    gevent.joinall(jobs)

if __name__ == '__main__':
    from timeit import Timer
    t = Timer(stmt="by_grequests()", setup="from __main__ import by_grequests")
    print 'by grequests: %s seconds' % t.timeit(number=3)
    t = Timer(stmt="by_urllib2()", setup="from __main__ import by_urllib2")
    print 'by urllib2: %s seconds' % t.timeit(number=3)
    sys.exit(0)