test gevent simple spider

simple_spider.py

__author__ = 'hzyuxin'

import requests
from gevent import monkey
monkey.patch_all()
import gevent

main_url = 'http://www.cellmap.cn/cellmap_gsm2gps_api.aspx'

def job(lac, cell):
    r = requests.get(main_url, params = {'lac':lac, 'cell':cell})
    print r.url
    print r.text

def param_generator():
    for x in xrange(0, 65536):
        for y in xrange(0, 65536):
            yield x, y

def main_process(thread_num, timeout = 1):
    jobs = []

    for x, y in param_generator():
        g = gevent.spawn(job, x, y)
        jobs.append(g)

        if len(jobs) == 100:
            gevent.joinall(jobs, timeout = timeout)
            jobs = []


main_process(100)