send gevent http request with second limit

example.py

from gevent import monkey

monkey.patch_all()
import gevent
from gevent.pool import Pool
from gevent.lock import Semaphore
import time
from urllib.request import urlopen


class SecondLimit:
    """
Request limit; n times per second
Example:
    semaphore = SecondLimit(10) # 10 requests per second
    with semaphore:
        some_request()
    """

    def __init__(self, n, interval=0.01):
        self.interval = interval
        self.time_sequence = []
        self.semaphore = Semaphore(n)

    def acquire(self):
        self.semaphore.acquire()
        self.time_sequence.append(time.time())

    def release(self):
        while not self.time_sequence or time.time() - self.time_sequence[0] < 1:
            gevent.sleep(self.interval)
        self.time_sequence.pop(0)
        self.semaphore.release()

    def __enter__(self):
        self.acquire()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.release()


semaphore = SecondLimit(20)  # 20 requests per second


def download(url):
    with semaphore:
        return urlopen(url).read()


def main():
    pool = Pool(20)
    urls = ['http://httpbin.org/get'] * 100
    print(pool.map(download, urls))


if __name__ == '__main__':
    before = time.time()
    main()
    print('{:0.2f} sec'.format(time.time() - before))