A silly class that lets you use Grequests in a somewhat similar manner as the ruby gem Typhoeus

request_queue.py

import grequests
from collections import deque
class RequestQueue(object):
    """
    This is a lame imitation of a Typhoeus Hydra using GRequests.
    
    The main thing this allows is building up a queue of requests and then
    executing them, and potentially adding requests to the queue in a callback
    so that you can build requests that depend on other requests more naturally.

    This shouldn't be as fast as Typhoeus in theory, but I think the interface
    is slightly more convenient and it should be fast enough until I feel like
    building a proper Typhoeus replacement using PyCurl.
    """
    def __init__(self):
        self.queue = deque()
        self.requests = deque()
        self.max_concurrent = 10
        self.running = False
        
    def add(self, request):
        if not self.running or len(self.requests) < self.max_concurrent:
            self.requests.append(request)
        else:
            self.queue.append(request)
                
    def run(self):
        """
        Runs grequests.map repeatedly until the queue is depleted.
        """
        self.running = True
        result = []
        while self.requests:
            result.extend(grequests.map(self.requests))
            self.requests = deque() # Clear the request queue
            while self.queue and len(self.requests) < self.max_concurrent:
                self.requests.append(self.queue.popleft())
        self.running = False
        return result
                
    
    def _pop(self):
        """
        TODO: Attempt to use grequests.imap to see if that also works and how
        it affects performance.
        """
        while self.requests:
            yield self.requests.popleft()
            if self.queue:
                self.requests.append(self.queue.popleft())
        
    def build_callback(self, fxn):
        """
        This forms a closure over the queue for use in a Requests callback   
        """
        # TODO could do the queue management here somewhat like Typhoeus does
        def single_arg_callback(arg):
            fxn(arg, self)
        return single_arg_callback

import grequests
def testQueue(queue, initial=10):
    from random import randint
    initials = [randint(0,10) for x in xrange(initial)]
    
    def build_funky_callback(count):
        def lolback(response, q):
            if count > 0:
                q.add(grequests.get("http://httpbin.org/get", hooks={"response":build_funky_callback(count-1)}) )
            else:
                q.add(grequests.get("http://httpbin.org/get"))
            
        return queue.build_callback(lolback)
    
    for x in initials:
        req = grequests.get("http://httpbin.org/get", hooks={"response":build_funky_callback(x)})
        queue.add(req)
        
    ret = queue.run()
    return ret
        
rq=RequestQueue()
res=testQueue(rq, 100)
print len(res)