DeferredSemaphoreHttpPoolExample.py

# Authors: Ingmar Steen / Sander Ferdinand

from twisted.internet import defer, task
from twisted.web.http_headers import Headers
from treq import text_content, request


class HttpPool(object):
    def __init__(self):
        self._sem = defer.DeferredSemaphore(20)  # no more than 20 at once
        self.cookies = {}
        self.headers = Headers({
            'User-Agent': ['python-requests/2.4.3 CPython/2.7.6 Darwin/14.0.0'],
            'Accept': ['*/*'],
            'Accept-Encoding': ['gzip, deflate'],
            'Connection': ['keep-alive']
        })

    _n = 0
    def fetch(self, method, url, allow_redirects=True):
        def _request(*a, **kw):
            self._n += 1

            f = request(*a, **kw)
            f.addCallback(self._hit)
            return f

        def release_and_return(r):
            """
            Used with addBoth to guarantee the semaphore being released.

            If treq.request returns an error, r will be a failure,
            otherwise it will be the IResponder instance. Either way, it
            is just returned so the normal callback/errback flow continues.
            """

            self._n -= 1
            self._sem.release()
            return r

        return self._sem.acquire().addCallback(
            lambda _, *a, **kw: _request(*a, **kw),
            method,
            url,
            headers=self.headers,
            allow_redirects=allow_redirects
        ).addBoth(release_and_return)

    def _hit(self, data):
        return data

A shortified example of the above code by 'asdf' from #twisted on freenode.

from twisted.internet import defer, task
from twisted.web.http_headers import Headers
from treq import text_content, request


class HttpPool(object):
    def __init__(self):
        self._sem = defer.DeferredSemaphore(20)  # no more than 20 at once
        self.cookies = {}
        self.headers = Headers({
            'User-Agent': ['python-requests/2.4.3 CPython/2.7.6 Darwin/14.0.0'],
            'Accept': ['*/*'],
            'Accept-Encoding': ['gzip, deflate'],
            'Connection': ['keep-alive']
        })

    _n = 0
    def fetch(self, method, url, allow_redirects=True):
        return self._sem.run(request,
            method,
            url,
            headers=self.headers,
            allow_redirects=allow_redirects
        ).addCallback(self._hit)

    def _hit(self, data):
        return data