Twisted proxy checker

proxy_cheker.py

# -*- coding: utf-8 -*-
from pprint import pformat
from urllib2 import _parse_proxy

from twisted.python.log import err
from twisted.web.client import ProxyAgent
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint


class DataReceiver(Protocol):
    def __init__(self, checker, proxy, response, finished):
        self.checker = checker
        self.proxy = proxy
        self.response = response
        self.finished = finished
        self.remaining = 1024 * 10
        self.data = list()

    def dataReceived(self, bytes):
        if self.remaining:
            chunk = bytes[:self.remaining]
            self.data.append(chunk)
            self.remaining -= len(chunk)

    def connectionLost(self, reason):
        body = ''.join(self.data)
        setattr(self.response, '_body', body)
        # print 'Finished receiving body:', reason.getErrorMessage()
        self.finished.callback(dict(response=self.response, reason=reason, proxy=self.proxy))

    def __del__(self):
        self.checker.done(self.proxy)


class Checker(object):

    def __init__(self, timeout=10):
        self.wait = set()
        self.timeout = timeout

    def done(self, proxy):
        if proxy in self.wait:
            self.wait.remove(proxy)

        print 'WAIT LEN', len(self.wait)

        # threadpool = reactor.getThreadPool()
        if not self.wait:
            reactor.stop()

    def process_request(self, response, proxy, finished):
        # print 'Response version:', response.version
        # print 'Response code:', response.code
        # print 'Response phrase:', response.phrase
        # print 'Response headers:'
        # print pformat(list(response.headers.getAllRawHeaders()))
        response.deliverBody(DataReceiver(self, proxy, response, finished))

    def process_request_err(self, failure, proxy, finished):
        self.done(proxy)
        finished.callback(dict(failure=failure, proxy=proxy))

    def queue_proxy(self, proxy, callback):
        if proxy in self.wait:
            return
        proxy_type, user, password, hostport = _parse_proxy(proxy)
        host, port = hostport.split(':')
        endpoint = TCP4ClientEndpoint(reactor, host, int(port), timeout=self.timeout)
        agent = ProxyAgent(endpoint)
        d = agent.request("GET", "https://wtfismyip.com/json")
        self.wait.add(proxy)
        finished = Deferred()
        finished.addCallbacks(callback, callback)
        cb_kwargs = dict(proxy=proxy, finished=finished)
        d.addCallbacks(self.process_request, self.process_request_err, callbackKeywords=cb_kwargs, errbackKeywords=cb_kwargs)

    def queue_seq(self, proxies_seq, callback):
        for proxy in proxies_seq:
            self.queue_proxy(proxy=proxy, callback=callback)
        reactor.run()

usage.py

import json

filtrated_proxies = set()

def twisted_callback(result):
    if 'failure' in result:
        print(repr(result))
    else:
        response = result['response']
        # print 'Response version:', response.version
        # print 'Response code:', response.code
        # print 'Response phrase:', response.phrase
        # print 'Response headers:'
        # print list(response.headers.getAllRawHeaders())
        # print result
        if response.code == 200:
            try:
                data = json.loads(response._body)
                if 'YourFuckingIPAddress' in data:
                    filtrated_proxies.add(result['proxy'])
                    print('Pre filtrated, GOOD %s' % result['proxy'])
            except Exception:
                pass

checker = Checker(8.0)
checker.queue_seq([
    'https://109.185.116.199:8080',
    'https://121.52.146.117:8080',
    'https://218.75.205.72:9999',
    ], twisted_callback)