crawler.py

from bs4 import BeautifulSoup as bs
from datetime import datetime
import requests
import httplib
import socket
import ssl

init = datetime.now()

def connect_patched(self):
    "Connect to a host on a given (SSL) port."

    sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address)
    if self._tunnel_host:
        self.sock = sock
        self._tunnel()
        
    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)


httplib.HTTPSConnection.connect = connect_patched

# to get an valip IP with a HTTP connection
conn = httplib.HTTPConnection("au.proxymesh.com", 31280)
conn.request("GET", "http://www.example.com")
req = conn.getresponse()

# set the initial headers with the getted IP
headers = {
    'X-ProxyMesh-IP': req.getheader('x-proxymesh-ip'),
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
}

for i in range(10):
    print headers
    
    # making the HTTPS connection
    conn = httplib.HTTPSConnection("au.proxymesh.com", 31280)
    conn.request("GET", "https://www.example.com")
    req = conn.getresponse()
    print req.getheaders()