map(GET(url)) over Tor exit nodes

tormap.py

#!/usr/bin/env python

TORPROXY=("localhost", 9050)
CONTROLPORT=9051
COOKIE="/var/lib/tor/control_auth_cookie"

URL="http://showip.net"
OUT="scan/"
RELAY="firsthop"
EXITS="exit-addresses"
# curl -s "https://check.torproject.org/exit-addresses" | sed -n "s/^ExitNode //p" >exit-addresses

import time, struct
import socket, select, ssl

import stem.connection
from stem.control import Controller

def get_exits(controller):
    IP = socket.gethostbyname(URL.split("://", 1)[1].split("/", 1)[0]) or None
    P = [443] if URL.startswith("https://") else [80, 443]
    for exit in [x.strip() for x in open(EXITS)]:
        try:
            desc = controller.get_server_descriptor(exit)
            if any(desc.exit_policy.can_exit_to(IP, port) for port in P):
                yield desc
        except stem.DescriptorUnavailable as ex:
            pass

context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False

def GET(url, allow_redirects=3):
    protocol, host = url.split("://", 1)
    host, path = (host.split("/", 1) + [""])[:2]
    path = "/" + path

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0, None)
    try:
        sock.settimeout(2*4.20)
        sock.connect(TORPROXY)

        if sock.send(b"\x05\x01\x00") != 3 or sock.recv(2) != b"\x05\x00":
            raise Exception("SOCKS5 handshake failed")

        # DNS resolve through Tor
        #msg = b"\x05\xF0\x00\x03" \
        #    + chr(len(host)).encode() \
        #    + host.encode() \
        #    + b"\x00\x00"
        #if sock.send(msg) < len(msg) or sock.recv(2) != b"\x05\x00":
        #    raise Exception("SOCKS5 resolve failed")
        #IP = socket.inet_ntoa(sock.recv(8)[2:6])

        msg = b"\x05\x01\x00\x03" \
            + chr(len(host)).encode() \
            + host.encode() \
            + struct.pack(">H", 80 if protocol != "https" else 443)
        if sock.send(msg) < len(msg) or sock.recv(3) != b"\x05\x00\x00":
            raise Exception("SOCKS5 connect failed")
        sock.recv(10)

        if protocol == "https":
            sock = context.wrap_socket(sock)

        msg = b"GET " + path.encode() + b" HTTP/1.1\r\n" \
            + b"Host: " + host.encode() + b"\r\n" \
            + b"User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:45.0) Gecko/20100101 Firefox/45.0\r\n" \
            + b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" \
            + b"Accept-Language: en-US,en;q=0.5\r\n" \
            + b"Accept-Encoding: gzip, deflate\r\n" \
            + b"Connection: close\r\n" \
            + b"\r\n"
        sock.send(msg)
        data = b""
        while len(select.select([sock], [], [])[0]) == 1:
            ret = sock.recv(4096)
            if not ret: break
            data += ret
    finally:
        sock.close()

    headers = data[:data.index(b"\r\n\r\n") + 4]
    content = data[len(headers):]
    status = int(headers.split(b" ", 2)[1].decode("ascii"))
    if status in [301, 302] and allow_redirects > 0:
        for hdr in headers.split(b"\r\n"):
            if hdr.startswith(b"Location: "):
                location = hdr[hdr.index(b" ")+1:].strip().decode("ascii")
                h, content = GET(location, allow_redirects-1)
                headers += h
    elif status == 200:
        if b"Transfer-Encoding: chunked" in headers:
            chunks, content, i = content, b"", 0
            while i < len(chunks):
                digits = chunks[i:].index(b"\r\n")
                length = int(chunks[i:i+digits] or "0", 16)
                i += digits + len("\r\n")
                content += chunks[i:i+length]
                i += length + len("\r\n")

    return headers, content

def torify(f, route=None):
    def attach(stream):
        try:
            if stream.status == "NEW":
                if exit.exit_policy.can_exit_to(port=stream.target_port):
                    controller.attach_stream(stream.id, circuit)
                else: raise
            elif stream.status == "NEWRESOLVE":
                controller.attach_stream(stream.id, circuit)
        except:
            try: controller.close_stream(stream.id)
            except: pass
    circuit = controller.new_circuit(route, await_build=True)
    controller.set_conf("__LeaveStreamsUnattached", "1")
    controller.add_event_listener(attach, stem.control.EventType.STREAM)
    try: f()
    finally:
        controller.reset_conf("__LeaveStreamsUnattached")
        controller.remove_event_listener(attach)
        try: controller.close_circuit(circuit)
        except stem.InvalidArguments: pass

with Controller.from_port(port=CONTROLPORT) as controller:
    stem.connection.authenticate_cookie(controller, COOKIE)
    relay = controller.get_server_descriptor(RELAY)
    exits = list(get_exits(controller))
    for i, exit in enumerate(exits):
        route = [relay.fingerprint, exit.fingerprint]
        filename = exit.fingerprint + "-" + exit.nickname
        timer = (lambda s: (lambda: time.time() - s))(time.time())
        stamp = lambda: "(%d/%d) %.2fs" % (i+1, len(exits), timer())

        def get_content_and_headers():
            headers, content = GET(URL)
            with open(OUT + filename + ".hdr", 'wb') as f:
                f.write(headers)
            try:
                encoder = headers.index(b"Content-Encoding: ")
                encoder = headers[encoder:].split(None, 2)[1].decode()
                with open(OUT + filename + ".html." + encoder, 'wb') as f:
                    f.write(content)
                if encoder == "gzip": from gzip import decompress
                elif encoder == "deflate": from zlib import decompress
                else: raise Exception("unknown content encoder: " + encoder)
                content = decompress(content)
            except ValueError: pass
            with open(OUT + filename + ".html", 'wb') as f:
                f.write(content)

        try:
            torify(lambda: get_content_and_headers(), route)
            print("[+]", exit.nickname, stamp())
        except Exception as ex:
            print("[-]", exit.nickname, stamp(), ex)