SP, TP hosts survey

gistfile1.py

# vim: fileencoding=utf-8

import socket
import httplib2
import re
import operator

http_client = httplib2.Http(".cache")

def parse_indextxt(url):
    (headers, content) = http_client.request(url + str(r"index.txt"))
    result = []
    lines = content.splitlines()
    for n, i in enumerate(lines):
        # print(str(i).split("<>")[2])
        print("{0}/{1}".format(n+1, len(lines)))
        r = re.search(r"(.+):.*", str(i).split("<>")[2])
        if r is None:
            continue
        addr = r.group(1)
        try:
            host = socket.gethostbyaddr(addr)
        except (socket.gaierror, OSError, ValueError):
            continue
        result.append(host)
    return result

if __name__ == "__main__":
    result = []
    sp = r"http://bayonet.ddo.jp/sp/"
    tp = r"http://temp.orz.hm/yp/"
    result.extend(parse_indextxt(sp))
    result.extend(parse_indextxt(tp))
    f = open("host.txt", "w")
    for i in result:
        print(i)
        f.write(i[0] + "\n")
    c = {}
    r_2 = re.compile(r".+\.(.+\..+\..{2})$")
    r_3 = re.compile(r".+\.(.+\..{3})$")
    for i in result:
        g = r_2.search(i[0])
        if g is None:
            g = r_3.search(i[0])
        if g is None:
            continue
        if not g.group(1) in c:
            c[g.group(1)] = 1
        else:
            c[g.group(1)] += 1
    fc = open("count.txt", "w")
    for k, v in sorted(c.items(), key=operator.itemgetter(1), reverse=True):
        fc.write("{} : {}\n".format(k, v))