Quick and dirty hostsfile generator from multiple host sources

hostsgen.py

import urllib2
import socket
import sys
from StringIO import StringIO
import gzip

SINKHOLE = '127.0.0.1'
SOURCES =\
(
	"http://adaway.org/hosts.txt",
	"http://winhelp2002.mvps.org/hosts.txt",
	"http://someonewhocares.org/hosts/zero/hosts",
	"http://hosts-file.net/ad_servers.txt",
	"http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext&useip=0.0.0.0",
	"http://www.malwaredomainlist.com/hostslist/hosts.txt"
)
LOCALHOST_ADDR = ('::1', '127.0.0.1')
LOCALHOST_ALIAS = ('localhost', 'localhost.localdomain')
USER_AGENT = "Mozilla/5.0 (X11; Linux i686; rv:38.0) Gecko/20100101 Firefox/38.0"

def valid_ip(address):
	try: 
		socket.inet_aton(address)
	except socket.error:
		return False
	return True

def preprocess(response, lastmod=''):
	head = True
	headlines = []
	hostaliases = []
	if not lastmod:
		lastmod = "Unknown"
	headlines.append("Last modified: %s" % lastmod)
	for line in response:
		l = line.strip()
		if head and l.startswith('#'):
			headlines.append(l)
		else:
			head = False
		spl = l.split()
		if len(spl) > 1 and valid_ip(spl[0]) and spl[1] != 'localhost' and '.' in spl[1]:
			hostaliases.append(spl[1])
	return (headlines, hostaliases)

def run():
	header = []
	entries = []
	for hostsrc in SOURCES:
		print >>sys.stderr, hostsrc,
		try:
			req = urllib2.Request(hostsrc, headers={'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip'})
			resp = urllib2.urlopen(req, timeout=30)
			if resp.info().getheader('Content-Encoding') == 'gzip':
				buf = StringIO(resp.read())
				f = gzip.GzipFile(fileobj=buf)
			else:
				f = resp
			data = preprocess(f, resp.info().getheader('Last-Modified'))
			resp.close()
		except StandardError:
			print >>sys.stderr, "Error"
		else:
			print >>sys.stderr, "OK", len(data[1]), "lines"
			header.extend(data[0])
			entries.extend(data[1])
	uniq = list(set(entries))
	uniq.sort()
	print >>sys.stderr, "Total entries:", len(uniq)
	sys.stdout.write('\n'.join(header))
	sys.stdout.write('\n' * 3)
	sys.stdout.write(''.join(map(lambda x: '%s %s\n' % (x, ' '.join(LOCALHOST_ALIAS)), LOCALHOST_ADDR)))
	sys.stdout.write('\n' * 2)
	sys.stdout.write(''.join(map(lambda x: '%s %s\n' % (SINKHOLE, x), uniq)))
	sys.stdout.flush()
	
if __name__ == "__main__":
	run()