The python 3 script which will go through list of domains and find interesting http response headers for each domain. This script will go through list of domains defined here:”D:\Data\sites_ba_domain.txt” and as a result you will get .csv file “D:\server_info.txt” which can be easily transformed into excel spreadsheet.

get_headers.py

#!/usr/bin/env python

__author__ = 'Alen Komljen'

import urllib.request, re, os.path, os
from socket import timeout

server_info = open("D:\server_info.txt", "w")
server_info.write("url, server_name, server_version, server_os \
					, server_misc, poweredby_engine, poweredby_version \
					, aspnet_version, aspnetmvc_version, sharepoint \
					, set_cookie")

debian_os = ("lenny", "squeeze", "etch", "dotdeb")

with open('D:\Data\sites_ba_domain.txt') as f:
    baDomains = [line.strip() for line in f]

print ("\nSearch for server headers in list of: " + str(len(baDomains)) + " sites...")

for x in baDomains:
	try:
		request = urllib.request.urlopen("http://" + x, timeout=10)
	except urllib.error.URLError:
		continue
	except timeout:
		continue
	except:
		continue

	server_header = request.info().get("Server")

	if server_header:
		server_name_match = re.search("(.*?)(\/.*)", server_header)
		server_version_match = re.search("(.*?)\/([0-9\.]*).*", server_header)
		server_misc_match = re.search("(.*\()([a-zA-Z]*)\)(.*)", server_header)
		server_os_match = re.search("(.*\()([a-zA-Z]*)\).*", server_header)

		if server_name_match is not None:
				server_name = server_name_match.group(1)
		else:
			server_name_match = re.search("(.*)", server_header)
			if server_name_match is not None:
				server_name = server_name_match.group(1)
			else:
				server_name = ""

		if server_version_match is not None:
			server_version = server_version_match.group(2)
		else:
			server_version = ""
		if server_misc_match is not None:
			server_misc = server_misc_match.group(3)
		else:
			server_misc = ""
		if server_os_match is not None:
			server_os = server_os_match.group(2)
		else:
			server_os = ""

		if server_os == "" and server_misc != "":
			for os in debian_os:
				debian = re.search(os, server_misc)
				if debian is not None:
					server_os = "Debian"
			ubuntu = re.search("ubuntu", server_misc)
			if ubuntu is not None:
				server_os = "Ubuntu"
			gentoo = re.search("gentoo", server_misc)
			if gentoo is not None:
				server_os = "Gentoo"

		if server_name == "Microsoft-IIS":
			server_os = "Win"

	poweredby_header = request.info().get("X-Powered-By")
	if poweredby_header:
		poweredby_engine_match = re.search("(.*?)(\/.*)", poweredby_header)
		poweredby_version_match = re.search("(.*?)\/([0-9\.]*)(.*)", poweredby_header)

		if poweredby_engine_match is not None:
			poweredby_engine = poweredby_engine_match.group(1)
		else:
			poweredby_engine_match = re.search("(.*)", poweredby_header)
			if poweredby_engine_match is not None:
				poweredby_engine = poweredby_engine_match.group(1)
			else:
				poweredby_engine = ""

		if poweredby_version_match is not None:
			poweredby_version = poweredby_version_match.group(2)
			if server_os == "":
				server_os_from_engine_version = poweredby_version_match.group(3)
				for os in debian_os:
					debian = re.search(os, server_os_from_engine_version)
					if debian is not None:
						server_os = "Debian"
				ubuntu = re.search("ubuntu", server_os_from_engine_version)
				if ubuntu is not None:
					server_os = "Ubuntu"
				gentoo = re.search("gentoo", server_os_from_engine_version)
				if gentoo is not None:
					server_os = "Gentoo"
		else:
			poweredby_version = ""
	else:
		poweredby_engine = ""
		poweredby_version = ""

	aspnet_version_header = request.info().get("X-AspNet-Version")
	if aspnet_version_header:
		aspnet_version = aspnet_version_header
	else:
		aspnet_version = ""

	aspnetmvc_version_header = request.info().get("X-AspNetMvc-Version")
	if aspnetmvc_version_header:
		aspnetmvc_version = aspnetmvc_version_header
	else:
		aspnetmvc_version = ""

	sharepoint_header = request.info().get("MicrosoftSharePointTeamServices")
	if sharepoint_header:
		sharepoint = sharepoint_header
	else:
		sharepoint = ""

	set_cookie_header = request.info().get("Set-cookie")
	if set_cookie_header:
		httponly_match = re.search("HttpOnly", set_cookie_header)

		if httponly_match is not None:
			set_cookie = httponly_match.group()
		else:
			set_cookie = ""
	else:
		set_cookie = ""

	server_info.write ("\n" + x + ", " + server_name + ", " + server_version + ", " + server_os \
								 + ", " + server_misc + ", " + poweredby_engine + ", " + poweredby_version \
								 + ", " + aspnet_version + ", " + aspnetmvc_version + ", " + sharepoint \
								 + ", " + set_cookie)

	print ("Checking URL: " + x +" finished...")

server_info.close()
print ("Completed")