getPage_twisted_with_headers.py

# built-in twisted getPage return deferred
# but i want to get response_headers
# this funct MyGetPage return HTTPClientFactory object
# instead of deferred 
# so after callback we can get headers

from twisted.internet import reactor
from twisted.python.compat import nativeString
from twisted.web.client import URI, HTTPClientFactory


def MyGetPage(url, contextFactory=None, *args, **kwargs):
	"""
	Instead return deffered like `twisted.web.client.getPage`
	this func return factory, which there is information about
	response_headers
	"""
	uri = URI.fromBytes(url)
	factory = HTTPClientFactory(url, *args, **kwargs)
	if uri.scheme == b'https':
		from twisted.internet import ssl
		if contextFactory is None:
			contextFactory = ssl.ClientContextFactory()
		reactor.connectSSL(
			nativeString(uri.host), uri.port, factory, contextFactory)
	else:
		reactor.connectTCP(nativeString(uri.host), uri.port, factory)
	
	return factory


if __name__ == "__main__":
	page = MyGetPage("http://www.google.com")
	
	def _success(args):
		reactor.stop()
		print len(args)
	
	def _error(err):
		print err
	
	d = page.deferred
	d.addCallbacks(_success, _error)
		
	reactor.run()
	
	# headers what we want
	print page.response_headers