Crawler

Crawler.py

# bug: investigate looping wen crawling www.wired.com

import os

class Config(object):
	def __init__(self):
		# force to download https
		self.forceHttps = True

import hashlib
import time

# used to filter for already downloaded websites by hash
class HashedUrlList(object):
	def __init__(self):
		self._hashtable = {}

	def has(self, url):
		try:
			hashed = hashlib.sha224(url.encode('utf-8')).hexdigest()
		except UnicodeDecodeError:
			# just ignore the error
			# we return false because we pretend that we don't have the key
			return False

		return hashed in self._hashtable

	def put(self, url):
		try:
			hashed = hashlib.sha224(url.encode('utf-8')).hexdigest()
		except UnicodeDecodeError:
			return # ignore, we hope that tis won't hamper the crawler

		self._hashtable[hashed] = True

class Logger(object):
	def __init__(self):
		self._filehandle = open("log.log", "a+")

	def log(self, message):
		self._filehandle.write(message + "\n")

# used to enqueue urls which need to be crawled
# 
# store to file and append, while reading in the middle of the file
class CrawlingQueue(object):
	def __init__(self):
		self._filehandle = open("queue", "a+")
		self._seek = 0

	def retNext(self):
		# check if at end of file
		self._filehandle.seek(0, 2)
		fileSize = self._filehandle.tell()

		isAtEndOfFile = self._seek >= fileSize

		if isAtEndOfFile:
			return None

		# read and advance
		self._filehandle.seek(self._seek)
		lines = self._filehandle.readline()
		self._seek += len(lines)

		splitedLines = lines.split("\n")
		splitedLines = splitedLines[:-1] # remove last line which is always empty
		assert len(splitedLines) == 1

		return splitedLines[0]

	def enqueue(self, url):
		self._filehandle.write((url + "\n"))

	def cleanup(self):
		# TODO< remove file >

		pass


config = Config()

class DumpFailed(BaseException):
	def __self__(self):
		pass

# returns the lines
def dumpWithLynx(filename):
	try:
		lines = os.popen("lynx -dump " + filename).readlines()
	except UnicodeDecodeError:
		raise DumpFailed() # propagate error

	lines = [i.replace("\n","") for i in lines]
	return lines

def downloadFile(url, filename, timeout = 20):
	lines = os.popen("wget -q  -N -k -T " + str(timeout) + " -O " + filename + " " + url).readlines()

	# TODO< detect timeout and return False if Timeout did occur


# returns filtered references as text
def retReferencedFromDump(content):
	filtered = []

	enabled = False
	for iContent in content:
		if enabled:
			filtered.append(iContent)
		else:
			enabled = iContent == 'References'

	return filtered

def retReferences(content):
	result = []

	for iContent in content:
		idx = iContent.find(". ")
		if idx == -1:
			continue

		result.append(iContent[idx+2:])

	return result

# remove the http protocol from the url or return None if it is not http
def removeHttpProtocol(url):
	if url[0:5+2] == "http://":
		return url[5+2:]
	elif url[0:6+2] == "https://":
		return url[6+2:]
	else:
		return None

def removeProtocol(url):
	# TODO< write cleaner way >
	# quick hack!
	return removeHttpProtocol(url)

# checks if the url is http and from the domain
def checkDomain(url, domain):
	withoutProtocol = removeProtocol(url)
	if withoutProtocol == None:
		return None

	if len(withoutProtocol) < len(domain):
		return False

	return withoutProtocol[:len(domain)] == domain

logger = Logger()

import sys
import datetime

# domain to be crawled
domain = sys.argv[1]

# hashed list of already crawed urls
# we just need to store the hash of the url because we don't need to know the url
# and 
alreadyCrawledUrls = HashedUrlList()


queue = CrawlingQueue()
queue.enqueue("https://" + domain + "/")

waitTime = 0.2

isFirstSite = True

while True:
	candidateUrl = queue.retNext()
	if candidateUrl == None:
		break

	if alreadyCrawledUrls.has(candidateUrl):
		continue

	currentUrl = candidateUrl
	del candidateUrl

	if not isFirstSite: # don't remove / from domain!
		if currentUrl[-1] == "/":
			currentUrl = currentUrl[:-1]

	logger.log("log: fetch " + currentUrl)
	print("log: fetch " + currentUrl)

	if isFirstSite:
		filename = "index.html"
	else:
		splitedUrl = currentUrl.rsplit('/', 1)
		# for debugging
		print(splitedUrl)

		# we need special handling because it may fail
		filename = None
		if len(splitedUrl) > 1:
			filename = currentUrl.rsplit('/', 1)[1]

		if filename != None and not filename.endswith(".html"):
			filename += ".html"



	print("filename=" + filename)

	# remember that we have already downloaded the url
	alreadyCrawledUrls.put(currentUrl)


	if filename == None:
		continue # ignore url if the url wasn't valid

	downloadFile(currentUrl, filename)


	try:
		contentFromInvocation = dumpWithLynx("./" + filename)
	except DumpFailed:
		if True:
			logger.log("warning: " + "dumping of " + currentUrl + " failed!")
			print("warning: " + "dumping of " + currentUrl + " failed!")

		continue

	isFirstSite = False

	filteredReferences = retReferencedFromDump(contentFromInvocation)
	references = retReferences(filteredReferences)

	# filter references so it is html or a relative link
	references = list(filter(lambda url: (len(url) >= 1 and url[-1] == '/') or (len(url) >= 5+1 and url[-5:] == ".html"), references))

	#print("references=")
	#print(references)


	# filter by domain
	referencesOnDomain = list(filter(lambda url: checkDomain(url, domain), references))


	# for each referenced url
	for iReference in referencesOnDomain:
		if alreadyCrawledUrls.has(iReference):
			continue

		# add to queue so it can be crawled later on
		queue.enqueue(iReference)




	

	# we don't wait if the target server is probably under low load
	targetServerUnderLowLoad = False

	if True:
		targetServerUnderLowLoad = (datetime.datetime.now().hour > 0 and datetime.datetime.now().hour < 10)
		targetServerUnderLowLoad = targetServerUnderLowLoad or (datetime.datetime.now().hour > 21)

	#if not targetServerUnderLowLoad:
	#	# force wait to not overwhelm server(s)
	#	time.sleep(waitTime)



# give the queue a chance to cleanup memory and temporary file(s)
queue.cleanup()

logger.log("log: completed crawling of " + domain)
print("log: completed crawling of " + domain)