building a non-blocking requests tool on top of Requests and Twisted

trequests.py

from time import time
import requests



languages_list = [
	"English", 
	"French",
	"German",
	"Swahili",
	"Tagalog", 
	"Hebrew", 
	"Spanish",
	"Arabic",
	"Hindi",
	"Persian"
	]

start_time = time()

# def language_scraper(languages_list):
# 	languages_requests = []
# 	for l in languages_list:
# 		r = requests.get('http://en.wikipedia.org/wiki/%s' % l)  
# 		languages_requests.append(r)
# 	return languages_requests

languages_responses = dict()




def keep_running_check(languages_list, languages_responses):
	if len(languages_list) == len(languages_responses):
		kill_reactor()
	else:
		pass

def non_blocking_language_scraper(language, languages_responses):
	url = 'http://en.wikipedia.org/wiki/%s' % language
	r = requests.get(url)
	# languages_responses[language] = r
	keep_running_check(languages_list, languages_responses)


def kill_reactor():
	from twisted.internet import reactor
	reactor.stop()
	print "async: ", time() - start_time



from twisted.internet import reactor


reactor.callWhenRunning(non_blocking_language_scraper, "English", languages_responses)

for language in languages_list:
	reactor.callWhenRunning(non_blocking_language_scraper, language, languages_responses)


reactor.run()

print languages_responses

# blocking code:

# start = time()

# print language_scraper(languages_list)

# end = time()

# print end-start