multithread downloader

download.py

#!/usr/bin/python

import sys, urllib, threading
from Queue import Queue

base_url = 'http://www.keiyac.org/files/'

class FileGetter(threading.Thread):
	def __init__(self, url):
		self.url = url
		self.result = None
		threading.Thread.__init__(self)
 
	def get_result(self):
		return self.result
 
	def run(self):
		try:
			print '[START] %s' % self.url
			f = urllib.urlretrieve(base_url + self.url,self.url)
			print '[COMPL] %s' % self.url
			self.result = f
		except IOError:
			print "Could not open document: %s" % self.url
 
def get_files(files):
	def producer(q, files):
		for file in files:
			thread = FileGetter(file)
			thread.start()
			q.put(thread, True)
 
	finished = []
	def consumer(q, total_files):
		while len(finished) < total_files:
			thread = q.get(True)
			thread.join()
			finished.append(thread.get_result())
 
	q = Queue(3)
	prod_thread = threading.Thread(target=producer, args=(q, files))
	cons_thread = threading.Thread(target=consumer, args=(q, len(files)))
	prod_thread.start()
	cons_thread.start()
	prod_thread.join()
	cons_thread.join()

if __name__ == '__main__':
	urls = ['dartmoor.mp4','dartmoor.webm','dartmoor.ogv','clannad.ogg','021c0217640039db8d1acef131a38748.png']
	get_files(urls)