urlchecker

urlcheck.py

#!/usr/bin/env python
"""
This is a multiprocess URLchecker
"""
import urllib
from multiprocessing import Process, Queue
def checker(url):
    try:
        code = urllib.urlopen(url).getcode()
        return code
    except Exception,e:
        return str(e)
# Function run by worker processes
def worker(input, output):
    for func in iter(input.get, 'STOP'):
        result = checker(func)
        output.put(func+"\t\t:\t\t"+str(result))
def main():
    """Runs everything"""
    # Task queue
    task_queue = Queue()
    #Result queque
    result_queue = Queue()
    #submit tasks
    for url in urls:
        task_queue.put(url)
    #Start worker processes
    for i in range(NUMBER_OF_PROCESSES):
        Process(target=worker, args=(task_queue, result_queue)).start()
     # Get and print results
    for i in range(len(urls)):
        print "\t",result_queue.get()
    # Tell child processes to stop
    for i in xrange(1,NUMBER_OF_PROCESSES):
        task_queue.put('STOP')
if __name__ == "__main__":
    #urls to check
    urls = ["http://www.baidu.com/", "http://xsc2131.com/","http://www.baidu.com/",\
            "http://www.baidu.com/","http://www.baidu.com/","http://www.baidu.com/",\
            "http://www.baidu.com/","http://www.baidu.com/","http://www.baidu.com/", \
            "http://xsc2131.com/","http://www.baidu.com/","http://www.baidu.com/",\
            "http://www.baidu.com/","http://www.baidu.com/","http://www.baidu.com/",
            "http://www.baidu.com/123fuck13.html",\
            "http://so.360.cn/123123123"]
    NUMBER_OF_PROCESSES = 10   
    main()