gistfile1.py

import mechanize
import os, sys

KILOBYTE = 1024
MEGABYTE = KILOBYTE * KILOBYTE
BUFFER = MEGABYTE / 2

NOISY = False

def noisyPrint(text, rn=True):
    if NOISY:
        if rn:
            text += "\r\n"
            
        sys.stdout.write(text)

def makeCookie(name, value, domain, **opts):
    opts["name"] = name
    opts["value"] = value
    opts["domain"] = domain
    
    defaults = {
        "version": 0,
        "port": None,
        "port_specified": False,
        "domain_specified": False, 
        "domain_initial_dot": False,
        "path": "/",
        "path_specified": True,
        "secure": False,
        "expires": None,
        "discard": True,
        "comment": None,
        "comment_url": None,
        "rest": {'HttpOnly': None},
        "rfc2109": False
    }
    
    for name, value in defaults.iteritems():
        if not name in opts:
            opts[name] = value
            
    ck = mechanize.Cookie(**opts)
        
    return ck

def download(url, outfile=None, cookies=None, cookiejar=None, headers=None, retries=0):
    if not cookiejar:
        cookiejar = mechanize.CookieJar()
    
    if cookies and isinstance(cookies, (tuple, list, mechanize.Cookie)):
        if isinstance(cookies, mechanize.Cookie):
            cookies = [cookies,]
            
        for cookie in cookies:
            if isinstance(cookie, mechanize.Cookie):
                cookiejar.set_cookie(cookie)
                
    opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookiejar))
    
    if headers:
        for name, value in headers.iteritems():
            opener.add_header(name, value)

    if not outfile:
        outfile = os.tmpfile()
        
    done = False
    while not done:
        noisyPrint("Attempting download...")
        
        try:
            stream = opener.open(url, timeout=5)
            noisyPrint("Connected...")
        except Exception, e:
            noisyPrint("Error: %s"%e)
            
            if retries > 0:
                retries -= 1
                noisyPrint("Retrying... (%d left)"%retries)
            elif retries < 0:
                noisyPrint("Retrying...")
                
            if retries: 
                continue
            else:
                break
                
        while True:
            try:
                # Read bit by bit, to keep memory usage down.
                data = stream.read(BUFFER)
                if not data:
                    done = True
                    break
                    
                noisyPrint("Received %d bytes."%len(data))
                
            except Exception, e:
                noisyPrint("Error: %s\r\n"%e)
                
                if retries > 0:
                    retries -= 1
                    noisyPrint("Retrying. (%d left)"%retries)
                elif retries < 0:
                    noisyPrint("Retrying...")
                        
                if retries:
                    # Clear out the local file, so we can start from scratch.
                    outfile.truncate()
                    break
                else:
                    # Give up.
                    done = True
                    noisyPrint("Retries exhausted. Giving up.")
                    break
            
            outfile.write(data)
        
    outfile.seek(0)
    noisyPrint("Done.")
    
    return outfile