gistfile1.py

import urllib2
import os, sys
import cookielib

KILOBYTE = 1024
MEGABYTE = KILOBYTE * KILOBYTE
BUFFER = MEGABYTE / 2

def makeCookie(name, value, domain, **opts):
    opts["name"] = name
    opts["value"] = value
    opts["domain"] = domain
    
    defaults = {
        "version": 0,
        "port": None,
        "port_specified": False,
        "domain_specified": False, 
        "domain_initial_dot": False,
        "path": "/",
        "path_specified": True,
        "secure": False,
        "expires": None,
        "discard": True,
        "comment": None,
        "comment_url": None,
        "rest": {'HttpOnly': None},
        "rfc2109": False
    }
    
    for name, value in defaults.iteritems():
        if not name in opts:
            opts[name] = value
            
    ck = cookielib.Cookie(**opts)
        
    return ck

def download(url, file, cookies=None, retries=0):
    print "Checking and adding cookies"
    if cookies and isinstance(cookies, (tuple, list, cookielib.Cookie)):
        cj = cookielib.LWPCookieJar()
        
        if isinstance(cookies, cookielib.Cookie):
            cookies = [cookies,]
            
        for cookie in cookies:
            if isinstance(cookie, cookielib.Cookie):
                cj.set_cookie(cookie)
                
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        urllib2.install_opener(opener)
    
    done = False
    while not done:
        print "Attempting download..."
        stream = urllib2.urlopen(url, timeout=2)
        
        while True:
            try:
                data = stream.read(BUFFER)
                print "Received %d bytes." % len(data)
            except Exception, e:
                print "Error: %s\r\nRetrying." % e
                if retries != 0: # retries < 0 will cause infinite retries
                    file.truncate()
                    
                    if retries > 0:
                        retry -= 1
                        
                    continue
                else:
                    print "Retries exhausted."
                    break
            if not data: 
                break
            
            file.write(data)
        
        done = True
        
    file.seek(0)
    print "Done."
    return file