http_request.py

def http_request(url, 
                data = {}, 
                header = {
                        # 默认伪装成IE9
                        "User-Agent"   : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 5.1; Trident/4.0)",
                        "Connection"   : "Close",
                        # 避免proxy cache页面
                        "Pragma"       : "no-cache",
                        "Cache-Control": "no-cache, max-age=0, must-revalidate"
                        }):
    """通用http请求"""
    import urllib,urllib2

    # 设置http头
    opener = urllib2.build_opener()
    if header.has_key("User-Agent"):
        opener.addheaders = [("User-Agent", header["User-Agent"])]
    for k in header.keys(): # 设置http header
        opener.addheaders.append((k, header[k])) 
    urllib2.install_opener(opener)

    if data: # 有data就是post，否则就是get
        enData = urllib.urlencode(data)
        return urllib2.urlopen(url, enData).read()
    else:
        return urllib2.urlopen(url).read()