One-off script for collecting nearest-matching source tarballs for a set of eggs

fetch_sdists_from_eggs.py

import os, glob
import httplib2, lxml.html

base = "http://files.turbogears.org/eggs/"

http = httplib2.Http()

def fetch(url):
    response, content = http.request(url)
    
    if response['status'] != '200':
        print "%s: %s" % (response['status'], url)
        return None

    return content

def look_on_pypi(name, requirement):
    try:
        doc = lxml.html.parse("http://pypi.python.org/simple/%s" % name)
    except IOError:
        return

    doc = doc.getroot()
    doc.make_links_absolute()
    for element, attribute, link, pos in doc.iterlinks():
        name = element.text
        if not name.endswith(".tar.gz"):
            continue
        index = -1 * len(".tar.gz")
        version = name[:index].split("-")[-1]
        if version == requirement:
            print "Found a match on pypi: downloading %s" % link
            return fetch(link)

eggs = glob.glob(os.path.join(os.path.dirname(__file__),'eggs','*.egg'))
tarballs = os.listdir("requirements/src")

from pprint import pformat
for egg in eggs:
    egg = egg.replace("eggs/", '', 1)
    egg, build = egg.split("-py2.5")

    if egg + ".tar.gz" in tarballs:
        print "** already satisfied: %s" % egg
        continue
    close = glob.glob('requirements/src/%s*.tar.gz' % egg)
    if len(close) > 0:
        print "**** looks close enough to %s: %s" % (egg, pformat(close))
        continue
    name, version = egg.split("-", 1)
    maybe = glob.glob("requirements/src/%s-*.tar.gz" % name)
    if len(maybe) > 0:
        print "******* looks like a different version from %s: %s" % (egg, pformat(maybe))
        continue

    url = base + egg + ".tar.gz"

    content = fetch(url)

    if content is None:
        content = fetch(url.replace("_", "-"))

    if content is None:
        name, version = egg.split("-", 1)
        print name, version
        content = look_on_pypi(name, version)

    if content is not None:
        print ":) downloading %s.tar.gz" % egg
        fp = open("requirements/src/%s.tar.gz" % egg, 'w')
        fp.write(content)
        fp.close()
    else:
        print ":( Couldn't find anything like", egg