Download all potential satisfying dependencies as per a requirements.txt file.

download_dependencies.py

import distlib
import distlib.index
import distlib.version
import requests
import pip.req
import pip.download
import os
import sys
import urlparse

import logging as LOG
LOG.basicConfig(level=LOG.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

PACKAGE_INDEX = 'https://pypi.python.org/pypi'
DOWNLOAD_DIR = "downloads"

def package_info(pkg):
    r = requests.get('{}/{}/json'.format(PACKAGE_INDEX, pkg))
    if not r.status_code == 200:
        return r.status_code, r.request.url

    data = r.json()
    if not 'name' in data:
        data['name'] = pkg
    return data


def candidates(pkg_info, needed):
    results = set()
    name = pkg_info['name']

    versions = pkg_info['releases'].keys()
    if not needed:
        return set(versions)

    version_str = "{} ({})".format(name, needed)
    satisfier = distlib.version.NormalizedMatcher(version_str)
    for version in versions:
        try:
            if satisfier.match(distlib.version.NormalizedVersion(version)):
                results.add(version)
        except distlib.version.UnsupportedVersionError:
            LOG.warn("%s is an unsupported version format for %s", version, name)

    LOG.info("%s of %s are candidates for %s", len(results), len(versions), name)
    return results

def download_urls(pkg_info, candidates):
    for version in candidates:
        release = pkg_info['releases'][version]
        for item in release:
            if 'url' in item:
                yield item['url']


def download(url):
    filename = os.path.join(DOWNLOAD_DIR, url.split('/')[-1])
    r = requests.get(url, stream=True)
    with open(filename, 'wb') as out:
        for chunk in r.iter_content(chunk_size=8192):
            if chunk:
                out.write(chunk)

    return filename

def dependencies(requirements_txt):
    session = pip.download.PipSession()
    for requirement in pip.req.parse_requirements(requirements_txt, session=session):
        yield(requirement.name, str(requirement.specifier))


if __name__ == "__main__":
    if not os.path.exists(DOWNLOAD_DIR):
        os.makedirs(DOWNLOAD_DIR)

    for arg in sys.argv[1:]:
        LOG.info("Reading requirements from: %s", arg)
        for pkg, spec in dependencies(arg):
            info = package_info(pkg)
            versions = candidates(info, spec)
            for url in download_urls(info, versions):
                outfile = download(url)
                LOG.info("%s saved to %s", url, outfile)