parallelpip demo

core.py

#! -*- Coding: utf-8 -*-
from gevent import monkey
monkey.patch_all()

import gevent
import time 
from envoy import run
from sys import exit, argv
import subprocess
import pip

def syntax():
    print "Usage:python core.py normal|parallel [libraries]"

def normal_download(lib = None):
    try:
        if lib:
            start = time.time()
            print "normal download started"
            for l in lib:
                print "Trying to install %s"%l
                run("pip install %s"%l)
            return time.time() - start
        else:
            syntax()
            exit()
    except:
        print "Unhandled exception"
        exit()

def parallel_download(lib = None):
    try:
        if lib:
            print "spawning using gevent"
            jobs = [gevent.spawn(pip.call_subprocess, ["pip","install",l]) \
                                                      for l in lib]
            start = time.time()
            print "joined all gevent, d/l started"
            gevent.joinall(jobs)
            for job in jobs:
                print job.value
            return time.time() - start
        else:
            syntax()
            exit()
    except Exception, e:
        print "unhandled exception", e
        exit()

if __name__ == "__main__":
    if argv[1] == 'parallel':
        print(parallel_download(argv[2:])," seconds for parallel d/l")
    elif argv[1] == 'normal':
        print(normal_download(argv[2:]), "seconds for normal d/l")
    else:
        syntax()
        exit()

how to test

1. Create two virtualenv named normalpip and parallelpip.
2. Install envoy, gevent in both the env(pip install envoy gevent)
3. . normalpip/bin/activate
4.clone the code and name as core.py.
5.python core.py normal flask requests
6. . parallelpip/bin/activate
7.python core.py parallel flask requests
sample output
=============
(normalpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py normal flask requests
normal download started
Trying to install flask
Trying to install requests
(146.34172201156616, 'seconds for normal d/l')

(testparallelpip)kracekumar@python-lover:~/codes/python/asyncpip$ python core.py parallel flask requests
spawning using gevent
joined all gevent, d/l started
Downloading/unpacking flask
  Downloading Flask-0.8.tar.gz (494Kb): 494Kb downloaded
  Running setup.py egg_info for package flask
    
    warning: no files found matching '*' under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'docs'
    warning: no previously-included files matching '*.pyo' found under directory 'docs'
    warning: no previously-included files matching '*.pyc' found under directory 'tests'
    warning: no previously-included files matching '*.pyo' found under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'examples'
    warning: no previously-included files matching '*.pyo' found under directory 'examples'
    no previously-included directories found matching 'docs/_build'
    no previously-included directories found matching 'docs/_themes/.git'
Downloading/unpacking Werkzeug>=0.6.1 (from flask)
  Downloading Werkzeug-0.8.3.tar.gz (1.1Mb): 1.1Mb downloaded
  Running setup.py egg_info for package Werkzeug
    
    warning: no files found matching '*' under directory 'werkzeug/debug/templates'
    warning: no files found matching '*' under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'docs'
    warning: no previously-included files matching '*.pyo' found under directory 'docs'
    warning: no previously-included files matching '*.pyc' found under directory 'tests'
    warning: no previously-included files matching '*.pyo' found under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'examples'
    warning: no previously-included files matching '*.pyo' found under directory 'examples'
    no previously-included directories found matching 'docs/_build'
Downloading/unpacking Jinja2>=2.4 (from flask)
  Downloading Jinja2-2.6.tar.gz (389Kb): 389Kb downloaded
  Running setup.py egg_info for package Jinja2
    
    warning: no previously-included files matching '*' found under directory 'docs/_build'
    warning: no previously-included files matching '*.pyc' found under directory 'jinja2'
    warning: no previously-included files matching '*.pyc' found under directory 'docs'
    warning: no previously-included files matching '*.pyo' found under directory 'jinja2'
    warning: no previously-included files matching '*.pyo' found under directory 'docs'
Installing collected packages: flask, Werkzeug, Jinja2
  Running setup.py install for flask
    
    warning: no files found matching '*' under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'docs'
    warning: no previously-included files matching '*.pyo' found under directory 'docs'
    warning: no previously-included files matching '*.pyc' found under directory 'tests'
    warning: no previously-included files matching '*.pyo' found under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'examples'
    warning: no previously-included files matching '*.pyo' found under directory 'examples'
    no previously-included directories found matching 'docs/_build'
    no previously-included directories found matching 'docs/_themes/.git'
  Running setup.py install for Werkzeug
    
    warning: no files found matching '*' under directory 'werkzeug/debug/templates'
    warning: no files found matching '*' under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'docs'
    warning: no previously-included files matching '*.pyo' found under directory 'docs'
    warning: no previously-included files matching '*.pyc' found under directory 'tests'
    warning: no previously-included files matching '*.pyo' found under directory 'tests'
    warning: no previously-included files matching '*.pyc' found under directory 'examples'
    warning: no previously-included files matching '*.pyo' found under directory 'examples'
    no previously-included directories found matching 'docs/_build'
  Running setup.py install for Jinja2
    
    warning: no previously-included files matching '*' found under directory 'docs/_build'
    warning: no previously-included files matching '*.pyc' found under directory 'jinja2'
    warning: no previously-included files matching '*.pyc' found under directory 'docs'
    warning: no previously-included files matching '*.pyo' found under directory 'jinja2'
    warning: no previously-included files matching '*.pyo' found under directory 'docs'
Successfully installed flask Werkzeug Jinja2
Cleaning up...
Downloading/unpacking requests
  Downloading requests-0.10.6.tar.gz (61Kb): 61Kb downloaded
  Running setup.py egg_info for package requests
    
    warning: no files found matching 'test_requests.py'
Downloading/unpacking certifi>=0.0.7 (from requests)
  Downloading certifi-0.0.8.tar.gz (118Kb): 118Kb downloaded
  Running setup.py egg_info for package certifi
    
Downloading/unpacking chardet>=1.0.0 (from requests)
  Downloading chardet-1.0.1.tar.gz (156Kb): 156Kb downloaded
  Running setup.py egg_info for package chardet
    
Installing collected packages: requests, certifi, chardet
  Running setup.py install for requests
    
    warning: no files found matching 'test_requests.py'
  Running setup.py install for certifi
    
  Running setup.py install for chardet
    
Successfully installed requests certifi chardet
Cleaning up...
None
None
(83.12853598594666, ' seconds for parallel d/l') 

The parallel version gives me errors when I run with many dependencies that have a common set of transitive dependencies. The error pops up when multiple projects try to simultaneously installed a transitive dependency. Any ideas how to alleviate/fix that problem?