req-to-local.sh

#!/bin/bash

#
# Please note: This script is likely very brittle...
#

ORIGIN=$1
OUT_DIR=$2

if [ "$2" == "" ] || [ "$1" == "" ]; then echo "Please specify origin requirements file and output directory"; exit 4; fi

if [ ! -f "$ORIGIN" ]; then echo "$1 doesnt seem to be a file"; exit 1; fi

if [ -e "$OUT_DIR" ]; then echo "$2 exists, cowardly refusing to overwrite"; exit 2; fi

mkdir -p $OUT_DIR
if [ $? -ne 0 ]; then echo "Failed creating directory $2"; exit 3; fi

# We need pip >= 1.3 (i think) to get dependencies from the "pip show" command
pip install --upgrade pip==1.5.4 1>/dev/null 2>/dev/null
if [ $? -ne 0 ]; then echo "Could not run 'pip install pip==1.5.4'"; exit 6; fi

function resolve() {
	for d in $(pip show $1 | grep Requires: | cut -d' ' -f2- | sed -r -e 's/, */ /g' | grep -Ev '^$'); do
		echo $FOUND | grep -q '#'$d'#'
		if [ $? -eq 0 ]; then continue; fi
		echo -e "\t\tdependency: $d" 1>&2
		FOUND="$FOUND#$d#"
		echo "$d"
		resolve $d
	done
}

function resolve_reqs() {
	echo "Resolving top level requirements..." 1>&2
	export FOUND=""
	for i in $(cat $ORIGIN | cut -d= -f1); do
		echo $FOUND | grep -q '#'$i'#'
		if [ $? -eq 0 ]; then
			echo -e "\tfound $i but we have already added it elsewhere" 1>&2
			continue
		fi
		echo -e "\tfound $i" 1>&2
		FOUND="$FOUND#$i#"
		resolve $i
		echo $i
	done
}

function download_reqs() {
	# Step 2... Compute a list of all dependencies
	P=""
	for w in $(resolve_reqs); do
		let found=$found+1
		# Step 3... Download all of those dependencies
		WANT=$(grep "$w==" $ORIGIN)
		if [ "$WANT" = "" ]; then WANT=$(pip freeze 2>/dev/null | grep "$w=="); fi
		P="$P $WANT"
	done
	echo -e "\n\npip install --download $OUT_DIR --no-install $P\n\n" 1>&2
	for F in $(pip install --download $OUT_DIR --no-install $P 2>/dev/null | grep -E '^\s*Saved\s' | sed -r -e 's/^\s*Saved\s*//g'); do
		echo $(basename $F)
	done
}

# Step 1... to do introspection deeply (dependencies) we need to actually install all this crap
pip install --upgrade --requirement $ORIGIN 1>/dev/null 2>/dev/null

export found=0

# Step 4... Write out new requirements file
download_reqs > $OUT_DIR/local-requirements

# Step 5... A little bit of sanity
if [ $(ls -1 $OUT_DIR | grep -v local-requirements | wc -l) -ne $found ]; then
	echo "Something went wrong. Expected $found downloaded files and got  $(ls -1 $OUT_DIR | grep -v local-requirements | wc -l)"
	exit 5
fi

echo "Okay... You should now be able to copy"
echo "$OUT_DIR"
echo "to another machine, and then (from inside"
echo "the directory) run the following:"
echo
echo -e "\tpip install --upgrade -r ./local-requirements\n\n"

Given the requirements file

Django==1.4
backports.ssl-match-hostname==3.4.0.2
bson==0.3.3
dogslow==0.9.7
gevent==1.0
greenlet==0.4.1
msgpack-python==0.4.0
nose==1.3.0
pymongo==2.6.3
python-memcached==1.53
pytz==2013.8
pyzmq==14.0.1
requests==2.1.0
simplejson==3.3.1
wsgiref==0.1.2

running the following

./req-to-local.sh ./requirements ./out

produces this output

Resolving top level requirements...
    found pip
    found Django
    found backports.ssl-match-hostname
    found bson
        dependency: pytz
    found dogslow
    found gevent
        dependency: greenlet
    found greenlet but we have already added it elsewhere
    found msgpack-python
    found nose
    found pymongo
    found python-memcached
    found pytz but we have already added it elsewhere
    found pyzmq
    found requests
    found simplejson
    found wsgiref


pip install --download ./out --no-install  pip==1.5.4 Django==1.4 backports.ssl-match-hostname==3.4.0.2 pytz==2013.8 bson==0.3.3 dogslow==0.9.7 greenlet==0.4.1 gevent==1.0 msgpack-python==0.4.0 nose==1.3.0 pymongo==2.6.3 python-memcached==1.53 pyzmq==14.0.1 requests==2.1.0 simplejson==3.3.1 wsgiref==0.1.2


Okay... You should now be able to copy
./out
to another machine, and then (from inside
the directory) run the following:

    pip install --upgrade -r ./local-requirements

And a ./out directory containing:

backports.ssl_match_hostname-3.4.0.2.tar.gz
bson-0.3.3.tar.gz
Django-1.4.tar.gz
dogslow-0.9.7.tar.gz
gevent-1.0.tar.gz
greenlet-0.4.1.zip
local-requirements
msgpack-python-0.4.0.tar.gz
nose-1.3.0.tar.gz
pip-1.5.4-py2.py3-none-any.whl
pymongo-2.6.3.tar.gz
python-memcached-1.53.tar.gz
pytz-2013.8.tar.bz2
pyzmq-14.0.1.tar.gz
requests-2.1.0-py2.py3-none-any.whl
simplejson-3.3.1.tar.gz
wsgiref-0.1.2.zip

where the local-requirements file looks like:

pip-1.5.4-py2.py3-none-any.whl
Django-1.4.tar.gz
backports.ssl_match_hostname-3.4.0.2.tar.gz
pytz-2013.8.tar.bz2
bson-0.3.3.tar.gz
dogslow-0.9.7.tar.gz
greenlet-0.4.1.zip
gevent-1.0.tar.gz
msgpack-python-0.4.0.tar.gz
nose-1.3.0.tar.gz
pymongo-2.6.3.tar.gz
python-memcached-1.53.tar.gz
pyzmq-14.0.1.tar.gz
requests-2.1.0-py2.py3-none-any.whl
simplejson-3.3.1.tar.gz
wsgiref-0.1.2.zip