Exploring various pypi repo/cache options

devpi.sh

# http://doc.devpi.net/latest/

# Basic host package set
apt-get update && \
  apt-get purge -y nano && \
  apt-get install -y git vim tmux fail2ban

# Install packages
apt-get install -y python-pip gcc git-core libssl-dev libffi-dev python-pyasn1 python-openssl python-ndg-httpsclient gzip python-dev lsb-release python-virtualenv
pip install devpi-server devpi-web devpi-client

# Initialise and start the service
devpi-server --init
devpi-server --start

# Create user for uploading packages
devpi use http://localhost:3141
devpi user -c testuser password=123
devpi login testuser --password=123

# Create index called 'openstack' and make it use the pypi mirror as a base
devpi index -c openstack bases=root/pypi
devpi use testuser/openstack

# Prepare git sources
mkdir ~/git
cd ~/git
git clone https://github.com/openstack/requirements
git clone https://github.com/openstack/keystone
git clone https://github.com/openstack/cinder

# Upload the packages
for svc in keystone cinder; do
  cd ~/git/${svc}
  devpi upload
done

# Build the venvs
mkdir ~/venvs
cd ~/venvs
for svc in keystone cinder; do
  virtualenv ${svc}
  pushd ${svc}
  source bin/activate
  pip install --constraint ~/git/requirements/upper-constraints.txt --index-url=http://localhost:3141/testuser/openstack ${svc}
  deactivate
  popd
done

nginx.sh

# Inspiration:
# https://gist.github.com/dctrwatson/5785638
# http://reinout.vanrees.org/weblog/2009/11/09/eggproxy-plus-private-packages.html

# Basic host package set
apt-get update && \
  apt-get purge -y nano && \
  apt-get install -y git vim tmux fail2ban

# Install packages
apt-get install -y nginx

### BEGIN: /etc/nginx/sites-enabled/default ###
server {
    listen 80 default_server;
    listen [::]:80 default_server;

    root /var/www/html;

    server_name _;

    proxy_cache pypi;
    proxy_cache_key $uri;
    proxy_cache_lock on;
    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;

    proxy_http_version 1.1;
    proxy_set_header Host pypi.python.org;
    proxy_set_header Connection "";
    proxy_set_header Accept-Encoding "";

    # Rewrite any http redirects to use relative to proxy
    proxy_redirect ~https?://pypi.python.org(.*) $1;

    location @pypi {
      add_header X-Cache2 $upstream_cache_status;

      proxy_pass https://pypi;
    }

    location ^~ /simple {
      alias /var/www/pypi;
      autoindex on;

      proxy_cache_valid any 5m;

      try_files $uri $uri/ @pypi;
    }

    location ^~ /packages {
      proxy_cache_valid any 1M;

      try_files $uri @pypi;
    }

    location / {
      # Replace any reference to actual pypi w/ caching proxy
      sub_filter 'https://pypi.python.org' $scheme://$host;
      sub_filter_once off;
      proxy_cache off;
      proxy_pass https://pypi;
    }

}
### END: /etc/nginx/sites-enabled/default ###

### BEGIN: /etc/nginx/conf.d/pypi.conf ###
# Cache 100G worth of packages for up to 1 month
proxy_cache_path /var/lib/nginx/pypi levels=1:2 keys_zone=pypi:16m inactive=1M max_size=100G;

# Multiple server definitions makes nginx retry on errors
upstream pypi {
    server pypi.python.org:443;
    server pypi.python.org:443;
    keepalive 16;
}
### END: /etc/nginx/conf.d/pypi.conf ###

# Setup the appropriate directories
mkdir -p /var/lib/nginx/pypi
chown www-data:www-data /var/lib/nginx/pypi
mkdir -p /var/www/pypi

# Restart nginx for config to take effect
systemctl restart nginx

# Install packages to build wheels
apt-get install -y python-pip gcc git-core libssl-dev libffi-dev python-pyasn1 python-openssl python-ndg-httpsclient gzip python-dev lsb-release python-virtualenv

# Prepare git sources
mkdir ~/git
cd ~/git
git clone https://github.com/openstack/requirements
git clone https://github.com/openstack/glance

# Build the wheel for glance (and only that wheel)
mkdir -p /var/www/pypi/glance
cd /var/www/pypi/glance
pip wheel --no-deps --index-url http://localhost/simple --constraint ~/git/requirements/upper-constraints.txt ~/git/glance/

# Build the venv
mkdir ~/venvs
virtualenv ~/venvs/glance
cd ~/venvs/glance
source bin/activate
pip install --constraint ~/git/requirements/upper-constraints.txt --index-url=http://localhost/simple/ glance
deactivate

pypi-server.sh

# https://github.com/mosquito/pypi-server

# Basic host package set
apt-get update && \
  apt-get purge -y nano && \
  apt-get install -y git vim tmux fail2ban

# Install packages
apt-get install -y python-pip gcc git-core libssl-dev libffi-dev python-pyasn1 python-openssl python-ndg-httpsclient gzip python-dev lsb-release python-virtualenv
pip install pypi-server
mkdir ~/packages
pypi-server &

# Testing a simple installation process caused repeated failures
# which appeared due to the inability to cope with downloading
# the wheels from pypi. The console would exhibit HTTP 500 errors
# but repeating the install request would get it past there and
# fail on a later one.
mkdir ~/venvs
cd ~/venvs
virtualenv glance
pushd glance
source bin/activate
pip install --index-url=http://localhost:8080/simple -c https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/pike --requirement https://git.openstack.org/cgit/openstack/glance/plain/requirements.txt?h=stable/pike

pypiserver.sh

# https://pypiserver.readthedocs.io/en/latest/

# Basic host package set
apt-get update && \
  apt-get purge -y nano && \
  apt-get install -y git vim tmux fail2ban

# Install packages
apt-get install -y python-pip gcc git-core libssl-dev libffi-dev python-pyasn1 python-openssl python-ndg-httpsclient gzip python-dev lsb-release python-virtualenv
pip install pypiserver
mkdir ~/packages
pypi-server -p 8080 ~/packages &

# Prepare git sources
mkdir ~/git
cd ~/git
git clone https://github.com/openstack/requirements
git clone https://github.com/openstack/keystone
git clone https://github.com/openstack/cinder

# Prepare to build keystone wheel (ensure required packages are installed)
pip install bindep
for svc in keystone cinder; do
  BINDEP_PKGS=$(bindep -b -f ~/git/${svc}/bindep.txt)
  apt-get -q --option "Dpkg::Options::=--force-confold" --assume-yes install $BINDEP_PKGS
done

# Build the wheels for keystone/cinder
cd ~/packages
pip wheel -c ~/git/requirements/upper-constraints.txt --find-links=~/packages ~/git/keystone/
pip wheel -c ~/git/requirements/upper-constraints.txt --find-links=~/packages ~/git/cinder/
pip wheel -c https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/pike --find-links=~/packages --extra-index http://tarballs.openstack.org/ --trusted-host tarballs.openstack.org 'glance>=15,<=16'

# Build the venvs
mkdir ~/venvs
cd ~/venvs
for svc in keystone cinder glance; do
  virtualenv ${svc}
  pushd ${svc}
  source bin/activate
  pip install --constraint ~/git/requirements/upper-constraints.txt --index-url=http://localhost:8080/simple/ ${svc}
  deactivate
  popd
done