Example of release day bug in pip 19.0 with the --no-cache-dir option (https://github.com/pypa/pip/issues/6158)

README.md

Demonstration of pypa/pip Issue 6158

If you make either the Ubutnu 18.04 or the CentOS 7 Docker images

make

and then run them with this directory bindmounted to their /root/data

docker run --rm -it -v $PWD:/root/data pip-19-example:ubuntu

then attempting to install the contents of requirements.txt with fail.sh will fail given the --no-cache-dir issue

bash fail.sh

but running pass.sh which omits --no-cache-dir installs the libraries correctly

bash pass.sh

Dockerfile

FROM ubuntu:bionic

USER root
WORKDIR /root

SHELL [ "/bin/bash", "-c" ]

ARG PYTHON_VERSION_TAG=3.6.8
ARG LINK_PYTHON_TO_PYTHON3=0

# Existing lsb_release causes issues with modern installations of Python3
# https://github.com/pypa/pip/issues/4924#issuecomment-435825490
RUN apt-get -qq -y update && \
    apt-get -qq -y upgrade && \
    apt-get -qq -y install \
        gcc \
        g++ \
        zlibc \
        zlib1g-dev \
        libssl-dev \
        libbz2-dev \
        libsqlite3-dev \
        wget \
        curl \
        git \
        make \
        sudo \
        bash-completion \
        tree \
        jq \
        software-properties-common && \
    mv /usr/bin/lsb_release /usr/bin/lsb_release.bak && \
    apt-get -y autoclean && \
    apt-get -y autoremove

ADD install_python.sh install_python.sh
RUN bash install_python.sh ${PYTHON_VERSION_TAG} ${LINK_PYTHON_TO_PYTHON3} && \
    rm -r install_python.sh Python-${PYTHON_VERSION_TAG}
ADD requirements.txt /root/data/requirements.txt

# Enable tab completion by uncommenting it from /etc/bash.bashrc
# The relevant lines are those below the phrase "enable bash completion in interactive shells"
RUN export SED_RANGE="$(($(sed -n '\|enable bash completion in interactive shells|=' /etc/bash.bashrc)+1)),$(($(sed -n '\|enable bash completion in interactive shells|=' /etc/bash.bashrc)+7))" && \
    sed -i -e "${SED_RANGE}"' s/^#//' /etc/bash.bashrc && \
    unset SED_RANGE

# Start the image not in /root to protect config files
WORKDIR /root/data

# Ignore the fact that we're leaving the user as ROOT as this is a test

# Start the image with BASH by default
CMD ["/bin/bash"]

Dockerfile.centos7

FROM centos:centos7

# Perform the installation as root:
USER root
WORKDIR /root

ARG PYTHON_VERSION_TAG=3.6.8
ARG LINK_PYTHON_TO_PYTHON3=0

RUN yum upgrade -y -q && \
    yum install -y \
        gcc \
        gcc-c++ \
        zlib-devel \
        openssl-devel \
        sqlite-devel \
        make \
        wget \
        git \
        which \
        tar \
        sudo \
        tree \
        epel-release && \
    yum install -y \
        jq \
        patch \
        ncurses-devel \
        readline-devel \
        bash-completion \
        bash-completion-extras && \
    yum clean all


ADD install_python.sh install_python.sh
RUN bash install_python.sh ${PYTHON_VERSION_TAG} ${LINK_PYTHON_TO_PYTHON3} && \
    rm -r install_python.sh Python-${PYTHON_VERSION_TAG}
ADD requirements.txt /root/data/requirements.txt

# Start the image not in /root to protect config files
WORKDIR /root/data

# Ignore the fact that we're leaving the user as ROOT as this is a test

# Start the image with BASH by default
CMD ["/bin/bash"]

fail.sh

#!/usr/bin/env bash

pip3 --version
# The following passes for pip v18.1
pip3 install --upgrade --no-cache-dir -r requirements.txt

install_python.sh

#!/usr/bin/env bash

set -e

# This is being run as root and so sudo is not needed

CXX_VERSION="$(which gcc)"

function download_cpython () {
    # 1: the version tag
    printf "\n### Downloading CPython source as Python-${1}.tgz\n"
    wget "https://www.python.org/ftp/python/${1}/Python-${1}.tgz" &> /dev/null
    tar -xvzf "Python-${1}.tgz" > /dev/null
    rm "Python-${1}.tgz"
}

function set_num_processors {
    # Set the number of processors used for build
    # to be 1 less than are available
    if [[ -f "$(which nproc)" ]]; then
        NPROC="$(nproc)"
    else
        NPROC="$(grep -c '^processor' /proc/cpuinfo)"
    fi
    echo `expr "${NPROC}" - 1`
}

function build_cpython () {
    # 1: the prefix to be passed to configure
    #    c.f. https://docs.python.org/3/using/unix.html#python-related-paths-and-files
    # 2: the path to the version of gcc to be used

    # https://docs.python.org/3/using/unix.html#building-python
    # https://github.com/python/cpython/blob/3.6/README.rst
    printf "\n### ./configure\n"
    ./configure --prefix="${1}" \
        --exec_prefix="${1}" \
        --with-cxx-main="${2}" \
        --enable-optimizations \
        --with-lto \
        --enable-loadable-sqlite-extensions \
        CXX="${2}"
    printf "\n### make -j${NPROC}\n"
    make -j${NPROC}
    printf "\n### make install\n"
    make install
}

function update_pip {
    # Update pip, setuptools, and wheel
    if [[ "$(id -u)" -eq 0 ]]; then
        # If root
        printf "\n### pip3 install --upgrade --no-cache-dir pip setuptools wheel\n"
        pip3 install --upgrade --no-cache-dir pip setuptools wheel
    else
        printf "\n### pip3 install --user --upgrade --no-cache-dir pip setuptools wheel\n"
        pip3 install --user --upgrade --no-cache-dir pip setuptools wheel
    fi
}

function symlink_python_to_python3 {
    local python_version="$(python3 --version)"
    local which_python="$(which python3)${python_version:8:-2}"
    local which_pip="$(which pip3)"

    # symlink python to python3
    printf "\n### ln -s -f ${which_python} ${which_python::-3}\n"
    ln -s -f "${which_python}" "${which_python::-3}"

    # symlink pip to pip3 if no pip exists or it is a different version than pip3
    if [[ ! -z "$(which pip)" ]]; then
        if [[ "$(pip --version)" = "$(pip3 --version)" ]]; then
            return 0
        fi
    fi
    printf "\n### ln -s -f ${which_pip} ${which_pip::-1}\n"
    ln -s -f "${which_pip}" "${which_pip::-1}"
    return 0
}

function main() {
    # 1: the Python version tag
    # 2: bool of if should symlink python and pip to python3 versions

    PYTHON_VERSION_TAG=3.6.8 # Switch to 3.7 once Tensorflow is out for it
    LINK_PYTHON_TO_PYTHON3=0 # By default don't link so as to reserve python for Python 2
    if [[ $# -gt 0 ]]; then
        PYTHON_VERSION_TAG="${1}"

        if [[ $# -gt 1 ]]; then
            LINK_PYTHON_TO_PYTHON3="${2}"
        fi
    fi

    NPROC="$(set_num_processors)"
    download_cpython "${PYTHON_VERSION_TAG}"
    cd Python-"${PYTHON_VERSION_TAG}"
    build_cpython /usr "${CXX_VERSION}"
    update_pip

    if [[ "${LINK_PYTHON_TO_PYTHON3}" -eq 1 ]]; then
        symlink_python_to_python3
    fi
}

main "$@" || exit 1

Makefile

default: all

all: example_ubuntu example_centos

example_ubuntu:
	docker build -f Dockerfile \
	--build-arg PYTHON_VERSION_TAG=3.6.8 \
	--build-arg LINK_PYTHON_TO_PYTHON3=1 \
	-t pip-19-example:ubuntu \
	--compress .

example_centos:
	docker build -f Dockerfile.centos7 \
	--build-arg PYTHON_VERSION_TAG=3.6.8 \
	--build-arg LINK_PYTHON_TO_PYTHON3=0 \
	-t pip-19-example:centos \
	--compress .

pass.sh

#!/usr/bin/env bash

pip3 --version
pip3 install --upgrade -r requirements.txt

requirements.txt

numpy>=1.16
tensorflow>=1.12
keras>=2.2
torch>=1.0
uproot>=3.3
matplotlib>=3.0
jupyter>=1.0