Last active
November 30, 2022 08:56
-
-
Save wassname/18895c4d62ed842fba1f to your computer and use it in GitHub Desktop.
Cache your pip wheels in docker using devpi, with graceful fails
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# your code here | |
if [ "$1" = '' ]; then | |
run_my_app.sh | |
fi | |
if [ "$1" = 'devpiupload' ]; then | |
# this will build and upload project requirements to devpi cache container | |
# install build requirements | |
cd requirements | |
apt-get update | |
apt-get install $(cat system.build.reqs.txt|grep -v '^#'|xargs) | |
# download packages and build wheels in the deploy environment | |
# they will be in ./requirements/packages and ./requirements/wheelhouse | |
pip install --download=packages -r requirements.txt | |
pip wheel --wheel-dir=wheelhouse -r requirements.txt | |
# upload to devpi, making a user and index if needed | |
pip install "devpi-client>=2.3.0" | |
# since it's published ports we can access it from the host_ip:port | |
export HOST_IP=$(ip route| awk '/^default/ {print $3}') | |
if devpi use http://$HOST_IP:3141>/dev/null; then | |
echo devpi detected on http://$HOST_IP:3141 | |
else | |
echo No started devpi container found at http://$HOST_IP:3141; | |
fi | |
# check that a user and index have been created else create them | |
if devpi login app --password=$DEVPI_PASSWORD > /dev/null; then | |
echo devpi user (app) lready created | |
else | |
devpi user -c app password=$DEVPI_PASSWORD | |
devpi index -c app/dev bases= | |
devpi login app --password=$DEVPI_PASSWORD | |
fi | |
# upload to devpi | |
devpi use http://$HOST_IP:3141/app/dev --set-cfg | |
devpi upload --from-dir --formats=* ./wheelhouse ./packages; | |
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Build and serve pip packages and wheels to speed development | |
devpi: | |
image: scrapinghub/docker-devpi | |
volumes: | |
- /mnt | |
- /var/lib/devpi | |
ports: | |
- 3141:3141 | |
env: | |
- DEVPI_PASSWORD=$(echo $(lscpu)|md5sum) # replace this password, this is just a hash of some system info | |
restart: always |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This code will use the devpi cache if it can be found | |
# otherwise fail with a few complaints | |
RUN export HOST_IP=$(ip route| awk '/^default/ {print $3}') \ | |
&& mkdir -p ~/.pip \ | |
&& echo [global] >> ~/.pip/pip.conf \ | |
&& echo extra-index-url = http://$HOST_IP:3141/app/dev/+simple >> ~/.pip/pip.conf \ | |
&& echo [install] >> ~/.pip/pip.conf \ | |
&& echo trusted-host = $HOST_IP >> ~/.pip/pip.conf \ | |
&& cat ~/.pip/pip.conf | |
# if it's working you will see out put including: | |
# 2 location(s) to search for versions of ignoredd: | |
# * https://pypi.python.org/simple/ignoredd/ | |
# * http://172.17.0.1:3141/app/dev/+simple/ignoredd/ | |
RUN pip install -v ignoredd | |
# you can add code to your entrypoint to build and upload packages from the container env | |
ENTRYPOINT ["./app_entry_points.sh"] | |
CMD ["devpiupload"] # this can be overridden i nthe yml file |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Thank you, this is just what I needed!