Script to automatically upgrade all python packages given a requirements.txt or a pyproject.toml (works over conda)

upgrade_py_pkgs.sh

#!/bin/bash

# If a requirements.txt file is supplied, it will be updated with the latest versions of each package.
# If a pyproject.toml is required, package upgrades will just be printed.

echo "Careful:"
echo " - Conda is required"
echo " - If any of your PYPY packages contain the strin 'http', it is possible that it gets astray, manually inspect it"

env_name="pkgs_upgrade"
dev_deps=""
reqs_file=""
pyproj_file=""

function usage(){
    echo "upgrade_py_pkgs [-r|--requirements <requirements.txt>] [-p|--pyproject <pyproject.toml>] [-d|--dev]"
}

function convert_pyproj(){
    file=$1; shift
    dev=$1; shift
    python3 -m venv /tmp/pyprojectconvert >/dev/null
    /tmp/pyprojectconvert/bin/pip3 install toml pip-tools >/dev/null
    /tmp/pyprojectconvert/bin/python3 -c "$(cat <<EOF
import toml
from packaging.requirements import InvalidRequirement, Requirement
proj = toml.load(open("${file}"))
if "${dev}" == "":
    deps = proj["project"]["dependencies"]
else:
    deps = proj["project"]["optional-dependencies"]["dev"]

parsed_deps = []
for item in deps:
    try:
        # only accept valid dependencies and those that do not depend on a url (i.e.: from pypi)
        req = Requirement(item)
        if req.url is None:
            parsed_deps.append(item)
    except:
        pass 

print("\n".join(parsed_deps))
EOF
)" > /tmp/pyprojectconvert_reqs.txt
    echo "/tmp/pyprojectconvert_reqs.txt"
}

if [[ $# -eq 0 ]]; then
    usage
    exit 0
fi

while [[ $# -gt 0 ]]; do
    case $1 in
        -r|--requirements)
            if [[ -n "${pyproj_file}" ]]; then
                echo "Specify pyproject or requirements, but not both!"
                exit 1
            fi
            reqs_file="$2"
            shift; shift
            ;;
        -p|--pyproject)
            if [[ -n "${reqs_file}" ]]; then
                echo "Specify pyproject or requirements, but not both!"
                exit 1
            fi
            pyproj_file="$2"
            shift; shift
            ;;
        -d|--dev)
            dev_deps="dev"
            shift;
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        *)
            echo "Unknown option $1"
            exit 1
            ;;
    esac
done

set -e
set -o xtrace

if [[ -n "${pyproj_file}" ]]; then
    reqs_file=$(convert_pyproj $pyproj_file $dev_deps)
fi

conda create -y -n "${env_name}" python=3.9 pip
# remove versions
cat $reqs_file | sed -E 's/([~><=]=[0-9.?]+)//' > /tmp/reqs_no_ver.txt
# create a patterns file for grep (excluding https packages, which cannot be upgraded)
# can fail if a package name contains http
cat /tmp/reqs_no_ver.txt | grep -v "+http" | sed 's/$/=/' > /tmp/reqs_no_ver_patts.txt
# get http non-upgradable packages
set +e
cat /tmp/reqs_no_ver.txt | grep "+http" > /tmp/nupgr_reqs.txt
set -e
# install up-to-date packages, let pip solve versions
conda run --name "${env_name}" pip install -r /tmp/reqs_no_ver.txt
conda run --name "${env_name}" pip freeze | grep -f /tmp/reqs_no_ver_patts.txt > /tmp/upgr_reqs_no_http.txt
cat /tmp/nupgr_reqs.txt /tmp/upgr_reqs_no_http.txt > "${reqs_file}"
conda env remove -n "${env_name}"

cat ${reqs_file}