Update mozilla/ichnaea

make_update_script.py

#!/usr/bin/env python3
# Usage: python make_update_script.py > update.sh; bash update.sh

import sys


def parse_reqs(requirements_content):
    all_lines = requirements_content.splitlines()
    modules = []
    for raw_line in all_lines:
        if type(raw_line) == bytes:
            line = raw_line.decode('utf8').strip()
        else:
            line = raw_line.strip()
        if not line:
            continue
        if line.startswith('#') or line.startswith('-'):
            continue
        assert '=' in line, line
        mod_name = line.split('=')[0]
        modules.append(mod_name)
    modules.sort()
    return modules


def update_script(modules, req_module_dict, check_script=None):
    script_lines = """\
#!/bin/bash
set -e

# Update requirements in mozilla/ichnaea

function set_title {
    # Set the title of the window (macOS specific?)
    text=$1
    echo -n -e "\033]0;$text\007"
}

function update_req {
    # Update to the (latest) version of a requirement
    module=$1
    requirements=$2
    set_title "Updating $module"
    hashin $module -r $requirements
}

function commit_req {
    # If there are changes, commit and test
    module=$1
    set_title "Commiting $module"
    if ! git diff --exit-code
    then
        git add -u
        git commit -m "Update $module"
        set_title "Checking $module"
        check
    fi
}

function update_and_commit_req {
    # Update and commit a stand-alone module
    module=$1
    requirements=$2
    update_req $module $requirements
    commit_req $module
}

function check {\
""".split('\n')

    # Add check script
    if check_script:
        if type(check_script) == str:
            script_lines.append('    ' + check_script)
        else:
            for line in check_script:
                if line:
                    script_lines.append('    ' + line)
    script_lines.extend(["}", ""])

    for module in modules:
        req_path = None
        for req_candidate, req_modules in req_module_dict.items():
            if module in req_modules:
                req_path = req_candidate
                break
            else:
                # Look for modules with package[variant] format
                for rmodule in req_modules:
                    if '[' in rmodule and rmodule.startswith(module):
                        module = rmodule
                        req_path = req_candidate
                        break
        if req_path is None:
            print(f'Skipping "{module}", not in requirements files.',
                  file=sys.stderr)
        else:
            script_lines.append(
                f'update_and_commit_req "{module}" "{req_path}"')
    return '\n'.join(script_lines)

CHECK_SCRIPT = """\
# Check that the project still works
pip install -r requirements/default.txt
make build
make test
make docs
cd docs/build/html
git reset @{u} --hard
cd ../../..
"""

if __name__ == "__main__":
    import subprocess
    installed_modules = parse_reqs(subprocess.check_output(['pip', 'freeze']))
    default_path = 'requirements/default.txt'
    constraints_path = 'requirements/constraints.txt'
    default_modules = parse_reqs(open(default_path).read())
    constraint_modules = parse_reqs(open(constraints_path).read())
    req_module_dict = {
        default_path: set(default_modules),
        constraints_path: set(constraint_modules)
    }
    script = update_script(
        installed_modules, req_module_dict,
        check_script=CHECK_SCRIPT.split('\n'))
    print(script)

update.sh

#!/bin/bash
set -e

# Update requirements in mozilla/ichnaea

function set_title {
    # Set the title of the window (macOS specific?)
    text=$1
    echo -n -e "\033]0;$text\007"
}

function update_req {
    # Update to the (latest) version of a requirement
    module=$1
    requirements=$2
    set_title "Updating $module"
    hashin $module -r $requirements
}

function commit_req {
    # If there are changes, commit and test
    module=$1
    set_title "Commiting $module"
    if ! git diff --exit-code
    then
        git add -u
        git commit -m "Update $module"
        set_title "Checking $module"
        check
    fi
}

function update_and_commit_req {
    # Update and commit a stand-alone module
    module=$1
    requirements=$2
    update_req $module $requirements
    commit_req $module
}

function check {
    # Check that the project still works
    pip install -r requirements/default.txt
    make build
    make test
    make docs
    cd docs/build/html
    git reset @{u} --hard
    cd ../../..
}

# Prepare a fresh branch
git checkout origin/master
git branch -D update-reqs || echo "no existing update-reqs branch to delete"
git checkout -b update-reqs
check

update_req "requests[security]" "requirements/default.txt"
update_req "certifi" "requirements/default.txt"
update_req "chardet" "requirements/constraints.txt"
update_req "idna" "requirements/constraints.txt"
update_req "urllib3[secure]" "requirements/constraints.txt"
update_req "requests-mock" "requirements/default.txt"
commit_req "requests, certifi, urllib3"

update_req "pyOpenSSL" "requirements/constraints.txt"
update_req "ndg-httpsclient" "requirements/default.txt"
update_req "asn1crypto" "requirements/constraints.txt"
update_req "cffi" "requirements/constraints.txt"
update_req "cryptography" "requirements/constraints.txt"
update_req "pyasn1" "requirements/constraints.txt"
update_req "pycparser" "requirements/constraints.txt"
commit_req "pyOpenSSL, ndg-httpsclient"

update_req "scipy" "requirements/default.txt"
update_req "numpy" "requirements/default.txt"
commit_req "scipy, numpy"

update_req "celery[redis]" "requirements/default.txt"
update_req "amqp" "requirements/constraints.txt"
update_req "vine" "requirements/constraints.txt"
update_req "billiard" "requirements/constraints.txt"
update_req "kombu" "requirements/constraints.txt"
update_req "pytz" "requirements/default.txt"
update_req "importlib-metadata" "requirements/constraints.txt"
update_req "zipp" "requirements/constraints.txt"
update_req "more-itertools" "requirements/constraints.txt"
commit_req "celery"

update_and_commit_req "PyMySQL" "requirements/default.txt"

update_req "alembic" "requirements/default.txt"
update_req "SQLAlchemy" "requirements/default.txt"
update_req "Mako" "requirements/constraints.txt"
update_req "MarkupSafe" "requirements/constraints.txt"
update_req "python-dateutil" "requirements/constraints.txt"
update_req "python-editor" "requirements/constraints.txt"
update_req "six" "requirements/constraints.txt"
commit_req "alembic, SQLAlchemy"

update_req "boto3" "requirements/default.txt"
update_req "botocore" "requirements/default.txt"
update_req "docutils" "requirements/constraints.txt"
update_req "jmespath" "requirements/constraints.txt"
update_req "s3transfer" "requirements/constraints.txt"
commit_req "boto3, botocore"

update_req "colander" "requirements/default.txt"
update_req "iso8601" "requirements/constraints.txt"
update_req "translationstring" "requirements/constraints.txt"
commit_req "colander"

update_and_commit_req "Cython" "requirements/default.txt"

update_req "datadog" "requirements/default.txt"
update_req "decorator" "requirements/constraints.txt"
commit_req "datadog"

update_req "factory-boy" "requirements/default.txt"
update_req "Faker" "requirements/constraints.txt"
update_req "text-unidecode" "requirements/constraints.txt"
commit_req "factory-boy"

update_req "flake8" "requirements/default.txt"
update_req "entrypoints" "requirements/constraints.txt"
update_req "mccabe" "requirements/constraints.txt"
update_req "pycodestyle" "requirements/constraints.txt"
update_req "pyflakes" "requirements/constraints.txt"
commit_req "flake8"

update_and_commit_req "genc" "requirements/default.txt"
update_and_commit_req "mobile-codes" "requirements/default.txt"

update_req "geoip2" "requirements/default.txt"
update_req "maxminddb" "requirements/default.txt"
commit_req "geoip2, maxminddb"

update_req "gunicorn" "requirements/default.txt"
update_req "gevent" "requirements/default.txt"
update_req "greenlet" "requirements/constraints.txt"
update_req "setproctitle" "requirements/default.txt"
commit_req "gunicorn"

update_req "redis[hiredis]" "requirements/default.txt"
update_req "hiredis" "requirements/constraints.txt"
commit_req "redis"

update_req "pyramid" "requirements/default.txt"
update_req "pyramid-chameleon" "requirements/default.txt"
update_req "Chameleon" "requirements/constraints.txt"
update_req "hupper" "requirements/constraints.txt"
update_req "PasteDeploy" "requirements/constraints.txt"
update_req "plaster" "requirements/constraints.txt"
update_req "plaster-pastedeploy" "requirements/constraints.txt"
update_req "venusian" "requirements/constraints.txt"
update_req "WebOb" "requirements/constraints.txt"
update_req "zope.deprecation" "requirements/constraints.txt"
update_req "zope.interface" "requirements/constraints.txt"
commit_req "pyramid, Chameleon"

update_and_commit_req "repoze.lru" "requirements/default.txt"

update_req "pytest" "requirements/default.txt"
update_req "pytest-cov" "requirements/default.txt"
update_req "coverage" "requirements/constraints.txt"
update_req "more-itertools" "requirements/constraints.txt"
update_req "importlib-metadata" "requirements/constraints.txt"
update_req "zipp" "requirements/constraints.txt"
update_req "pluggy" "requirements/constraints.txt"
update_req "atomicwrites" "requirements/constraints.txt"
update_req "wcwidth" "requirements/constraints.txt"
update_req "py" "requirements/constraints.txt"
commit_req "pytest, -cov"

update_req "raven" "requirements/default.txt"
update_req "contextlib2" "requirements/constraints.txt"
commit_req "raven"

update_and_commit_req "Rtree" "requirements/default.txt"
update_and_commit_req "Shapely[vectorized]" "requirements/default.txt"

update_req "Sphinx" "requirements/default.txt"
update_req "sphinxcontrib-applehelp" "requirements/constraints.txt"
update_req "sphinxcontrib-htmlhelp" "requirements/constraints.txt"
update_req "sphinxcontrib-serializinghtml" "requirements/constraints.txt"
update_req "sphinxcontrib-qthelp" "requirements/constraints.txt"
update_req "sphinxcontrib-jsmath" "requirements/constraints.txt"
update_req "sphinxcontrib-devhelp" "requirements/constraints.txt"
update_req "alabaster" "requirements/constraints.txt"
update_req "Babel" "requirements/constraints.txt"
update_req "Jinja2" "requirements/constraints.txt"
update_req "Pygments" "requirements/constraints.txt"
update_req "snowballstemmer" "requirements/constraints.txt"
update_req "sphinx-rtd-theme" "requirements/default.txt"
update_req "sphinxcontrib-websupport" "requirements/constraints.txt"
update_req "imagesize" "requirements/constraints.txt"
commit_req "Sphinx"

update_req "WebTest" "requirements/default.txt"
update_req "beautifulsoup4" "requirements/constraints.txt"
update_req "soupsieve" "requirements/constraints.txt"
update_req "waitress" "requirements/constraints.txt"
commit_req "WebTest"