A script for automatically updating dependencies in conda-forge bot branches using grayskull and pypi

grayskull_autoupdate.py

#!/usr/bin/env python
import argparse
import os
import shutil
import subprocess
import packaging.version
import grayskull.strategy
from importlib.resources import open_binary
import yaml


def clone_feedstock(feedstock, fork):
    if os.path.exists(feedstock):
        return

    cwd = os.getcwd()
    os.makedirs(feedstock)
    os.chdir(feedstock)

    args = ['git', 'clone', f'[email protected]:conda-forge/{feedstock}.git']
    print_run_and_check(args)
    os.chdir(feedstock)

    args = ['git', 'remote', 'rename', 'origin', f'conda-forge/{feedstock}']
    print_run_and_check(args)

    for org in [fork]:
        args = ['git', 'remote', 'add', f'{org}/{feedstock}',
                f'[email protected]:{org}/{feedstock}.git']
        print_run_and_check(args)

    args = ['git', 'fetch', '--all', '-p']
    print_run_and_check(args)
    os.chdir(cwd)


def update_feedstock(feedstock):
    args = ['git', 'fetch', '--all', '-p']
    print_run_and_check(args)

    args = ['git', 'reset', '--hard', f'conda-forge/{feedstock}/main']
    print_run_and_check(args)


def get_current_version():
    version = None
    with open(f'recipe/meta.yaml', 'r') as file:
        for line in file.readlines():
            if line.startswith('{% set version = '):
                version = line
                break
    version = version[len('{% set version = "'):-len('" %}\n')]
    return version


def check_out_latest_bot_branch(feedstock, fork):

    args = ['git', 'ls-remote', f'{fork}/{feedstock}']
    output = subprocess.check_output(args).decode('utf-8')
    newest_branch = None
    newest_version = None
    newest_version_string = None
    for line in output.split('\n'):
        parts = line.split()
        if len(parts) != 2:
            continue
        branch = parts[1]
        if 'refs/heads/' not in branch:
            continue
        branch = branch[len('refs/heads/'):]
        try:
            version_string = branch
            version = packaging.version.Version(version_string)
        except packaging.version.InvalidVersion:
            if '_h' not in branch:
                continue

            parts = branch.split('_h')
            if len(parts) != 2:
                continue

            version_string = parts[0]
            try:
                version = packaging.version.Version(version_string)
            except packaging.version.InvalidVersion:
                continue

        if newest_version is None or version > newest_version:
            newest_branch = branch
            newest_version = version
            newest_version_string = version_string

    if newest_branch is None:
        raise ValueError(f'No bot branch found for {feedstock}')

    print(f'\nNewest bot branch: {newest_branch}\n')

    worktree = f'../{newest_branch}'
    if not os.path.exists(worktree):
        args = ['git', 'worktree', 'add', worktree]
        print_run_and_check(args)

    cwd = os.getcwd()
    os.chdir(worktree)

    args = ['git', 'reset', '--hard',
            f'{fork}/{feedstock}/{newest_branch}']
    print_run_and_check(args)

    os.chdir(cwd)

    return newest_branch, newest_version_string


def get_quirks():
    with open_binary(grayskull.strategy, 'config.yaml') as fp:
        gs_quirks = yaml.load(fp, Loader=yaml.Loader)

    quirks = dict()
    for pypi_name in gs_quirks:
        cf_name = gs_quirks[pypi_name]['conda_forge']
        quirks[cf_name] = pypi_name

    quirks['python-eccodes'] = 'eccodes'

    return quirks

def run_grayskull(package, version):

    quirks = get_quirks()

    outdir = f'{package}-{version}'
    if os.path.exists(outdir):
        return
    for dir in [package, outdir]:
        try:
            shutil.rmtree(dir)
        except FileNotFoundError:
            pass
    if package in quirks:
        package = quirks[package]
    args = ['grayskull', 'pypi', f'{package}={version}']
    print_run_and_check(args)
    os.rename(package, outdir)


def update_from_grayskull(package, old_version, new_version, feedstock,
                          fork, branch):

    start_hash = get_current_hash()

    shutil.copy(f'{package}-{old_version}/meta.yaml', 'recipe/meta.yaml')

    args = ['git', 'commit', '--allow-empty', '-m',
            'Revert recipe to grayskull', 'recipe/meta.yaml']
    print_run_and_check(args)

    shutil.copy(f'{package}-{new_version}/meta.yaml', 'recipe/meta.yaml')

    args = ['git', 'commit', '-m',
            'Update recipe with my grayskull autoupdate script',
            'recipe/meta.yaml']
    print_run_and_check(args)

    new_hash = get_current_hash()

    args = ['git', 'reset', '--hard', start_hash]
    print_run_and_check(args)

    args = ['git', 'cherry-pick', new_hash]
    try:
        print_run_and_check(args)
    except subprocess.CalledProcessError:
        args = ['git', '--no-pager', 'diff']
        print_run_and_check(args)
        args = ['git', 'diff', '--quiet']
        try:
            print_run_and_check(args)
            # presumably there's nothing to commit, so we want to make a
            # comment
            comment_no_changes(feedstock, fork, branch, new_version,
                               old_version, package)
        except subprocess.CalledProcessError:
            # there were conflicts that we need to resolve
            args = ['vim', 'recipe/meta.yaml']
            print_run_and_check(args)
            args = ['git', 'add', 'recipe/meta.yaml']
            print_run_and_check(args)

            args = ['git', '--no-pager', 'diff', '--staged']
            print_run_and_check(args)
            args = ['git', 'diff', '--staged', '--quiet']
            try:
                print_run_and_check(args)
                # presumably there's nothing to commit, so we want to make a
                # comment
                comment_no_changes(feedstock, fork, branch, new_version,
                                   old_version, package)
            except subprocess.CalledProcessError:
                args = ['git', 'cherry-pick',  '--continue', '--no-edit']
                print_run_and_check(args)


def comment_no_changes(feedstock, fork, branch, new_version, old_version,
                       package):
    args = ['gh', 'pr', 'comment',
            f'{fork}:{branch}',
            '-R', f'https://github.com/conda-forge/{feedstock}',
            '-b', f'**message from my grayskull autoupdate script:** All '
                  f'changes that I found between the current ({old_version}) '
                  f'and the new ({new_version}) versions of {package} are '
                  f'already included in this branch.']
    print_run_and_check(args)


def get_current_hash():
    args = ['git', 'rev-parse', 'HEAD']
    hash = subprocess.check_output(args).decode('utf-8').split('\n')[0]
    return hash


def push_changes(package, branch, fork):

    args = ['git', 'push', f'{fork}/{package}-feedstock',
            branch]
    subprocess.check_call(args)


def pop_startswith(lines, startswith):
    pop = None
    for line in lines:
        if line.startswith(startswith):
            pop = line
            break
    if pop is not None:
        lines.remove(pop)
    return pop


def print_run_and_check(args):
    print(' '.join(args))
    subprocess.run(args, check=True)

def add_bot_grayskull_update():
    with open('conda-forge.yml') as fp:
        conda_forge_yaml = yaml.load(fp, Loader=yaml.Loader)

    if 'bot' in conda_forge_yaml and 'inspection' in conda_forge_yaml['bot'] \
            and conda_forge_yaml['bot']['inspection'] == 'update-grayskull':
        # we're already all set
        return

    if 'bot' not in conda_forge_yaml:
        conda_forge_yaml['bot'] = dict()

    conda_forge_yaml['bot']['inspection'] = 'update-grayskull'
    with open('conda-forge.yml', 'w') as fp:
        yaml.dump(conda_forge_yaml, fp)

    args = ['git', 'commit', '-m', 'Add bot inspection: update-grayskull',
            'conda-forge.yml']
    print_run_and_check(args)


def update_package(package, fork='regro-cf-autotick-bot',
                   add_grayskull_update=False):
    cwd = os.getcwd()
    print(f'\n\n{package}\n')

    feedstock = f'{package}-feedstock'
    print(f'\n\n{feedstock}\n')
    clone_feedstock(feedstock, fork)

    os.chdir(f'{feedstock}/{feedstock}')
    update_feedstock(feedstock)

    old_version = get_current_version()

    branch, new_version = check_out_latest_bot_branch(feedstock, fork)

    if packaging.version.Version(old_version) > \
            packaging.version.Version(new_version):
        raise ValueError('Latest bot branch has an older version than the '
                         'current main branch.')

    if packaging.version.Version(old_version) == \
            packaging.version.Version(new_version):
        raise ValueError('Latest bot branch has the same version than the '
                         'current main branch.')

    os.chdir(f'../{branch}')

    if add_grayskull_update:
        add_bot_grayskull_update()

    run_grayskull(package, old_version)
    run_grayskull(package, new_version)

    update_from_grayskull(package, old_version, new_version, feedstock, fork,
                          branch)

    push_changes(package, branch, fork)
    os.chdir(cwd)


def main():
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('-p', '--packages', nargs='*', dest='packages',
                        help='A list of packages to update')
    parser.add_argument('-f', '--fork', default='regro-cf-autotick-bot',
                        dest='fork', help='The fork to take the branch from')
    parser.add_argument('-a', '--add_grayskull_update', action='store_true',
                        dest='add_grayskull_update',
                        help='Add bot inspection: grayskull-update')

    args = parser.parse_args()
    packages = args.packages
    fork = args.fork

    for package in packages:
        update_package(package, fork, args.add_grayskull_update)


if __name__ == '__main__':
    main()