Script to synchronize ros-gbp devel branches with the entries from rosdistro

sync-ros-gbp-devel-branch.py

import argparse
import git
import github
import keyring
import os
import sys
import tempfile
import urllib.request
import yaml

gh_body = """This PR from an automated script updates the devel_branch for {ros_distro} to match the source branch as specified in https://github.com/ros/rosdistro/{ros_distro}/distribution.yaml ."""
commit_message = """Change the devel_branch for {ros_distro}.

This makes it match the source entry in https://github.com/ros/rosdistro/{ros_distro}/distribution.yaml
"""

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('distribution', nargs=1, help='Which ROS distribution to do the sync for', action='store')
    args = parser.parse_args()

    key = keyring.get_password('github-open-prs', 'may-open-prs')
    if key is None:
        raise RuntimeError('Failed to get GitHub API key')

    gh = github.Github(key)

    ros_distro = args.distribution[0]

    # First get the rosdistro distribution.yaml, which we will use as the source
    # of the devel_branch we should use.
    rosdistro_url = 'https://raw.githubusercontent.com/ros/rosdistro/master/{ros_distro}/distribution.yaml'.format(ros_distro=ros_distro)

    with urllib.request.urlopen(rosdistro_url) as response:
        ros_distro_data = response.read()

    ros_distro_yaml = yaml.safe_load(ros_distro_data)

    # Now get the ros2.repos corresponding to this release, which we will use
    # to constrain the list of packages that we consider to be "core".
    ros2_repos_url = 'https://raw.githubusercontent.com/ros2/ros2/{ros_distro}/ros2.repos'.format(ros_distro=ros_distro)

    with urllib.request.urlopen(ros2_repos_url) as response:
        ros2_repos_data = response.read()

    ros2_repos_yaml = yaml.safe_load(ros2_repos_data)

    # Now build up the constrained list of packages to look at.
    constrained_list = []
    for repo in ros_distro_yaml['repositories']:
        repo_dict = ros_distro_yaml['repositories'][repo]
        if not 'source' in repo_dict:
            print("Package '{repo}' has no source entry, skipping".format(repo=repo))
            continue
        source_url = repo_dict['source']['url']

        item_to_delete = None
        for ros2_repo in ros2_repos_yaml['repositories']:
            ros2_repos_package_url = ros2_repos_yaml['repositories'][ros2_repo]['url']
            if ros2_repos_package_url == source_url:
                # OK, we found what we were looking for.  We are going to break
                # out of here and remove this from the list either way, but we
                # will only add it to the constrained_list if it has both a
                # 'release' section and it is on github.
                item_to_delete = ros2_repo
                if not 'release' in repo_dict:
                    print("No release section for package '{repo}', skipping".format(repo=repo))
                    break

                release_url = repo_dict['release']['url']
                if not release_url.startswith('https://github.com'):
                    print("Release URL {release_url} for package '{repo}' is not on GitHub, do not know how to fetch tracks.yaml data".format(release_url=release_url, repo=repo))
                    break

                constrained_list.append(repo_dict)
                break

        if item_to_delete is not None:
            del ros2_repos_yaml['repositories'][item_to_delete]

    # Now that we have the list of repositories constrained, iterate over each
    # one, comparing what is in the tracks.yaml in the release repository to
    # what is in the source entry in the <distro>/distribution.yaml
    for repo in constrained_list:
        release_url = repo['release']['url']

        release_end = release_url[19:-4]
        tracks_url = 'https://raw.githubusercontent.com/' + release_end + '/master/tracks.yaml'

        with urllib.request.urlopen(tracks_url) as response:
            tracks_data = response.read()

        tracks_yaml = yaml.safe_load(tracks_data)
        tracks_yaml_distro = tracks_yaml['tracks'][ros_distro]

        if tracks_yaml_distro['devel_branch'] != repo['source']['version']:
            print("Package '{reponame}' rosdistro source branch ({source_branch}) does not match release branch ({release_branch})".format(reponame=tracks_yaml_distro['name'], source_branch=repo['source']['version'], release_branch=tracks_yaml_distro['devel_branch']))

            branch_name = '{ros_distro}/sync-devel-branch'.format(ros_distro=ros_distro)

            with tempfile.TemporaryDirectory() as tmpdirname:
                print(tmpdirname)
                gitrepo = git.Repo.clone_from(release_url, tmpdirname)
                branch = gitrepo.create_head(branch_name)
                branch.checkout()
                with open(os.path.join(tmpdirname, 'tracks.yaml'), 'r') as infp:
                    local_tracks_data = infp.read()
                local_tracks_yaml = yaml.safe_load(local_tracks_data)
                local_tracks_yaml['tracks'][ros_distro]['devel_branch'] = repo['source']['version']
                with open(os.path.join(tmpdirname, 'tracks.yaml'), 'w') as outfp:
                    yaml.dump(local_tracks_yaml, outfp)
                gitrepo.git.add(A=True)
                gitrepo.index.commit(commit_message.format(ros_distro=ros_distro))
                try:
                    gitrepo.git.push('--set-upstream', gitrepo.remote(), gitrepo.head.ref)
                except git.exc.GitCommandError:
                   print('Could not push to release repo for {ros_distro}: {reponame}, skipping...'.format(ros_distro=ros_distro, reponame=tracks_yaml_distro['name']))
                   continue

            gh_title = 'Update {ros_distro} devel_branch to match rosdistro source entry'.format(ros_distro=ros_distro)
            gh_repo = gh.get_repo(release_end)
            pull = gh_repo.create_pull(title=gh_title, head=branch_name, base='master', body=gh_body)

    return 0

if __name__ == '__main__':
    sys.exit(main())