get-stable-branches,py

#!/usr/bin/env python

from __future__ import print_function

import argparse
import copy
import functools
import json
import os
import requests
import requests_file
import subprocess
import sys
import yaml


# This is needed to that 'requests' understands file:// urls which is nice
# for reading objects from local git clones rather than http{,s}
requests_session = requests.Session()
requests_session.mount('file://', requests_file.FileAdapter())

repo_to_project_map = dict()
look_at_repos = set()
all_repos = set()
stable_repos = set()
tagged_repos = set()
eol_repos = set()
total_changes = 0


def cache_results(fname):
    def decorator(f):
        try:
            cache = json.load(open(fname))
        except (IOError, ValueError):
            cache = dict()

        @functools.wraps(f)
        def wrapped(*args, **kwargs):
            key = '-'.join([a for a in args if isinstance(a, str)])
            if key not in cache:
                cache[key] = f(*args, **kwargs)
                json.dump(cache, open(fname, 'w'))
            return cache[key]
        return wrapped
    return decorator


class GitConfigBase(object):
    def __init__(self, git_base, ref_prefix, tag_prefix, cgit_base):
        self.git_base = git_base
        self.ref_prefix = ref_prefix
        self.tag_prefix = tag_prefix
        self.cgit_base = cgit_base

    def cgit_url(self, repo, path, branch='master'):
        raise NotImplementedError()

    @property
    def projects_url(self):
        return self.cgit_url('openstack/governance',
                             'reference/projects.yaml')

    @property
    def layout_url(self):
        return self.cgit_url('openstack-infra/project-config',
                             'zuul/layout.yaml')

    def _git_query_remote(self, repo, refs):
        remote = '%s/%s' % (self.git_base, repo)
        cmd = ['git', 'ls-remote', remote] + refs
        try:
            out, err = subprocess.Popen(cmd,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE).communicate()
        except subprocess.CalledProcessError:
            out = ''
        return out != ''

    @cache_results('.cache.branches.json')
    def check_has_branch(self, repo, release):
        refs = ['%s/stable/%s' % (self.ref_prefix, release),
                '%s/%s' % (self.ref_prefix, release)]
        return self._git_query_remote(repo, refs)

    @cache_results('.cache.tags.json')
    def check_has_tag(self, repo, tag):
        prefix = 'refs/tags'
        refs = ['%s/%s' % (prefix, tag)]
        return self._git_query_remote(repo, refs)


class GitConfigLocal(GitConfigBase):
    def __init__(self, git_base):
        super(GitConfigLocal, self).__init__(git_base,
                                             'refs/remotes/origin',
                                             'refs/tags',
                                             git_base)

    def cgit_url(self, repo, path, branch='master'):
        return "file://%s" % (os.path.join(self.git_base, repo, path))


class GitConfigRemote(GitConfigBase):
    def __init__(self, git_base):
        super(GitConfigRemote, self).__init__(git_base,
                                              'refs/heads',
                                              'refs/tags',
                                              'http://git.openstack.org/cgit')

    def cgit_url(self, repo, path, branch='master'):
        return "%s/%s/plain/%s?h=%s" % (self.cgit_base, repo, path, branch)


@cache_results('.cache.changes.json')
def check_for_changes(repo, release):
    host = 'review.openstack.org'
    if 'master' != release:
        branch = 'stable/%s' % release
    url = ('https://%s/changes/?q=is:open+project:%s+branch:%s'
           % (host, repo, branch))
    return json.loads(requests_session.get(url).text[4:])


def dump_report(repos, release, fmt="%-35s %18s %7s %-17s", eol_tag=None):
    global total_changes
    print(fmt % ('',     '',        'Open',    ''))
    print(fmt % ('Repo', 'Project', 'Reviews', 'Notes'))
    print(fmt % ('----', '-------', '-------', '-----'))
    for repo in sorted(repos):
        project_name = repo_to_project_map.get(repo, 'BigTent')
        changes = check_for_changes(repo, release)
        nr_changes = ''
        notes = ''
        total_changes += len(changes)
        if len(changes) and repo in tagged_repos:
            notes += 'ERROR!'
        if eol_tag and repo in tagged_repos:
            notes += 'tagged %s' % (eol_tag)
        if len(changes):
            nr_changes = str(len(changes))
        print(fmt % (repo, project_name, nr_changes, notes))


def opt_ins(fname):
    try:
        with open(fname) as f:
            return set(yaml.safe_load(f))
    except IOError:
        return set()

# FIXME: add a decription
# FIXME: add help text to options
parser = argparse.ArgumentParser(description='')
parser.add_argument('--release', dest='release', required=True)
parser.add_argument('--local', dest='local', action='store_true')
parser.set_defaults(local=False)
parser.add_argument('--git-base', dest='git_base',
                    default='git://git.openstack.org')
parser.add_argument('--quick-test', dest='quick_test', default=False)
parser.add_argument('--opt-ins', dest='opt_ins', default='opt-ins.yaml')
args, extras = parser.parse_known_args()

eol_tag = "%s-eol" % (args.release)
look_at_repos.update(extras)
look_at_repos.update(opt_ins(args.opt_ins))

if args.local:
    gitconfig = GitConfigLocal(os.path.expanduser(args.git_base))
else:
    gitconfig = GitConfigRemote(args.git_base)

print('Loading : %s' % (gitconfig.projects_url), file=sys.stderr)
projects = yaml.safe_load(requests_session.get(gitconfig.projects_url).text)
for project_name in projects:
    project_data = projects[project_name]
    for deliverable_name, data in project_data['deliverables'].items():
        repos = data.get('repos', [])
        for repo in repos:
            repo_to_project_map[repo] = project_name
            all_repos.add(repo)

        tags = data.get('tags', [])
        if set(['release:managed', 'stable:follows-policy']) & set(tags):
            look_at_repos.update(repos)
print('Done', file=sys.stderr)

print('Loading : %s' % (gitconfig.layout_url), file=sys.stderr)
layout = yaml.safe_load(requests_session.get(gitconfig.layout_url).text)
for project in layout.get('projects', []):
    repo = project['name']
    template = project.get('template', [])
    if 'check-requirements' in [job.get('name', '') for job in template]:
        look_at_repos.add(repo)
    all_repos.add(repo)
print('Done', file=sys.stderr)

if args.quick_test:
    look_at_repos = set(['openstack-dev/devstack', 'openstack-dev/grenade',
                         'openstack/requirements', 'openstack/nova',
                         'openstack/oslotest',
                         'openstack/python-ceilometerclient',
                         'openstack/requirements'])
    all_repos = copy.copy(look_at_repos)
    all_repos.update(['openstack/openstack-ansible'])

print('Checking %d repos for stable branches' % (len(all_repos)),
      file=sys.stderr)
for repo in all_repos:
    if gitconfig.check_has_branch(repo, args.release):
        stable_repos.add(repo)
    if repo in look_at_repos and repo in stable_repos:
        eol_repos.add(repo)
    if gitconfig.check_has_tag(repo, eol_tag):
        tagged_repos.add(repo)
print('Done', file=sys.stderr)

dump_report(eol_repos, args.release, eol_tag=eol_tag)
print('-'*80)
print('Repos below are (currently) not going to be EOL\'d')
print('-'*80)
dump_report(stable_repos ^ eol_repos, args.release,
            fmt="%-45s %18s %7s %-17s", eol_tag=eol_tag)
print("")

fmt = "%-25s : %3d"
print(fmt % ('All Repos', len(all_repos)))
print(fmt % ('Checked Repos', len(look_at_repos)))
print(fmt % ('Repos with %s branches' % (args.release), len(stable_repos)))
print(fmt % ('EOL Repos', len(eol_repos)))
print(fmt % ('NOT EOL Repos', len(stable_repos ^ eol_repos)))
print(fmt % ('Tagged Repos', len(tagged_repos)))
print(fmt % ('Open Reviews', total_changes))

with open('eol_repos.yaml', 'w') as f:
    l = sorted(list(eol_repos ^ tagged_repos))
    f.write(yaml.dump(l, default_flow_style=False,
                      explicit_start=True))