generate-modulemd.py

#!/usr/bin/python3
from collections import OrderedDict, defaultdict
from functools import reduce
import sys

import koji
import solv
import yaml
from yaml import SafeDumper
from yaml.representer import SafeRepresenter

class literal(str):
    pass

class flow_list(list):
    pass

class LocalRepresenter(SafeRepresenter):
    def represent_ordered_dict(self, od):
        return self.represent_dict(od.items())
    def represent_flow_list(self, data):
        return self.represent_sequence('tag:yaml.org,2002:seq', data, flow_style=True)
    def represent_literal(self, s):
        if '\n' in s:
            return self.represent_scalar('tag:yaml.org,2002:str', s, style='|')
        return self.represent_str(s)

class LocalDumper(LocalRepresenter, SafeDumper):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.add_representer(OrderedDict, type(self).represent_ordered_dict)
        self.add_representer(flow_list, type(self).represent_flow_list)
        self.add_representer(literal, type(self).represent_literal)

def get_sourcepkg(p):
    if p.arch in {'src', 'nosrc'}:
        return p
    s = p.lookup_sourcepkg()[:-4]
    sel = p.pool.select(s, solv.Selection.SELECTION_CANON | solv.Selection.SELECTION_SOURCE_ONLY)
    if sel.isempty():
        raise Exception(s)
    return sel.solvables()[0]

def nvr(p):
    return f'{p.name}-{p.evr}'

def toposort2(data):
    # https://rosettacode.org/wiki/Topological_sort#Python
    for k, v in data.items():
        v.discard(k) # Ignore self dependencies
    extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys())
    data.update({item: set() for item in extra_items_in_deps})
    while True:
        ordered = set(item for item, dep in data.items() if not dep)
        if not ordered:
            break
        yield ordered
        data = {item: (dep - ordered) for item, dep in data.items()
                if item not in ordered}
    if data:
        raise Exception(f'A cyclic dependency exists amongst {data}')

pool = solv.Pool()
pool.setarch()

rbin = pool.add_repo('binary')
f = solv.xfopen('/var/cache/dnf/rawhide.solv')
rbin.add_solv(f)
f.close()

rsrc = pool.add_repo('source')
f = solv.xfopen('/var/cache/dnf/rawhide-source.solv')
rsrc.add_solv(f)
f.close()

#import glob
#rlocal = pool.add_repo('local')
#f = solv.xfopen(glob.glob('/home/brain/tmp/repo/repodata/*-primary.xml.gz')[0])
#rlocal.add_rpmmd(f, None)
#f.close()

pool.addfileprovides()
pool.createwhatprovides()

sel = pool.select(sys.argv[1], solv.Selection.SELECTION_NAME)
solvables = pool.best_solvables(sel.solvables())
opkg = pkg = solvables[0]
ospkg = get_sourcepkg(pkg)

mmd = OrderedDict({
    'document': 'modulemd',
    'version': 2,
    'data': OrderedDict({
        'summary': pkg.lookup_str(solv.SOLVABLE_SUMMARY),
        'description': literal(pkg.lookup_str(solv.SOLVABLE_DESCRIPTION)),
        'license': {
            'module': flow_list(['MIT']),
        },
        'dependencies': [
            OrderedDict({
                'buildrequires': {
                    'platform': [],
                    'rust': flow_list(['stable']),
                },
                'requires': {
                    'platform': [],
                },
            }),
        ],
        'api': {
            'rpms': [
                pkg.name,
            ],
        },
        'profiles': {
            'default': {
                'rpms': [
                    pkg.name,
                ],
            },
        },
        'filter': {
            'rpms': [],
        },
        'components': {
            'rpms': OrderedDict({}),
        },
    }),
})

done = set()
candq = {pkg}
g = defaultdict(set)
solver = pool.Solver()
while candq:
    candq_n = set()
    for pkg in candq:
        job = pool.Job(solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_INSTALL, pkg.id)
        problems = solver.solve([job])
        if problems:
            for i, problem in enumerate(problems, start=1):
                print(f'Problem #{i}:', file=sys.stderr)
                for rule in problem.findallproblemrules():
                    for info in rule.allinfos():
                        print(f'- {info.problemstr()}', file=sys.stderr)
            sys.exit(1)
        pkgs = set()
        for p in solver.transaction().newsolvables():
            if not p.name.startswith('rust-') or p.name in {'rust-srpm-macros', 'rust-pacakging', 'python3-rust2rpm'}:
                # short-circuit
                continue
            s = get_sourcepkg(p)
            if not s.name.startswith('rust-') or s.name in {'rust-srpm-macros', 'rust-packaging'}:
                continue
            candq_n.add(p)
            g[pkg].add(s)
        if pkg.arch not in {'src', 'nosrc'}:
            spkg = get_sourcepkg(pkg)
            candq_n.add(spkg)
            g[pkg].add(spkg)
        done.add(pkg)
    candq = candq_n - done

sel = pool.Selection()
for p in done:
    sel.add_raw(solv.Job.SOLVER_SOLVABLE, p.id)
depinfo = defaultdict(set)
for p in done:
    if p.arch in {'src', 'nosrc'}:
        continue
    tmp = sel.clone()
    tmp.matchsolvable(p, 0, solv.SOLVABLE_REQUIRES)
    spkg = get_sourcepkg(p)
    for s in tmp.solvables():
        if spkg == get_sourcepkg(s):
            continue
        depinfo[spkg].add(s)

batches = []
for batch in toposort2(g):
    srcs = [p for p in batch if p.arch in {'src', 'nosrc'}]
    if srcs:
        batches.append(srcs)

ks = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
ks.multicall = True
for p in g:
    ks.getBuild(nvr(p))
ret = ks.multiCall(strict=True)
ks.multicall = True
nvrs = {}
for build in ret:
    build = build[0]
    if build is not None:
        nvrs[build['task_id']] = build['nvr']
        ks.getTaskInfo(build['task_id'], request=True)
ret = ks.multiCall(strict=True)
hashes = {}
for task in ret:
    task = task[0]
    hashes[nvrs[task['id']]] = task['request'][0].split('#')[-1]

pkgs = set(str(p) for p in g)
filters = set()
for p in pool.solvables:
    if p.arch in {'src', 'nosrc'}:
        continue
    if p.lookup_sourcepkg()[:-4] in pkgs and p != opkg:
        filters.add(p.name)

mmd['data']['filter']['rpms'] = list(sorted(filters))

rpms = mmd['data']['components']['rpms']
for l, batch in enumerate(batches):
    for pkg in sorted(batch, key=lambda x: str(x)):
        if pkg == ospkg:
            rationale = 'Main component.'
        else:
            rationale = 'Dependency of other components.'
            deps = depinfo[pkg]
            builddep_of = {p for p in deps if p.arch in {'src', 'nosrc'}}
            dep_of = deps - builddep_of
            for d, t in ((builddep_of, 'BuildRequired'),
                         (dep_of, 'Required')):
                if d:
                    tmp = []
                    for p in sorted(d, key=lambda x: str(x)):
                        tmp.append(f'* {p.name}')
                        if p.arch not in {'src', 'nosrc'}:
                            tmp.append(f'  (built by {get_sourcepkg(p).name})')
                    formatted_deps = "\n".join(tmp)
                    rationale = f'''{rationale}

{t} by:
{formatted_deps}'''

        rpms[pkg.name] = {
            'buildorder': l * 10,
            'rationale': literal(rationale),
            'ref': f'{hashes.get(nvr(pkg), "XXX")} # {nvr(pkg)}',
        }

print(yaml.dump(mmd, Dumper=LocalDumper, default_flow_style=False), end='')