Create osbuild pipelines from OSTree Treefile-like inputs

osbuild-config.py

#!/usr/bin/python3

import argparse
import configparser
import json
import os
import sys
import tempfile
import urllib.request
import yaml

import dnf
import dnf.conf
import dnf.conf.read
import hawkey


def log(msg):
    print(msg, file=sys.stderr)


class DepSolver:
    def __init__(self, arch, relver, dirs):
        self.base = dnf.Base()
        self.arch = arch
        conf = self.base.conf
        conf.config_file_path = "/dev/null"
        conf.persistdir = dirs["persistdir"]
        conf.cachedir = dirs["cachedir"]
        conf.substitutions["arch"] = arch
        conf.substitutions["basearch"] = dnf.rpm.basearch(arch)
        conf.substitutions["releasever"] = relver
        conf.reposdir = [dirs["repodir"]]
        self.repos = self.read_repos()

    def read_repos(self):
        conf = self.base.conf
        reader = dnf.conf.read.RepoReader(conf, {})
        return {r.id: r for r in reader}

    def resolve(self, manifest):

        base = self.base
        base.reset(goal=True, repos=True, sack=True)

        gpgkeys = []

        for repo_id in manifest["repos"]:
            if repo_id not in self.repos:
                raise ValueError("fUnknown repo: {repo_id}")
            repo = self.repos[repo_id]
            base.repos.add(repo)
            for key in repo.gpgkey:
                with urllib.request.urlopen(key) as r:
                    gpgkeys.append(r.read().decode("utf-8"))

        base.fill_sack(load_system_repo=False)

        include = manifest["packages"]
        exclude = manifest.get("packages-exclude", [])
        base.install_specs(include, exclude=exclude)
        base.resolve()

        deps = []
        for tsi in base.transaction:
            # avoid using the install_set() helper, as it does not guarantee a stable order
            if tsi.action not in dnf.transaction.FORWARD_ACTIONS:
                continue
            package = tsi.pkg

            checksum_id = hawkey.chksum_name(package.chksum[0])
            checksum = package.chksum[1].hex()
            deps.append({
                "name": package.name,
                "epoch": package.epoch,
                "version": package.version,
                "release": package.release,
                "arch": package.arch,
                "repo_id": package.reponame,
                "path": package.relativepath,
                "remote_location": package.remote_location(),
                "checksum": f"{checksum_id}:{checksum}",
            })

        ret = {
            "dependencies": deps,
            "gpgkeys": gpgkeys
        }
        return ret


def gen_rpm_stage_data(data):
    checksums = []
    urls = {}
    for pkg in data["dependencies"]:
        checksum = pkg["checksum"]
        checksums.append(checksum)
        urls[checksum] = pkg["remote_location"]

    return urls, checksums


def manifest_update_packages(manifest, pkgs):
    urls, checksums = gen_rpm_stage_data(pkgs)

    sources = manifest["sources"]
    sources["org.osbuild.files"]["urls"].update(urls)

    pipeline = manifest["pipeline"]
    rpmstage = None
    for stage in pipeline["stages"]:
        if stage["name"] == "org.osbuild.rpm":
            rpmstage = stage
            break
    if not rpmstage:
        raise RuntimeError("Failed to find rpm stage")

    rpmstage["options"]["gpgkeys"] = pkgs["gpgkeys"]
    rpmstage["options"]["packages"] = checksums


def manifest_add_build_pipeline(manifest, builder):
    pipeline = builder["pipeline"]
    runner = builder["runner"]
    urls = builder["sources"]["org.osbuild.files"]["urls"]

    manifest["pipeline"]["build"]["pipeline"] = pipeline
    manifest["pipeline"]["build"]["runner"] = runner
    manifest["sources"]["org.osbuild.files"]["urls"].update(urls)


def prepare_build_manifest(treefile, solver):
    log("Preparing the build pipeline manifest:")

    manifest = {
        "pipeline": {
            "stages": [
                {
                    "name": "org.osbuild.rpm",
                    "options": {}
                }
            ]
        },
        "runner": None,
        "sources": {
            "org.osbuild.files": {
                "urls": {}
            }
        },
    }

    arch = solver.arch
    with open(treefile) as f:
        treespec = yaml.safe_load(f)

    log("  Generating package list and sources")
    pkgs = solver.resolve(treespec)
    manifest_update_packages(manifest, pkgs)

    manifest["runner"] = treespec["runner"]
    return manifest


def prepare_manifest(templatefile, treespec, solver):
    log("Preparing the manifest:")

    arch = solver.arch
    section = f"commit-{arch}"

    with open(templatefile, "r") as fp:
        manifest = json.load(fp)

    log("  Generating package list and sources")
    pkgs = solver.resolve(treespec)

    manifest_update_packages(manifest, pkgs)
    return manifest


def main():
    parser = argparse.ArgumentParser(description="Build operating system images")
    parser.add_argument("manifest_path", metavar="MANIFEST",
                        help="json file containing the manifest that should be used")
    parser.add_argument("treefile_path", metavar="TREEFILE",
                        help="json file containing the treefile that should be used")
    parser.add_argument("--arch", metavar="ARCH", default="x86_64",
                        help="Architecture to use (default x86_64)")
    parser.add_argument("--release", metavar="RELEASE", default="31",
                        help="Release to use (31)")
    parser.add_argument("--cachedir", metavar="DIRECTORY", type=os.path.abspath,
                        default=".osbuild",
                        help="directory where cache files are stored in")
    parser.add_argument("--repodir", metavar="DIRECTORY", type=os.path.abspath,
                        default=os.path.curdir,
                        help="directory where repo files are to be found")
    parser.add_argument("--buildtree", metavar="TREEFILE", default=None,
                        help="Build tree file for build pipeline")
    args = parser.parse_args()

    os.makedirs(args.cachedir, exist_ok=True)
    cachedir = args.cachedir
    arch = args.arch
    relver = args.release
    tree_file = args.treefile_path
    manifest_path = args.manifest_path
    buildtree = args.buildtree

    with open(tree_file, "r") as f:
        treespec = yaml.unsafe_load(f)

    with tempfile.TemporaryDirectory(dir=cachedir) as persistdir:
        dirs = {
            "repodir": args.repodir,
            "cachedir": cachedir,
            "persistdir": persistdir
        }

        solver = DepSolver(arch, str(relver), dirs)
        manifest = prepare_manifest(manifest_path, treespec, solver)
        if buildtree:
            builder = prepare_build_manifest(buildtree, solver)
            manifest_add_build_pipeline(manifest, builder)

    json.dump(manifest, sys.stdout, indent=2)


if __name__ == "__main__":
    sys.exit(main())