build a conda-forge recipe.yaml for morph-kgc

build_morph_kgc_recipe.py

import os
import tarfile
import subprocess
import re
from io import StringIO
from difflib import unified_diff
from urllib.request import urlretrieve
from pathlib import Path
from ruamel.yaml import YAML as YAML_
from typing import Any
from tempfile import TemporaryDirectory

import sys

try:
    import tomllib
except ImportError:
    import tomli as tomllib

YAML = YAML_(typ="rt")
YAML.width = 4096
YAML.preserve_quotes = True
YAML.default_flow_style = False
YAML.explicit_start = False
YAML.indent(mapping=2, sequence=4, offset=2)

PPT = "pyproject.toml"

UTF8 = dict(encoding="utf-8")
PKG_NAME = "morph-kgc"
PKG_MOD = PKG_NAME.replace("-", "_")
PKG_PREFIX = f"{PKG_NAME}-with"
RECIPE_DIR = Path(__file__).parent

RECIPE_YAML = RECIPE_DIR / "recipe.yaml"
RECIPE_TEXT = RECIPE_YAML.read_text(**UTF8).strip()
RECIPE = YAML.load(RECIPE_TEXT)
SKIP_EXTRAS = {
    "all": "custom package",
    "kuzu": "missing depedencies",
    "neo4j": "missing depedencies",
    "test": "custom package",
}

RE_PYPI_CONDA = {
    r"^(.*?)(?=[=<>])": r"\1 ",
    r"psycopg\[binary\]": "psycopg",
    r"^duckdb": "python-duckdb",
    ", ": ",",
}
PYTHON_MIN_EXACT = "python ${{ python_min }}.*"
PIP_INSTALL = "${{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation --disable-pip-version-check"
TEST_SKIPS = [
    # broken as of https://github.com/conda-forge/staged-recipes/pull/27298
    "issue_316_a",
    "issue_316_b",
    "math_floor",
    "math_gcd",
    "RMLIMTC0000",
    "RMLTC0000",
    "RMLTC0001a",
    "RMLTC0001a",
    "RMLTC0001b",
    "RMLTC0001b",
    "RMLTC0002a",
    "RMLTC0002a",
    "RMLTC0002b",
    "RMLTC0002b",
    "RMLTC0003c",
    "RMLTC0003c",
    "RMLTC0004a",
    "RMLTC0004a",
    "RMLTC0005a",
    "RMLTC0005a",
    "RMLTC0007a",
    "RMLTC0007a",
    "RMLTC0007b",
    "RMLTC0007b",
    "RMLTC0007c",
    "RMLTC0007c",
    "RMLTC0007d",
    "RMLTC0007d",
    "RMLTC0007e",
    "RMLTC0007e",
    "RMLTC0007f",
    "RMLTC0007f",
    "RMLTC0007g",
    "RMLTC0007g",
    "RMLTC0008a",
    "RMLTC0008a",
    "RMLTC0008b",
    "RMLTC0008b",
    "RMLTC0008c",
    "RMLTC0008c",
    "RMLTC0009a",
    "RMLTC0009a",
    "RMLTC0009b",
    "RMLTC0009b",
    "RMLTC0010a",
    "RMLTC0010a",
    "RMLTC0010b",
    "RMLTC0010b",
    "RMLTC0010c",
    "RMLTC0010c",
    "RMLTC0011b",
    "RMLTC0011b",
    "RMLTC0012a",
    "RMLTC0012a",
    "RMLTC0012b",
    "RMLTC0012b",
    "RMLTC0013a",
    "RMLTC0015a",
    "RMLTC0015a",
    "RMLTC0016a",
    "triples_map_without_po",
]
PYTEST_ARGS = """
    -vv --tb=long --color=yes -k "not (${{ test_skips | split | sort | join(" or ") }})"
""".strip()
COV_FAIL_UNDER = 57
TEST_ALL_SCRIPTS = [
    f"coverage run --source={PKG_MOD} --branch -m pytest {PYTEST_ARGS}",
    f"coverage report --show-missing --skip-covered --fail-under={COV_FAIL_UNDER}",
]


def get_pyproject_data() -> dict[str, Any]:
    src_dir = os.environ.get("SRC_DIR")
    if src_dir:
        ppt_text = (Path(src_dir) / "pyproject.toml").read_text(**UTF8)
    else:
        url = RECIPE["source"]["url"].replace(
            "${{ version }}", RECIPE["context"]["version"]
        )
        with TemporaryDirectory() as td:
            tdp = Path(td)
            tgz = tdp / Path(url).name
            urlretrieve(url, filename=f"{tgz}")
            with tarfile.open(tgz, "r:gz") as tf:
                for member in tf.getmembers():
                    if not member.name.endswith(PPT):
                        continue
                    ppt_text = tf.extractfile(member).read().decode(**UTF8)
                    break

    return tomllib.loads(ppt_text)


def to_conda(all_raw: list[str], extra_conda: list[str] | None = None) -> str:
    conda: list[str] = []
    for raw in all_raw:
        raw = raw.lower()
        for pattern, repl in RE_PYPI_CONDA.items():
            raw = re.sub(pattern, repl, raw)
        conda += [raw]
    return sorted([*conda, *(extra_conda or [])])


def pip_check() -> dict[str, Any]:
    return {
        "python": {
            "imports": PKG_MOD,
            "pip_check": True,
            "python_version": ["${{ python_min }}.*", "${{ python_check_max }}.*"],
        }
    }


def pin_subpackage(other: str) -> str:
    return """${{ pin_subpackage("%s", exact=True) }}""" % other


def build_core(build_deps: list[str], run_deps: list[str]) -> dict[str, Any]:
    return {
        "package": {"name": PKG_NAME},
        "build": {"noarch": "python", "script": [PIP_INSTALL]},
        "requirements": {
            "host": to_conda(build_deps, [PYTHON_MIN_EXACT, "pip"]),
            "run": to_conda(run_deps, ["python >=${{ python_min }}"]),
        },
        "tests": [pip_check()],
    }


def build_extra(extra: str, run_deps: list[str]) -> dict[str, Any]:
    return {
        "package": {"name": f"{PKG_PREFIX}-{extra}"},
        "build": {"noarch": "generic"},
        "requirements": {"run": to_conda(run_deps, [pin_subpackage(PKG_NAME)])},
        "tests": [pip_check()],
        "about": {
            "summary": f"""{RECIPE["about"]["summary"]} (with [{extra}])""",
            "license": RECIPE["about"]["license"],
            "license_file": RECIPE["about"]["license_file"],
        },
    }


def build_all(
    outputs: list[dict[str, str]],
    test_deps: list[str],
) -> dict[str, Any]:
    output_name = f"{PKG_PREFIX}-all"
    with_outputs = [
        pin_subpackage(output["package"]["name"])
        for output in outputs
        if output["package"]["name"] not in [output_name]
    ]

    return {
        "package": {"name": output_name},
        "build": {"noarch": "generic"},
        "requirements": {"run": sorted(with_outputs)},
        "tests": [
            pip_check(),
            {
                "requirements": {
                    "run": to_conda(test_deps, [PYTHON_MIN_EXACT, "pytest-cov"])
                },
                "files": {"source": ["test/"]},
                "script": TEST_ALL_SCRIPTS,
            },
        ],
        "about": {
            "summary": f"""{RECIPE["about"]["summary"]} (with [all])""",
            "license": RECIPE["about"]["license"],
            "license_file": RECIPE["about"]["license_file"],
        },
    }


def main(*, write: bool = False, lint: bool = False, build: bool = False) -> int:
    ppt = get_pyproject_data()
    RECIPE["context"]["summary"] = ppt["project"]["description"]
    RECIPE["about"]["license"] = ppt["project"]["license"]
    RECIPE["context"]["test_skips"] = " ".join(TEST_SKIPS)
    proj = ppt["project"]
    extras = {
        k: v for k, v in proj["optional-dependencies"].items() if k not in SKIP_EXTRAS
    }

    outputs = [
        build_core(ppt["build-system"]["requires"], proj["dependencies"]),
        *[build_extra(extra, deps) for extra, deps in extras.items()],
    ]

    RECIPE["outputs"] = [
        *outputs,
        build_all([*outputs], proj["optional-dependencies"]["test"]),
    ]

    io = StringIO()
    YAML.dump(RECIPE, io)
    new_text = io.getvalue().strip()

    diff = "\n".join(unified_diff(RECIPE_TEXT.splitlines(), new_text.splitlines()))

    rc = 1
    if not diff:
        print("... no change")
        rc = 0
    else:
        print(diff)

        if write:
            RECIPE_YAML.write_text(new_text + "\n", **UTF8)
            print("... wrote", RECIPE_YAML)
        else:
            print("\n!!! run again with --write to update recipe.yaml")
            return 1

    if not rc and lint:
        rc = subprocess.call(["pixi", "run", "lint"])

    if not rc and build:
        rc = subprocess.call(["pixi", "run", "build-linux", "linux64"])

    return rc


if __name__ == "__main__":
    sys.exit(
        main(
            write="--write" in sys.argv,
            lint="--lint" in sys.argv,
            build="--build" in sys.argv,
        )
    )