Draft-yt-notes.md

draft_yt_notes.py

A script for initially drafting release notes for yt.

Most of the code is from @neutrinoceros, with some minor updates from @chrishavlin.

It's designed to run with uv. To run directly off of this gist, use the url you get when vewing the raw draft_yt_release_notes.py and supply --help to see the command line options:

uv run https://gist.githubusercontent.com/chrishavlin/248adea4296abb7bcdbaac952f304cf0/raw/fc3443b281fea40f195c563aeddbc722fac4408a/draft_yt_release_notes.py --help 

You can also download the draft_yt_release_notes.py and run it locally with

uv run draft_yt_release_notes.py --help 

The initial run requires a github user and password be passed in as arguments.

Maybe some day this will be refined and in the yt repo somewhere... ​

draft_yt_release_notes.py

# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "github3-py==4.0.1",
#     "loguru==0.7.2",
# ]
# ///
# the rate limit for unauthenticated users is 60 requests/h
# which in practice is more than enough for this script (as long as it's used once)
#
# For arguments and example usage:
#
# uv run draft_yt_release_notes.py --help 
from __future__ import annotations

import argparse
import re
import json
import sys
from pathlib import Path
from typing import TYPE_CHECKING, TypedDict

from github3 import login
from loguru import logger
from copy import deepcopy

if TYPE_CHECKING:
    from typing import Literal

    from github3 import ShortIssue

logger.remove()
logger.add(sys.stderr, colorize=True, format="<level>{level:<8} {message}</level>")


# github_context fields are set from the command line
github_context = {'user': None, 
                  'pw': None, 
                  'milestone_id': None}
owner = "yt-project"
repository = "yt"

# output files
db_file = Path(__file__).parent / "db.json"  # store of github api output, created on initial fetch
md_file = Path(__file__).parent / "release_notes.md"  # output draft notes, over-written every run

TAG_REGEXP = re.compile(r"^\[?[A-Z]{3,4}\]?:?")

# PRs from the following are not included in notes
author_blacklist = ['meeseeksmachine',
                    'github-actions[bot]', 
                    'dependabot[bot]',
                    ]

# the following gets inserted before the categorized
# PR/issue list
release_preamble = """# Summary 

<<<< INSERT RELEASE SUMMARY HERE >>>> 

How to upgrade

To upgrade from PyPI, run

```shell
python -m pip install --upgrade yt
```

or, with conda

```shell
conda update --channel conda-forge yt
```

## :star2: Highlights

<<<< WRITE ANY HIGHLIGHTS IF ANY HERE >>>>

"""

def get_issues_and_prs() -> list[ShortIssue]:
    gh = login(github_context['user'], github_context['pw'])
    logger.info("Logged as {}", github_context['user'])

    return list(
        gh.issues_on(
            username=owner,
            repository=repository,
            milestone=github_context['milestone_id'],
            state="closed",
        )
    )


def normalize_title(title: str):
    if (match := TAG_REGEXP.match(title)) is not None:
        title = title[len(match.group()) :]
    title = title.strip()
    return (
        title.capitalize()
        .replace("cython", "Cython")
        .replace("cpython", "CPython")
        .replace(" pep", " PEP")
        .replace(" gil", " GIL")
        .replace("ipython", "IPython")
        .replace("Yt ", "yt ")
    )


class Item(TypedDict):
    number: int
    title: str
    author: str
    labels: list[str]


def serialize_group(
    issues_and_prs: list[ShortIssue], type: Literal["issues", "pulls"]
) -> list[Item]:
    return [
        {
            "number": obj.number,
            "title": obj.title,
            "author": str(obj.user),
            "labels": [label.name for label in obj.original_labels],
        }
        for obj in issues_and_prs
        if type in obj.html_url
    ]


def categorize_pulls(pulls: list[Item]) -> dict[str, list[Item]]:
    breaking = []
    frontends = []
    fix = []
    perf = []
    rfc = []
    doc = []
    build = []
    tests = []
    misc = []
    retv = {
        "breaking changes": breaking,
        "bug fixes": fix,
        "frontend-specific changes": frontends,
        "performance": perf,
        "refactors": rfc,
        "documentation": doc,
        "building": build,
        "testing": tests,
        "other changes": misc,
    }
    for pull in pulls:
        if pull['author'] in author_blacklist: 
            continue

        labels = pull["labels"]
        if "backwards incompatible" in labels:
            breaking.append(pull)
        elif "code frontends" in labels:
            frontends.append(pull)
        elif "bug" in labels:
            fix.append(pull)
        elif "performance" in labels:
            perf.append(pull)
        elif "refactor" in labels:
            rfc.append(pull)
        elif "docs" in labels:
            doc.append(pull)
        elif "build" in labels:
            build.append(pull)
        elif "test: running tests" in labels or "TST" in pull["title"]:
            tests.append(pull)
        else:
            misc.append(pull)

    for cat, lst in retv.items():
        retv[cat] = sorted(lst, key=lambda pull: pull["number"])

    return retv


emoji_map = {
    'bug fixes': ':bug: ',
    'frontend-specific changes': ':control_knobs: ',     
    'performance': ':performing_arts: ', 
    'refactors': ':factory: ',
    'documentation': ':books: ', 
    'building': ':jigsaw: ',
    'testing': ':robot: '
}




def render_item(it: Item) -> str:
    return f"#{it['number']} {normalize_title(it['title'])}, by @{it['author']}"


    
def make_raw_note(data: dict[str, list[Item]]) -> str:
    cats = categorize_pulls(data["pulls"])
    lines = []
    for category, pulls in cats.items():        
        emoji_char = emoji_map.get(category, '')
        print((category, emoji_char, category in emoji_map))
        if len(pulls) > 0:
            lines.append(f"## {emoji_char}{category.capitalize()}\n")
            lines.extend([render_item(pull) for pull in pulls])
            lines.append("\n")

    lines.append("## Issues resolved\n")
    lines.extend([render_item(issue) for issue in sorted(data["issues"], key=lambda issue: issue["number"]) if issue["author"] not in author_blacklist])

    return "\n".join(lines)

def insert_preamble(raw_note: str) -> str:    
    return release_preamble + raw_note
    
def main(force_fetch: bool = False) -> int:
    data : dict[str, list[Item]]
    if db_file.is_file() and force_fetch is False:
        logger.info("Loading cached database from {}", db_file)
        with db_file.open() as fdr:
            data = json.load(fdr)
    else:
        
        if github_context["milestone_id"] is None:
            raise ValueError("You must provide an integer value for the target milestone when fetching from github")
            
        logger.info("requesting data")
        issues_and_prs = get_issues_and_prs()

        logger.info("serializing data")
        data = {
            "issues": serialize_group(issues_and_prs, type="issues"),
            "pulls": serialize_group(issues_and_prs, type="pull"),
        }

        logger.info("saving data to {}", db_file)
        with db_file.open("w") as fdw:
            json.dump(data, fdw, indent=2)

    raw_note = make_raw_note(data)
    raw_note = insert_preamble(raw_note)

    print(raw_note)

    logger.info("writing raw note to {}", md_file)
    with md_file.open("w") as release_notes:
        release_notes.write(raw_note)
    return 0


if __name__ == "__main__":

    example_text = """example usage:

    First run: fetches from GitHub API and writes db.json:

        uv run draft_yt_release_notes.py -user myGithubUser -pw mySecretGithubPassword -milestone 33

    Regenerate notes after db.json has been created from GitHub API (will not 
    fetch from GitHub again):

        uv run draft_yt_release_notes.py

    Force a re-build of db.json (WILL fetch from GithHub API again):

        uv run draft_yt_release_notes.py -user myGithubUser -pw mySecretGithubPassword -milestone 33 --fetch
    """


    parser = argparse.ArgumentParser(
        prog="draft_yt_release_notes",
        description="draft release notes for yt, recommended that you run it with uv.",
        epilog=example_text,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    
    parser.add_argument("-user", help="your github username, required if fetching data again", default="")
    parser.add_argument("-pw", help="your github password, required if fetching data again", default="")
    helpmsg = 'The integer milestone id, required if fetching data again. Milestone id is part of the ' 
    helpmsg += 'milestone url:  e.g., yt 4.4.1 id is 33, url is  https://github.com/yt-project/yt/milestone/33 '
    parser.add_argument('-milestone', type=int, help=helpmsg, default=None)
    helpmsg = "fetch data again, overwriting local files (note github API rate limit of 60 requests/h, "
    helpmsg += "this script makes no attempt to rate limit.)"
    parser.add_argument("--fetch", action='store_true', help=helpmsg)

    args = parser.parse_args() 
        
    github_context['user'] = args.user
    github_context['pw'] = args.pw
    github_context['milestone_id'] = args.milestone

    raise SystemExit(main(args.fetch))