Analyze lib-python failures

analyze_failures.py

#!/usr/bin/env python3
import re
import sys
import urllib.request

BUILD_NUMBER = 11645

BASE_URL = (
    "https://buildbot.pypy.org/builders/pypy-c-jit-linux-x86-64"
    "/builds/{build}/steps/shell_6/logs/stdio/text"
)


def fetch_log(build_number):
    url = BASE_URL.format(build=build_number)
    with urllib.request.urlopen(url) as resp:
        return resp.read().decode("utf-8", errors="replace")


def is_separator(line):
    stripped = line.strip()
    # separator is two blocks of underscores separated by spaces
    return bool(re.match(r'^_{10,}\s+_{10,}$', stripped))


def parse_failed_line(line):
    """Parse 'FAILED (failures=3, errors=1, ...)' -> (failures, errors)."""
    failures = errors = 0
    m = re.search(r'failures=(\d+)', line)
    if m:
        failures = int(m.group(1))
    m = re.search(r'errors=(\d+)', line)
    if m:
        errors = int(m.group(1))
    return failures, errors


def parse_log(text):
    lines = text.splitlines()

    # Split into sections by separator lines
    sections = []
    current = []
    for line in lines:
        if is_separator(line):
            if current:
                sections.append(current)
            current = []
        else:
            current.append(line)
    if current:
        sections.append(current)

    results = []
    for section in sections:
        test_name = None
        failures = errors = 0
        found_failed_marker = False

        for i, line in enumerate(section):
            if line.strip() == "1 test failed:":
                found_failed_marker = True
                # test name is on the next non-empty line
                for j in range(i + 1, len(section)):
                    candidate = section[j].strip()
                    if candidate:
                        test_name = candidate
                        break
            if line.startswith("FAILED"):
                f, e = parse_failed_line(line)
                failures += f
                errors += e

        if found_failed_marker and test_name:
            results.append({
                "name": test_name,
                "lines": len(section),
                "failures": failures,
                "errors": errors,
                "total": failures + errors,
            })

    return results


def print_table(results):
    results = sorted(results, key=lambda r: r["total"])

    name_w = max(len(r["name"]) for r in results)
    name_w = max(name_w, len("test name"))

    header = (
        f"{'test name':<{name_w}}  {'lines':>6}  {'failures':>8}  "
        f"{'errors':>6}  {'total':>5}"
    )
    sep = "-" * len(header)
    print(header)
    print(sep)
    for r in results:
        print(
            f"{r['name']:<{name_w}}  {r['lines']:>6}  {r['failures']:>8}  "
            f"{r['errors']:>6}  {r['total']:>5}"
        )
    print(sep)
    total_lines = sum(r["lines"] for r in results)
    total_f = sum(r["failures"] for r in results)
    total_e = sum(r["errors"] for r in results)
    total_t = sum(r["total"] for r in results)
    print(
        f"{'TOTAL':<{name_w}}  {total_lines:>6}  {total_f:>8}  "
        f"{total_e:>6}  {total_t:>5}"
    )


def main():
    build_number = int(sys.argv[1]) if len(sys.argv) > 1 else BUILD_NUMBER
    print(f"Fetching build {build_number}...", flush=True)
    text = fetch_log(build_number)
    results = parse_log(text)
    print(f"Found {len(results)} failed tests.\n")
    print_table(results)


if __name__ == "__main__":
    main()