selfhosted_runners_watchdog.py

selfhosted_runners_watchdog.py

#!/usr/bin/env python3
"""
Watchdog for persistent self-hosted GitHub Actions runners.

Motivation
----------
This script exists for a failure mode that is easy to miss if you only look at
the local process supervisor. On one self-hosted runner host, GitHub showed
individual runners as "offline" even though the corresponding systemd services
were still active and the Runner.Listener processes were still alive.

In the observed cases, systemd reported healthy services like:

    actions.runner.<org>.<runner-name>.service: active (running)

The runner processes also still had established TLS connections to GitHub
Actions broker endpoints. But the GitHub Actions runner API reported those same
runners as:

    status=offline, busy=false

The local runner diagnostic logs showed the listener getting stuck around broker
or token handling after accepting a runner request. Because the process did not
exit, systemd's normal Restart=on-failure policy could not help. From systemd's
point of view, nothing had failed.

That made the GitHub API the only reliable health signal. If GitHub says a
runner is offline while the local service is still around, the safest recovery
action is to restart just that runner's systemd unit and let it reconnect.

What this script does
---------------------
1. Queries the GitHub Actions runner API for an organization.
2. Filters for the expected local runner names.
3. Logs the current GitHub-visible status for each runner.
4. If a runner is not online, captures recent journal output for that runner.
5. Restarts only that runner's systemd service.
6. Applies a per-runner cooldown so a flapping runner does not get restarted on
   every watchdog tick.
7. Supports --dry-run so the detection path can be tested without restarting
   anything.

This is intended to run from a systemd timer as root, or from another service
account that has narrowly scoped permission to restart the runner units.

Prerequisites
-------------
- The GitHub CLI (`gh`) must be installed and authenticated.
- The token used by `gh` must be able to read organization Actions runners.
- The local runner services must follow the GitHub runner service naming scheme:
  actions.runner.<org>.<runner-name>.service
"""

import argparse
import json
import logging
import subprocess
import time
from pathlib import Path


ORG = "wonderlydotcom"
RUNNER_PREFIX = "omarchy3-internal-tools-"
RUNNER_NUMBERS = range(1, 10)
COOLDOWN_SECONDS = 10 * 60
STATE_FILE = Path("/var/lib/github-runner-watchdog/state.json")
LOG_DIR = Path("/var/log/github-runner-watchdog")


def run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
    """Run a command and return the completed process with captured output."""
    return subprocess.run(cmd, text=True, capture_output=True, check=check)


def load_state(state_file: Path) -> dict:
    """Load cooldown state. Invalid state should not prevent recovery."""
    try:
        return json.loads(state_file.read_text())
    except FileNotFoundError:
        return {}
    except json.JSONDecodeError:
        logging.warning("State file is invalid JSON; starting with empty state")
        return {}


def save_state(state_file: Path, state: dict) -> None:
    """Persist the last restart time per runner."""
    state_file.parent.mkdir(parents=True, exist_ok=True)
    state_file.write_text(json.dumps(state, indent=2, sort_keys=True))


def expected_runner_names() -> list[str]:
    return [f"{RUNNER_PREFIX}{number}" for number in RUNNER_NUMBERS]


def fetch_github_runners() -> dict[str, dict]:
    """Fetch all matching org runners, handling gh's paginated JSON output."""
    result = run(["gh", "api", f"/orgs/{ORG}/actions/runners", "--paginate", "--slurp"])
    pages = json.loads(result.stdout)
    runners = {}

    for page in pages:
        for runner in page.get("runners", []):
            name = runner.get("name", "")
            if name.startswith(RUNNER_PREFIX):
                runners[name] = runner

    return runners


def service_name(runner_name: str) -> str:
    return f"actions.runner.{ORG}.{runner_name}.service"


def capture_recent_journal(runner_name: str, log_dir: Path) -> None:
    """Capture context before restarting so the evidence is not lost."""
    log_dir.mkdir(parents=True, exist_ok=True)
    svc = service_name(runner_name)
    log_path = log_dir / f"{runner_name}.journal.log"

    result = subprocess.run(
        ["journalctl", "-u", svc, "-n", "120", "--no-pager", "-o", "short-iso"],
        text=True,
        capture_output=True,
    )

    with log_path.open("a") as file:
        file.write(f"\n===== {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} =====\n")
        file.write(result.stdout)
        if result.stderr:
            file.write("\n[stderr]\n")
            file.write(result.stderr)


def restart_runner(runner_name: str, log_dir: Path, dry_run: bool) -> bool:
    """Restart one runner service, or report what would happen in dry-run mode."""
    svc = service_name(runner_name)

    if dry_run:
        logging.warning("DRY RUN: would capture journal and restart %s", svc)
        return True

    logging.warning("Restarting %s", svc)
    capture_recent_journal(runner_name, log_dir)

    result = subprocess.run(["systemctl", "restart", svc], text=True, capture_output=True)
    if result.returncode != 0:
        logging.error("Failed to restart %s: %s", svc, result.stderr.strip())
        return False

    return True


def should_restart(state: dict, runner_name: str, now: int, cooldown_seconds: int) -> bool:
    """Return false if this runner was restarted recently."""
    last_restart = int(state.get(runner_name, {}).get("last_restart", 0))
    return now - last_restart >= cooldown_seconds


def update_restart_state(state: dict, runner_name: str, status: str, now: int) -> None:
    state[runner_name] = {"last_restart": now, "last_status": status}


def verify_runner_online(runner_name: str) -> None:
    """Re-query GitHub after restart and log the result."""
    time.sleep(10)
    runners = fetch_github_runners()
    runner = runners.get(runner_name)

    if not runner:
        logging.error("%s missing from GitHub after restart", runner_name)
        return

    logging.info(
        "%s after restart: status=%s busy=%s",
        runner_name,
        runner.get("status"),
        runner.get("busy"),
    )


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Restart wedged local GitHub Actions runners.")
    parser.add_argument("--dry-run", action="store_true", help="Log actions without restarting services.")
    parser.add_argument("--state-file", type=Path, default=STATE_FILE)
    parser.add_argument("--log-dir", type=Path, default=LOG_DIR)
    parser.add_argument("--cooldown-seconds", type=int, default=COOLDOWN_SECONDS)
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")

    now = int(time.time())
    state = load_state(args.state_file)

    try:
        runners = fetch_github_runners()
    except Exception:
        logging.exception("Failed to fetch GitHub runners")
        return 2

    changed = False
    for runner_name in expected_runner_names():
        runner = runners.get(runner_name)
        if runner is None:
            logging.error("%s is missing from GitHub API results", runner_name)
            continue

        status = runner.get("status")
        busy = runner.get("busy")
        logging.info("%s: status=%s busy=%s", runner_name, status, busy)

        if status == "online":
            continue

        if not should_restart(state, runner_name, now, args.cooldown_seconds):
            logging.warning("%s is %s but restart cooldown is active", runner_name, status)
            continue

        if restart_runner(runner_name, args.log_dir, args.dry_run):
            update_restart_state(state, runner_name, status, now)
            changed = True
            if not args.dry_run:
                verify_runner_online(runner_name)

    if changed and not args.dry_run:
        save_state(args.state_file, state)

    return 0


if __name__ == "__main__":
    raise SystemExit(main())