Created
July 16, 2026 20:32
-
-
Save DavidMetcalfe/a69cc57999648027efaacabe6ce247fb to your computer and use it in GitHub Desktop.
PR #51493 rework: pick_safe_default_model + parameterized tests
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/hermes_cli/models.py b/hermes_cli/models.py | |
| index 69f958d..688558c 100644 | |
| --- a/hermes_cli/models.py | |
| +++ b/hermes_cli/models.py | |
| @@ -1263,6 +1263,62 @@ _PROVIDER_SILENT_DEFAULT_OVERRIDES: dict[str, str] = { | |
| } | |
| +def _is_anthropic_frontier_tier(model_id: Optional[str]) -> bool: | |
| + """Return True for Anthropic frontier-tier models (Opus + Fable). | |
| + | |
| + Both tiers carry the priciest Anthropic pricing and are unsafe defaults | |
| + for a freshly-authenticated provider picker — paid users landing on a | |
| + frontier tier from one-click onboarding have no opportunity to opt out | |
| + before the choice pins their main model and inherits into every cron | |
| + job that doesn't override ``model.provider``. | |
| + | |
| + Anchored on the ``claude-`` family prefix and the ``opus-`` or | |
| + ``fable-`` tier substring (after vendor-strip and lowercase | |
| + normalization) so the predicate survives future Anthropic releases | |
| + (``claude-opus-5-0``, ``claude-fable-6``, etc.) and rejects community | |
| + / distill models whose slug merely *contains* the substring ``opus`` | |
| + (e.g. ``qwopus3.6-27b-coder``). Sonnet and Haiku are deliberately NOT | |
| + classified as frontier here — they are reasonable defaults. | |
| + """ | |
| + raw = _strip_vendor_prefix(str(model_id or "")) | |
| + base = raw.split(":")[0].lower() | |
| + if not base.startswith("claude-"): | |
| + return False | |
| + return ("opus-" in base) or ("fable-" in base) | |
| + | |
| + | |
| +def pick_safe_default_model(model_ids: list[str], provider: Optional[str]) -> str: | |
| + """Pick a cost-safe default model from a (Portal-augmented) model list. | |
| + | |
| + Implements the shared explicit policy for silent/onboarding defaults: | |
| + | |
| + 1. If ``provider`` has an override in | |
| + :data:`_PROVIDER_SILENT_DEFAULT_OVERRIDES` AND that override is | |
| + present in ``model_ids``, return the override. | |
| + 2. Else, return the first entry in ``model_ids`` that is NOT an | |
| + Anthropic frontier tier (Opus / Fable). This skips the priciest | |
| + flagships while preserving curated order so a freshly-released | |
| + cheaper model still wins over a leftover Sonnet. | |
| + 3. Else (every entry is a frontier tier), return ``model_ids[0]`` so | |
| + the picker is never empty. | |
| + 4. Else (empty list), return ``""``. | |
| + | |
| + Replaces the prior inline Opus-filter in | |
| + ``get_recommended_default_model`` so the cost-safe intent lives in one | |
| + place alongside :func:`get_default_model_for_provider` (which already | |
| + applies the same override to the *non-interactive* silent default). | |
| + """ | |
| + if not model_ids: | |
| + return "" | |
| + override = _PROVIDER_SILENT_DEFAULT_OVERRIDES.get(provider or "") | |
| + if override and override in model_ids: | |
| + return override | |
| + non_frontier = [mid for mid in model_ids if not _is_anthropic_frontier_tier(mid)] | |
| + if non_frontier: | |
| + return non_frontier[0] | |
| + return model_ids[0] | |
| + | |
| + | |
| def get_default_model_for_provider(provider: str) -> str: | |
| """Return a cost-safe default model for a provider, or "" if unknown. | |
| @@ -2061,29 +2117,6 @@ def _is_anthropic_fast_model(model_id: Optional[str]) -> bool: | |
| return "opus-4-6" in base or "opus-4.6" in base | |
| -def _is_anthropic_opus_tier(model_id: Optional[str]) -> bool: | |
| - """Return True if ``model_id`` is an Anthropic Opus-tier model. | |
| - | |
| - Opus is the most expensive Anthropic tier (~$15/$75 per MTok on Opus 4.x) | |
| - and is therefore an unsafe default for a freshly-authenticated provider | |
| - picker — paid users landing on Opus from a one-click onboarding have | |
| - no opportunity to opt out before the choice pins their main model and | |
| - inherits into every cron job that doesn't override ``model.provider``. | |
| - | |
| - Anchored on the ``claude-opus-`` prefix (after vendor-strip and lowercase | |
| - normalization) so the predicate survives future Anthropic releases | |
| - (``claude-opus-5-0``, ``claude-opus-5``, etc. all match without code | |
| - changes) and rejects community / distill models whose slug merely | |
| - *contains* the substring ``opus`` (e.g. ``qwopus3.6-27b-coder``). Sonnet | |
| - and Haiku are deliberately NOT classified as Opus here — Sonnet is the | |
| - default the rest of the codebase treats as reasonable, and Haiku is | |
| - cheap enough that it is not a billing hazard. | |
| - """ | |
| - raw = _strip_vendor_prefix(str(model_id or "")) | |
| - base = raw.split(":")[0].lower() | |
| - return base.startswith("claude-opus-") | |
| - | |
| - | |
| def resolve_fast_mode_overrides(model_id: Optional[str]) -> dict[str, Any] | None: | |
| """Return request_overrides for fast/priority mode, or None if unsupported. | |
| diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py | |
| index 5cb26c6..b9d4976 100644 | |
| --- a/hermes_cli/web_server.py | |
| +++ b/hermes_cli/web_server.py | |
| @@ -3714,21 +3714,19 @@ def get_recommended_default_model(provider: str = ""): | |
| model_ids, pricing, portal_url | |
| ) | |
| - # The curated Nous list currently puts ``anthropic/claude-opus-*`` at | |
| - # position 0 because it's the strongest Anthropic offering on Nous | |
| - # Portal, but Opus-tier is an unsafe one-click default — paid users | |
| - # have no opt-out before the choice pins their main model + every | |
| - # cron job that doesn't override ``model.provider`` (which is most | |
| - # of them). Pick the first non-Opus entry as the recommended default | |
| - # so onboarding lands on Sonnet / Haiku / a non-Anthropic model | |
| - # instead; Opus is still in the picker for users who want it. Fall | |
| - # back to the head of the list if every curated entry happens to be | |
| - # Opus so the picker is never empty. | |
| - from hermes_cli.models import _is_anthropic_opus_tier | |
| - | |
| - non_opus_defaults = [mid for mid in model_ids if not _is_anthropic_opus_tier(mid)] | |
| - default_candidates = non_opus_defaults or model_ids | |
| - model = default_candidates[0] if default_candidates else "" | |
| + # Use the shared cost-safe default policy from ``models.py`` so the | |
| + # interactive picker applies the same override mechanism as the | |
| + # non-interactive silent fallback (see ``_PROVIDER_SILENT_DEFAULT_ | |
| + # OVERRIDES`` — added after the 863-request billing incident where | |
| + # a missing default escalated to Opus). The policy returns the | |
| + # per-provider override if present in the Portal-augmented list, | |
| + # else the first non-frontier (Opus / Fable) entry, else | |
| + # ``model_ids[0]`` as a defensive fallback so the picker is never | |
| + # empty. Independent of catalog ordering — works for both the | |
| + # current Opus-first ordering and a future Fable-first one. | |
| + from hermes_cli.models import pick_safe_default_model | |
| + | |
| + model = pick_safe_default_model(model_ids, "nous") | |
| return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} | |
| except Exception: | |
| _log.exception("GET /api/model/recommended-default (nous) failed") | |
| diff --git a/tests/cli/test_fast_command.py b/tests/cli/test_fast_command.py | |
| index 7e040b2..7745737 100644 | |
| --- a/tests/cli/test_fast_command.py | |
| +++ b/tests/cli/test_fast_command.py | |
| @@ -354,90 +354,6 @@ class TestAnthropicFastMode(unittest.TestCase): | |
| assert _is_anthropic_fast_model("gpt-5.4") is False | |
| assert _is_anthropic_fast_model("") is False | |
| - | |
| -class TestIsAnthropicOpusTier: | |
| - """Unit tests for the ``_is_anthropic_opus_tier`` helper that gates the | |
| - Nous recommended-default filter. Covers Nous (dash) and OpenRouter / | |
| - native Anthropic (dot) slug variants and the non-Opus Claude families | |
| - (Sonnet, Haiku) that must NOT be filtered out.""" | |
| - | |
| - def test_matches_opus_in_every_supported_slug_form(self): | |
| - from hermes_cli.models import _is_anthropic_opus_tier | |
| - | |
| - # Nous / dash form | |
| - assert _is_anthropic_opus_tier("claude-opus-4-8") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-4-7") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-4-6") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-3-5") is True | |
| - | |
| - # OpenRouter / dot form | |
| - assert _is_anthropic_opus_tier("claude-opus-4.8") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-4.7") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-4.6") is True | |
| - | |
| - # Vendor prefix + colon-suffixed variants | |
| - assert _is_anthropic_opus_tier("anthropic/claude-opus-4.8") is True | |
| - assert _is_anthropic_opus_tier("anthropic/claude-opus-4-8") is True | |
| - assert _is_anthropic_opus_tier("anthropic/claude-opus-4.8:thinking") is True | |
| - | |
| - def test_does_not_match_non_opus_claude_families(self): | |
| - from hermes_cli.models import _is_anthropic_opus_tier | |
| - | |
| - # Sonnet, Haiku — cheap tiers that must remain as default candidates. | |
| - assert _is_anthropic_opus_tier("claude-sonnet-4-6") is False | |
| - assert _is_anthropic_opus_tier("claude-sonnet-4.6") is False | |
| - assert _is_anthropic_opus_tier("claude-haiku-4-5") is False | |
| - assert _is_anthropic_opus_tier("claude-haiku-4.5") is False | |
| - assert _is_anthropic_opus_tier("anthropic/claude-sonnet-4-6") is False | |
| - | |
| - def test_does_not_match_non_anthropic_models(self): | |
| - from hermes_cli.models import _is_anthropic_opus_tier | |
| - | |
| - assert _is_anthropic_opus_tier("openai/gpt-5.5") is False | |
| - assert _is_anthropic_opus_tier("google/gemini-3-pro-preview") is False | |
| - assert _is_anthropic_opus_tier("minimax/minimax-m3") is False | |
| - assert _is_anthropic_opus_tier("") is False | |
| - assert _is_anthropic_opus_tier(None) is False | |
| - | |
| - def test_is_case_insensitive(self): | |
| - from hermes_cli.models import _is_anthropic_opus_tier | |
| - | |
| - assert _is_anthropic_opus_tier("CLAUDE-OPUS-4-8") is True | |
| - assert _is_anthropic_opus_tier("Anthropic/Claude-Opus-4.8") is True | |
| - | |
| - def test_forward_compat_and_community_distill(self): | |
| - """Regression tests for the failure modes the prefix anchor prevents. | |
| - | |
| - Three forward-compat cases pin the behavior across Anthropic slug | |
| - variants and the next Opus generation — the substring-based | |
| - implementation would have silently regressed on ``claude-opus-5`` | |
| - because it only matched ``opus-3`` / ``opus-4`` substrings, and | |
| - the same substring implementation matched community / distill | |
| - slugs whose lowercase form merely *contains* the bytes ``opus`` | |
| - (e.g. Jackrong's ``Qwopus3.6`` family — Qwen + Opus-reasoning | |
| - distill, not Anthropic pricing). | |
| - """ | |
| - from hermes_cli.models import _is_anthropic_opus_tier | |
| - | |
| - # Forward-compat: shipping + hypothetical future Anthropic Opus. | |
| - # claude-opus-4-9 is the next shipping family on the dash slug; | |
| - # claude-opus-4.9 is the same family on the dot slug (OpenRouter / | |
| - # native Anthropic); claude-opus-5-0 is the next-generation prefix | |
| - # the substring implementation would have rejected. | |
| - assert _is_anthropic_opus_tier("claude-opus-4-9") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-4.9") is True | |
| - assert _is_anthropic_opus_tier("claude-opus-5-0") is True | |
| - assert _is_anthropic_opus_tier("anthropic/claude-opus-5-0") is True | |
| - | |
| - # Negative regression: community / distill slugs that lowercased | |
| - # merely contain the substring ``opus`` and must NOT be classified | |
| - # as Anthropic Opus-tier (the substring implementation matched | |
| - # these as a side effect of ``opus-3.`` falling inside ``qwopus3.``). | |
| - assert _is_anthropic_opus_tier("qwopus3.6-27b-coder") is False | |
| - assert _is_anthropic_opus_tier("jackrong/qwopus3.6-27b-coder") is False | |
| - assert _is_anthropic_opus_tier("someorg/opus-4-clone") is False | |
| - assert _is_anthropic_opus_tier("opus-4") is False | |
| - | |
| def test_fast_command_exposed_for_anthropic_model(self): | |
| cli_mod = _import_cli() | |
| stub = SimpleNamespace( | |
| diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py | |
| index a73666a..2835e5e 100644 | |
| --- a/tests/hermes_cli/test_web_server.py | |
| +++ b/tests/hermes_cli/test_web_server.py | |
| @@ -2640,25 +2640,58 @@ class TestWebServerEndpoints: | |
| assert data["model"] == "top/model" | |
| assert data["free_tier"] is False | |
| - def test_recommended_default_nous_paid_skips_opus_tier(self, monkeypatch): | |
| - """A paid Nous user must not be defaulted onto an Opus-tier model. | |
| - | |
| - Regression for the one-click onboarding foot-gun where the curated | |
| - Nous list led with ``anthropic/claude-opus-*`` (Opus = ~$15/$75 per | |
| - MTok), pinning the most expensive Anthropic offering as the user's | |
| - main model with no opt-out. Sonnet, Haiku, and non-Anthropic models | |
| - are all acceptable defaults; only Opus is filtered out. | |
| + @pytest.mark.parametrize( | |
| + "model_ordering,expected_model", | |
| + [ | |
| + # Current Opus-first ordering (PR-branch state): skip Opus. | |
| + ( | |
| + [ | |
| + "anthropic/claude-opus-4.8", | |
| + "anthropic/claude-sonnet-4.6", | |
| + "openai/gpt-5.5", | |
| + ], | |
| + "anthropic/claude-sonnet-4.6", | |
| + ), | |
| + # Future Fable-first ordering (current main state): skip BOTH | |
| + # Fable and Opus. The cost-safe policy must skip every frontier | |
| + # tier regardless of relative ordering. | |
| + ( | |
| + [ | |
| + "anthropic/claude-fable-5", | |
| + "anthropic/claude-opus-4.8", | |
| + "anthropic/claude-sonnet-5", | |
| + "anthropic/claude-haiku-4.5", | |
| + ], | |
| + "anthropic/claude-sonnet-5", | |
| + ), | |
| + # Override beats ordering: if the per-provider cost-safe | |
| + # override is in the (Portal-augmented) list, it wins outright. | |
| + ( | |
| + [ | |
| + "anthropic/claude-opus-4.8", | |
| + "deepseek/deepseek-v4-flash", | |
| + "anthropic/claude-sonnet-4.6", | |
| + ], | |
| + "deepseek/deepseek-v4-flash", | |
| + ), | |
| + ], | |
| + ids=["opus_first_pr_branch", "fable_first_main", "override_beats_ordering"], | |
| + ) | |
| + def test_recommended_default_nous_paid_cost_safe_policy( | |
| + self, monkeypatch, model_ordering, expected_model, | |
| + ): | |
| + """Regression for Teknium's automated review feedback on PR #51493. | |
| + | |
| + The picker must apply the cost-safe default policy | |
| + (``pick_safe_default_model``) for both the current Opus-first and a | |
| + future Fable-first catalog ordering. The override | |
| + (``deepseek/deepseek-v4-flash`` for Nous) wins outright when present | |
| + in the Portal-augmented list — independent of relative ordering. | |
| """ | |
| import hermes_cli.models as models_mod | |
| monkeypatch.setattr( | |
| - models_mod, "get_curated_nous_model_ids", | |
| - lambda: [ | |
| - "anthropic/claude-opus-4.8", | |
| - "anthropic/claude-sonnet-4.6", | |
| - "openai/gpt-5.5", | |
| - "minimax/minimax-m3", | |
| - ], | |
| + models_mod, "get_curated_nous_model_ids", lambda: list(model_ordering), | |
| ) | |
| monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda provider: {}) | |
| monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda *, force_fresh=False: False) | |
| @@ -2671,21 +2704,19 @@ class TestWebServerEndpoints: | |
| assert resp.status_code == 200 | |
| data = resp.json() | |
| assert data["provider"] == "nous" | |
| - assert data["model"] == "anthropic/claude-sonnet-4.6", ( | |
| - "Opus-tier must be skipped; Sonnet is the next curated entry and the " | |
| - "expected non-Opus default for a paid Nous user." | |
| - ) | |
| + assert data["model"] == expected_model | |
| assert data["free_tier"] is False | |
| - def test_recommended_default_nous_paid_falls_back_when_all_opus(self, monkeypatch): | |
| - """If every curated entry is Opus, fall back to the head of the list | |
| - so the picker is never empty (defensive — curated list currently | |
| - always has non-Opus entries, but the contract is non-empty).""" | |
| + def test_recommended_default_nous_paid_falls_back_when_all_frontier(self, monkeypatch): | |
| + """If every curated entry is a frontier tier (Opus / Fable), fall | |
| + back to the head of the list so the picker is never empty. Defensive | |
| + — curated list currently always has non-frontier entries, but the | |
| + contract is non-empty.""" | |
| import hermes_cli.models as models_mod | |
| monkeypatch.setattr( | |
| models_mod, "get_curated_nous_model_ids", | |
| - lambda: ["anthropic/claude-opus-4.8", "anthropic/claude-opus-4.7"], | |
| + lambda: ["anthropic/claude-opus-4.8", "anthropic/claude-fable-5"], | |
| ) | |
| monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda provider: {}) | |
| monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda *, force_fresh=False: False) | |
| @@ -2699,6 +2730,17 @@ class TestWebServerEndpoints: | |
| data = resp.json() | |
| assert data["model"] == "anthropic/claude-opus-4.8" | |
| + def test_pick_safe_default_model_empty_list_returns_empty_string(self): | |
| + """Empty model list must return ``""`` per the helper contract — the | |
| + caller degrades gracefully when there's nothing to recommend. | |
| + Regression test pinning the empty-list branch in | |
| + ``pick_safe_default_model``.""" | |
| + from hermes_cli.models import pick_safe_default_model | |
| + | |
| + assert pick_safe_default_model([], "nous") == "" | |
| + assert pick_safe_default_model([], "") == "" | |
| + assert pick_safe_default_model([], "openrouter") == "" | |
| + | |
| def test_recommended_default_handles_failure_gracefully(self, monkeypatch): | |
| """Endpoint never 500s — returns empty model on internal error.""" | |
| import hermes_cli.models as models_mod |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment