Skip to content

Instantly share code, notes, and snippets.

@DavidMetcalfe
Created July 16, 2026 20:32
Show Gist options
  • Select an option

  • Save DavidMetcalfe/a69cc57999648027efaacabe6ce247fb to your computer and use it in GitHub Desktop.

Select an option

Save DavidMetcalfe/a69cc57999648027efaacabe6ce247fb to your computer and use it in GitHub Desktop.
PR #51493 rework: pick_safe_default_model + parameterized tests
diff --git a/hermes_cli/models.py b/hermes_cli/models.py
index 69f958d..688558c 100644
--- a/hermes_cli/models.py
+++ b/hermes_cli/models.py
@@ -1263,6 +1263,62 @@ _PROVIDER_SILENT_DEFAULT_OVERRIDES: dict[str, str] = {
}
+def _is_anthropic_frontier_tier(model_id: Optional[str]) -> bool:
+ """Return True for Anthropic frontier-tier models (Opus + Fable).
+
+ Both tiers carry the priciest Anthropic pricing and are unsafe defaults
+ for a freshly-authenticated provider picker — paid users landing on a
+ frontier tier from one-click onboarding have no opportunity to opt out
+ before the choice pins their main model and inherits into every cron
+ job that doesn't override ``model.provider``.
+
+ Anchored on the ``claude-`` family prefix and the ``opus-`` or
+ ``fable-`` tier substring (after vendor-strip and lowercase
+ normalization) so the predicate survives future Anthropic releases
+ (``claude-opus-5-0``, ``claude-fable-6``, etc.) and rejects community
+ / distill models whose slug merely *contains* the substring ``opus``
+ (e.g. ``qwopus3.6-27b-coder``). Sonnet and Haiku are deliberately NOT
+ classified as frontier here — they are reasonable defaults.
+ """
+ raw = _strip_vendor_prefix(str(model_id or ""))
+ base = raw.split(":")[0].lower()
+ if not base.startswith("claude-"):
+ return False
+ return ("opus-" in base) or ("fable-" in base)
+
+
+def pick_safe_default_model(model_ids: list[str], provider: Optional[str]) -> str:
+ """Pick a cost-safe default model from a (Portal-augmented) model list.
+
+ Implements the shared explicit policy for silent/onboarding defaults:
+
+ 1. If ``provider`` has an override in
+ :data:`_PROVIDER_SILENT_DEFAULT_OVERRIDES` AND that override is
+ present in ``model_ids``, return the override.
+ 2. Else, return the first entry in ``model_ids`` that is NOT an
+ Anthropic frontier tier (Opus / Fable). This skips the priciest
+ flagships while preserving curated order so a freshly-released
+ cheaper model still wins over a leftover Sonnet.
+ 3. Else (every entry is a frontier tier), return ``model_ids[0]`` so
+ the picker is never empty.
+ 4. Else (empty list), return ``""``.
+
+ Replaces the prior inline Opus-filter in
+ ``get_recommended_default_model`` so the cost-safe intent lives in one
+ place alongside :func:`get_default_model_for_provider` (which already
+ applies the same override to the *non-interactive* silent default).
+ """
+ if not model_ids:
+ return ""
+ override = _PROVIDER_SILENT_DEFAULT_OVERRIDES.get(provider or "")
+ if override and override in model_ids:
+ return override
+ non_frontier = [mid for mid in model_ids if not _is_anthropic_frontier_tier(mid)]
+ if non_frontier:
+ return non_frontier[0]
+ return model_ids[0]
+
+
def get_default_model_for_provider(provider: str) -> str:
"""Return a cost-safe default model for a provider, or "" if unknown.
@@ -2061,29 +2117,6 @@ def _is_anthropic_fast_model(model_id: Optional[str]) -> bool:
return "opus-4-6" in base or "opus-4.6" in base
-def _is_anthropic_opus_tier(model_id: Optional[str]) -> bool:
- """Return True if ``model_id`` is an Anthropic Opus-tier model.
-
- Opus is the most expensive Anthropic tier (~$15/$75 per MTok on Opus 4.x)
- and is therefore an unsafe default for a freshly-authenticated provider
- picker — paid users landing on Opus from a one-click onboarding have
- no opportunity to opt out before the choice pins their main model and
- inherits into every cron job that doesn't override ``model.provider``.
-
- Anchored on the ``claude-opus-`` prefix (after vendor-strip and lowercase
- normalization) so the predicate survives future Anthropic releases
- (``claude-opus-5-0``, ``claude-opus-5``, etc. all match without code
- changes) and rejects community / distill models whose slug merely
- *contains* the substring ``opus`` (e.g. ``qwopus3.6-27b-coder``). Sonnet
- and Haiku are deliberately NOT classified as Opus here — Sonnet is the
- default the rest of the codebase treats as reasonable, and Haiku is
- cheap enough that it is not a billing hazard.
- """
- raw = _strip_vendor_prefix(str(model_id or ""))
- base = raw.split(":")[0].lower()
- return base.startswith("claude-opus-")
-
-
def resolve_fast_mode_overrides(model_id: Optional[str]) -> dict[str, Any] | None:
"""Return request_overrides for fast/priority mode, or None if unsupported.
diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py
index 5cb26c6..b9d4976 100644
--- a/hermes_cli/web_server.py
+++ b/hermes_cli/web_server.py
@@ -3714,21 +3714,19 @@ def get_recommended_default_model(provider: str = ""):
model_ids, pricing, portal_url
)
- # The curated Nous list currently puts ``anthropic/claude-opus-*`` at
- # position 0 because it's the strongest Anthropic offering on Nous
- # Portal, but Opus-tier is an unsafe one-click default — paid users
- # have no opt-out before the choice pins their main model + every
- # cron job that doesn't override ``model.provider`` (which is most
- # of them). Pick the first non-Opus entry as the recommended default
- # so onboarding lands on Sonnet / Haiku / a non-Anthropic model
- # instead; Opus is still in the picker for users who want it. Fall
- # back to the head of the list if every curated entry happens to be
- # Opus so the picker is never empty.
- from hermes_cli.models import _is_anthropic_opus_tier
-
- non_opus_defaults = [mid for mid in model_ids if not _is_anthropic_opus_tier(mid)]
- default_candidates = non_opus_defaults or model_ids
- model = default_candidates[0] if default_candidates else ""
+ # Use the shared cost-safe default policy from ``models.py`` so the
+ # interactive picker applies the same override mechanism as the
+ # non-interactive silent fallback (see ``_PROVIDER_SILENT_DEFAULT_
+ # OVERRIDES`` — added after the 863-request billing incident where
+ # a missing default escalated to Opus). The policy returns the
+ # per-provider override if present in the Portal-augmented list,
+ # else the first non-frontier (Opus / Fable) entry, else
+ # ``model_ids[0]`` as a defensive fallback so the picker is never
+ # empty. Independent of catalog ordering — works for both the
+ # current Opus-first ordering and a future Fable-first one.
+ from hermes_cli.models import pick_safe_default_model
+
+ model = pick_safe_default_model(model_ids, "nous")
return {"provider": "nous", "model": model, "free_tier": bool(free_tier)}
except Exception:
_log.exception("GET /api/model/recommended-default (nous) failed")
diff --git a/tests/cli/test_fast_command.py b/tests/cli/test_fast_command.py
index 7e040b2..7745737 100644
--- a/tests/cli/test_fast_command.py
+++ b/tests/cli/test_fast_command.py
@@ -354,90 +354,6 @@ class TestAnthropicFastMode(unittest.TestCase):
assert _is_anthropic_fast_model("gpt-5.4") is False
assert _is_anthropic_fast_model("") is False
-
-class TestIsAnthropicOpusTier:
- """Unit tests for the ``_is_anthropic_opus_tier`` helper that gates the
- Nous recommended-default filter. Covers Nous (dash) and OpenRouter /
- native Anthropic (dot) slug variants and the non-Opus Claude families
- (Sonnet, Haiku) that must NOT be filtered out."""
-
- def test_matches_opus_in_every_supported_slug_form(self):
- from hermes_cli.models import _is_anthropic_opus_tier
-
- # Nous / dash form
- assert _is_anthropic_opus_tier("claude-opus-4-8") is True
- assert _is_anthropic_opus_tier("claude-opus-4-7") is True
- assert _is_anthropic_opus_tier("claude-opus-4-6") is True
- assert _is_anthropic_opus_tier("claude-opus-3-5") is True
-
- # OpenRouter / dot form
- assert _is_anthropic_opus_tier("claude-opus-4.8") is True
- assert _is_anthropic_opus_tier("claude-opus-4.7") is True
- assert _is_anthropic_opus_tier("claude-opus-4.6") is True
-
- # Vendor prefix + colon-suffixed variants
- assert _is_anthropic_opus_tier("anthropic/claude-opus-4.8") is True
- assert _is_anthropic_opus_tier("anthropic/claude-opus-4-8") is True
- assert _is_anthropic_opus_tier("anthropic/claude-opus-4.8:thinking") is True
-
- def test_does_not_match_non_opus_claude_families(self):
- from hermes_cli.models import _is_anthropic_opus_tier
-
- # Sonnet, Haiku — cheap tiers that must remain as default candidates.
- assert _is_anthropic_opus_tier("claude-sonnet-4-6") is False
- assert _is_anthropic_opus_tier("claude-sonnet-4.6") is False
- assert _is_anthropic_opus_tier("claude-haiku-4-5") is False
- assert _is_anthropic_opus_tier("claude-haiku-4.5") is False
- assert _is_anthropic_opus_tier("anthropic/claude-sonnet-4-6") is False
-
- def test_does_not_match_non_anthropic_models(self):
- from hermes_cli.models import _is_anthropic_opus_tier
-
- assert _is_anthropic_opus_tier("openai/gpt-5.5") is False
- assert _is_anthropic_opus_tier("google/gemini-3-pro-preview") is False
- assert _is_anthropic_opus_tier("minimax/minimax-m3") is False
- assert _is_anthropic_opus_tier("") is False
- assert _is_anthropic_opus_tier(None) is False
-
- def test_is_case_insensitive(self):
- from hermes_cli.models import _is_anthropic_opus_tier
-
- assert _is_anthropic_opus_tier("CLAUDE-OPUS-4-8") is True
- assert _is_anthropic_opus_tier("Anthropic/Claude-Opus-4.8") is True
-
- def test_forward_compat_and_community_distill(self):
- """Regression tests for the failure modes the prefix anchor prevents.
-
- Three forward-compat cases pin the behavior across Anthropic slug
- variants and the next Opus generation — the substring-based
- implementation would have silently regressed on ``claude-opus-5``
- because it only matched ``opus-3`` / ``opus-4`` substrings, and
- the same substring implementation matched community / distill
- slugs whose lowercase form merely *contains* the bytes ``opus``
- (e.g. Jackrong's ``Qwopus3.6`` family — Qwen + Opus-reasoning
- distill, not Anthropic pricing).
- """
- from hermes_cli.models import _is_anthropic_opus_tier
-
- # Forward-compat: shipping + hypothetical future Anthropic Opus.
- # claude-opus-4-9 is the next shipping family on the dash slug;
- # claude-opus-4.9 is the same family on the dot slug (OpenRouter /
- # native Anthropic); claude-opus-5-0 is the next-generation prefix
- # the substring implementation would have rejected.
- assert _is_anthropic_opus_tier("claude-opus-4-9") is True
- assert _is_anthropic_opus_tier("claude-opus-4.9") is True
- assert _is_anthropic_opus_tier("claude-opus-5-0") is True
- assert _is_anthropic_opus_tier("anthropic/claude-opus-5-0") is True
-
- # Negative regression: community / distill slugs that lowercased
- # merely contain the substring ``opus`` and must NOT be classified
- # as Anthropic Opus-tier (the substring implementation matched
- # these as a side effect of ``opus-3.`` falling inside ``qwopus3.``).
- assert _is_anthropic_opus_tier("qwopus3.6-27b-coder") is False
- assert _is_anthropic_opus_tier("jackrong/qwopus3.6-27b-coder") is False
- assert _is_anthropic_opus_tier("someorg/opus-4-clone") is False
- assert _is_anthropic_opus_tier("opus-4") is False
-
def test_fast_command_exposed_for_anthropic_model(self):
cli_mod = _import_cli()
stub = SimpleNamespace(
diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py
index a73666a..2835e5e 100644
--- a/tests/hermes_cli/test_web_server.py
+++ b/tests/hermes_cli/test_web_server.py
@@ -2640,25 +2640,58 @@ class TestWebServerEndpoints:
assert data["model"] == "top/model"
assert data["free_tier"] is False
- def test_recommended_default_nous_paid_skips_opus_tier(self, monkeypatch):
- """A paid Nous user must not be defaulted onto an Opus-tier model.
-
- Regression for the one-click onboarding foot-gun where the curated
- Nous list led with ``anthropic/claude-opus-*`` (Opus = ~$15/$75 per
- MTok), pinning the most expensive Anthropic offering as the user's
- main model with no opt-out. Sonnet, Haiku, and non-Anthropic models
- are all acceptable defaults; only Opus is filtered out.
+ @pytest.mark.parametrize(
+ "model_ordering,expected_model",
+ [
+ # Current Opus-first ordering (PR-branch state): skip Opus.
+ (
+ [
+ "anthropic/claude-opus-4.8",
+ "anthropic/claude-sonnet-4.6",
+ "openai/gpt-5.5",
+ ],
+ "anthropic/claude-sonnet-4.6",
+ ),
+ # Future Fable-first ordering (current main state): skip BOTH
+ # Fable and Opus. The cost-safe policy must skip every frontier
+ # tier regardless of relative ordering.
+ (
+ [
+ "anthropic/claude-fable-5",
+ "anthropic/claude-opus-4.8",
+ "anthropic/claude-sonnet-5",
+ "anthropic/claude-haiku-4.5",
+ ],
+ "anthropic/claude-sonnet-5",
+ ),
+ # Override beats ordering: if the per-provider cost-safe
+ # override is in the (Portal-augmented) list, it wins outright.
+ (
+ [
+ "anthropic/claude-opus-4.8",
+ "deepseek/deepseek-v4-flash",
+ "anthropic/claude-sonnet-4.6",
+ ],
+ "deepseek/deepseek-v4-flash",
+ ),
+ ],
+ ids=["opus_first_pr_branch", "fable_first_main", "override_beats_ordering"],
+ )
+ def test_recommended_default_nous_paid_cost_safe_policy(
+ self, monkeypatch, model_ordering, expected_model,
+ ):
+ """Regression for Teknium's automated review feedback on PR #51493.
+
+ The picker must apply the cost-safe default policy
+ (``pick_safe_default_model``) for both the current Opus-first and a
+ future Fable-first catalog ordering. The override
+ (``deepseek/deepseek-v4-flash`` for Nous) wins outright when present
+ in the Portal-augmented list — independent of relative ordering.
"""
import hermes_cli.models as models_mod
monkeypatch.setattr(
- models_mod, "get_curated_nous_model_ids",
- lambda: [
- "anthropic/claude-opus-4.8",
- "anthropic/claude-sonnet-4.6",
- "openai/gpt-5.5",
- "minimax/minimax-m3",
- ],
+ models_mod, "get_curated_nous_model_ids", lambda: list(model_ordering),
)
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda provider: {})
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda *, force_fresh=False: False)
@@ -2671,21 +2704,19 @@ class TestWebServerEndpoints:
assert resp.status_code == 200
data = resp.json()
assert data["provider"] == "nous"
- assert data["model"] == "anthropic/claude-sonnet-4.6", (
- "Opus-tier must be skipped; Sonnet is the next curated entry and the "
- "expected non-Opus default for a paid Nous user."
- )
+ assert data["model"] == expected_model
assert data["free_tier"] is False
- def test_recommended_default_nous_paid_falls_back_when_all_opus(self, monkeypatch):
- """If every curated entry is Opus, fall back to the head of the list
- so the picker is never empty (defensive — curated list currently
- always has non-Opus entries, but the contract is non-empty)."""
+ def test_recommended_default_nous_paid_falls_back_when_all_frontier(self, monkeypatch):
+ """If every curated entry is a frontier tier (Opus / Fable), fall
+ back to the head of the list so the picker is never empty. Defensive
+ — curated list currently always has non-frontier entries, but the
+ contract is non-empty."""
import hermes_cli.models as models_mod
monkeypatch.setattr(
models_mod, "get_curated_nous_model_ids",
- lambda: ["anthropic/claude-opus-4.8", "anthropic/claude-opus-4.7"],
+ lambda: ["anthropic/claude-opus-4.8", "anthropic/claude-fable-5"],
)
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda provider: {})
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda *, force_fresh=False: False)
@@ -2699,6 +2730,17 @@ class TestWebServerEndpoints:
data = resp.json()
assert data["model"] == "anthropic/claude-opus-4.8"
+ def test_pick_safe_default_model_empty_list_returns_empty_string(self):
+ """Empty model list must return ``""`` per the helper contract — the
+ caller degrades gracefully when there's nothing to recommend.
+ Regression test pinning the empty-list branch in
+ ``pick_safe_default_model``."""
+ from hermes_cli.models import pick_safe_default_model
+
+ assert pick_safe_default_model([], "nous") == ""
+ assert pick_safe_default_model([], "") == ""
+ assert pick_safe_default_model([], "openrouter") == ""
+
def test_recommended_default_handles_failure_gracefully(self, monkeypatch):
"""Endpoint never 500s — returns empty model on internal error."""
import hermes_cli.models as models_mod
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment