anhtnm1andClaude Opus 5
96bec976e7
EPIC R03 (Team Duy) - one provider catalogue, one routing flow, one usage seam.
R03-T01 tests/contracts/test_providers.py
29 contract tests every provider must satisfy: canonical assistant message,
streamed text == returned content, reasoning never joins the answer, parsed
tool arguments, ProviderError for every failure. Real adapters exercised
offline by stubbing Provider._request.
R03-T02 domain/models/provider_descriptor.py
infrastructure/providers/provider_registry.py
Provider facts declared once (was split across providers/factory.py,
DEFAULT_CONFIG and PROVIDER_LABELS). ProviderRegistry.build() also stamps the
descriptor id onto the instance, so ollama/github_copilot/codex usage is no
longer all attributed to "openai_compat", and never mutates the caller config.
R03-T03 application/model_routing/routing_application_service.py
Pure-Python routing policy with four modes: Off, Auto, Manual and the new
Fallback (switch only AFTER the current model fails). Depends on a RoutingPort
protocol; production wires the existing core.routing engine underneath.
R03-T04/T05 ui/chat_panel.py, ui/co4e_tab.py, ui/folder_tab.py
Three near-identical routing copies (~40 lines each) replaced by a call to
ctx.routing_application() plus a confirm callback. Mode vocabulary now lives
in one place (normalize_mode/is_valid_mode) instead of four literal tuples.
R03-T06 infrastructure/telemetry/usage_sink.py
Token usage extracted from both providers into UsageEvent + UsageEventSink.
Estimation pinned against core.usage_tracker so no recorded number changes.
Also fixes a deadlock introduced while wiring AppContext: routing_application()
held _routing_lock and called routing(), which takes the same non-reentrant lock.
Suite: 186 passed, 1.22s. check_imports: PASS. All new files < 400 LOC.
2 pre-existing failures remain in test_config_security.py (EPIC R02/Team Nam).
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
347 lines
13 KiB
Python
347 lines
13 KiB
Python
"""Unit tests for :mod:`application.model_routing` (R03-T03).
|
|
|
|
These run against a hand-written fake router rather than ``core.routing``: the
|
|
point of the service is the DECISION policy around the engine (mode handling,
|
|
the manual confirm, never-raise behaviour, failure fallback), and mixing in the
|
|
real scorer would test the wrong thing and drag the suite over its time budget.
|
|
|
|
No Qt, no config, no network - the whole file runs in milliseconds, which is the
|
|
concrete payoff of moving this logic out of ``ui/chat_panel.py``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any, List, Optional, Tuple
|
|
|
|
import pytest
|
|
|
|
from cowork_local.application.model_routing import (
|
|
RoutingApplicationService,
|
|
RoutingDecision,
|
|
RoutingMode,
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Test doubles shaped like core.routing's RouteResult / SwitchDecision
|
|
# --------------------------------------------------------------------------- #
|
|
@dataclass
|
|
class _TaskType:
|
|
value: str
|
|
|
|
|
|
@dataclass
|
|
class _Decision:
|
|
score_gain: float = 0.0
|
|
reason: str = ""
|
|
|
|
|
|
@dataclass
|
|
class _RouteResult:
|
|
should_switch: bool
|
|
to: Optional[Tuple[str, str]] = None
|
|
task_type: Any = None
|
|
decision: Any = None
|
|
|
|
def target(self) -> Optional[Tuple[str, str]]:
|
|
return self.to
|
|
|
|
|
|
class _FakeRouter:
|
|
"""Records every route() call and replays a canned result."""
|
|
|
|
def __init__(self, result: Any = None, raises: bool = False) -> None:
|
|
self._result = result or _RouteResult(should_switch=False, decision=_Decision())
|
|
self._raises = raises
|
|
self.calls: List[dict] = []
|
|
|
|
def route(self, surface, prompt, current_provider, current_model, **kwargs):
|
|
self.calls.append({"surface": surface, "prompt": prompt,
|
|
"provider": current_provider, "model": current_model, **kwargs})
|
|
if self._raises:
|
|
raise RuntimeError("assessment store is corrupt")
|
|
return self._result
|
|
|
|
|
|
def _switch_to(provider: str, model: str, gain: float = 0.2, task: str = "coding") -> _RouteResult:
|
|
return _RouteResult(
|
|
should_switch=True, to=(provider, model), task_type=_TaskType(task),
|
|
decision=_Decision(score_gain=gain, reason=f"{task} fit beats current by {gain}"),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Mode parsing
|
|
# --------------------------------------------------------------------------- #
|
|
@pytest.mark.parametrize("raw,expected", [
|
|
("off", RoutingMode.OFF),
|
|
("AUTO", RoutingMode.AUTO),
|
|
(" manual ", RoutingMode.MANUAL),
|
|
("fallback", RoutingMode.FALLBACK),
|
|
])
|
|
def test_parse_accepts_the_config_spellings(raw, expected):
|
|
assert RoutingMode.parse(raw) is expected
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["", None, "nonsense", 0])
|
|
def test_parse_degrades_unknown_values_to_off(raw):
|
|
"""A corrupt setting must leave the user's own model alone rather than
|
|
silently moving their work onto another model."""
|
|
assert RoutingMode.parse(raw) is RoutingMode.OFF
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# OFF
|
|
# --------------------------------------------------------------------------- #
|
|
def test_off_never_consults_the_engine():
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", "hi", "openai_compat", "gpt-4o-mini",
|
|
mode="off")
|
|
|
|
assert router.calls == [] # not even scored: OFF costs nothing
|
|
assert decision.switched is False
|
|
assert decision.target() == ("openai_compat", "gpt-4o-mini")
|
|
|
|
|
|
def test_blank_prompt_is_never_routed():
|
|
"""An empty message carries no signal to classify; all three legacy copies
|
|
guarded this and the guard has to survive the move."""
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", " ", "openai_compat", "m", mode="auto")
|
|
|
|
assert router.calls == []
|
|
assert decision.switched is False
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# AUTO
|
|
# --------------------------------------------------------------------------- #
|
|
def test_auto_switches_silently_and_reports_the_target():
|
|
router = _FakeRouter(_switch_to("anthropic", "claude-sonnet-4-6", gain=0.31))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", "write a function", "openai_compat", "gpt-4o-mini",
|
|
mode="auto")
|
|
|
|
assert decision.switched is True
|
|
assert decision.target() == ("anthropic", "claude-sonnet-4-6")
|
|
assert decision.task_type == "coding"
|
|
assert decision.score_gain == pytest.approx(0.31)
|
|
assert decision.should_notify is True
|
|
|
|
|
|
def test_auto_keeps_the_current_model_when_no_candidate_wins():
|
|
router = _FakeRouter(_RouteResult(should_switch=False, decision=_Decision(reason="no gain")))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", "hello", "openai_compat", "gpt-4o-mini", mode="auto")
|
|
|
|
assert decision.switched is False
|
|
# The decision still names a model to run on, so the call site never has to
|
|
# re-derive the fallback itself - the exact drift the three copies suffered.
|
|
assert decision.target() == ("openai_compat", "gpt-4o-mini")
|
|
assert decision.should_notify is False
|
|
|
|
|
|
def test_auto_never_asks_for_confirmation():
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
asked: List[RoutingDecision] = []
|
|
service = RoutingApplicationService(router)
|
|
|
|
service.route_turn("cowork", "q", "openai_compat", "m", mode="auto",
|
|
confirm=lambda d: asked.append(d) or True)
|
|
|
|
assert asked == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# MANUAL
|
|
# --------------------------------------------------------------------------- #
|
|
def test_manual_switches_only_after_the_user_approves():
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
seen: List[RoutingDecision] = []
|
|
|
|
def confirm(proposal: RoutingDecision) -> bool:
|
|
seen.append(proposal)
|
|
return True
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "m",
|
|
mode="manual", confirm=confirm)
|
|
|
|
assert decision.switched is True
|
|
assert decision.target() == ("anthropic", "claude")
|
|
# The dialog is handed the full proposal so it can explain the trade-off.
|
|
assert seen[0].model == "claude"
|
|
assert seen[0].score_gain > 0
|
|
|
|
|
|
def test_manual_keeps_the_current_model_when_declined():
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "gpt-4o-mini",
|
|
mode="manual", confirm=lambda d: False)
|
|
|
|
assert decision.switched is False
|
|
assert decision.declined is True
|
|
assert decision.target() == ("openai_compat", "gpt-4o-mini")
|
|
|
|
|
|
def test_manual_without_a_confirm_callback_does_not_switch():
|
|
"""A headless caller (scheduler) has nobody to ask, so Manual must behave as
|
|
"not approved" rather than as "approved by default"."""
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "m", mode="manual")
|
|
|
|
assert decision.switched is False
|
|
assert decision.declined is True
|
|
|
|
|
|
def test_a_confirm_dialog_that_raises_counts_as_declined():
|
|
"""If the modal blows up (window closing mid-turn) the safe reading is that
|
|
the user did NOT consent to running on another model."""
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
def confirm(_proposal):
|
|
raise RuntimeError("dialog destroyed")
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "m",
|
|
mode="manual", confirm=confirm)
|
|
|
|
assert decision.switched is False
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# FALLBACK
|
|
# --------------------------------------------------------------------------- #
|
|
def test_fallback_does_not_switch_up_front():
|
|
"""The whole point of the mode: honour the user's model choice until it
|
|
actually fails."""
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "gpt-4o-mini",
|
|
mode="fallback")
|
|
|
|
assert router.calls == []
|
|
assert decision.switched is False
|
|
assert decision.target() == ("openai_compat", "gpt-4o-mini")
|
|
|
|
|
|
def test_fallback_switches_after_a_failure():
|
|
router = _FakeRouter(_switch_to("anthropic", "claude", gain=0.4))
|
|
service = RoutingApplicationService(router)
|
|
|
|
decision = service.fallback_after_failure("cowork", "q", "openai_compat", "gpt-4o-mini",
|
|
mode="fallback")
|
|
|
|
assert decision is not None
|
|
assert decision.switched is True
|
|
assert decision.target() == ("anthropic", "claude")
|
|
assert "failed" in decision.reason
|
|
|
|
|
|
def test_fallback_never_returns_the_model_that_just_failed():
|
|
"""Retrying the model that just went down would spin on the outage."""
|
|
router = _FakeRouter(_switch_to("openai_compat", "gpt-4o-mini"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
assert service.fallback_after_failure(
|
|
"cowork", "q", "openai_compat", "gpt-4o-mini", mode="fallback") is None
|
|
|
|
|
|
def test_fallback_returns_none_when_there_is_no_alternative():
|
|
router = _FakeRouter(_RouteResult(should_switch=False, decision=_Decision()))
|
|
service = RoutingApplicationService(router)
|
|
|
|
assert service.fallback_after_failure("cowork", "q", "openai_compat", "m",
|
|
mode="auto") is None
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["off", "manual"])
|
|
def test_off_and_manual_do_not_auto_recover_from_a_failure(mode):
|
|
"""Both modes exist to keep the user in control of which model runs their
|
|
work; moving it on failure would break that promise silently."""
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
service = RoutingApplicationService(router)
|
|
|
|
assert service.fallback_after_failure("cowork", "q", "openai_compat", "m",
|
|
mode=mode) is None
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Robustness - routing must never break a chat turn
|
|
# --------------------------------------------------------------------------- #
|
|
def test_engine_failure_degrades_to_keeping_the_current_model():
|
|
service = RoutingApplicationService(_FakeRouter(raises=True))
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "gpt-4o-mini", mode="auto")
|
|
|
|
assert decision.switched is False
|
|
assert decision.target() == ("openai_compat", "gpt-4o-mini")
|
|
|
|
|
|
def test_engine_failure_during_fallback_returns_none():
|
|
"""A broken router must not mask the original provider error with its own."""
|
|
service = RoutingApplicationService(_FakeRouter(raises=True))
|
|
|
|
assert service.fallback_after_failure("cowork", "q", "p", "m", mode="auto") is None
|
|
|
|
|
|
def test_a_malformed_route_result_is_treated_as_no_switch():
|
|
"""The engine is a legacy module still under refactor; a missing attribute
|
|
must degrade, not raise into the middle of a turn."""
|
|
class _Garbage:
|
|
should_switch = True # claims a switch but exposes no target()
|
|
|
|
service = RoutingApplicationService(_FakeRouter(_Garbage()))
|
|
|
|
decision = service.route_turn("cowork", "q", "openai_compat", "m", mode="auto")
|
|
|
|
assert decision.switched is False
|
|
assert decision.target() == ("openai_compat", "m")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Per-surface mode lookup
|
|
# --------------------------------------------------------------------------- #
|
|
def test_mode_is_read_per_surface_when_not_passed_explicitly():
|
|
"""Each screen has its own Off/Auto/Manual toggle, and workspaces override
|
|
it - so the surface, not a global setting, decides."""
|
|
router = _FakeRouter(_switch_to("anthropic", "claude"))
|
|
modes = {"cowork": "auto", "ai_edit": "off"}
|
|
service = RoutingApplicationService(router, mode_reader=modes.get)
|
|
|
|
assert service.route_turn("cowork", "q", "p", "m").switched is True
|
|
assert service.route_turn("ai_edit", "q", "p", "m").switched is False
|
|
|
|
|
|
def test_a_failing_mode_reader_falls_back_to_off():
|
|
def broken(_surface):
|
|
raise KeyError("config not loaded yet")
|
|
|
|
service = RoutingApplicationService(_FakeRouter(_switch_to("a", "b")),
|
|
mode_reader=broken)
|
|
|
|
assert service.route_turn("cowork", "q", "p", "m").switched is False
|
|
|
|
|
|
def test_required_capabilities_are_passed_through_to_the_engine():
|
|
"""An image turn must only be routed to a vision-capable model; the filter
|
|
has to reach the scorer or the constraint is silently dropped."""
|
|
router = _FakeRouter()
|
|
service = RoutingApplicationService(router)
|
|
|
|
service.route_turn("cowork", "describe this", "p", "m", mode="auto",
|
|
required_capabilities=["vision"])
|
|
|
|
assert router.calls[0]["required_capabilities"] == ["vision"]
|