135 lines
4.6 KiB
Python
135 lines
4.6 KiB
Python
"""Tests for the fit-score formula and policy weights."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from cowork_local.core.routing.models import ModelMetadata, Policy, ProbeResult
|
|
from cowork_local.core.routing.scorer import POLICY_WEIGHTS, compute_fit_score
|
|
|
|
|
|
def _meta(**kw) -> ModelMetadata:
|
|
base = dict(
|
|
provider="anthropic",
|
|
model_id="claude-x",
|
|
cost_per_1k_input=0.001,
|
|
cost_per_1k_output=0.003,
|
|
max_context=200000,
|
|
)
|
|
base.update(kw)
|
|
return ModelMetadata(**base)
|
|
|
|
|
|
def _probe(**kw) -> ProbeResult:
|
|
base = dict(latency_ms=500.0, success=True, quality_score=0.8, tokens_out=100)
|
|
base.update(kw)
|
|
return ProbeResult(**base)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Failure / availability short-circuits
|
|
# --------------------------------------------------------------------------- #
|
|
def test_failed_probe_scores_zero():
|
|
probe = _probe(success=False, quality_score=0.9, error="boom")
|
|
for policy in Policy:
|
|
assert compute_fit_score(_meta(), probe, policy) == 0.0
|
|
|
|
|
|
def test_unavailable_model_scores_zero():
|
|
meta = _meta(available=False)
|
|
for policy in Policy:
|
|
assert compute_fit_score(meta, _probe(), policy) == 0.0
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Range + monotonicity
|
|
# --------------------------------------------------------------------------- #
|
|
def test_score_within_unit_interval():
|
|
for policy in Policy:
|
|
s = compute_fit_score(_meta(), _probe(), policy)
|
|
assert 0.0 <= s <= 1.0
|
|
|
|
|
|
def test_higher_quality_scores_higher():
|
|
lo = compute_fit_score(_meta(), _probe(quality_score=0.2), Policy.QUALITY)
|
|
hi = compute_fit_score(_meta(), _probe(quality_score=0.9), Policy.QUALITY)
|
|
assert hi > lo
|
|
|
|
|
|
def test_lower_latency_scores_higher_under_latency_policy():
|
|
slow = compute_fit_score(_meta(), _probe(latency_ms=5000), Policy.LATENCY)
|
|
fast = compute_fit_score(_meta(), _probe(latency_ms=100), Policy.LATENCY)
|
|
assert fast > slow
|
|
|
|
|
|
def test_cheaper_scores_higher_under_cost_policy():
|
|
cheap = compute_fit_score(
|
|
_meta(cost_per_1k_input=0.0001, cost_per_1k_output=0.0002),
|
|
_probe(),
|
|
Policy.COST,
|
|
)
|
|
pricey = compute_fit_score(
|
|
_meta(cost_per_1k_input=0.05, cost_per_1k_output=0.15),
|
|
_probe(),
|
|
Policy.COST,
|
|
)
|
|
assert cheap > pricey
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Policy weighting behaviour
|
|
# --------------------------------------------------------------------------- #
|
|
def test_all_policy_rows_sum_to_one():
|
|
for policy, weights in POLICY_WEIGHTS.items():
|
|
assert abs(sum(weights) - 1.0) < 1e-9, policy
|
|
|
|
|
|
def test_quality_policy_favors_smart_slow_model_over_fast_dumb():
|
|
"""Under QUALITY, a smart-but-slow model beats a fast-but-weak one."""
|
|
smart_slow = compute_fit_score(
|
|
_meta(), _probe(quality_score=0.95, latency_ms=4000), Policy.QUALITY
|
|
)
|
|
fast_dumb = compute_fit_score(
|
|
_meta(), _probe(quality_score=0.3, latency_ms=100), Policy.QUALITY
|
|
)
|
|
assert smart_slow > fast_dumb
|
|
|
|
|
|
def test_latency_policy_favors_fast_dumb_over_smart_slow():
|
|
"""Under LATENCY, the ordering flips — speed dominates."""
|
|
smart_slow = compute_fit_score(
|
|
_meta(), _probe(quality_score=0.95, latency_ms=8000), Policy.LATENCY
|
|
)
|
|
fast_dumb = compute_fit_score(
|
|
_meta(), _probe(quality_score=0.5, latency_ms=50), Policy.LATENCY
|
|
)
|
|
assert fast_dumb > smart_slow
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Unknown-cost handling
|
|
# --------------------------------------------------------------------------- #
|
|
def test_unknown_cost_does_not_beat_known_cheap_model_under_cost_policy():
|
|
"""A model with unknown price must not be handed a free cost advantage."""
|
|
known_cheap = compute_fit_score(
|
|
_meta(cost_per_1k_input=0.0001, cost_per_1k_output=0.0001),
|
|
_probe(),
|
|
Policy.COST,
|
|
)
|
|
unknown = compute_fit_score(
|
|
_meta(cost_per_1k_input=None, cost_per_1k_output=None,
|
|
metadata_incomplete=True),
|
|
_probe(),
|
|
Policy.COST,
|
|
)
|
|
# Both are usable; the genuinely-cheap known model should not score below
|
|
# the unknown-price one (no fabricated cost=0 advantage).
|
|
assert known_cheap >= unknown
|
|
|
|
|
|
def test_quality_score_clamped():
|
|
"""A judge returning >1 or <0 must not push fit outside [0,1]."""
|
|
over = compute_fit_score(_meta(), _probe(quality_score=5.0), Policy.QUALITY)
|
|
under = compute_fit_score(_meta(), _probe(quality_score=-3.0), Policy.QUALITY)
|
|
assert 0.0 <= over <= 1.0
|
|
assert 0.0 <= under <= 1.0
|