chore(repo): initialize Cowork Local Gitea repository

core/routing/orchestrator.py

@@ -0,0 +1,165 @@
+"""Orchestrate a full assessment run: enrich → probe → score → store.
+
+``check_and_update`` is the single entry point the API/scheduler call. It:
+
+1. Enriches each candidate's static metadata (price / context / capabilities).
+2. Probes every candidate on every task type concurrently, bounded per provider
+   (delegated to ``prober.probe_candidates``), grading each answer with one
+   fixed judge.
+3. Computes fit scores per task type under the active policy.
+4. Persists the results atomically, backing up the previous version to history
+   first (so a model that *degrades* between runs can be spotted).
+
+Cost-awareness: probing spends real tokens, so this runs only on a schedule,
+when a model is added, or on an explicit reassess — never per chat turn. The
+number of API calls made is logged so the cost is visible.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Callable, Dict, List, Optional, Tuple
+
+from .clients import ProbeClient
+from .metadata import LLMDeclarer, enrich
+from .models import ModelAssessment, Policy, TaskType, candidate_key
+from .prober import JudgeFn, make_judge, probe_candidates
+from .scorer import compute_fit_score
+from .store import AssessmentStore, utc_now_iso
+
+logger = logging.getLogger("cowork_local.routing")
+
+# A candidate to assess: (provider, model_id, tier|None).
+Candidate = Tuple[str, str, Optional[str]]
+
+
+def build_assessment(
+    provider: str,
+    model_id: str,
+    tier: Optional[str],
+    probes: Dict[str, "object"],
+    policy: Policy,
+    *,
+    config=None,
+    task_types: Optional[List[TaskType]] = None,
+    llm_declarer: Optional[LLMDeclarer] = None,
+) -> ModelAssessment:
+    """Assemble one :class:`ModelAssessment` from its probe results.
+
+    Pure except for metadata enrichment (which may read the config price
+    sheet). Kept separate from I/O so it's unit-testable without any network.
+    """
+    from .models import ProbeResult
+
+    task_types = task_types or list(TaskType)
+    meta = enrich(provider, model_id, config=config, tier=tier, llm_declarer=llm_declarer)
+
+    # A model that failed EVERY probe is effectively unavailable this run.
+    typed_probes: Dict[str, ProbeResult] = {}
+    any_success = False
+    for tt in task_types:
+        probe = probes.get(tt.value)
+        if isinstance(probe, ProbeResult):
+            typed_probes[tt.value] = probe
+            any_success = any_success or probe.success
+    if typed_probes and not any_success:
+        meta.available = False
+
+    fit_scores: Dict[str, float] = {}
+    for tt in task_types:
+        probe = typed_probes.get(tt.value)
+        if probe is not None:
+            fit_scores[tt.value] = compute_fit_score(meta, probe, policy)
+
+    return ModelAssessment(
+        metadata=meta,
+        probes=typed_probes,
+        fit_scores=fit_scores,
+        assessed_at=utc_now_iso(),
+    )
+
+
+def check_and_update(
+    candidates: List[Candidate],
+    client: ProbeClient,
+    *,
+    judge: Optional[JudgeFn] = None,
+    judge_provider: str = "",
+    judge_model: str = "",
+    store: Optional[AssessmentStore] = None,
+    config=None,
+    policy: Policy = Policy.BALANCED,
+    task_types: Optional[List[TaskType]] = None,
+    per_provider_concurrency: int = 2,
+    max_workers: int = 8,
+    llm_declarer: Optional[LLMDeclarer] = None,
+    persist: bool = True,
+) -> Dict[str, ModelAssessment]:
+    """Assess every candidate and (optionally) persist the results.
+
+    Provide either a ready ``judge`` callable, or ``judge_provider`` +
+    ``judge_model`` to build the standard rubric judge from ``client``.
+
+    Returns ``{candidate_key: ModelAssessment}``. ``persist=False`` skips the
+    store write (used by tests / dry runs).
+    """
+    task_types = task_types or list(TaskType)
+    if not candidates:
+        logger.info("routing.reassess: no candidates configured — nothing to do")
+        return {}
+
+    if judge is None:
+        if not (judge_provider and judge_model):
+            raise ValueError("check_and_update needs either `judge` or judge_provider+judge_model")
+        judge = make_judge(client, judge_provider, judge_model)
+
+    judge_key = candidate_key(judge_provider, judge_model) if judge_provider else None
+    if judge_key and any(candidate_key(p, m) == judge_key for p, m, _ in candidates):
+        # The judge is also a candidate — its own answers are self-graded. We
+        # keep it routable (it may genuinely be a fine cheap model) but flag the
+        # bias so it's not mistaken for an independent score.
+        logger.warning(
+            "routing.reassess: judge model %s is also a candidate — its quality "
+            "scores are self-judged and may be optimistic", judge_key,
+        )
+
+    # --- count API calls so the cost of a reassess is visible ------------- #
+    call_count = {"n": 0}
+
+    def _tick() -> None:
+        call_count["n"] += 1
+
+    pairs = [(p, m) for (p, m, _tier) in candidates]
+    probe_map = probe_candidates(
+        client, pairs, task_types, judge,
+        per_provider_concurrency=per_provider_concurrency,
+        max_workers=max_workers,
+        call_counter=_tick,
+    )
+
+    assessments: Dict[str, ModelAssessment] = {}
+    for (provider, model_id, tier) in candidates:
+        key = candidate_key(provider, model_id)
+        assessments[key] = build_assessment(
+            provider, model_id, tier,
+            probe_map.get(key, {}), policy,
+            config=config, task_types=task_types, llm_declarer=llm_declarer,
+        )
+
+    # ~1 probe call + 1 judge call per (candidate, task). The counter above only
+    # counts probe calls (judge calls happen inside probe_model), so report both.
+    probe_calls = call_count["n"]
+    logger.info(
+        "routing.reassess: %d candidate(s) × %d task(s) → ~%d probe calls "
+        "(+~%d judge calls), policy=%s",
+        len(candidates), len(task_types), probe_calls, probe_calls, policy.value,
+    )
+
+    if persist:
+        store = store or AssessmentStore()
+        store.save(assessments, policy)
+        store.prune_history(keep=30)
+
+    return assessments
+
+
+__all__ = ["check_and_update", "build_assessment", "Candidate"]