merge: kéo Delta epic-R04 (gồm cả R01 và R03) vào gamma/refactor
Nam chốt: không chờ Delta merge vào main, lấy sớm để va chạm nhỏ và sửa ngay, thay vì dồn một cục lúc cả hai cùng lên main. R04 chứa trọn R01 và R03 nên một lần merge là đủ cả ba: 96 file, +8260 dòng. Xung đột chỉ 5 file, đều là __init__.py add/add — hai team cùng dựng khung thư mục nên đụng docstring. Giữ docstring của Gamma (nói rõ ràng buộc "không import PySide6"), giữ mọi phần code của Delta. Riêng tests/fakes/__init__.py: bỏ hai dòng import háo hức của Delta (fake_provider, fake_tool_executor). fake_provider dùng `from providers.base import ...` — import tuyệt đối, chỉ chạy được khi cwd là gốc repo — nên nó làm đứt bài test "dùng fake mà không nạp config thật". Không ai import ở cấp package; test của Delta gọi thẳng module nên bỏ đi không ảnh hưởng họ. Đã ghi lý do vào docstring của gói. Delta cũng xoá preview-desktop và "requirements (cloud copy).txt". 430 test xanh sau merge. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Nam Pham Dinh Thanh
@@ -0,0 +1,157 @@
|
||||
"""Characterization tests for core/chat_agent.py (run_chat and run_cowork runtime seams).
|
||||
|
||||
These tests capture existing behavior as an executable baseline specification,
|
||||
ensuring that future refactoring to ConversationApplicationService does not alter
|
||||
core turn semantics, event emissions, or file handling.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from cowork_local.core import chat_agent
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
|
||||
|
||||
def test_run_chat_characterization() -> None:
|
||||
"""Capture baseline behavior of run_chat: system prompt insertion, streaming, and message persistence."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Hello there!", chunks=["Hello ", "there!"])
|
||||
|
||||
messages: List[Dict[str, Any]] = [{"role": "user", "content": "Hi assistant"}]
|
||||
emitted_events: List[Dict[str, Any]] = []
|
||||
|
||||
def emit(event: Dict[str, Any]) -> None:
|
||||
emitted_events.append(event)
|
||||
|
||||
result = chat_agent.run_chat(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
emit=emit,
|
||||
)
|
||||
|
||||
# 1. Verify system prompt was injected at position 0
|
||||
assert messages[0]["role"] == "system"
|
||||
assert "Cowork Local" in messages[0]["content"]
|
||||
|
||||
# 2. Verify returned assistant message
|
||||
assert result["role"] == "assistant"
|
||||
assert result["content"] == "Hello there!"
|
||||
|
||||
# 3. Verify assistant message was appended to messages list
|
||||
assert messages[-1] == result
|
||||
|
||||
# 4. Verify emitted events sequence
|
||||
text_deltas = [e["delta"] for e in emitted_events if e["type"] == "text"]
|
||||
assert "".join(text_deltas) == "Hello there!"
|
||||
assert any(e["type"] == "assistant_done" for e in emitted_events)
|
||||
|
||||
|
||||
def test_run_cowork_save_file_characterization(tmp_path: Path) -> None:
|
||||
"""Capture baseline behavior of run_cowork: tool execution loop and file production."""
|
||||
output_dir = tmp_path / "output"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
provider = FakeProvider()
|
||||
# Step 1: Model requests save_file tool
|
||||
provider.queue_response(
|
||||
content="Saving your requested report.",
|
||||
tool_calls=[{
|
||||
"id": "call_save_1",
|
||||
"name": "save_file",
|
||||
"arguments": {
|
||||
"filename": "report.md",
|
||||
"content": "# Executive Summary\nAll systems nominal.",
|
||||
},
|
||||
}],
|
||||
)
|
||||
# Step 2: Model finishes after tool result
|
||||
provider.queue_response(
|
||||
content="I have created report.md in your output directory.",
|
||||
chunks=["I have created report.md in your output directory."],
|
||||
)
|
||||
|
||||
messages: List[Dict[str, Any]] = [{"role": "user", "content": "Export report to markdown file"}]
|
||||
emitted_events: List[Dict[str, Any]] = []
|
||||
|
||||
def emit(event: Dict[str, Any]) -> None:
|
||||
emitted_events.append(event)
|
||||
|
||||
final_messages = chat_agent.run_cowork(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
output_dir=output_dir,
|
||||
emit=emit,
|
||||
enforce_rules=False,
|
||||
)
|
||||
|
||||
# 1. Verify file was created in output directory with expected content
|
||||
created_file = output_dir / "report.md"
|
||||
assert created_file.exists()
|
||||
assert created_file.read_text(encoding="utf-8") == "# Executive Summary\nAll systems nominal."
|
||||
|
||||
# 2. Verify message history contains user -> assistant (tool_calls) -> tool -> assistant
|
||||
roles = [m["role"] for m in final_messages]
|
||||
assert "system" in roles
|
||||
assert "user" in roles
|
||||
assert "tool" in roles
|
||||
|
||||
# 3. Verify tool result message content
|
||||
tool_msg = next(m for m in final_messages if m["role"] == "tool")
|
||||
assert tool_msg["name"] == "save_file"
|
||||
assert "Saved report.md" in tool_msg["content"]
|
||||
|
||||
|
||||
def test_run_cowork_cancellation_characterization(tmp_path: Path) -> None:
|
||||
"""Capture cancellation behavior in run_cowork."""
|
||||
output_dir = tmp_path / "output_cancel"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Working...")
|
||||
|
||||
is_cancelled = True
|
||||
|
||||
def check_cancel() -> bool:
|
||||
return is_cancelled
|
||||
|
||||
emitted_events: List[Dict[str, Any]] = []
|
||||
messages: List[Dict[str, Any]] = [{"role": "user", "content": "Please start"}]
|
||||
|
||||
chat_agent.run_cowork(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
output_dir=output_dir,
|
||||
emit=lambda e: emitted_events.append(e),
|
||||
cancel=check_cancel,
|
||||
enforce_rules=False,
|
||||
)
|
||||
|
||||
# Provider should not have executed turns if cancelled right away
|
||||
assert provider.call_count == 0
|
||||
|
||||
|
||||
def test_cleanup_turn_output_characterization(tmp_path: Path) -> None:
|
||||
"""Capture behavior of temporary .scratch folder cleanup and artifact preservation."""
|
||||
output_dir = tmp_path / "output_cleanup"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
scratch_dir = output_dir / ".scratch"
|
||||
scratch_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create a generator script and a deliverable inside scratch
|
||||
generator_script = scratch_dir / "gen.py"
|
||||
generator_script.write_text("print('generating')", encoding="utf-8")
|
||||
deliverable = scratch_dir / "data.csv"
|
||||
deliverable.write_text("a,b,c\n1,2,3", encoding="utf-8")
|
||||
|
||||
before_snapshot = chat_agent._snapshot(output_dir)
|
||||
removed, moved = chat_agent._cleanup_cowork_intermediates(output_dir, before_snapshot, cancelled=False)
|
||||
|
||||
# .scratch directory should be removed
|
||||
assert not scratch_dir.exists()
|
||||
# deliverable should be moved to output root
|
||||
root_csv = output_dir / "data.csv"
|
||||
assert root_csv.exists()
|
||||
# script should not be in output root
|
||||
assert not (output_dir / "gen.py").exists()
|
||||
|
||||
+69
-4
@@ -1,10 +1,75 @@
|
||||
"""Make the repository package importable when pytest runs from the repo root."""
|
||||
"""Make THIS checkout importable as the ``cowork_local`` package during tests.
|
||||
|
||||
Why this is not just a ``sys.path`` insert
|
||||
------------------------------------------
|
||||
Test modules import the app in two different styles:
|
||||
|
||||
* top-level (``from providers.base import ...``) — resolved by the repository
|
||||
root already sitting on ``sys.path`` when pytest is launched from it;
|
||||
* fully qualified (``from cowork_local.core.routing.service import ...``) —
|
||||
which only resolves when a directory literally named ``cowork_local`` is
|
||||
importable.
|
||||
|
||||
Simply appending the repository's PARENT directory to ``sys.path`` (the previous
|
||||
behaviour) makes the second style resolve against *whatever* sibling folder
|
||||
happens to be called ``cowork_local`` — on a developer machine that is often an
|
||||
unrelated older checkout, so the whole suite silently exercises the wrong code
|
||||
while still reporting green. Instead we bind the name ``cowork_local`` in
|
||||
``sys.modules`` to the package rooted at THIS repository, so both import styles
|
||||
always reach the working copy under test regardless of the checkout's directory
|
||||
name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_PARENT = Path(__file__).resolve().parents[2]
|
||||
if str(REPOSITORY_PARENT) not in sys.path:
|
||||
sys.path.insert(0, str(REPOSITORY_PARENT))
|
||||
# .../<checkout>/tests/conftest.py -> .../<checkout>
|
||||
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
||||
PACKAGE_NAME = "cowork_local"
|
||||
|
||||
# The repository root must stay importable so the top-level import style
|
||||
# (``providers``/``domain``/``application``/``tests``) keeps working.
|
||||
if str(PACKAGE_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PACKAGE_ROOT))
|
||||
|
||||
|
||||
def _bind_checkout_as_package() -> None:
|
||||
"""Register this checkout in ``sys.modules`` under the canonical package name.
|
||||
|
||||
Executed at import time of the conftest (i.e. before any test module is
|
||||
imported) so that a stale same-named directory elsewhere on ``sys.path`` can
|
||||
never win the lookup. A no-op when the package is already bound to this very
|
||||
directory, which keeps repeated conftest loads (pytest-xdist, sub-sessions)
|
||||
idempotent.
|
||||
"""
|
||||
existing = sys.modules.get(PACKAGE_NAME)
|
||||
if existing is not None:
|
||||
# Already bound. Only rebind when it points at a DIFFERENT checkout,
|
||||
# otherwise re-executing the package __init__ would duplicate module
|
||||
# state that tests may already hold references to.
|
||||
origin = getattr(existing, "__file__", "") or ""
|
||||
if Path(origin).resolve().parent == PACKAGE_ROOT:
|
||||
return
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
PACKAGE_NAME,
|
||||
PACKAGE_ROOT / "__init__.py",
|
||||
# Declaring the search locations is what turns the module into a real
|
||||
# package, so ``cowork_local.core.routing`` and friends resolve as
|
||||
# sub-modules of this directory.
|
||||
submodule_search_locations=[str(PACKAGE_ROOT)],
|
||||
)
|
||||
if spec is None or spec.loader is None: # pragma: no cover — defensive
|
||||
return
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
# Insert BEFORE executing so that a circular ``import cowork_local`` from
|
||||
# inside the package body resolves to the partially-initialised module
|
||||
# instead of restarting the import (standard CPython import semantics).
|
||||
sys.modules[PACKAGE_NAME] = module
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
|
||||
_bind_checkout_as_package()
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Contract tests: one shared specification every interchangeable adapter must satisfy.
|
||||
|
||||
Unlike unit tests (which pin ONE implementation's behaviour), a contract test is
|
||||
parametrised over every implementation of an interface, so adding a new provider
|
||||
means adding a row — not writing a new test file — and a provider that quietly
|
||||
breaks the canonical shape fails here rather than in production.
|
||||
"""
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Offline transport doubles + per-protocol stream scripts for the provider contract tests.
|
||||
|
||||
Kept in its own module so ``test_providers.py`` stays a readable list of
|
||||
assertions instead of a wall of SSE fixtures, and so the LOC ceiling (400 lines
|
||||
per production file, applied here too) is comfortably met by both halves.
|
||||
|
||||
Nothing in here touches the network: :class:`FakeStreamResponse` mimics just
|
||||
enough of ``requests.Response`` for the streaming loops in
|
||||
``providers/openai_compat.py`` and ``providers/anthropic.py`` — status code,
|
||||
mutable ``encoding``, ``iter_lines`` and ``close``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Canonical turn every protocol script below must produce, so the contract test
|
||||
# can assert one expected result no matter which provider produced it.
|
||||
EXPECTED_TEXT = "Hello world"
|
||||
EXPECTED_TOOL_CALL = {"id": "call-1", "name": "read_file", "arguments": {"path": "a.txt"}}
|
||||
EXPECTED_INPUT_TOKENS = 11
|
||||
EXPECTED_OUTPUT_TOKENS = 7
|
||||
EXPECTED_CACHED_TOKENS = 3
|
||||
|
||||
|
||||
class FakeStreamResponse:
|
||||
"""A minimal stand-in for a streaming ``requests.Response``.
|
||||
|
||||
``iter_lines`` replays pre-baked SSE lines; ``closed`` records that the
|
||||
provider released the connection, which the contract asserts because a
|
||||
provider that leaks the response leaks a socket per turn.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
lines: Optional[List[str]] = None,
|
||||
status_code: int = 200,
|
||||
body: str = "",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self._lines = list(lines or ())
|
||||
self.text = body
|
||||
self.headers = dict(headers or {})
|
||||
self._payload = payload
|
||||
self.closed = False
|
||||
# Providers force UTF-8 on the response before reading it; the attribute
|
||||
# simply has to exist and be writable.
|
||||
self.encoding = None
|
||||
|
||||
def iter_lines(self, decode_unicode: bool = False):
|
||||
for line in self._lines:
|
||||
yield line
|
||||
|
||||
def json(self) -> Any:
|
||||
if self._payload is None:
|
||||
raise ValueError("no JSON payload configured on this fake response")
|
||||
return self._payload
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _sse(payload: Dict[str, Any]) -> str:
|
||||
"""One SSE ``data:`` line carrying a JSON event."""
|
||||
return "data: " + json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def openai_stream_lines() -> List[str]:
|
||||
"""A complete OpenAI Chat Completions stream: text, one tool call, usage.
|
||||
|
||||
Split across several deltas on purpose — chunk boundaries are where naive
|
||||
stream parsers break, so the contract exercises them.
|
||||
"""
|
||||
return [
|
||||
_sse({"choices": [{"delta": {"content": "Hello "}}]}),
|
||||
_sse({"choices": [{"delta": {"content": "world"}}]}),
|
||||
_sse({"choices": [{"delta": {"tool_calls": [{
|
||||
"index": 0, "id": "call-1",
|
||||
"function": {"name": "read_file", "arguments": '{"path":'},
|
||||
}]}}]}),
|
||||
# Arguments arrive fragmented; the provider must concatenate before parsing.
|
||||
_sse({"choices": [{"delta": {"tool_calls": [{
|
||||
"index": 0, "function": {"arguments": '"a.txt"}'},
|
||||
}]}}]}),
|
||||
_sse({
|
||||
"choices": [{"delta": {}}],
|
||||
"usage": {
|
||||
"prompt_tokens": EXPECTED_INPUT_TOKENS,
|
||||
"completion_tokens": EXPECTED_OUTPUT_TOKENS,
|
||||
"prompt_tokens_details": {"cached_tokens": EXPECTED_CACHED_TOKENS},
|
||||
},
|
||||
}),
|
||||
"data: [DONE]",
|
||||
]
|
||||
|
||||
|
||||
def anthropic_stream_lines() -> List[str]:
|
||||
"""The same canonical turn expressed as an Anthropic Messages stream."""
|
||||
return [
|
||||
_sse({"type": "message_start", "message": {"usage": {
|
||||
"input_tokens": EXPECTED_INPUT_TOKENS,
|
||||
"cache_read_input_tokens": EXPECTED_CACHED_TOKENS,
|
||||
}}}),
|
||||
_sse({"type": "content_block_start", "index": 0,
|
||||
"content_block": {"type": "text"}}),
|
||||
_sse({"type": "content_block_delta", "index": 0,
|
||||
"delta": {"type": "text_delta", "text": "Hello "}}),
|
||||
_sse({"type": "content_block_delta", "index": 0,
|
||||
"delta": {"type": "text_delta", "text": "world"}}),
|
||||
_sse({"type": "content_block_start", "index": 1, "content_block": {
|
||||
"type": "tool_use", "id": "call-1", "name": "read_file"}}),
|
||||
_sse({"type": "content_block_delta", "index": 1,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"path":'}}),
|
||||
_sse({"type": "content_block_delta", "index": 1,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '"a.txt"}'}}),
|
||||
_sse({"type": "message_delta",
|
||||
"usage": {"output_tokens": EXPECTED_OUTPUT_TOKENS}}),
|
||||
_sse({"type": "message_stop"}),
|
||||
]
|
||||
|
||||
|
||||
# Per wire protocol: how to script a successful turn, and the model-list payload
|
||||
# ``list_models()`` expects. Keyed by the descriptor's wire protocol value so a
|
||||
# new provider that reuses an existing protocol needs no new entry here.
|
||||
PROTOCOL_FIXTURES = {
|
||||
"openai_compat": {
|
||||
"stream_lines": openai_stream_lines,
|
||||
"models_payload": {"data": [{"id": "gpt-4o-mini"}, {"id": "gpt-4o"}]},
|
||||
"expected_models": ["gpt-4o-mini", "gpt-4o"],
|
||||
},
|
||||
"anthropic": {
|
||||
"stream_lines": anthropic_stream_lines,
|
||||
"models_payload": {"data": [{"id": "claude-sonnet-4-6"}]},
|
||||
"expected_models": ["claude-sonnet-4-6"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ScriptedTransport:
|
||||
"""Replaces ``Provider._request`` and hands back scripted responses.
|
||||
|
||||
Records every call so a test can assert *how* the provider talked to the
|
||||
endpoint (method, url, JSON payload) without a socket ever being opened.
|
||||
"""
|
||||
|
||||
def __init__(self, responses: List[FakeStreamResponse]) -> None:
|
||||
self._responses = list(responses)
|
||||
self.calls: List[Dict[str, Any]] = []
|
||||
|
||||
def __call__(self, method: str, url: str, **kwargs) -> FakeStreamResponse:
|
||||
self.calls.append({"method": method, "url": url, **kwargs})
|
||||
if not self._responses:
|
||||
raise AssertionError(f"unexpected extra request: {method} {url}")
|
||||
# Pop in order: a provider that retries gets the NEXT scripted response,
|
||||
# which is how the retry/error paths are driven.
|
||||
return self._responses.pop(0)
|
||||
|
||||
@property
|
||||
def last_payload(self) -> Dict[str, Any]:
|
||||
"""The JSON body of the most recent request."""
|
||||
return self.calls[-1].get("json") or {}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EXPECTED_CACHED_TOKENS",
|
||||
"EXPECTED_INPUT_TOKENS",
|
||||
"EXPECTED_OUTPUT_TOKENS",
|
||||
"EXPECTED_TEXT",
|
||||
"EXPECTED_TOOL_CALL",
|
||||
"FakeStreamResponse",
|
||||
"PROTOCOL_FIXTURES",
|
||||
"ScriptedTransport",
|
||||
"anthropic_stream_lines",
|
||||
"openai_stream_lines",
|
||||
]
|
||||
@@ -0,0 +1,279 @@
|
||||
"""R03-T01 — the contract every LLM provider adapter must satisfy.
|
||||
|
||||
Parametrised over EVERY provider in the central registry
|
||||
(``infrastructure/providers/provider_registry.py``), so registering a new
|
||||
provider automatically subjects it to the same specification and a provider that
|
||||
drifts from the canonical shapes fails here.
|
||||
|
||||
The contract, in one list:
|
||||
|
||||
* construction — the registry builds a real ``Provider`` for every id;
|
||||
* ``chat()`` — canonical signature, canonical assistant message, streamed text
|
||||
delivered through ``on_text``, tool calls normalised to
|
||||
``{"id", "name", "arguments": dict}``, response always closed;
|
||||
* tool schema translation matches the adapter's wire protocol;
|
||||
* failures raise ``ProviderError`` — never a bare transport exception;
|
||||
* ``list_models()`` / ``test_connection()`` report a reason instead of a silent
|
||||
empty list;
|
||||
* telemetry — exactly one ``UsageEvent`` per turn (R03-T06), with the real
|
||||
counts when the stream reports them.
|
||||
|
||||
Everything runs offline: ``Provider._request`` is replaced by a scripted
|
||||
transport, so the suite needs no network, no API key and no Qt event loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from cowork_local.infrastructure.providers.provider_registry import (
|
||||
BUILTIN_DESCRIPTORS,
|
||||
ProviderRegistry,
|
||||
)
|
||||
from cowork_local.infrastructure.telemetry import usage_sink
|
||||
from cowork_local.providers.base import Provider, ProviderError, ToolSpec
|
||||
from cowork_local.tests.contracts.provider_stubs import (
|
||||
EXPECTED_CACHED_TOKENS,
|
||||
EXPECTED_INPUT_TOKENS,
|
||||
EXPECTED_OUTPUT_TOKENS,
|
||||
EXPECTED_TEXT,
|
||||
EXPECTED_TOOL_CALL,
|
||||
PROTOCOL_FIXTURES,
|
||||
FakeStreamResponse,
|
||||
ScriptedTransport,
|
||||
)
|
||||
|
||||
# Every provider id in the catalogue — the parametrisation that makes this a
|
||||
# contract suite rather than a per-adapter unit test.
|
||||
PROVIDER_IDS = [d.provider_id for d in BUILTIN_DESCRIPTORS]
|
||||
|
||||
# Minimal config: enough for any adapter to build a URL and headers offline.
|
||||
BASE_CONF = {"base_url": "https://gateway.test/v1", "api_key": "test-key"}
|
||||
|
||||
SAMPLE_MESSAGES = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Say hello"},
|
||||
]
|
||||
|
||||
SAMPLE_TOOL = ToolSpec(
|
||||
name="read_file",
|
||||
description="Read a file from disk",
|
||||
parameters={"type": "object", "properties": {"path": {"type": "string"}}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def registry() -> ProviderRegistry:
|
||||
"""A private registry per test so registrations never leak between tests."""
|
||||
return ProviderRegistry(BUILTIN_DESCRIPTORS)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def collected_usage(monkeypatch) -> usage_sink.InMemoryUsageSink:
|
||||
"""Swap the process-wide telemetry sink for an in-memory one.
|
||||
|
||||
Restored by monkeypatch after each test, so a contract run never appends to
|
||||
the developer's real ``~/.cowork_local/usage/`` files.
|
||||
"""
|
||||
sink = usage_sink.InMemoryUsageSink()
|
||||
monkeypatch.setattr(usage_sink, "_sink", usage_sink.CompositeUsageSink([sink]))
|
||||
return sink
|
||||
|
||||
|
||||
def _fixtures_for(registry: ProviderRegistry, provider_id: str) -> dict:
|
||||
"""The stream/model-list script matching this provider's wire protocol."""
|
||||
protocol = registry.get(provider_id).wire_protocol.value
|
||||
return PROTOCOL_FIXTURES[protocol]
|
||||
|
||||
|
||||
def _build(registry: ProviderRegistry, provider_id: str, transport=None) -> Provider:
|
||||
"""Build a provider and (optionally) replace its transport with a script."""
|
||||
provider = registry.build(provider_id, dict(BASE_CONF))
|
||||
if transport is not None:
|
||||
# Patch the INSTANCE, not the class: parallel parametrised cases must
|
||||
# not see each other's scripted transport.
|
||||
provider._request = transport
|
||||
return provider
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Construction & interface shape
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_registry_builds_a_provider_for_every_registered_id(registry, provider_id) -> None:
|
||||
"""Every catalogued provider must be constructible — a descriptor with no
|
||||
working adapter is a broken entry, not a feature flag."""
|
||||
provider = _build(registry, provider_id)
|
||||
|
||||
assert isinstance(provider, Provider)
|
||||
# The registry fills in the descriptor's default model when config omits it,
|
||||
# so a half-configured provider still names a concrete model.
|
||||
assert provider.model, f"{provider_id} built without a model id"
|
||||
assert provider.describe() == f"{provider.name}:{provider.model}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_chat_signature_is_uniform(registry, provider_id) -> None:
|
||||
"""All adapters accept the same call, so the agent runtime can swap
|
||||
providers without knowing which one it holds."""
|
||||
import inspect
|
||||
|
||||
provider = _build(registry, provider_id)
|
||||
params = list(inspect.signature(provider.chat).parameters)
|
||||
|
||||
assert params == ["messages", "tools", "on_text", "cancel", "on_reasoning"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_tool_schema_matches_the_wire_protocol(registry, provider_id) -> None:
|
||||
"""A ToolSpec must translate into the exact shape the endpoint expects."""
|
||||
descriptor = registry.get(provider_id)
|
||||
|
||||
if descriptor.wire_protocol.value == "anthropic":
|
||||
translated = SAMPLE_TOOL.to_anthropic()
|
||||
assert translated["input_schema"] == SAMPLE_TOOL.parameters
|
||||
assert translated["name"] == "read_file"
|
||||
else:
|
||||
translated = SAMPLE_TOOL.to_openai()
|
||||
assert translated["type"] == "function"
|
||||
assert translated["function"]["parameters"] == SAMPLE_TOOL.parameters
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The turn itself
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_chat_returns_the_canonical_assistant_message(registry, provider_id, collected_usage) -> None:
|
||||
"""Whatever the wire format, one turn yields the same canonical result."""
|
||||
fixtures = _fixtures_for(registry, provider_id)
|
||||
response = FakeStreamResponse(lines=fixtures["stream_lines"]())
|
||||
transport = ScriptedTransport([response])
|
||||
provider = _build(registry, provider_id, transport)
|
||||
|
||||
streamed: list = []
|
||||
result = provider.chat(
|
||||
SAMPLE_MESSAGES, tools=[SAMPLE_TOOL], on_text=streamed.append,
|
||||
)
|
||||
|
||||
assert result["role"] == "assistant"
|
||||
assert result["content"] == EXPECTED_TEXT
|
||||
# Text must arrive incrementally, not only in the final message — the chat
|
||||
# UI streams from these callbacks.
|
||||
assert "".join(streamed) == EXPECTED_TEXT
|
||||
assert len(streamed) >= 2
|
||||
# Tool calls are normalised: parsed arguments, never the raw JSON fragments.
|
||||
assert result["tool_calls"] == [EXPECTED_TOOL_CALL]
|
||||
assert response.closed, "provider left the streaming response open"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_chat_publishes_exactly_one_usage_event(registry, provider_id, collected_usage) -> None:
|
||||
"""R03-T06: a turn reports its token usage through the telemetry sink, with
|
||||
the server's real counts when the stream carried them."""
|
||||
fixtures = _fixtures_for(registry, provider_id)
|
||||
transport = ScriptedTransport([FakeStreamResponse(lines=fixtures["stream_lines"]())])
|
||||
provider = _build(registry, provider_id, transport)
|
||||
|
||||
provider.chat(SAMPLE_MESSAGES, tools=[SAMPLE_TOOL])
|
||||
|
||||
events = collected_usage.snapshot()
|
||||
assert len(events) == 1, "a turn must publish exactly one usage event"
|
||||
event = events[0]
|
||||
assert event.provider == provider.name
|
||||
assert event.model == provider.model
|
||||
assert event.input_tokens == EXPECTED_INPUT_TOKENS
|
||||
assert event.output_tokens == EXPECTED_OUTPUT_TOKENS
|
||||
assert event.cached_tokens == EXPECTED_CACHED_TOKENS
|
||||
# Real counts were available, so the event must NOT be flagged as a guess.
|
||||
assert event.estimated is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_usage_is_estimated_when_the_stream_reports_none(registry, provider_id, collected_usage) -> None:
|
||||
"""Gateways that never send usage still produce a dashboard row — clearly
|
||||
flagged as an estimate rather than silently recorded as zero."""
|
||||
# Only text; no usage block anywhere in the stream.
|
||||
silent_stream = ['data: ' + '{"choices": [{"delta": {"content": "hi"}}]}', "data: [DONE]"]
|
||||
if registry.get(provider_id).wire_protocol.value == "anthropic":
|
||||
silent_stream = [
|
||||
'data: {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}',
|
||||
'data: {"type": "content_block_delta", "index": 0,'
|
||||
' "delta": {"type": "text_delta", "text": "hi"}}',
|
||||
]
|
||||
transport = ScriptedTransport([FakeStreamResponse(lines=silent_stream)])
|
||||
provider = _build(registry, provider_id, transport)
|
||||
|
||||
provider.chat(SAMPLE_MESSAGES)
|
||||
|
||||
events = collected_usage.snapshot()
|
||||
assert len(events) == 1
|
||||
assert events[0].estimated is True
|
||||
# An estimate still has to be a positive number to be worth showing.
|
||||
assert events[0].total_tokens > 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Failure behaviour
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_http_error_becomes_provider_error(registry, provider_id, collected_usage) -> None:
|
||||
"""Callers handle exactly one exception type; adapters must not leak
|
||||
transport- or JSON-level errors past their boundary."""
|
||||
failing = FakeStreamResponse(status_code=401, body='{"error": {"message": "bad key"}}')
|
||||
transport = ScriptedTransport([failing])
|
||||
provider = _build(registry, provider_id, transport)
|
||||
|
||||
with pytest.raises(ProviderError):
|
||||
provider.chat(SAMPLE_MESSAGES)
|
||||
|
||||
assert failing.closed, "provider left a failed response open"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_list_models_and_test_connection_report_a_reason(registry, provider_id) -> None:
|
||||
"""A failed model load must explain itself: ``last_error`` is what Settings
|
||||
shows instead of an unexplained empty dropdown."""
|
||||
def _boom(*_args, **_kwargs):
|
||||
# A transport failure, i.e. what actually happens when the gateway is
|
||||
# unreachable — adapters translate this class of error, not arbitrary
|
||||
# programming errors, which must still surface as bugs.
|
||||
raise requests.ConnectionError("network down")
|
||||
|
||||
provider = _build(registry, provider_id, _boom)
|
||||
|
||||
models = provider.list_models()
|
||||
|
||||
assert provider.last_error, f"{provider_id} swallowed a model-load failure"
|
||||
ok, message = provider.test_connection()
|
||||
assert ok is False
|
||||
assert message
|
||||
# Anthropic answers with a built-in fallback catalogue; a gateway answers
|
||||
# with nothing. Both are acceptable — the contract is only that a failure is
|
||||
# never reported as success.
|
||||
assert isinstance(models, list)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_list_models_returns_ids_on_success(registry, provider_id) -> None:
|
||||
"""The happy path returns plain model-id strings, not raw API objects."""
|
||||
fixtures = _fixtures_for(registry, provider_id)
|
||||
transport = ScriptedTransport([
|
||||
FakeStreamResponse(status_code=200, payload=fixtures["models_payload"]),
|
||||
])
|
||||
provider = _build(registry, provider_id, transport)
|
||||
|
||||
models = provider.list_models()
|
||||
|
||||
assert models == fixtures["expected_models"]
|
||||
assert provider.last_error == ""
|
||||
assert all(isinstance(m, str) for m in models)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider_id", PROVIDER_IDS)
|
||||
def test_strip_think_removes_inline_reasoning(registry, provider_id) -> None:
|
||||
"""Reasoning must never leak into a final answer, whichever adapter ran."""
|
||||
provider = _build(registry, provider_id)
|
||||
|
||||
cleaned = provider.strip_think("<think>secret plan</think>Visible answer")
|
||||
|
||||
assert cleaned == "Visible answer"
|
||||
+14
-1
@@ -1 +1,14 @@
|
||||
"""Test double dùng chung cho cả 3 team — không phụ thuộc Qt."""
|
||||
"""Test double dùng chung cho cả 3 team — không phụ thuộc Qt.
|
||||
|
||||
Gói này cố ý **không** import sẵn fake nào. Import ở đây là import háo hức:
|
||||
chạm vào bất kỳ fake nào là kéo theo mọi phụ thuộc của nó, nên chỉ cần một
|
||||
fake lỡ import module cần sys.path đặc biệt là cả gói hỏng trong môi trường
|
||||
cô lập. Đã xảy ra thật khi merge Delta: `fake_provider` dùng
|
||||
`from providers.base import ...` (import tuyệt đối) làm đứt bài kiểm
|
||||
"dùng fake mà không nạp config thật".
|
||||
|
||||
Import thẳng module cần dùng:
|
||||
|
||||
from cowork_local.tests.fakes.fake_config import FakeConfigRepository
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Fake LLM Provider for offline unit, contract, and characterization testing.
|
||||
|
||||
Provides deterministic responses, stream simulation, tool-call dispatching,
|
||||
and fault injection without requiring any external network access or API keys.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from providers.base import CancelFn, Provider, ProviderError, TextCallback, ToolSpec
|
||||
|
||||
|
||||
class FakeProvider(Provider):
|
||||
"""Deterministic test double mimicking real LLM Providers (OpenAI, Anthropic, Ollama)."""
|
||||
|
||||
name = "fake"
|
||||
supports_vision = True
|
||||
|
||||
def __init__(self, conf: Optional[Dict[str, Any]] = None) -> None:
|
||||
# Initialize base provider with default configuration if none provided
|
||||
super().__init__(conf or {"model": "fake-model-v1"})
|
||||
# History of all message batches sent across all chat calls
|
||||
self.call_history: List[List[Dict[str, Any]]] = []
|
||||
# Queue of programmed assistant responses to return sequentially
|
||||
self.response_queue: List[Dict[str, Any]] = []
|
||||
# Queue of exceptions to raise on corresponding calls
|
||||
self.error_queue: List[Exception] = []
|
||||
# Default text returned when response queue is empty
|
||||
self.default_text: str = "Fake model response."
|
||||
# Total number of chat invocations
|
||||
self.call_count: int = 0
|
||||
# Recorded tool specs passed into each turn
|
||||
self.last_tools: Optional[List[ToolSpec]] = None
|
||||
|
||||
def queue_response(
|
||||
self,
|
||||
content: str = "",
|
||||
tool_calls: Optional[List[Dict[str, Any]]] = None,
|
||||
reasoning: Optional[str] = None,
|
||||
chunks: Optional[List[str]] = None,
|
||||
) -> FakeProvider:
|
||||
"""Enqueue a pre-configured response structure for upcoming chat turns."""
|
||||
self.response_queue.append({
|
||||
"content": content,
|
||||
"tool_calls": tool_calls or [],
|
||||
"reasoning": reasoning,
|
||||
"chunks": chunks or ([content] if content else []),
|
||||
})
|
||||
return self
|
||||
|
||||
def queue_error(self, exc: Exception) -> FakeProvider:
|
||||
"""Enqueue an exception to simulate network/API errors on the next turn."""
|
||||
self.error_queue.append(exc)
|
||||
return self
|
||||
|
||||
def chat(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
tools: Optional[List[ToolSpec]] = None,
|
||||
on_text: Optional[TextCallback] = None,
|
||||
cancel: Optional[CancelFn] = None,
|
||||
on_reasoning: Optional[TextCallback] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Simulate single LLM turn with full streaming and tool-call support."""
|
||||
self.call_count += 1
|
||||
self.call_history.append([dict(m) for m in messages])
|
||||
self.last_tools = tools
|
||||
|
||||
# 1. Check for injected errors
|
||||
if self.error_queue:
|
||||
raise self.error_queue.pop(0)
|
||||
|
||||
# 2. Check early cancellation before processing
|
||||
if cancel and cancel():
|
||||
raise ProviderError("Execution aborted by user cancel signal before response generation.")
|
||||
|
||||
# 3. Retrieve queued response or construct default response
|
||||
if self.response_queue:
|
||||
resp_spec = self.response_queue.pop(0)
|
||||
content = resp_spec.get("content", "")
|
||||
tool_calls = resp_spec.get("tool_calls", [])
|
||||
reasoning = resp_spec.get("reasoning")
|
||||
chunks = resp_spec.get("chunks", [content] if content else [])
|
||||
else:
|
||||
content = self.default_text
|
||||
tool_calls = []
|
||||
reasoning = None
|
||||
chunks = [content]
|
||||
|
||||
# 4. Stream reasoning chunks if provided
|
||||
if reasoning and on_reasoning:
|
||||
on_reasoning(reasoning)
|
||||
|
||||
# 5. Stream text chunks, checking cancellation between fragments
|
||||
for chunk in chunks:
|
||||
if cancel and cancel():
|
||||
raise ProviderError("Execution cancelled during text chunk streaming.")
|
||||
if on_text and chunk:
|
||||
on_text(chunk)
|
||||
|
||||
# 6. Return canonical assistant message payload
|
||||
assistant_msg: Dict[str, Any] = {
|
||||
"role": "assistant",
|
||||
"content": content,
|
||||
}
|
||||
if tool_calls:
|
||||
assistant_msg["tool_calls"] = tool_calls
|
||||
|
||||
return assistant_msg
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
"""Return available mock models for settings and validation tests."""
|
||||
return ["fake-model-v1", "fake-reasoner-pro", "fake-vision-plus"]
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Fake Tool Executor for isolated, offline agent tool-call verification.
|
||||
|
||||
Allows tests to verify tool invocation arguments, mock tool return values,
|
||||
and simulate failures/delays without performing unsafe host disk or OS operations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
class FakeToolExecutor:
|
||||
"""Mock execution engine for agent tool-call dispatching."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# History of all executed tool invocations: List of {"name": str, "args": dict, "result": dict}
|
||||
self.call_log: List[Dict[str, Any]] = []
|
||||
# Custom handlers registered per tool name
|
||||
self.handlers: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {}
|
||||
# Pre-programmed fixed responses keyed by tool name
|
||||
self.mock_responses: Dict[str, Dict[str, Any]] = {}
|
||||
# Default response when no specific handler or response is found
|
||||
self.default_result: Dict[str, Any] = {"ok": True, "output": "Fake tool executed successfully."}
|
||||
|
||||
def register_handler(
|
||||
self,
|
||||
tool_name: str,
|
||||
handler: Callable[[Dict[str, Any]], Dict[str, Any]],
|
||||
) -> FakeToolExecutor:
|
||||
"""Register a dynamic handler function for a specific tool name."""
|
||||
self.handlers[tool_name] = handler
|
||||
return self
|
||||
|
||||
def set_mock_response(
|
||||
self,
|
||||
tool_name: str,
|
||||
result: Dict[str, Any],
|
||||
) -> FakeToolExecutor:
|
||||
"""Set a static return payload for a specific tool name."""
|
||||
self.mock_responses[tool_name] = result
|
||||
return self
|
||||
|
||||
def execute(self, tool_name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Execute a tool call using registered mocks and record invocation details."""
|
||||
# 1. Resolve result from handler, preset response, or default fallback
|
||||
if tool_name in self.handlers:
|
||||
result = self.handlers[tool_name](arguments)
|
||||
elif tool_name in self.mock_responses:
|
||||
result = self.mock_responses[tool_name]
|
||||
else:
|
||||
result = dict(self.default_result)
|
||||
result["tool"] = tool_name
|
||||
result["received_args"] = arguments
|
||||
|
||||
# 2. Record execution trace for post-test assertions
|
||||
self.call_log.append({
|
||||
"name": tool_name,
|
||||
"args": dict(arguments),
|
||||
"result": dict(result),
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def get_calls_for(self, tool_name: str) -> List[Dict[str, Any]]:
|
||||
"""Retrieve all recorded calls for a given tool name."""
|
||||
return [call for call in self.call_log if call["name"] == tool_name]
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear recorded logs and registered mock responses."""
|
||||
self.call_log.clear()
|
||||
self.handlers.clear()
|
||||
self.mock_responses.clear()
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Offline test doubles for the R04 turn runtime seams.
|
||||
|
||||
Sits beside ``fake_provider.py``/``fake_tool_executor.py`` (R01-T02) and plays
|
||||
the same role one level up: those fake a *provider*, these fake the ports
|
||||
``ConversationApplicationService`` is driven through
|
||||
(``application/conversations/turn_runtime.py``).
|
||||
|
||||
Deliberately dumb — they record what they were asked and return canned answers.
|
||||
A failing test then points at the service under test rather than at a mock
|
||||
framework's configuration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from cowork_local.domain.agents.agent_event import ToolPreview
|
||||
from cowork_local.domain.agents.conversation_execution_request import (
|
||||
ConversationExecutionRequest,
|
||||
)
|
||||
|
||||
|
||||
class FakeSpec:
|
||||
"""An advertised tool. The service only ever reads ``.name`` off a spec."""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
|
||||
|
||||
class FakeReply:
|
||||
"""One programmed provider answer."""
|
||||
|
||||
def __init__(self, content: str = "", tool_calls=None, chunks=None, reasoning: str = ""):
|
||||
self.content = content
|
||||
self.tool_calls = tool_calls or []
|
||||
# Default to streaming the whole content as a single chunk, which is what
|
||||
# a non-streaming gateway effectively does.
|
||||
self.chunks = chunks if chunks is not None else ([content] if content else [])
|
||||
self.reasoning = reasoning
|
||||
|
||||
|
||||
class FakeModelCall:
|
||||
""":class:`ModelCallPort` returning programmed replies in order.
|
||||
|
||||
A programmed entry may be an exception instead of a reply, which is how a
|
||||
test simulates the gateway dying mid-turn.
|
||||
"""
|
||||
|
||||
def __init__(self, replies: List[Any]) -> None:
|
||||
self.replies = list(replies)
|
||||
self.calls: List[Dict[str, Any]] = []
|
||||
|
||||
def call(self, messages, tools, on_text=None, on_reasoning=None, cancel=None):
|
||||
# Snapshot the messages: the service keeps mutating its own list, so
|
||||
# storing it by reference would make every recorded call look identical.
|
||||
self.calls.append({"messages": [dict(m) for m in messages],
|
||||
"tool_names": [getattr(t, "name", "") for t in tools]})
|
||||
reply = self.replies.pop(0) if self.replies else FakeReply(content="(default)")
|
||||
if isinstance(reply, BaseException):
|
||||
raise reply
|
||||
if reply.reasoning and on_reasoning:
|
||||
on_reasoning(reply.reasoning)
|
||||
for chunk in reply.chunks:
|
||||
if on_text and chunk:
|
||||
on_text(chunk)
|
||||
assistant: Dict[str, Any] = {"role": "assistant", "content": reply.content}
|
||||
if reply.tool_calls:
|
||||
assistant["tool_calls"] = reply.tool_calls
|
||||
return assistant
|
||||
|
||||
|
||||
class FakeToolRuntime:
|
||||
""":class:`ToolRuntimePort` over an imaginary output folder."""
|
||||
|
||||
def __init__(self, specs=("save_file", "run_command", "update_plan"),
|
||||
results: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
removed: Tuple[str, ...] = (), added: Tuple[str, ...] = ()) -> None:
|
||||
self._specs = [FakeSpec(n) for n in specs]
|
||||
self._results = results or {}
|
||||
self._removed, self._added = removed, added
|
||||
self.executed: List[Tuple[str, Dict[str, Any]]] = []
|
||||
self.finalize_calls: List[Dict[str, Any]] = []
|
||||
# When set, every executed tool streams this string through ``on_output``.
|
||||
self.emit_output: Optional[str] = None
|
||||
|
||||
def specs(self, allowed_tools=None):
|
||||
if allowed_tools is None:
|
||||
return list(self._specs)
|
||||
return [s for s in self._specs if s.name in allowed_tools]
|
||||
|
||||
def preview(self, name, args):
|
||||
return ToolPreview(kind="info", title=name, text=str(args))
|
||||
|
||||
def execute(self, name, args, on_output=None, cancel=None):
|
||||
self.executed.append((name, dict(args)))
|
||||
if self.emit_output and on_output:
|
||||
on_output(self.emit_output)
|
||||
return dict(self._results.get(name, {"ok": True, "output": f"{name} ok"}))
|
||||
|
||||
def snapshot(self):
|
||||
return "before"
|
||||
|
||||
def finalize(self, before, cancelled=False):
|
||||
self.finalize_calls.append({"before": before, "cancelled": cancelled})
|
||||
return list(self._removed), list(self._added)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Small helpers shared by the turn tests.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def make_request(**overrides) -> ConversationExecutionRequest:
|
||||
"""A minimal valid request; each test overrides only what it exercises."""
|
||||
base: Dict[str, Any] = {"turn_id": "t1", "session_id": "s1", "prompt": "do it"}
|
||||
base.update(overrides)
|
||||
return ConversationExecutionRequest(**base)
|
||||
|
||||
|
||||
def run_turn(service, request=None, cancel=None):
|
||||
"""Execute a turn and return ``(result, events)``."""
|
||||
events: List[Any] = []
|
||||
result = service.execute(request or make_request(), events.append, cancel=cancel)
|
||||
return result, events
|
||||
|
||||
|
||||
def events_of_type(events, cls):
|
||||
"""Every emitted event of one type, in order."""
|
||||
return [e for e in events if isinstance(e, cls)]
|
||||
|
||||
|
||||
def tool_turn(tool_name: str = "save_file", args=None, **tool_kwargs):
|
||||
"""A turn that calls one tool and then answers — ``(model, tools)``."""
|
||||
calls = [{"id": "c1", "name": tool_name, "arguments": args or {"filename": "a.md"}}]
|
||||
model = FakeModelCall([FakeReply(content="working", tool_calls=calls),
|
||||
FakeReply(content="done")])
|
||||
return model, FakeToolRuntime(**tool_kwargs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FakeSpec", "FakeReply", "FakeModelCall", "FakeToolRuntime",
|
||||
"make_request", "run_turn", "events_of_type", "tool_turn",
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Integration tests: several real layers wired together, still fully offline.
|
||||
|
||||
Where unit tests pin one class against fakes and contract tests pin an interface
|
||||
across implementations, these exercise a real path end to end — e.g. the
|
||||
application routing service on top of the real ``core/routing`` engine — so a
|
||||
seam that only works against a mock is caught here.
|
||||
"""
|
||||
@@ -0,0 +1,113 @@
|
||||
"""R04-T02 — the typed event vocabulary vs. what the real runtime emits.
|
||||
|
||||
The unit tests pin each event against the shape I *read* out of
|
||||
``core/chat_agent.py``. This one removes the reading: it runs the actual
|
||||
``run_cowork`` loop offline (FakeProvider, real tool execution, real cleanup)
|
||||
and asserts every dict it emits is recognised by :func:`from_legacy_dict` and
|
||||
survives a round trip byte-for-byte.
|
||||
|
||||
That makes it a guard against the two failure modes a hand-written vocabulary
|
||||
has: an event type nobody modelled, and a key that silently changes meaning.
|
||||
Either one would surface here as a failure instead of as a blank chat bubble
|
||||
after R04-T03 starts routing events through the typed layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
from cowork_local.core import chat_agent
|
||||
from cowork_local.domain.agents.agent_event_codec import from_legacy_dict
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
|
||||
|
||||
def _run_turn_and_collect(tmp_path: Path, provider: FakeProvider) -> List[Dict[str, Any]]:
|
||||
"""Run one real ``run_cowork`` turn offline and return every emitted dict."""
|
||||
output_dir = tmp_path / "output"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
emitted: List[Dict[str, Any]] = []
|
||||
|
||||
chat_agent.run_cowork(
|
||||
provider=provider,
|
||||
messages=[{"role": "user", "content": "make me a report"}],
|
||||
output_dir=output_dir,
|
||||
emit=emitted.append,
|
||||
# security_config=None disables the AI guardrail layers, which is the
|
||||
# documented behaviour for headless callers and keeps this test offline.
|
||||
security_config=None,
|
||||
title="Report",
|
||||
)
|
||||
return emitted
|
||||
|
||||
|
||||
def _reporting_turn(tmp_path: Path) -> List[Dict[str, Any]]:
|
||||
"""A turn that streams text, calls save_file, then answers — the common path."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(
|
||||
content="Writing it now.",
|
||||
chunks=["Writing ", "it now."],
|
||||
tool_calls=[{"id": "call_1", "name": "save_file",
|
||||
"arguments": {"filename": "report.md", "content": "# Report\n"}}],
|
||||
)
|
||||
provider.queue_response(content="Saved to report.md.", chunks=["Saved to report.md."])
|
||||
return _run_turn_and_collect(tmp_path, provider)
|
||||
|
||||
|
||||
def test_the_runtime_emits_only_event_types_the_domain_layer_models(tmp_path: Path) -> None:
|
||||
emitted = _reporting_turn(tmp_path)
|
||||
|
||||
unmodelled = sorted({e["type"] for e in emitted if from_legacy_dict(e) is None})
|
||||
|
||||
assert unmodelled == [], f"run_cowork emits event types R04-T02 does not model: {unmodelled}"
|
||||
|
||||
|
||||
def test_every_emitted_event_round_trips_without_losing_a_key(tmp_path: Path) -> None:
|
||||
emitted = _reporting_turn(tmp_path)
|
||||
assert emitted, "the turn produced no events at all — the fixture is wrong"
|
||||
|
||||
for raw in emitted:
|
||||
event = from_legacy_dict(raw)
|
||||
assert event is not None, raw
|
||||
assert event.to_legacy_dict() == raw, f"round trip changed the {raw['type']} event"
|
||||
|
||||
|
||||
def test_a_tool_using_turn_really_exercises_the_tool_events(tmp_path: Path) -> None:
|
||||
# Guards the test above from passing trivially: if the fixture ever stopped
|
||||
# calling a tool, the round-trip check would only cover text events.
|
||||
types = {e["type"] for e in _reporting_turn(tmp_path)}
|
||||
|
||||
assert {"text", "assistant_done", "tool_proposed", "tool_result"} <= types
|
||||
|
||||
|
||||
def test_reasoning_events_from_a_thinking_model_round_trip(tmp_path: Path) -> None:
|
||||
# A separate fixture because only reasoning models emit these, and the
|
||||
# common-path turn above would otherwise never cover the event.
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="42", chunks=["42"], reasoning="Let me think...")
|
||||
|
||||
emitted = _run_turn_and_collect(tmp_path, provider)
|
||||
|
||||
reasoning_events = [e for e in emitted if e["type"] == "reasoning"]
|
||||
assert reasoning_events, "a reasoning model produced no reasoning event"
|
||||
for raw in reasoning_events:
|
||||
assert from_legacy_dict(raw).to_legacy_dict() == raw
|
||||
|
||||
|
||||
def test_plan_events_from_the_real_update_plan_tool_round_trip(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(
|
||||
content="Planning.",
|
||||
tool_calls=[{"id": "call_1", "name": "update_plan",
|
||||
"arguments": {"steps": [{"title": "Draft", "status": "running"},
|
||||
{"title": "Review", "status": "pending"}]}}],
|
||||
)
|
||||
provider.queue_response(content="Done.")
|
||||
|
||||
emitted = _run_turn_and_collect(tmp_path, provider)
|
||||
|
||||
plan_events = [e for e in emitted if e["type"] == "plan_set"]
|
||||
assert plan_events, "update_plan did not produce a plan_set event"
|
||||
for raw in plan_events:
|
||||
assert from_legacy_dict(raw).to_legacy_dict() == raw
|
||||
@@ -0,0 +1,207 @@
|
||||
"""R04-T03 (c) — the service must behave exactly like ``run_cowork``.
|
||||
|
||||
The unit tests prove the loop follows the rules I wrote down. They cannot prove
|
||||
those rules are the ones the shipped runtime actually follows. This file does:
|
||||
each test scripts one provider, runs the SAME turn twice — once through
|
||||
``core/chat_agent.py::run_cowork``, once through
|
||||
``ConversationApplicationService`` wired by ``core_runtime_adapter`` — and
|
||||
compares the emitted event stream, the resulting conversation and the tool list
|
||||
the model was shown.
|
||||
|
||||
Anything the port got wrong (a missing event, a reordered guard, a different
|
||||
tool set, a changed message) fails here rather than in front of a user. The only
|
||||
allowed difference is the extra ``turn_completed`` event R04 introduces, which
|
||||
has no legacy consumer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from cowork_local.application.conversations.core_runtime_adapter import (
|
||||
build_cowork_conversation_service,
|
||||
legacy_event_sink,
|
||||
)
|
||||
from cowork_local.core import chat_agent
|
||||
from cowork_local.domain.agents.conversation_execution_request import (
|
||||
ConversationExecutionRequest,
|
||||
)
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
|
||||
_USER_TURN = [{"role": "user", "content": "make me a report"}]
|
||||
|
||||
|
||||
class _FakeGate:
|
||||
"""Stands in for ``core/permissions.py::PermissionGate``."""
|
||||
|
||||
def __init__(self, approve: bool) -> None:
|
||||
self.approve = approve
|
||||
self.requests: List[Dict[str, Any]] = []
|
||||
|
||||
def request(self, action: Dict[str, Any]) -> bool:
|
||||
self.requests.append(action)
|
||||
return self.approve
|
||||
|
||||
|
||||
def _normalise(events: List[Dict[str, Any]], out_dir: Path) -> List[Dict[str, Any]]:
|
||||
"""Replace the run's own output path with a placeholder.
|
||||
|
||||
The two runs write into different temp folders, so absolute paths in
|
||||
``tool_result``/``outputs_*`` events differ by construction. Everything else
|
||||
must match verbatim.
|
||||
"""
|
||||
marker, raw = "<OUT>", str(out_dir)
|
||||
|
||||
def scrub(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return value.replace(raw, marker).replace(raw.replace("\\", "/"), marker)
|
||||
if isinstance(value, list):
|
||||
return [scrub(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: scrub(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
return [scrub(e) for e in events]
|
||||
|
||||
|
||||
def _run_legacy(tmp_path: Path, provider: FakeProvider, *, allowed_tools=None,
|
||||
gate: Optional[_FakeGate] = None, max_steps: int = 30
|
||||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[str]]:
|
||||
"""Run the turn through the existing ``run_cowork``."""
|
||||
out_dir = tmp_path / "legacy"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
events: List[Dict[str, Any]] = []
|
||||
messages = [dict(m) for m in _USER_TURN]
|
||||
|
||||
chat_agent.run_cowork(
|
||||
provider, messages, out_dir, events.append, title="Report",
|
||||
security_config=None, allowed_tools=allowed_tools, gate=gate, max_steps=max_steps,
|
||||
)
|
||||
tool_names = [t.name for t in (provider.last_tools or [])]
|
||||
return _normalise(events, out_dir), messages, tool_names
|
||||
|
||||
|
||||
def _run_service(tmp_path: Path, provider: FakeProvider, *, allowed_tools=None,
|
||||
gate: Optional[_FakeGate] = None, max_steps: int = 30
|
||||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[str]]:
|
||||
"""Run the same turn through the application service."""
|
||||
out_dir = tmp_path / "service"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
events: List[Dict[str, Any]] = []
|
||||
|
||||
service = build_cowork_conversation_service(
|
||||
provider, out_dir, events.append, title="Report", security_config=None, gate=gate)
|
||||
request = ConversationExecutionRequest(
|
||||
turn_id="t1", session_id="s1",
|
||||
# run_cowork receives the user message already appended; the request
|
||||
# carries the history and this turn's prompt separately.
|
||||
messages=_USER_TURN[:-1], prompt=_USER_TURN[-1]["content"],
|
||||
output_dir=out_dir, allowed_tools=allowed_tools, max_steps=max_steps,
|
||||
gate_mode="confirm" if gate is not None else "auto",
|
||||
)
|
||||
result = service.execute(request, legacy_event_sink(events.append))
|
||||
|
||||
# The end-of-turn event is new in R04 and has no legacy counterpart.
|
||||
kept = [e for e in events if e.get("type") != "turn_completed"]
|
||||
tool_names = [t.name for t in (provider.last_tools or [])]
|
||||
return _normalise(kept, out_dir), list(result.messages), tool_names
|
||||
|
||||
|
||||
def _assert_parity(tmp_path: Path, script, *, approve: Optional[bool] = None, **kwargs) -> None:
|
||||
"""Script two identical providers, run both paths, compare everything."""
|
||||
legacy_provider, service_provider = FakeProvider(), FakeProvider()
|
||||
script(legacy_provider)
|
||||
script(service_provider)
|
||||
|
||||
legacy_gate = _FakeGate(approve) if approve is not None else None
|
||||
service_gate = _FakeGate(approve) if approve is not None else None
|
||||
|
||||
legacy_events, legacy_messages, legacy_tools = _run_legacy(
|
||||
tmp_path, legacy_provider, gate=legacy_gate, **kwargs)
|
||||
service_events, service_messages, service_tools = _run_service(
|
||||
tmp_path, service_provider, gate=service_gate, **kwargs)
|
||||
|
||||
assert service_events == legacy_events
|
||||
assert service_messages == legacy_messages
|
||||
assert service_tools == legacy_tools
|
||||
if legacy_gate is not None and service_gate is not None:
|
||||
assert [r["name"] for r in service_gate.requests] == \
|
||||
[r["name"] for r in legacy_gate.requests]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Scenarios.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_a_plain_answer_turn_behaves_identically(tmp_path: Path) -> None:
|
||||
def script(provider: FakeProvider) -> None:
|
||||
provider.queue_response(content="Here you go.", chunks=["Here ", "you go."])
|
||||
|
||||
_assert_parity(tmp_path, script)
|
||||
|
||||
|
||||
def test_a_save_file_turn_behaves_identically(tmp_path: Path) -> None:
|
||||
def script(provider: FakeProvider) -> None:
|
||||
provider.queue_response(
|
||||
content="Writing it.",
|
||||
tool_calls=[{"id": "c1", "name": "save_file",
|
||||
"arguments": {"filename": "report.md", "content": "# Report\n"}}],
|
||||
)
|
||||
provider.queue_response(content="Saved.")
|
||||
|
||||
_assert_parity(tmp_path, script)
|
||||
|
||||
|
||||
def test_an_update_plan_turn_behaves_identically(tmp_path: Path) -> None:
|
||||
def script(provider: FakeProvider) -> None:
|
||||
provider.queue_response(
|
||||
content="Planning.",
|
||||
tool_calls=[{"id": "c1", "name": "update_plan",
|
||||
"arguments": {"steps": [{"title": "Draft", "status": "running"},
|
||||
{"title": "Ship", "status": "pending"}]}}],
|
||||
)
|
||||
provider.queue_response(content="Done.")
|
||||
|
||||
_assert_parity(tmp_path, script)
|
||||
|
||||
|
||||
def test_a_reasoning_only_reply_behaves_identically(tmp_path: Path) -> None:
|
||||
def script(provider: FakeProvider) -> None:
|
||||
provider.queue_response(content="", reasoning="thinking hard")
|
||||
|
||||
_assert_parity(tmp_path, script)
|
||||
|
||||
|
||||
def test_restricting_the_tool_scope_advertises_the_same_tools(tmp_path: Path) -> None:
|
||||
def script(provider: FakeProvider) -> None:
|
||||
provider.queue_response(content="ok")
|
||||
|
||||
_assert_parity(tmp_path, script, allowed_tools=["save_file"])
|
||||
|
||||
|
||||
def test_a_rejected_command_behaves_identically(tmp_path: Path) -> None:
|
||||
# The security-critical path: the gate says no, so the command must never
|
||||
# run and the model must read back the same refusal in both designs.
|
||||
def script(provider: FakeProvider) -> None:
|
||||
provider.queue_response(
|
||||
content="Running it.",
|
||||
tool_calls=[{"id": "c1", "name": "run_command",
|
||||
"arguments": {"command": "echo hi"}}],
|
||||
)
|
||||
provider.queue_response(content="Understood.")
|
||||
|
||||
_assert_parity(tmp_path, script, approve=False)
|
||||
|
||||
|
||||
def test_hitting_the_step_ceiling_behaves_identically(tmp_path: Path) -> None:
|
||||
# The model never stops calling tools, so both paths must stop at the same
|
||||
# place and say so the same way.
|
||||
def script(provider: FakeProvider) -> None:
|
||||
for i in range(4):
|
||||
provider.queue_response(
|
||||
content=f"step {i}",
|
||||
tool_calls=[{"id": f"c{i}", "name": "save_file",
|
||||
"arguments": {"filename": f"f{i}.md", "content": "x"}}],
|
||||
)
|
||||
|
||||
_assert_parity(tmp_path, script, max_steps=2)
|
||||
@@ -0,0 +1,220 @@
|
||||
"""R04-T04 — the migrated Cowork call site, exercised end to end without Qt.
|
||||
|
||||
``CoworkTab.build_job`` only ever *reads attributes* off its widget, so the real
|
||||
production method can be invoked against a stand-in that supplies those
|
||||
attributes. That is what happens here: the actual ``build_job`` body runs, builds
|
||||
a request, wires the service through ``core_runtime_adapter``, and drives a real
|
||||
turn (real tool execution, real output-folder cleanup) against ``FakeProvider``.
|
||||
|
||||
Why it matters: this is the only automated check that the widget's contract with
|
||||
the service still holds — that the worker's list is appended to in place (the
|
||||
transcript re-render and history merge both read it), that events still arrive as
|
||||
legacy dicts, and that a produced file really lands in the turn's folder. None of
|
||||
it needs a display server, so it runs in CI like every other test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
from cowork_local.config import DEFAULT_CONFIG, AppConfig
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
|
||||
|
||||
class _FakeWorker:
|
||||
"""The parts of ``core/worker.py::AgentWorker`` a job actually touches."""
|
||||
|
||||
def __init__(self, approve_commands: bool = True) -> None:
|
||||
self.events: List[Dict[str, Any]] = []
|
||||
self.gate: Optional[Any] = None
|
||||
self._approve = approve_commands
|
||||
self.cancelled = False
|
||||
|
||||
def emit_event(self, event: Dict[str, Any]) -> None:
|
||||
self.events.append(event)
|
||||
|
||||
def is_cancelled(self) -> bool:
|
||||
return self.cancelled
|
||||
|
||||
def new_gate(self, mode: str, agent_role: str = "") -> Any:
|
||||
# Mirrors AgentWorker.new_gate: the gate is stored on the worker so the
|
||||
# UI thread can resolve it, and answers request() from the worker thread.
|
||||
worker = self
|
||||
|
||||
class _Gate:
|
||||
requests: List[Dict[str, Any]] = []
|
||||
|
||||
def request(self, action: Dict[str, Any]) -> bool:
|
||||
self.requests.append(action)
|
||||
return worker._approve
|
||||
|
||||
self.gate = _Gate()
|
||||
return self.gate
|
||||
|
||||
|
||||
class _FakeCtx:
|
||||
"""The ``AppContext`` surface ``build_job`` uses."""
|
||||
|
||||
def __init__(self, config: AppConfig, confirm_commands: bool = False) -> None:
|
||||
self.config = config
|
||||
self._confirm = confirm_commands
|
||||
|
||||
def project_confirm_commands(self) -> bool:
|
||||
return self._confirm
|
||||
|
||||
def build_mcp_tools(self):
|
||||
return [], None
|
||||
|
||||
|
||||
class _WidgetStub:
|
||||
"""Stands in for the CoworkTab instance ``build_job`` reads its state from."""
|
||||
|
||||
kind = "cowork"
|
||||
|
||||
def __init__(self, out_root: Path, ctx: _FakeCtx, provider: FakeProvider) -> None:
|
||||
self._out_root = out_root
|
||||
self.ctx = ctx
|
||||
self._provider = provider
|
||||
self.title = "Report"
|
||||
self.session_id = "s1"
|
||||
self.project_id = "" # the auto-seeded default workspace
|
||||
self._model = ""
|
||||
self._routed_provider = None
|
||||
self._routed_model = None
|
||||
|
||||
def _session_output_dir(self) -> Path:
|
||||
return self._out_root
|
||||
|
||||
def workspace_dir(self) -> Path:
|
||||
return self._out_root
|
||||
|
||||
def admin_agent_prompt(self) -> str:
|
||||
return ""
|
||||
|
||||
def build_provider(self) -> FakeProvider:
|
||||
return self._provider
|
||||
|
||||
|
||||
def _config() -> AppConfig:
|
||||
"""A real AppConfig that never touches ``~/.cowork_local``.
|
||||
|
||||
The AI security guardrails are switched off: they would call the model to
|
||||
review the prompt, which is a separate feature with its own tests and would
|
||||
make this one depend on what the fake answers.
|
||||
"""
|
||||
data = copy.deepcopy(DEFAULT_CONFIG)
|
||||
data["agent_security"]["enabled"] = False
|
||||
return AppConfig(data)
|
||||
|
||||
|
||||
def _run_turn(tmp_path: Path, provider: FakeProvider, messages: List[Dict[str, Any]],
|
||||
*, confirm_commands: bool = False, approve: bool = True):
|
||||
"""Invoke the real ``CoworkTab.build_job`` against the stub and run its job."""
|
||||
from cowork_local.ui.cowork_tab import CoworkTab
|
||||
|
||||
out_dir = tmp_path / ".turns" / "t1"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
widget = _WidgetStub(tmp_path, _FakeCtx(_config(), confirm_commands), provider)
|
||||
worker = _FakeWorker(approve_commands=approve)
|
||||
|
||||
job = CoworkTab.build_job(widget, "make me a report", messages, out_dir)
|
||||
result = job(worker)
|
||||
return result, worker
|
||||
|
||||
|
||||
def test_the_turn_runs_and_reports_its_folder(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Here you go.", chunks=["Here ", "you go."])
|
||||
messages = [{"role": "user", "content": "make me a report"}]
|
||||
|
||||
result, worker = _run_turn(tmp_path, provider, messages)
|
||||
|
||||
assert result["turn_dir"] == str(tmp_path / ".turns" / "t1")
|
||||
assert [e["type"] for e in worker.events] == [
|
||||
"text", "text", "assistant_done", "turn_completed"]
|
||||
|
||||
|
||||
def test_the_worker_list_is_appended_to_in_place(tmp_path: Path) -> None:
|
||||
# _reattach_running_turn replays from this very list while the turn runs, and
|
||||
# _finalize_turn slices it by the pre-turn length afterwards.
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Done.")
|
||||
user = {"role": "user", "content": "make me a report"}
|
||||
messages = [user]
|
||||
|
||||
result, _ = _run_turn(tmp_path, provider, messages)
|
||||
|
||||
assert result["messages"] is messages
|
||||
# Identity, not just equality: _reattach_running_turn locates the turn's user
|
||||
# message with ``m is ctx["user_msg"]`` to replay the steps after it.
|
||||
assert any(m is user for m in messages)
|
||||
# Several system blocks are expected — the tool prompt plus the tagged
|
||||
# skills/security-rules blocks the runtime refreshes on every turn.
|
||||
assert [m["role"] for m in messages if m["role"] != "system"] == ["user", "assistant"]
|
||||
assert messages[-1]["content"] == "Done."
|
||||
|
||||
|
||||
def test_a_saved_file_lands_in_the_turn_folder(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(
|
||||
content="Writing it.",
|
||||
tool_calls=[{"id": "c1", "name": "save_file",
|
||||
"arguments": {"filename": "report.md", "content": "# Report\n"}}],
|
||||
)
|
||||
provider.queue_response(content="Saved.")
|
||||
messages = [{"role": "user", "content": "make me a report"}]
|
||||
|
||||
_, worker = _run_turn(tmp_path, provider, messages)
|
||||
|
||||
produced = list((tmp_path / ".turns" / "t1").glob("*.md"))
|
||||
assert len(produced) == 1
|
||||
assert produced[0].read_text(encoding="utf-8") == "# Report\n"
|
||||
results = [e for e in worker.events if e["type"] == "tool_result"]
|
||||
assert results and results[0]["ok"] is True
|
||||
|
||||
|
||||
def test_auto_run_mode_never_creates_a_permission_gate(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="ok")
|
||||
|
||||
_, worker = _run_turn(tmp_path, provider, [{"role": "user", "content": "hi"}],
|
||||
confirm_commands=False)
|
||||
|
||||
assert worker.gate is None
|
||||
|
||||
|
||||
def test_confirm_mode_creates_the_gate_and_a_refusal_stops_the_command(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(
|
||||
content="Running it.",
|
||||
tool_calls=[{"id": "c1", "name": "run_command",
|
||||
"arguments": {"command": "echo hi"}}],
|
||||
)
|
||||
provider.queue_response(content="Understood.")
|
||||
|
||||
_, worker = _run_turn(tmp_path, provider, [{"role": "user", "content": "run it"}],
|
||||
confirm_commands=True, approve=False)
|
||||
|
||||
assert worker.gate is not None
|
||||
refusals = [e for e in worker.events
|
||||
if e["type"] == "tool_result" and e["output"] == "Rejected by user."]
|
||||
assert len(refusals) == 1
|
||||
|
||||
|
||||
def test_cancelling_before_the_turn_starts_calls_no_model(tmp_path: Path) -> None:
|
||||
from cowork_local.ui.cowork_tab import CoworkTab
|
||||
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="never")
|
||||
out_dir = tmp_path / ".turns" / "t1"
|
||||
out_dir.mkdir(parents=True)
|
||||
widget = _WidgetStub(tmp_path, _FakeCtx(_config()), provider)
|
||||
worker = _FakeWorker()
|
||||
worker.cancelled = True
|
||||
|
||||
CoworkTab.build_job(widget, "x", [{"role": "user", "content": "x"}], out_dir)(worker)
|
||||
|
||||
assert provider.call_count == 0
|
||||
@@ -0,0 +1,249 @@
|
||||
"""R03-T03/T04/T05 — the unified routing path over the REAL routing engine.
|
||||
|
||||
The unit tests drive ``RoutingApplicationService`` against fakes; this suite
|
||||
proves the same service produces correct outcomes on top of the actual
|
||||
``core/routing`` stack (classifier → assessment store → scorer → selector →
|
||||
switch controller), which is what the three chat surfaces now call.
|
||||
|
||||
Offline by construction: a fake probe client answers benchmarks and judging, and
|
||||
the assessment store is a temp file — no network, no Qt, no ``$HOME`` writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
from cowork_local.application.model_routing import (
|
||||
AppContextModeResolver,
|
||||
CoreRoutingEngine,
|
||||
RoutingApplicationService,
|
||||
RoutingMode,
|
||||
RoutingRequest,
|
||||
)
|
||||
from cowork_local.config import DEFAULT_CONFIG, AppConfig
|
||||
from cowork_local.core import projects as projects_mod
|
||||
from cowork_local.core.routing.clients import CompletionResult
|
||||
from cowork_local.core.routing.service import RoutingService
|
||||
from cowork_local.core.routing.store import AssessmentStore
|
||||
from cowork_local.state import AppContext
|
||||
|
||||
STRONG_ANSWER = "STRONG-DETAILED-CORRECT-ANSWER"
|
||||
WEAK_ANSWER = "weak"
|
||||
|
||||
|
||||
class FakeProbeClient:
|
||||
"""Deterministic stand-in for the provider layer used during assessment.
|
||||
|
||||
Mirrors ``tests/routing/test_service.py``'s client: benchmark prompts get a
|
||||
per-model canned answer, and judge prompts are graded by looking up that
|
||||
answer, so scores are stable and no model is ever really called.
|
||||
"""
|
||||
|
||||
def __init__(self, answers, quality) -> None:
|
||||
self.answers = answers
|
||||
self.quality = quality
|
||||
|
||||
def complete(self, provider, model_id, messages) -> CompletionResult:
|
||||
text = messages[0]["content"]
|
||||
if "grading an AI assistant" in text: # the judge rubric prompt
|
||||
score = 0.0
|
||||
for answer, value in self.quality.items():
|
||||
if answer and answer in text:
|
||||
score = value
|
||||
break
|
||||
return CompletionResult(text='{"score": %s}' % score)
|
||||
answer = self.answers.get((provider, model_id))
|
||||
if answer is None:
|
||||
return CompletionResult(error="unavailable")
|
||||
return CompletionResult(text=answer, tokens_out=len(answer) // 4)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def ctx(tmp_path, monkeypatch):
|
||||
"""An AppContext with two assessable models and temp-only persistence."""
|
||||
# Keep workspace load/save off the developer's real ~/.cowork_local.
|
||||
monkeypatch.setattr(projects_mod, "PROJECTS_DIR", tmp_path / "projects")
|
||||
data = copy.deepcopy(DEFAULT_CONFIG)
|
||||
data["providers"] = {
|
||||
"anthropic": {"base_url": "x", "api_key": "x", "model": "strong-model"},
|
||||
}
|
||||
data["routing"]["candidates"] = [
|
||||
{"provider": "anthropic", "model_id": "strong-model", "tier": "powerful"},
|
||||
{"provider": "anthropic", "model_id": "weak-model", "tier": "fast"},
|
||||
]
|
||||
data["routing"]["judge_provider"] = "anthropic"
|
||||
data["routing"]["judge_model"] = "judge-model"
|
||||
data["routing"]["policy"] = "quality"
|
||||
data["routing"]["min_score_gain"] = 0.05
|
||||
return AppContext(AppConfig(data=data, path=tmp_path / "config.json"))
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def routing_service(ctx, tmp_path) -> RoutingService:
|
||||
"""A real RoutingService with a populated assessment store."""
|
||||
client = FakeProbeClient(
|
||||
answers={
|
||||
("anthropic", "strong-model"): STRONG_ANSWER,
|
||||
("anthropic", "weak-model"): WEAK_ANSWER,
|
||||
},
|
||||
quality={STRONG_ANSWER: 0.95, WEAK_ANSWER: 0.35},
|
||||
)
|
||||
store = AssessmentStore(store_path=tmp_path / "assess.json",
|
||||
history_dir=tmp_path / "history")
|
||||
service = RoutingService(ctx, store=store, client=client)
|
||||
service.reassess() # populate real probe results + fit scores
|
||||
return service
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def app_service(ctx, routing_service) -> RoutingApplicationService:
|
||||
"""The application service wired exactly the way the UI wires it."""
|
||||
return RoutingApplicationService(
|
||||
CoreRoutingEngine(routing_service),
|
||||
AppContextModeResolver(ctx),
|
||||
confirm_timeout_sec=lambda: float(ctx.config.routing["confirm_timeout_sec"]),
|
||||
)
|
||||
|
||||
|
||||
def coding_request(**overrides) -> RoutingRequest:
|
||||
"""A coding turn currently pinned to the weaker model."""
|
||||
fields = dict(
|
||||
surface="cowork",
|
||||
prompt="Write a Python function to reverse a linked list",
|
||||
current_provider="anthropic",
|
||||
current_model="weak-model",
|
||||
)
|
||||
fields.update(overrides)
|
||||
return RoutingRequest(**fields)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Auto / Off / Manual over the real engine
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_auto_switches_to_the_better_assessed_model(app_service) -> None:
|
||||
"""The real scorer must rank the strong model first and the service must
|
||||
hand that model back as this turn's override."""
|
||||
outcome = app_service.resolve(coding_request(mode=RoutingMode.AUTO))
|
||||
|
||||
assert outcome.switched is True
|
||||
assert outcome.provider == "anthropic"
|
||||
assert outcome.model == "strong-model"
|
||||
assert outcome.task_type == "coding" # classified from the prompt
|
||||
assert outcome.score_gain > 0
|
||||
|
||||
|
||||
def test_off_keeps_the_pinned_model(app_service) -> None:
|
||||
"""Off must not switch even when a clearly better model is assessed."""
|
||||
outcome = app_service.resolve(coding_request(mode=RoutingMode.OFF))
|
||||
|
||||
assert outcome.switched is False
|
||||
assert outcome.provider is None
|
||||
|
||||
|
||||
def test_manual_asks_before_switching(app_service) -> None:
|
||||
"""The confirm callback receives the engine's own decision object, which is
|
||||
what ``ui/routing_toggle.py::confirm_switch`` renders."""
|
||||
seen: list = []
|
||||
|
||||
outcome = app_service.resolve(
|
||||
coding_request(mode=RoutingMode.MANUAL),
|
||||
confirm=lambda decision, timeout: seen.append((decision, timeout)) or True,
|
||||
)
|
||||
|
||||
assert outcome.switched is True
|
||||
decision, timeout = seen[0]
|
||||
assert decision.to_model == "anthropic/strong-model"
|
||||
assert decision.reason # human-readable explanation
|
||||
assert timeout == pytest.approx(60.0) # from DEFAULT_CONFIG
|
||||
|
||||
|
||||
def test_manual_decline_keeps_the_pinned_model(app_service) -> None:
|
||||
outcome = app_service.resolve(
|
||||
coding_request(mode=RoutingMode.MANUAL),
|
||||
confirm=lambda decision, timeout: False,
|
||||
)
|
||||
|
||||
assert outcome.switched is False
|
||||
assert outcome.declined is True
|
||||
|
||||
|
||||
def test_already_best_model_is_left_alone(app_service) -> None:
|
||||
"""No pointless churn: being on the best model is not a switch."""
|
||||
outcome = app_service.resolve(
|
||||
coding_request(mode=RoutingMode.AUTO, current_model="strong-model"))
|
||||
|
||||
assert outcome.switched is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fallback over the real engine
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_fallback_keeps_an_assessed_model_even_though_a_better_one_exists(app_service) -> None:
|
||||
"""weak-model IS usable (it has a real probe score), so Fallback stays put
|
||||
where Auto would switch — the behavioural difference between the modes."""
|
||||
outcome = app_service.resolve(coding_request(mode=RoutingMode.FALLBACK))
|
||||
|
||||
assert outcome.switched is False
|
||||
|
||||
|
||||
def test_fallback_rescues_a_model_the_engine_cannot_serve(app_service) -> None:
|
||||
"""A model absent from the ranking (never assessed / unavailable) is exactly
|
||||
the situation Fallback exists for."""
|
||||
outcome = app_service.resolve(
|
||||
coding_request(mode=RoutingMode.FALLBACK, current_model="ghost-model"))
|
||||
|
||||
assert outcome.switched is True
|
||||
assert outcome.model == "strong-model"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Surface parity — the point of R03-T04/T05
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("surface", ["cowork", "co4e", "ai_edit"])
|
||||
def test_every_surface_gets_the_same_decision(app_service, surface) -> None:
|
||||
"""Chat, Co4E and AI-Edit used to hold three copies of this logic. Given the
|
||||
same inputs they must now be indistinguishable."""
|
||||
outcome = app_service.resolve(coding_request(surface=surface, mode=RoutingMode.AUTO))
|
||||
|
||||
assert outcome.switched is True
|
||||
assert outcome.model == "strong-model"
|
||||
|
||||
|
||||
def test_ai_edit_pinned_task_type_reaches_the_engine(app_service) -> None:
|
||||
"""AI-Edit pins "coding" instead of classifying; the engine must honour it
|
||||
even when the instruction text reads like something else entirely."""
|
||||
outcome = app_service.resolve(coding_request(
|
||||
surface="ai_edit",
|
||||
prompt="Write a poem about the ocean", # classifier would say "creative"
|
||||
task_type="coding",
|
||||
mode=RoutingMode.AUTO,
|
||||
))
|
||||
|
||||
assert outcome.task_type == "coding"
|
||||
|
||||
|
||||
def test_mode_comes_from_the_workspace_when_not_pinned(ctx, app_service) -> None:
|
||||
"""With no explicit mode, the service reads the per-workspace setting — the
|
||||
lookup the widgets used to do themselves."""
|
||||
ctx.config.data["routing"]["switch_mode"] = "auto"
|
||||
|
||||
outcome = app_service.resolve(coding_request())
|
||||
|
||||
assert outcome.mode is RoutingMode.AUTO
|
||||
assert outcome.switched is True
|
||||
|
||||
|
||||
def test_fallback_mode_survives_a_round_trip_through_config(ctx) -> None:
|
||||
"""The new mode must be persistable, or the toggle could never select it."""
|
||||
ctx.config.set_routing_mode_for("cowork", "fallback")
|
||||
|
||||
assert ctx.config.routing_mode_for("cowork") == "fallback"
|
||||
assert ctx.project_routing_mode("cowork") == "fallback"
|
||||
|
||||
|
||||
def test_unknown_persisted_mode_degrades_to_off(ctx) -> None:
|
||||
"""A hand-edited config must not enable routing by accident."""
|
||||
ctx.config.routing["surface_modes"]["cowork"] = "turbo"
|
||||
|
||||
assert ctx.config.routing_mode_for("cowork") == "off"
|
||||
@@ -0,0 +1,191 @@
|
||||
"""R04-T05 — the Schedule Task runner's cowork branch, pinned before and after.
|
||||
|
||||
Written against the CURRENT ``_run_agent`` first, as the safety net for moving it
|
||||
onto ``ConversationApplicationService``: an unattended run has five behaviours the
|
||||
interactive path does not have (the plan reminder prefixed to the prompt, the
|
||||
session registered in History before the model starts, a re-save after every
|
||||
assistant message, the timeout notice, and the "did the agent's own checklist
|
||||
finish?" report), and none of them was covered by a test.
|
||||
|
||||
Everything is isolated from the user's real config: history goes to ``tmp_path``
|
||||
via ``history.custom_dir`` and the AI guardrails are off, so no run touches
|
||||
``~/.cowork_local`` or calls a model to review a prompt.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from cowork_local.config import DEFAULT_CONFIG, AppConfig
|
||||
from cowork_local.core import task_executors
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
|
||||
|
||||
class _FakeCtx:
|
||||
"""The ``AppContext`` surface ``_run_agent`` touches."""
|
||||
|
||||
def __init__(self, config: AppConfig, provider: FakeProvider) -> None:
|
||||
self.config = config
|
||||
self._provider = provider
|
||||
|
||||
def build_active_provider(self) -> FakeProvider:
|
||||
return self._provider
|
||||
|
||||
def build_provider_for(self, name=None, model=None) -> FakeProvider:
|
||||
return self._provider
|
||||
|
||||
|
||||
def _config(tmp_path: Path) -> AppConfig:
|
||||
data = copy.deepcopy(DEFAULT_CONFIG)
|
||||
# Keep the run entirely offline and off the real config dir.
|
||||
data["agent_security"]["enabled"] = False
|
||||
data["history"]["custom_dir"] = str(tmp_path / "history")
|
||||
return AppConfig(data)
|
||||
|
||||
|
||||
def _run(tmp_path: Path, provider: FakeProvider, *, prompt: str = "write the report",
|
||||
timeout_sec: Optional[int] = None, admin_agent: Any = None):
|
||||
"""Run one cowork task and return ``(result_tuple, events, config)``."""
|
||||
out_dir = tmp_path / "run"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
config = _config(tmp_path)
|
||||
events: List[Dict[str, Any]] = []
|
||||
|
||||
result = task_executors._run_agent(
|
||||
_FakeCtx(config, provider), "cowork", prompt, out_dir,
|
||||
events.append, lambda: False, title="Weekly report",
|
||||
timeout_sec=timeout_sec, admin_agent=admin_agent,
|
||||
)
|
||||
return result, events, config
|
||||
|
||||
|
||||
def _saved_conversation(config: AppConfig) -> Dict[str, Any]:
|
||||
"""The single conversation the run wrote into the isolated history folder."""
|
||||
files = list(Path(config.history_dir()).rglob("*.json"))
|
||||
assert len(files) == 1, f"expected one saved conversation, found {files}"
|
||||
return json.loads(files[0].read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_a_cowork_task_returns_the_final_answer(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Report is ready.")
|
||||
|
||||
(answer, timed_out, incomplete), _events, _config = _run(tmp_path, provider)
|
||||
|
||||
assert answer == "Report is ready."
|
||||
assert timed_out is False
|
||||
assert incomplete == ""
|
||||
|
||||
|
||||
def test_the_plan_reminder_is_prefixed_to_the_prompt(tmp_path: Path) -> None:
|
||||
# An unattended run has nobody watching, so the agent is pushed to keep its
|
||||
# own checklist honest. The reminder must lead the message.
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="ok")
|
||||
|
||||
_run(tmp_path, provider, prompt="write the report")
|
||||
|
||||
sent = provider.call_history[0][-1]["content"]
|
||||
assert sent.startswith("This runs unattended (Schedule Task)")
|
||||
assert sent.endswith("write the report")
|
||||
|
||||
|
||||
def test_an_admin_agent_persona_sits_between_the_reminder_and_the_prompt(
|
||||
tmp_path: Path) -> None:
|
||||
class _Agent:
|
||||
# An admin agent may pin its own provider/model; blank means "use the
|
||||
# machine's Settings default", which is what build_agent_provider reads.
|
||||
provider = ""
|
||||
model = ""
|
||||
|
||||
def effective_prompt(self) -> str:
|
||||
return "You are the reporting agent."
|
||||
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="ok")
|
||||
|
||||
_run(tmp_path, provider, prompt="write the report", admin_agent=_Agent())
|
||||
|
||||
sent = provider.call_history[0][-1]["content"]
|
||||
assert sent.index("This runs unattended") < sent.index("You are the reporting agent.")
|
||||
assert sent.index("You are the reporting agent.") < sent.index("write the report")
|
||||
|
||||
|
||||
def test_the_session_is_announced_once_it_exists_on_disk(tmp_path: Path) -> None:
|
||||
# The scheduler refreshes History on this event, so it must not fire before
|
||||
# the conversation is really there.
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="ok")
|
||||
|
||||
_result, events, config = _run(tmp_path, provider)
|
||||
|
||||
ready = [e for e in events if e["type"] == "history_ready"]
|
||||
assert len(ready) == 1
|
||||
assert ready[0]["session_id"]
|
||||
assert _saved_conversation(config)["session_id"] == ready[0]["session_id"]
|
||||
|
||||
|
||||
def test_the_saved_conversation_carries_the_answer_and_the_task_title(
|
||||
tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Report is ready.")
|
||||
|
||||
_result, _events, config = _run(tmp_path, provider)
|
||||
|
||||
saved = _saved_conversation(config)
|
||||
assert saved["title"] == "[Task] Weekly report"
|
||||
assert saved["messages"][-1] == {"role": "assistant", "content": "Report is ready."}
|
||||
|
||||
|
||||
def test_an_unfinished_checklist_is_reported_back_to_the_scheduler(
|
||||
tmp_path: Path) -> None:
|
||||
# The agent ticked no step to done, so the task must not be called finished
|
||||
# just because no exception was raised.
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(
|
||||
content="Working on it.",
|
||||
tool_calls=[{"id": "c1", "name": "update_plan",
|
||||
"arguments": {"steps": [{"title": "Draft", "status": "running"}]}}],
|
||||
)
|
||||
provider.queue_response(content="Stopping here.")
|
||||
|
||||
(_answer, _timed_out, incomplete), _events, _config = _run(tmp_path, provider)
|
||||
|
||||
assert incomplete
|
||||
assert "Draft" in incomplete
|
||||
|
||||
|
||||
def test_a_finished_checklist_reports_nothing_outstanding(tmp_path: Path) -> None:
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(
|
||||
content="Done.",
|
||||
tool_calls=[{"id": "c1", "name": "update_plan",
|
||||
"arguments": {"steps": [{"title": "Draft", "status": "done"}]}}],
|
||||
)
|
||||
provider.queue_response(content="All done.")
|
||||
|
||||
(_answer, _timed_out, incomplete), _events, _config = _run(tmp_path, provider)
|
||||
|
||||
assert incomplete == ""
|
||||
|
||||
|
||||
def test_running_out_of_time_appends_the_timeout_notice_to_the_conversation(
|
||||
tmp_path: Path) -> None:
|
||||
# A negative timeout puts the deadline in the past, which is the only
|
||||
# deterministic way to exercise a wall-clock branch in a unit test.
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="never gets there")
|
||||
|
||||
(answer, timed_out, incomplete), events, config = _run(
|
||||
tmp_path, provider, timeout_sec=-1)
|
||||
|
||||
assert timed_out is True
|
||||
assert incomplete == "" # a timeout is not an unfinished checklist
|
||||
assert "quá thời gian chờ" in answer
|
||||
assert any(e["type"] == "assistant_done" and "quá thời gian chờ" in e["content"]
|
||||
for e in events)
|
||||
assert "quá thời gian chờ" in _saved_conversation(config)["messages"][-1]["content"]
|
||||
@@ -1,17 +1,9 @@
|
||||
"""Pytest fixtures/shared helpers for the routing test suite.
|
||||
|
||||
Ensures the ``cowork_local`` package is importable when pytest is invoked from
|
||||
the package directory itself (so ``import cowork_local.core.routing...`` works
|
||||
regardless of the working directory the suite is launched from).
|
||||
Package importability is handled once and for all by ``tests/conftest.py``,
|
||||
which binds THIS checkout to the ``cowork_local`` name in ``sys.modules``.
|
||||
This file used to push the checkout's PARENT directory onto ``sys.path``, which
|
||||
let an unrelated sibling folder named ``cowork_local`` shadow the working copy —
|
||||
so that logic is intentionally gone; keep it that way.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# .../cowork_local/tests/routing/conftest.py → parent of the package dir
|
||||
_PKG_DIR = Path(__file__).resolve().parents[2] # .../cowork_local
|
||||
_REPO_ROOT = _PKG_DIR.parent # .../cowork_local_20260722
|
||||
for p in (str(_REPO_ROOT), str(_PKG_DIR)):
|
||||
if p not in sys.path:
|
||||
sys.path.insert(0, p)
|
||||
|
||||
@@ -14,7 +14,6 @@ from cowork_local.mcp_servers.project_context.registry import (
|
||||
)
|
||||
from cowork_local.mcp_servers.project_context.runtime import require_supported_python
|
||||
from cowork_local.mcp_servers.project_context.server import dispatch
|
||||
from mcp import types
|
||||
|
||||
EXPECTED_TOOLS = {
|
||||
"get_project_issue_context",
|
||||
@@ -88,6 +87,14 @@ def source() -> dict[str, str]:
|
||||
|
||||
|
||||
def test_template_exposes_exactly_three_provider_neutral_tools() -> None:
|
||||
# The MCP SDK is a RUNTIME dependency (requirements.txt) and is deliberately
|
||||
# absent from requirements-test.txt, which is all CI installs. Importing it at
|
||||
# module scope aborted collection for the ENTIRE suite, so the guard lives here,
|
||||
# inside the only test that touches the SDK. Guarding per-test rather than
|
||||
# per-module keeps the other cases -- pure-Python contract checks that need no
|
||||
# SDK -- running on CI instead of silently skipping with it.
|
||||
types = pytest.importorskip("mcp.types")
|
||||
|
||||
assert set(TOOL_NAMES) == EXPECTED_TOOLS
|
||||
declarations = tool_declarations()
|
||||
assert {item["name"] for item in declarations} == EXPECTED_TOOLS
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""R04-T02 — unit tests for the typed agent event stream.
|
||||
|
||||
The events replace the untyped ``{"type": ...}`` dicts the runtime emits today,
|
||||
but ``ui/chat_panel.py::_on_event`` still dispatches on those dicts until R08.
|
||||
So the contract under test is two-sided: each event must be a real typed value
|
||||
AND must serialise back to the exact legacy shape the widget already reads —
|
||||
same wire name, same keys, same optional-key behaviour.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
import pytest
|
||||
from cowork_local.domain.agents.agent_event import (
|
||||
AssistantMessageCompletedEvent,
|
||||
ErrorEvent,
|
||||
HistoryReadyEvent,
|
||||
NoticeEvent,
|
||||
OutputsAddedEvent,
|
||||
OutputsRemovedEvent,
|
||||
PlanStep,
|
||||
PlanUpdatedEvent,
|
||||
ReasoningChunkEvent,
|
||||
TextChunkEvent,
|
||||
ToolCallFinishedEvent,
|
||||
ToolCallStartedEvent,
|
||||
ToolOutputChunkEvent,
|
||||
ToolPreview,
|
||||
TurnCompletedEvent,
|
||||
)
|
||||
from cowork_local.domain.agents.agent_event_codec import from_legacy_dict
|
||||
|
||||
|
||||
# -- base contract --------------------------------------------------------- #
|
||||
def test_events_reject_mutation() -> None:
|
||||
event = TextChunkEvent(delta="hello")
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
event.delta = "goodbye"
|
||||
|
||||
|
||||
# -- legacy wire compatibility --------------------------------------------- #
|
||||
def test_text_chunk_serialises_as_the_legacy_text_event() -> None:
|
||||
assert TextChunkEvent(delta="hi").to_legacy_dict() == {"type": "text", "delta": "hi"}
|
||||
|
||||
|
||||
def test_reasoning_chunk_serialises_as_the_legacy_reasoning_event() -> None:
|
||||
assert ReasoningChunkEvent(delta="hmm").to_legacy_dict() == {
|
||||
"type": "reasoning", "delta": "hmm"}
|
||||
|
||||
|
||||
def test_assistant_message_completed_serialises_as_assistant_done() -> None:
|
||||
# Fires once per provider call, so several times in a tool-using turn — it
|
||||
# is NOT the end of the turn (that is TurnCompletedEvent).
|
||||
assert AssistantMessageCompletedEvent(content="done").to_legacy_dict() == {
|
||||
"type": "assistant_done", "content": "done"}
|
||||
|
||||
|
||||
def test_tool_call_started_serialises_with_the_legacy_id_and_args_keys() -> None:
|
||||
event = ToolCallStartedEvent(
|
||||
call_id="call_1", name="write_file", arguments={"path": "a.md"},
|
||||
preview=ToolPreview(kind="diff", title="Create file: a.md", text="+ hi"),
|
||||
)
|
||||
|
||||
assert event.to_legacy_dict() == {
|
||||
"type": "tool_proposed",
|
||||
"id": "call_1",
|
||||
"name": "write_file",
|
||||
"args": {"path": "a.md"},
|
||||
"preview": {"kind": "diff", "title": "Create file: a.md", "text": "+ hi"},
|
||||
}
|
||||
|
||||
|
||||
def test_tool_call_started_omits_the_preview_when_there_is_none() -> None:
|
||||
event = ToolCallStartedEvent(call_id="call_1", name="read_file")
|
||||
|
||||
assert "preview" not in event.to_legacy_dict()
|
||||
|
||||
|
||||
def test_tool_output_chunk_serialises_as_the_legacy_tool_output_event() -> None:
|
||||
event = ToolOutputChunkEvent(call_id="call_1", name="run_command", delta="line\n")
|
||||
|
||||
assert event.to_legacy_dict() == {
|
||||
"type": "tool_output", "id": "call_1", "name": "run_command", "delta": "line\n"}
|
||||
|
||||
|
||||
def test_tool_call_finished_serialises_as_the_legacy_tool_result_event() -> None:
|
||||
event = ToolCallFinishedEvent(
|
||||
call_id="call_1", name="save_file", ok=True, output="saved",
|
||||
path="C:/out/a.md", produced=["C:/out/b.pptx"],
|
||||
)
|
||||
|
||||
assert event.to_legacy_dict() == {
|
||||
"type": "tool_result",
|
||||
"id": "call_1",
|
||||
"name": "save_file",
|
||||
"ok": True,
|
||||
"output": "saved",
|
||||
"path": "C:/out/a.md",
|
||||
"produced": ["C:/out/b.pptx"],
|
||||
}
|
||||
|
||||
|
||||
def test_tool_call_finished_omits_path_and_produced_when_empty() -> None:
|
||||
# chat_agent only sets these keys when they exist; emitting them as None
|
||||
# would make ``ev.get("path")`` truthy checks read differently downstream.
|
||||
legacy = ToolCallFinishedEvent(call_id="c", name="read_file", ok=True).to_legacy_dict()
|
||||
|
||||
assert "path" not in legacy
|
||||
assert "produced" not in legacy
|
||||
|
||||
|
||||
def test_plan_updated_serialises_steps_back_to_title_status_dicts() -> None:
|
||||
event = PlanUpdatedEvent(steps=(PlanStep(title="Read config", status="done"),
|
||||
PlanStep(title="Patch it", status="running")))
|
||||
|
||||
assert event.to_legacy_dict() == {
|
||||
"type": "plan_set",
|
||||
"steps": [{"title": "Read config", "status": "done"},
|
||||
{"title": "Patch it", "status": "running"}],
|
||||
}
|
||||
|
||||
|
||||
def test_notice_serialises_with_its_level() -> None:
|
||||
assert NoticeEvent(text="reading page 2/9", level="progress").to_legacy_dict() == {
|
||||
"type": "notice", "level": "progress", "text": "reading page 2/9"}
|
||||
|
||||
|
||||
def test_notice_defaults_to_the_info_level() -> None:
|
||||
assert NoticeEvent(text="compacted").to_legacy_dict()["level"] == "info"
|
||||
|
||||
|
||||
def test_outputs_added_and_removed_serialise_their_path_lists() -> None:
|
||||
assert OutputsAddedEvent(paths=("a.md",)).to_legacy_dict() == {
|
||||
"type": "outputs_added", "paths": ["a.md"]}
|
||||
assert OutputsRemovedEvent(paths=("tmp.py",)).to_legacy_dict() == {
|
||||
"type": "outputs_removed", "paths": ["tmp.py"]}
|
||||
|
||||
|
||||
def test_history_ready_serialises_its_session_id() -> None:
|
||||
assert HistoryReadyEvent(session_id="s7").to_legacy_dict() == {
|
||||
"type": "history_ready", "session_id": "s7"}
|
||||
|
||||
|
||||
# -- events introduced by R04 (no legacy consumer) ------------------------- #
|
||||
def test_turn_completed_carries_the_final_answer_and_step_count() -> None:
|
||||
event = TurnCompletedEvent(final_text="all done", steps_used=3)
|
||||
|
||||
assert event.to_legacy_dict() == {
|
||||
"type": "turn_completed", "final_text": "all done", "steps_used": 3,
|
||||
"cancelled": False, "budget_exhausted": False}
|
||||
|
||||
|
||||
def test_error_event_is_fatal_unless_marked_recoverable() -> None:
|
||||
assert ErrorEvent(message="boom").recoverable is False
|
||||
assert ErrorEvent(message="rate limited", recoverable=True).recoverable is True
|
||||
|
||||
|
||||
# -- parsing legacy dicts back into events --------------------------------- #
|
||||
_ROUND_TRIP_CASES = [
|
||||
TextChunkEvent(delta="hi"),
|
||||
ReasoningChunkEvent(delta="hmm"),
|
||||
AssistantMessageCompletedEvent(content="done"),
|
||||
ToolCallStartedEvent(call_id="c", name="run_command", arguments={"command": "ls"},
|
||||
preview=ToolPreview(kind="command", title="Run", text="ls")),
|
||||
ToolCallStartedEvent(call_id="c", name="read_file"),
|
||||
ToolOutputChunkEvent(call_id="c", name="run_command", delta="out"),
|
||||
ToolCallFinishedEvent(call_id="c", name="save_file", ok=True, output="ok",
|
||||
path="a.md", produced=["b.md"]),
|
||||
ToolCallFinishedEvent(call_id="c", name="read_file", ok=False, output="missing"),
|
||||
PlanUpdatedEvent(steps=(PlanStep(title="Step", status="pending"),)),
|
||||
NoticeEvent(text="warned", level="warning"),
|
||||
OutputsAddedEvent(paths=("a.md",)),
|
||||
OutputsRemovedEvent(paths=("tmp.py",)),
|
||||
HistoryReadyEvent(session_id="s7"),
|
||||
TurnCompletedEvent(final_text="done", steps_used=2, cancelled=True),
|
||||
ErrorEvent(message="boom", recoverable=True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event", _ROUND_TRIP_CASES, ids=lambda e: type(e).__name__)
|
||||
def test_every_event_survives_a_round_trip_through_the_legacy_dict(event) -> None:
|
||||
assert from_legacy_dict(event.to_legacy_dict()) == event
|
||||
|
||||
|
||||
def test_unknown_event_types_parse_to_none_instead_of_raising() -> None:
|
||||
# Co4E emits its own vocabulary (node_status, stage_text, run_done) which R04
|
||||
# deliberately leaves alone; a bridge must be able to pass those through
|
||||
# untouched rather than crash on them.
|
||||
assert from_legacy_dict({"type": "node_status", "node_id": "n1"}) is None
|
||||
assert from_legacy_dict({"type": ""}) is None
|
||||
assert from_legacy_dict("not a dict") is None
|
||||
|
||||
|
||||
def test_missing_payload_keys_parse_to_empty_values() -> None:
|
||||
# Defensive: a truncated event from an older emitter must not kill the turn.
|
||||
assert from_legacy_dict({"type": "text"}) == TextChunkEvent(delta="")
|
||||
assert from_legacy_dict({"type": "tool_result", "id": "c", "name": "x"}) == (
|
||||
ToolCallFinishedEvent(call_id="c", name="x", ok=False, output=""))
|
||||
|
||||
|
||||
def test_plan_steps_from_legacy_drop_entries_without_a_title() -> None:
|
||||
# normalize_plan_steps already clamps upstream; this only guards the parse
|
||||
# path so a hand-written dict cannot produce a titleless step.
|
||||
event = from_legacy_dict({"type": "plan_set",
|
||||
"steps": [{"title": "Real", "status": "done"}, {"status": "done"}]})
|
||||
|
||||
assert event == PlanUpdatedEvent(steps=(PlanStep(title="Real", status="done"),))
|
||||
@@ -0,0 +1,101 @@
|
||||
"""R04-T03 (a) — unit tests for the value a finished turn returns.
|
||||
|
||||
Two callers need different things out of one turn today:
|
||||
``ui/chat_panel.py::_finalize_turn`` wants the message list, while
|
||||
``core/task_executors.py::_run_agent`` returns a
|
||||
``(answer_text, timed_out, incomplete_reason)`` tuple assembled by hand. This
|
||||
type is what both read instead, so "what happened in that turn?" has one answer
|
||||
with names on it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
import pytest
|
||||
from cowork_local.domain.agents.agent_event import PlanStep, TurnCompletedEvent
|
||||
from cowork_local.domain.agents.agent_result import AgentResult
|
||||
|
||||
|
||||
def test_result_rejects_mutation() -> None:
|
||||
result = AgentResult(steps_used=1)
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
result.steps_used = 2
|
||||
|
||||
|
||||
def test_messages_are_frozen_into_a_tuple() -> None:
|
||||
live = [{"role": "user", "content": "hi"}]
|
||||
|
||||
result = AgentResult(messages=live)
|
||||
live.append({"role": "assistant", "content": "later"})
|
||||
|
||||
assert result.messages == ({"role": "user", "content": "hi"},)
|
||||
|
||||
|
||||
def test_final_text_is_the_last_non_empty_assistant_message() -> None:
|
||||
# A turn ends on a tool message often enough (cancelled mid-loop) that the
|
||||
# answer cannot simply be messages[-1].
|
||||
result = AgentResult(messages=[
|
||||
{"role": "assistant", "content": "first pass"},
|
||||
{"role": "assistant", "content": "the answer"},
|
||||
{"role": "tool", "tool_call_id": "c", "name": "read_file", "content": "..."},
|
||||
])
|
||||
|
||||
assert result.final_text == "the answer"
|
||||
|
||||
|
||||
def test_final_text_skips_a_blank_assistant_message() -> None:
|
||||
result = AgentResult(messages=[
|
||||
{"role": "assistant", "content": "the answer"},
|
||||
{"role": "assistant", "content": " "},
|
||||
])
|
||||
|
||||
assert result.final_text == "the answer"
|
||||
|
||||
|
||||
def test_final_text_is_empty_when_the_model_never_answered() -> None:
|
||||
assert AgentResult(messages=[{"role": "user", "content": "hi"}]).final_text == ""
|
||||
|
||||
|
||||
def test_a_plain_finished_turn_is_ok() -> None:
|
||||
assert AgentResult(messages=[{"role": "assistant", "content": "done"}]).ok is True
|
||||
|
||||
|
||||
def test_a_cancelled_turn_is_not_ok() -> None:
|
||||
assert AgentResult(cancelled=True).ok is False
|
||||
|
||||
|
||||
def test_a_failed_turn_is_not_ok_and_keeps_its_message() -> None:
|
||||
result = AgentResult(error="SecurityBlocked: nope")
|
||||
|
||||
assert result.ok is False
|
||||
assert result.error == "SecurityBlocked: nope"
|
||||
|
||||
|
||||
def test_hitting_the_step_ceiling_is_reported_separately_from_cancelling() -> None:
|
||||
# "Stopped because the safety limit was reached" and "the user pressed Stop"
|
||||
# need different wording in the transcript, so they stay separate flags.
|
||||
result = AgentResult(budget_exhausted=True, steps_used=30)
|
||||
|
||||
assert result.budget_exhausted is True
|
||||
assert result.cancelled is False
|
||||
|
||||
|
||||
def test_result_converts_to_the_turn_completed_event() -> None:
|
||||
result = AgentResult(
|
||||
messages=[{"role": "assistant", "content": "done"}],
|
||||
steps_used=3, cancelled=False, budget_exhausted=True,
|
||||
)
|
||||
|
||||
assert result.to_turn_completed_event() == TurnCompletedEvent(
|
||||
final_text="done", steps_used=3, cancelled=False, budget_exhausted=True)
|
||||
|
||||
|
||||
def test_plan_steps_are_frozen_into_a_tuple() -> None:
|
||||
steps = [PlanStep(title="Draft", status="done")]
|
||||
|
||||
result = AgentResult(plan_steps=steps)
|
||||
steps.append(PlanStep(title="Review"))
|
||||
|
||||
assert result.plan_steps == (PlanStep(title="Draft", status="done"),)
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Unit tests for the Clean Architecture AST Import Guard (check_imports.py)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from scripts.check_imports import FORBIDDEN_MODULE_PREFIXES, scan_file
|
||||
|
||||
|
||||
def test_clean_python_file_passes(tmp_path: Path) -> None:
|
||||
"""Verify that pure Python code without GUI imports produces 0 violations."""
|
||||
clean_code = """
|
||||
import os
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import List
|
||||
|
||||
@dataclass
|
||||
class UserRequest:
|
||||
id: str
|
||||
prompt: str
|
||||
"""
|
||||
clean_file = tmp_path / "clean_service.py"
|
||||
clean_file.write_text(clean_code, encoding="utf-8")
|
||||
|
||||
violations = scan_file(clean_file, FORBIDDEN_MODULE_PREFIXES)
|
||||
assert len(violations) == 0
|
||||
|
||||
|
||||
def test_forbidden_pyside_import_detected(tmp_path: Path) -> None:
|
||||
"""Verify that PySide6 import is caught with correct line number."""
|
||||
dirty_code = """
|
||||
from dataclasses import dataclass
|
||||
from PySide6.QtWidgets import QWidget
|
||||
|
||||
class BadService:
|
||||
pass
|
||||
"""
|
||||
dirty_file = tmp_path / "bad_service.py"
|
||||
dirty_file.write_text(dirty_code, encoding="utf-8")
|
||||
|
||||
violations = scan_file(dirty_file, FORBIDDEN_MODULE_PREFIXES)
|
||||
assert len(violations) == 1
|
||||
assert violations[0].line_number == 3
|
||||
assert "PySide6" in violations[0].imported_module
|
||||
|
||||
|
||||
def test_forbidden_ui_and_app_import_detected(tmp_path: Path) -> None:
|
||||
"""Verify that importing concrete UI or app modules from domain is caught."""
|
||||
dirty_code = """
|
||||
import ui.chat_panel
|
||||
from app import MainWindow
|
||||
"""
|
||||
dirty_file = tmp_path / "cross_layer_leak.py"
|
||||
dirty_file.write_text(dirty_code, encoding="utf-8")
|
||||
|
||||
violations = scan_file(dirty_file, FORBIDDEN_MODULE_PREFIXES)
|
||||
assert len(violations) == 2
|
||||
modules = [v.imported_module for v in violations]
|
||||
assert "ui.chat_panel" in modules
|
||||
assert "app" in modules
|
||||
@@ -0,0 +1,293 @@
|
||||
"""R04-T03 (b) — the turn loop: composition, tool dispatch, budget, cancel.
|
||||
|
||||
Behaviour that used to be reachable only by running the real widget. Every
|
||||
dependency is a fake from ``tests/fakes/turn_runtime_fakes.py``, so the file
|
||||
runs in milliseconds and each test states one rule of the loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from cowork_local.application.conversations.conversation_application_service import (
|
||||
ConversationApplicationService,
|
||||
)
|
||||
from cowork_local.domain.agents.agent_event import (
|
||||
AssistantMessageCompletedEvent,
|
||||
PlanStep,
|
||||
PlanUpdatedEvent,
|
||||
TextChunkEvent,
|
||||
ToolCallFinishedEvent,
|
||||
ToolCallStartedEvent,
|
||||
ToolOutputChunkEvent,
|
||||
ToolPreview,
|
||||
TurnCompletedEvent,
|
||||
)
|
||||
from cowork_local.tests.fakes.turn_runtime_fakes import (
|
||||
FakeModelCall,
|
||||
FakeReply,
|
||||
FakeToolRuntime,
|
||||
events_of_type,
|
||||
make_request,
|
||||
run_turn,
|
||||
tool_turn,
|
||||
)
|
||||
|
||||
|
||||
def _service(model, tools, **overrides) -> ConversationApplicationService:
|
||||
return ConversationApplicationService(model, tools, **overrides)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The happy path.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_a_plain_answer_streams_text_then_reports_the_message_and_the_turn() -> None:
|
||||
model = FakeModelCall([FakeReply(content="Hello there", chunks=["Hello ", "there"])])
|
||||
|
||||
result, events = run_turn(_service(model, FakeToolRuntime()))
|
||||
|
||||
assert [e.delta for e in events_of_type(events, TextChunkEvent)] == ["Hello ", "there"]
|
||||
assert events_of_type(events, AssistantMessageCompletedEvent) == [
|
||||
AssistantMessageCompletedEvent(content="Hello there")]
|
||||
assert events_of_type(events, TurnCompletedEvent) == [
|
||||
TurnCompletedEvent(final_text="Hello there", steps_used=1)]
|
||||
assert result.final_text == "Hello there"
|
||||
assert result.ok is True
|
||||
|
||||
|
||||
def test_the_composed_user_message_is_appended_before_the_first_call() -> None:
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
request = make_request(prompt="ship it", instruction_prefix="RULES",
|
||||
session_notes="earlier: a.md",
|
||||
messages=[{"role": "user", "content": "previous"}])
|
||||
|
||||
run_turn(_service(model, FakeToolRuntime()), request)
|
||||
|
||||
sent = model.calls[0]["messages"]
|
||||
assert sent[-1] == {"role": "user",
|
||||
"content": "RULES\n\n---\n\nship it\n\nearlier: a.md"}
|
||||
assert sent[-2] == {"role": "user", "content": "previous"}
|
||||
|
||||
|
||||
def test_attachments_are_read_when_the_turn_runs_not_when_it_was_built() -> None:
|
||||
# Extraction can pip-install a parser or shell out to LibreOffice, so it must
|
||||
# happen here (worker thread), not while the UI was assembling the request.
|
||||
seen: List[Tuple[str, Tuple[str, ...]]] = []
|
||||
|
||||
def reader(prompt: str, attachments: Tuple[str, ...]) -> str:
|
||||
seen.append((prompt, attachments))
|
||||
return f"{prompt}\n\n<contents of {len(attachments)} file(s)>"
|
||||
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
request = make_request(prompt="summarise", attachments=["a.docx", "b.pdf"])
|
||||
|
||||
run_turn(_service(model, FakeToolRuntime(), attachment_reader=reader), request)
|
||||
|
||||
assert seen == [("summarise", ("a.docx", "b.pdf"))]
|
||||
assert "contents of 2 file(s)" in model.calls[0]["messages"][-1]["content"]
|
||||
|
||||
|
||||
def test_the_prompt_preparer_is_told_which_tools_the_turn_advertises() -> None:
|
||||
# The system prompt gains an MS365 paragraph only when ms365__* tools are
|
||||
# present, so the preparer has to see the real list.
|
||||
seen: List[Tuple[str, ...]] = []
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
tools = FakeToolRuntime(specs=("save_file", "ms365__send_mail"))
|
||||
|
||||
run_turn(_service(model, tools,
|
||||
prepare_prompt=lambda messages, names: seen.append(names)))
|
||||
|
||||
assert seen == [("save_file", "ms365__send_mail")]
|
||||
|
||||
|
||||
def test_only_the_allowed_tools_are_advertised() -> None:
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
tools = FakeToolRuntime(specs=("save_file", "run_command", "update_plan"))
|
||||
|
||||
run_turn(_service(model, tools), make_request(allowed_tools=("save_file", "update_plan")))
|
||||
|
||||
assert model.calls[0]["tool_names"] == ["save_file", "update_plan"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tool dispatch.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def tool_turn(tool_name: str = "save_file", args=None, **tool_kwargs):
|
||||
"""A turn that calls one tool, then answers."""
|
||||
calls = [{"id": "c1", "name": tool_name, "arguments": args or {"filename": "a.md"}}]
|
||||
model = FakeModelCall([FakeReply(content="working", tool_calls=calls),
|
||||
FakeReply(content="done")])
|
||||
return model, FakeToolRuntime(**tool_kwargs)
|
||||
|
||||
|
||||
def test_a_tool_call_is_announced_executed_and_answered_in_the_message_list() -> None:
|
||||
model, tools = tool_turn(results={"save_file": {"ok": True, "output": "saved",
|
||||
"path": "out/a.md"}})
|
||||
|
||||
result, events = run_turn(_service(model, tools))
|
||||
|
||||
assert events_of_type(events, ToolCallStartedEvent) == [ToolCallStartedEvent(
|
||||
call_id="c1", name="save_file", arguments={"filename": "a.md"},
|
||||
preview=ToolPreview(kind="info", title="save_file", text="{'filename': 'a.md'}"))]
|
||||
assert events_of_type(events, ToolCallFinishedEvent) == [ToolCallFinishedEvent(
|
||||
call_id="c1", name="save_file", ok=True, output="saved", path="out/a.md")]
|
||||
assert tools.executed == [("save_file", {"filename": "a.md"})]
|
||||
assert result.messages[-2] == {"role": "tool", "tool_call_id": "c1",
|
||||
"name": "save_file", "content": "saved"}
|
||||
|
||||
|
||||
def test_live_tool_output_is_streamed_while_the_tool_runs() -> None:
|
||||
model, tools = tool_turn("run_command", {"command": "ls"})
|
||||
tools.emit_output = "file-a\n"
|
||||
|
||||
_, events = run_turn(_service(model, tools))
|
||||
|
||||
assert events_of_type(events, ToolOutputChunkEvent) == [ToolOutputChunkEvent(
|
||||
call_id="c1", name="run_command", delta="file-a\n")]
|
||||
|
||||
|
||||
def test_the_loop_ends_as_soon_as_the_model_stops_calling_tools() -> None:
|
||||
model, tools = tool_turn()
|
||||
|
||||
result, _ = run_turn(_service(model, tools))
|
||||
|
||||
assert result.steps_used == 2
|
||||
assert result.budget_exhausted is False
|
||||
|
||||
|
||||
def test_the_plan_tool_reports_a_plan_update_and_no_tool_bubble() -> None:
|
||||
calls = [{"id": "c1", "name": "update_plan",
|
||||
"arguments": {"steps": [{"title": "Draft", "status": "running"}]}}]
|
||||
model = FakeModelCall([FakeReply(content="planning", tool_calls=calls), FakeReply(content="done")])
|
||||
tools = FakeToolRuntime(results={"update_plan": {
|
||||
"ok": True, "output": "Plan updated.",
|
||||
"plan_steps": [PlanStep(title="Draft", status="running")]}})
|
||||
|
||||
result, events = run_turn(_service(model, tools))
|
||||
|
||||
assert events_of_type(events, PlanUpdatedEvent) == [
|
||||
PlanUpdatedEvent(steps=(PlanStep(title="Draft", status="running"),))]
|
||||
assert events_of_type(events, ToolCallStartedEvent) == []
|
||||
assert events_of_type(events, ToolCallFinishedEvent) == []
|
||||
assert result.plan_steps == (PlanStep(title="Draft", status="running"),)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Budget, cancellation.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_running_out_of_steps_is_flagged_and_announced() -> None:
|
||||
# The model keeps calling tools forever; the ceiling must stop it visibly.
|
||||
forever = [FakeReply(content=f"step {i}",
|
||||
tool_calls=[{"id": f"c{i}", "name": "save_file", "arguments": {}}])
|
||||
for i in range(5)]
|
||||
model = FakeModelCall(forever)
|
||||
|
||||
result, events = run_turn(_service(model, FakeToolRuntime()), make_request(max_steps=2))
|
||||
|
||||
assert result.steps_used == 2
|
||||
assert result.budget_exhausted is True
|
||||
assert "2-step safety limit" in events_of_type(events, TextChunkEvent)[-1].delta
|
||||
# The note reaches the transcript but NOT the stored answer: a turn that hits
|
||||
# the ceiling always ends on a tool message, and the existing runtime only
|
||||
# merges the note when the last message is the assistant's. Pinned here so a
|
||||
# future change to that rule is a deliberate decision, not a silent drift.
|
||||
assert result.final_text == "step 1"
|
||||
|
||||
|
||||
def test_run_to_completion_uses_the_higher_ceiling() -> None:
|
||||
forever = [FakeReply(content="x", tool_calls=[{"id": "c", "name": "save_file", "arguments": {}}])
|
||||
for _ in range(6)]
|
||||
model = FakeModelCall(forever)
|
||||
|
||||
result, _ = run_turn(_service(model, FakeToolRuntime()),
|
||||
make_request(max_steps=2, completion_max_steps=5, run_to_completion=True))
|
||||
|
||||
assert result.steps_used == 5
|
||||
|
||||
|
||||
def test_a_turn_cancelled_before_it_starts_never_calls_the_model() -> None:
|
||||
model = FakeModelCall([FakeReply(content="never")])
|
||||
|
||||
result, events = run_turn(_service(model, FakeToolRuntime()), cancel=lambda: True)
|
||||
|
||||
assert model.calls == []
|
||||
assert result.cancelled is True
|
||||
assert result.budget_exhausted is False
|
||||
assert events_of_type(events, TurnCompletedEvent) == [TurnCompletedEvent(cancelled=True)]
|
||||
|
||||
|
||||
def test_cancelling_during_a_turn_stops_dispatching_the_remaining_tool_calls() -> None:
|
||||
calls = [{"id": "c1", "name": "save_file", "arguments": {}},
|
||||
{"id": "c2", "name": "save_file", "arguments": {}}]
|
||||
model = FakeModelCall([FakeReply(content="two tools", tool_calls=calls)])
|
||||
tools = FakeToolRuntime()
|
||||
stop = {"now": False}
|
||||
|
||||
def cancel() -> bool:
|
||||
return stop["now"]
|
||||
|
||||
original_execute = tools.execute
|
||||
|
||||
def execute(name, args, on_output=None, cancel=None):
|
||||
stop["now"] = True # cancel raised while the first tool runs
|
||||
return original_execute(name, args, on_output=on_output, cancel=cancel)
|
||||
|
||||
tools.execute = execute
|
||||
|
||||
result, _ = run_turn(_service(model, tools), cancel=cancel)
|
||||
|
||||
assert len(tools.executed) == 1
|
||||
assert result.cancelled is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Bring-your-own working list.
|
||||
#
|
||||
# ``ui/chat_panel.py`` holds the turn's message list in its own turn context and
|
||||
# reads it WHILE the worker appends (``_reattach_running_turn`` replays the steps
|
||||
# done so far when the user reopens a running conversation; ``_finalize_turn``
|
||||
# slices it by ``snapshot_len``). A service that built its own private list would
|
||||
# silently break both, so a caller can hand its list over instead.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_a_caller_supplied_list_is_appended_to_in_place() -> None:
|
||||
model, tools = tool_turn()
|
||||
live: List[Dict[str, Any]] = [{"role": "user", "content": "already composed"}]
|
||||
|
||||
result = ConversationApplicationService(model, tools).execute(
|
||||
make_request(), lambda event: None, messages=live)
|
||||
|
||||
roles = [m["role"] for m in live]
|
||||
assert roles == ["user", "assistant", "tool", "assistant"]
|
||||
assert result.messages == tuple(live)
|
||||
|
||||
|
||||
def test_a_caller_supplied_list_is_used_as_is_without_recomposing_the_prompt() -> None:
|
||||
# The widget already applied the skill prefix and the session notes when it
|
||||
# built its message; composing again would duplicate them.
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
user = {"role": "user", "content": "already composed"}
|
||||
live = [user]
|
||||
|
||||
ConversationApplicationService(model, FakeToolRuntime()).execute(
|
||||
make_request(prompt="typed text", instruction_prefix="RULES",
|
||||
session_notes="notes"),
|
||||
lambda event: None, messages=live)
|
||||
|
||||
assert live[0] is user
|
||||
assert live[0]["content"] == "already composed"
|
||||
assert [m["role"] for m in live].count("user") == 1
|
||||
|
||||
|
||||
def test_a_caller_supplied_list_skips_the_attachment_reader() -> None:
|
||||
# Reading the attachments is what produced the caller's message in the first
|
||||
# place; doing it again would re-parse every file.
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
calls: List[Any] = []
|
||||
|
||||
ConversationApplicationService(
|
||||
model, FakeToolRuntime(),
|
||||
attachment_reader=lambda prompt, attachments: calls.append(prompt) or prompt,
|
||||
).execute(make_request(attachments=["a.docx"]), lambda event: None,
|
||||
messages=[{"role": "user", "content": "composed"}])
|
||||
|
||||
assert calls == []
|
||||
@@ -0,0 +1,132 @@
|
||||
"""R04-T01 — unit tests for the immutable turn snapshot.
|
||||
|
||||
The snapshot exists so a turn already running cannot be altered by the UI the
|
||||
user keeps clicking on. These tests pin exactly that: the object refuses
|
||||
mutation, it copies the mutable collections handed to it at submit time, and it
|
||||
owns the prompt-composition rules that were inline in
|
||||
``ui/chat_panel.py::_start_turn``'s worker closure (prefix separator, session
|
||||
notes, model-switch review note).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cowork_local.domain.agents.conversation_execution_request import (
|
||||
ConversationExecutionRequest,
|
||||
)
|
||||
|
||||
|
||||
def _request(**overrides) -> ConversationExecutionRequest:
|
||||
"""A minimal valid request; each test overrides only what it exercises."""
|
||||
base = {"turn_id": "t1", "session_id": "s1"}
|
||||
base.update(overrides)
|
||||
return ConversationExecutionRequest(**base)
|
||||
|
||||
|
||||
# -- immutability ---------------------------------------------------------- #
|
||||
def test_request_rejects_mutation_after_construction() -> None:
|
||||
request = _request(model="gpt-4o-mini")
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
request.model = "claude-sonnet-4-6"
|
||||
|
||||
|
||||
def test_turn_id_is_required() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
ConversationExecutionRequest(turn_id="", session_id="s1")
|
||||
|
||||
|
||||
def test_session_id_is_required() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
ConversationExecutionRequest(turn_id="t1", session_id="")
|
||||
|
||||
|
||||
# -- snapshotting mutable UI state ---------------------------------------- #
|
||||
def test_attachments_are_snapshotted_away_from_the_caller_list() -> None:
|
||||
picked = ["a.docx"]
|
||||
|
||||
request = _request(attachments=picked)
|
||||
picked.append("b.pdf") # the composer clears/refills its own list next turn
|
||||
|
||||
assert request.attachments == ("a.docx",)
|
||||
|
||||
|
||||
def test_messages_are_snapshotted_away_from_the_live_history_list() -> None:
|
||||
history = [{"role": "user", "content": "earlier"}]
|
||||
|
||||
request = _request(messages=history)
|
||||
history.append({"role": "assistant", "content": "later"})
|
||||
|
||||
assert len(request.messages) == 1
|
||||
assert isinstance(request.messages, tuple)
|
||||
|
||||
|
||||
def test_allowed_tools_none_means_every_tool_stays_available() -> None:
|
||||
# None and () must stay distinguishable: None = no restriction, () = deny
|
||||
# every built-in tool. Coercing None to () would silently disarm the agent.
|
||||
assert _request().allowed_tools is None
|
||||
assert _request(allowed_tools=[]).allowed_tools == ()
|
||||
|
||||
|
||||
def test_output_paths_accept_strings_and_normalise_to_path() -> None:
|
||||
request = _request(output_dir="out/t1", home_output_root="out")
|
||||
|
||||
assert request.output_dir == Path("out/t1")
|
||||
assert request.home_output_root == Path("out")
|
||||
|
||||
|
||||
# -- derived turn policy --------------------------------------------------- #
|
||||
def test_effective_max_steps_uses_the_interactive_cap_by_default() -> None:
|
||||
assert _request(max_steps=30, completion_max_steps=200).effective_max_steps == 30
|
||||
|
||||
|
||||
def test_effective_max_steps_lifts_the_cap_when_running_to_completion() -> None:
|
||||
request = _request(max_steps=30, completion_max_steps=200, run_to_completion=True)
|
||||
|
||||
assert request.effective_max_steps == 200
|
||||
|
||||
|
||||
def test_permission_gate_is_required_only_in_confirm_mode() -> None:
|
||||
assert _request(gate_mode="confirm").requires_permission_gate is True
|
||||
assert _request(gate_mode="auto").requires_permission_gate is False
|
||||
|
||||
|
||||
def test_has_prompt_ignores_whitespace_only_input() -> None:
|
||||
assert _request(prompt=" \n ").has_prompt is False
|
||||
assert _request(prompt="do it").has_prompt is True
|
||||
|
||||
|
||||
# -- prompt composition (moved out of the widget's worker closure) --------- #
|
||||
def test_user_content_returns_the_body_unchanged_without_prefix_or_notes() -> None:
|
||||
assert _request().user_content("the body") == "the body"
|
||||
|
||||
|
||||
def test_user_content_separates_the_instruction_prefix_from_the_body() -> None:
|
||||
request = _request(instruction_prefix="SKILL RULES")
|
||||
|
||||
assert request.user_content("the body") == "SKILL RULES\n\n---\n\nthe body"
|
||||
|
||||
|
||||
def test_user_content_appends_session_notes_after_the_body() -> None:
|
||||
request = _request(session_notes="Files produced earlier: a.md")
|
||||
|
||||
assert request.user_content("the body") == "the body\n\nFiles produced earlier: a.md"
|
||||
|
||||
|
||||
def test_user_content_falls_back_to_session_notes_when_the_body_is_empty() -> None:
|
||||
# An attachment-only turn has no typed text, so the notes must not be
|
||||
# prefixed with a stray blank line.
|
||||
request = _request(session_notes="Files produced earlier: a.md")
|
||||
|
||||
assert request.user_content("") == "Files produced earlier: a.md"
|
||||
|
||||
|
||||
def test_user_content_puts_the_review_note_ahead_of_everything_else() -> None:
|
||||
request = _request(instruction_prefix="SKILL RULES", review_note="[Note: switched]")
|
||||
|
||||
content = request.user_content("the body")
|
||||
|
||||
assert content == "[Note: switched]\n\nSKILL RULES\n\n---\n\nthe body"
|
||||
@@ -0,0 +1,210 @@
|
||||
"""R04-T03 (b) — the turn loop: guards, permission gate, compaction, cleanup.
|
||||
|
||||
Split out of ``test_conversation_application_service.py`` to keep each file
|
||||
inside the 400-LOC limit. Same fakes, same service; this half pins the ORDER of
|
||||
the safety steps (guard before model, guard before execute, gate before execute)
|
||||
and the promise that the output sandbox is tidied on the way out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
from cowork_local.application.conversations.conversation_application_service import (
|
||||
ConversationApplicationService,
|
||||
)
|
||||
from cowork_local.domain.agents.agent_event import (
|
||||
ErrorEvent,
|
||||
OutputsAddedEvent,
|
||||
ReasoningChunkEvent,
|
||||
TextChunkEvent,
|
||||
ToolCallFinishedEvent,
|
||||
)
|
||||
from cowork_local.tests.fakes.turn_runtime_fakes import (
|
||||
FakeModelCall,
|
||||
FakeReply,
|
||||
FakeToolRuntime,
|
||||
events_of_type,
|
||||
make_request,
|
||||
run_turn,
|
||||
tool_turn,
|
||||
)
|
||||
|
||||
|
||||
def _service(model, tools, **overrides) -> ConversationApplicationService:
|
||||
return ConversationApplicationService(model, tools, **overrides)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Guards and the permission gate.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_the_prompt_guard_runs_before_the_model_is_ever_called() -> None:
|
||||
order: List[str] = []
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
model_call = model.call
|
||||
|
||||
def call(*a, **kw):
|
||||
order.append("model")
|
||||
return model_call(*a, **kw)
|
||||
|
||||
model.call = call
|
||||
|
||||
run_turn(_service(model, FakeToolRuntime(), prompt_guard=lambda messages: order.append("guard")))
|
||||
|
||||
assert order == ["guard", "model"]
|
||||
|
||||
|
||||
def test_a_blocked_prompt_propagates_before_the_output_folder_is_touched() -> None:
|
||||
model = FakeModelCall([FakeReply(content="never")])
|
||||
tools = FakeToolRuntime()
|
||||
events: List[Any] = []
|
||||
|
||||
def guard(messages) -> None:
|
||||
raise RuntimeError("SecurityBlocked: nope")
|
||||
|
||||
service = _service(model, tools, prompt_guard=guard)
|
||||
|
||||
with pytest.raises(RuntimeError, match="SecurityBlocked"):
|
||||
service.execute(make_request(), events.append)
|
||||
|
||||
assert model.calls == []
|
||||
assert events_of_type(events, ErrorEvent) == [ErrorEvent(message="SecurityBlocked: nope")]
|
||||
# Cleanup is NOT a read-only operation (it deletes a stale .scratch and every
|
||||
# empty sub-folder), so a turn rejected before it started must not run it.
|
||||
assert tools.finalize_calls == []
|
||||
|
||||
|
||||
def test_output_cleanup_still_runs_when_the_turn_fails_mid_loop() -> None:
|
||||
# Once the turn has started producing files, the sandbox must be tidied on
|
||||
# the way out no matter how the turn ends.
|
||||
model = FakeModelCall([RuntimeError("gateway exploded")])
|
||||
tools = FakeToolRuntime()
|
||||
events: List[Any] = []
|
||||
|
||||
with pytest.raises(RuntimeError, match="gateway exploded"):
|
||||
_service(model, tools).execute(make_request(), events.append)
|
||||
|
||||
assert tools.finalize_calls == [{"before": "before", "cancelled": False}]
|
||||
assert events_of_type(events, ErrorEvent) == [ErrorEvent(message="gateway exploded")]
|
||||
|
||||
|
||||
def test_the_command_guard_runs_before_the_tool_executes() -> None:
|
||||
order: List[str] = []
|
||||
model, tools = tool_turn("run_command", {"command": "ls"})
|
||||
original = tools.execute
|
||||
|
||||
def execute(name, args, on_output=None, cancel=None):
|
||||
order.append("execute")
|
||||
return original(name, args, on_output=on_output, cancel=cancel)
|
||||
|
||||
tools.execute = execute
|
||||
|
||||
run_turn(_service(model, tools,
|
||||
command_guard=lambda name, args: order.append(f"guard:{name}")))
|
||||
|
||||
assert order == ["guard:run_command", "execute"]
|
||||
|
||||
|
||||
def test_disabling_rule_enforcement_skips_both_guards() -> None:
|
||||
# Co4E flow steps run inside the workspace sandbox and opt out on purpose.
|
||||
calls: List[str] = []
|
||||
model, tools = tool_turn("run_command", {"command": "ls"})
|
||||
|
||||
run_turn(_service(model, tools,
|
||||
prompt_guard=lambda messages: calls.append("prompt"),
|
||||
command_guard=lambda name, args: calls.append("command")),
|
||||
make_request(enforce_rules=False))
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_the_permission_gate_is_asked_only_for_command_tools() -> None:
|
||||
asked: List[str] = []
|
||||
model, tools = tool_turn("save_file", {"filename": "a.md"})
|
||||
|
||||
run_turn(_service(model, tools,
|
||||
permission_request=lambda action: asked.append(action["name"]) or True),
|
||||
make_request(gate_mode="confirm"))
|
||||
|
||||
assert asked == [] # save_file writes into the sandbox: never gated
|
||||
|
||||
|
||||
def test_a_command_tool_in_confirm_mode_asks_before_running() -> None:
|
||||
asked: List[Dict[str, Any]] = []
|
||||
model, tools = tool_turn("run_command", {"command": "ls"})
|
||||
|
||||
def approve(action: Dict[str, Any]) -> bool:
|
||||
asked.append(action)
|
||||
return True
|
||||
|
||||
run_turn(_service(model, tools, permission_request=approve), make_request(gate_mode="confirm"))
|
||||
|
||||
assert [a["name"] for a in asked] == ["run_command"]
|
||||
assert tools.executed == [("run_command", {"command": "ls"})]
|
||||
|
||||
|
||||
def test_a_rejected_command_is_reported_as_a_failed_tool_and_never_runs() -> None:
|
||||
model, tools = tool_turn("run_command", {"command": "rm -rf /"})
|
||||
|
||||
result, events = run_turn(_service(model, tools, permission_request=lambda action: False),
|
||||
make_request(gate_mode="confirm"))
|
||||
|
||||
assert tools.executed == []
|
||||
assert events_of_type(events, ToolCallFinishedEvent) == [ToolCallFinishedEvent(
|
||||
call_id="c1", name="run_command", ok=False, output="Rejected by user.")]
|
||||
assert result.messages[-2]["content"] == "Rejected by user."
|
||||
|
||||
|
||||
def test_auto_mode_never_asks_even_for_a_command() -> None:
|
||||
model, tools = tool_turn("run_command", {"command": "ls"})
|
||||
|
||||
def refuse(action): # would block the turn if it were consulted
|
||||
raise AssertionError("the gate must not be consulted in auto mode")
|
||||
|
||||
run_turn(_service(model, tools, permission_request=refuse), make_request(gate_mode="auto"))
|
||||
|
||||
assert tools.executed == [("run_command", {"command": "ls"})]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Context compaction, reasoning, output cleanup.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_the_conversation_is_offered_for_compaction_before_every_call() -> None:
|
||||
compactions: List[int] = []
|
||||
model, tools = tool_turn()
|
||||
|
||||
run_turn(_service(model, tools,
|
||||
compact=lambda messages, cancel: compactions.append(len(messages))))
|
||||
|
||||
assert len(compactions) == 2 # once per provider call
|
||||
|
||||
|
||||
def test_reasoning_is_streamed_as_its_own_event() -> None:
|
||||
model = FakeModelCall([FakeReply(content="42", reasoning="thinking...")])
|
||||
|
||||
_, events = run_turn(_service(model, FakeToolRuntime()))
|
||||
|
||||
assert events_of_type(events, ReasoningChunkEvent) == [ReasoningChunkEvent(delta="thinking...")]
|
||||
|
||||
|
||||
def test_a_reasoning_only_reply_gets_a_visible_note_in_the_transcript() -> None:
|
||||
# Otherwise a Schedule Task run reads back an empty answer and writes
|
||||
# "(no output)" into its report.
|
||||
model = FakeModelCall([FakeReply(content="", reasoning="thought hard")])
|
||||
|
||||
result, events = run_turn(_service(model, FakeToolRuntime()))
|
||||
|
||||
assert "only its reasoning" in events_of_type(events, TextChunkEvent)[-1].delta
|
||||
assert "only its reasoning" in result.final_text
|
||||
|
||||
|
||||
def test_promoted_and_discarded_output_files_are_reported_at_the_end() -> None:
|
||||
model = FakeModelCall([FakeReply(content="ok")])
|
||||
tools = FakeToolRuntime(added=("out/report.pptx",))
|
||||
|
||||
_, events = run_turn(_service(model, tools))
|
||||
|
||||
assert events_of_type(events, OutputsAddedEvent) == [
|
||||
OutputsAddedEvent(paths=("out/report.pptx",))]
|
||||
assert tools.finalize_calls == [{"before": "before", "cancelled": False}]
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Unit tests for the adapters that bridge the routing engine to the app service.
|
||||
|
||||
The integration suite covers the happy path over the real engine; this file pins
|
||||
the translation edge cases that are hard to provoke there — malformed task
|
||||
types, a missing ranking, and the service-caching contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from cowork_local.application.model_routing import (
|
||||
AppContextModeResolver,
|
||||
CoreRoutingEngine,
|
||||
RoutingApplicationService,
|
||||
RoutingMode,
|
||||
RoutingRequest,
|
||||
)
|
||||
from cowork_local.application.model_routing.core_routing_adapter import (
|
||||
build_routing_application_service,
|
||||
)
|
||||
from cowork_local.core.routing.models import SwitchDecision, SwitchMode, TaskType
|
||||
|
||||
|
||||
class FakeRanking:
|
||||
"""Just enough of ``selector.Ranking`` for the adapter's usability check."""
|
||||
|
||||
def __init__(self, scores) -> None:
|
||||
self._scores = dict(scores)
|
||||
|
||||
def score_of(self, key: str) -> float:
|
||||
return self._scores.get(key, 0.0)
|
||||
|
||||
|
||||
class FakeRouteResult:
|
||||
"""Stands in for ``core.routing.service.RouteResult``."""
|
||||
|
||||
def __init__(self, decision, task_type=TaskType.CODING, ranking=None, target=None) -> None:
|
||||
self.decision = decision
|
||||
self.task_type = task_type
|
||||
self.ranking = ranking
|
||||
self._target = target
|
||||
|
||||
@property
|
||||
def should_switch(self) -> bool:
|
||||
return self.decision.should_switch
|
||||
|
||||
def target(self):
|
||||
return self._target
|
||||
|
||||
|
||||
class FakeRoutingService:
|
||||
"""Records the arguments the adapter forwards to the engine."""
|
||||
|
||||
def __init__(self, result: FakeRouteResult) -> None:
|
||||
self.result = result
|
||||
self.calls: list = []
|
||||
|
||||
def route(self, surface, prompt, current_provider, current_model, **kwargs):
|
||||
self.calls.append({"surface": surface, "prompt": prompt,
|
||||
"current_provider": current_provider,
|
||||
"current_model": current_model, **kwargs})
|
||||
return self.result
|
||||
|
||||
|
||||
def make_decision(**overrides) -> SwitchDecision:
|
||||
fields = dict(
|
||||
should_switch=True,
|
||||
from_model="anthropic/weak-model",
|
||||
to_model="anthropic/strong-model",
|
||||
score_gain=0.3,
|
||||
reason="coding fit 0.9 > current 0.6",
|
||||
mode=SwitchMode.AUTO,
|
||||
task_type="coding",
|
||||
)
|
||||
fields.update(overrides)
|
||||
return SwitchDecision(**fields)
|
||||
|
||||
|
||||
def make_request(**overrides) -> RoutingRequest:
|
||||
fields = dict(surface="cowork", prompt="Fix this bug",
|
||||
current_provider="anthropic", current_model="weak-model")
|
||||
fields.update(overrides)
|
||||
return RoutingRequest(**fields)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CoreRoutingEngine translation
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_engine_flattens_the_route_result() -> None:
|
||||
"""No ``core.routing`` type may leak past the adapter — the application
|
||||
service and the widgets only ever see plain fields."""
|
||||
service = FakeRoutingService(FakeRouteResult(
|
||||
make_decision(),
|
||||
ranking=FakeRanking({"anthropic/weak-model": 0.6}),
|
||||
target=("anthropic", "strong-model"),
|
||||
))
|
||||
|
||||
evaluation = CoreRoutingEngine(service).evaluate(make_request(), RoutingMode.AUTO)
|
||||
|
||||
assert evaluation.task_type == "coding" # str, not TaskType
|
||||
assert evaluation.should_switch is True
|
||||
assert evaluation.target_provider == "anthropic"
|
||||
assert evaluation.target_model == "strong-model"
|
||||
assert evaluation.score_gain == pytest.approx(0.3)
|
||||
assert evaluation.current_is_usable is True
|
||||
|
||||
|
||||
def test_engine_forwards_the_mode_as_a_plain_string() -> None:
|
||||
"""``RoutingService.route`` takes the mode as a string; handing it an enum
|
||||
would silently fall through to its "unknown mode -> off" branch."""
|
||||
service = FakeRoutingService(FakeRouteResult(make_decision(should_switch=False)))
|
||||
|
||||
CoreRoutingEngine(service).evaluate(make_request(), RoutingMode.AUTO)
|
||||
|
||||
assert service.calls[0]["mode_override"] == "auto"
|
||||
|
||||
|
||||
def test_engine_reports_an_unranked_model_as_unusable() -> None:
|
||||
"""This is the signal Fallback acts on: absent from the ranking means the
|
||||
selector already rejected it (unavailable / no probe / failed probe)."""
|
||||
service = FakeRoutingService(FakeRouteResult(
|
||||
make_decision(),
|
||||
ranking=FakeRanking({"anthropic/strong-model": 0.9}), # current is absent
|
||||
target=("anthropic", "strong-model"),
|
||||
))
|
||||
|
||||
evaluation = CoreRoutingEngine(service).evaluate(make_request(), RoutingMode.AUTO)
|
||||
|
||||
assert evaluation.current_is_usable is False
|
||||
|
||||
|
||||
def test_engine_assumes_usable_without_a_ranking() -> None:
|
||||
"""No ranking (routing off, or the engine's own error path) is absence of
|
||||
evidence — it must not trigger a surprise Fallback switch."""
|
||||
service = FakeRoutingService(FakeRouteResult(make_decision(), ranking=None))
|
||||
|
||||
evaluation = CoreRoutingEngine(service).evaluate(make_request(), RoutingMode.AUTO)
|
||||
|
||||
assert evaluation.current_is_usable is True
|
||||
|
||||
|
||||
def test_engine_assumes_usable_when_the_ranking_misbehaves() -> None:
|
||||
"""A broken ranking object must not fail the turn."""
|
||||
class BrokenRanking:
|
||||
def score_of(self, key):
|
||||
raise RuntimeError("corrupt ranking")
|
||||
|
||||
service = FakeRoutingService(FakeRouteResult(make_decision(), ranking=BrokenRanking()))
|
||||
|
||||
evaluation = CoreRoutingEngine(service).evaluate(make_request(), RoutingMode.AUTO)
|
||||
|
||||
assert evaluation.current_is_usable is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw, expected",
|
||||
[("coding", TaskType.CODING), ("QA", TaskType.QA), (None, None), ("nonsense", None)],
|
||||
)
|
||||
def test_task_type_strings_are_coerced_or_dropped(raw, expected) -> None:
|
||||
"""A pinned task type is honoured; an unknown one falls back to letting the
|
||||
engine classify the prompt rather than raising mid-turn."""
|
||||
service = FakeRoutingService(FakeRouteResult(make_decision(should_switch=False)))
|
||||
|
||||
CoreRoutingEngine(service).evaluate(make_request(task_type=raw), RoutingMode.AUTO)
|
||||
|
||||
assert service.calls[0]["task_type"] == expected
|
||||
|
||||
|
||||
def test_required_capabilities_are_passed_as_a_list_or_none() -> None:
|
||||
"""``rank_models`` filters on a list; an empty tuple must become None so it
|
||||
is treated as "no filter" rather than "require nothing, but filter"."""
|
||||
service = FakeRoutingService(FakeRouteResult(make_decision(should_switch=False)))
|
||||
engine = CoreRoutingEngine(service)
|
||||
|
||||
engine.evaluate(make_request(required_capabilities=("vision",)), RoutingMode.AUTO)
|
||||
engine.evaluate(make_request(), RoutingMode.AUTO)
|
||||
|
||||
assert service.calls[0]["required_capabilities"] == ["vision"]
|
||||
assert service.calls[1]["required_capabilities"] is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Mode resolver + wiring
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_mode_resolver_reads_the_per_workspace_mode() -> None:
|
||||
"""Per-workspace routing keeps working now that the lookup left the widgets."""
|
||||
class StubCtx:
|
||||
def project_routing_mode(self, surface):
|
||||
return "fallback" if surface == "co4e" else "off"
|
||||
|
||||
resolver = AppContextModeResolver(StubCtx())
|
||||
|
||||
assert resolver.mode_for("co4e") is RoutingMode.FALLBACK
|
||||
assert resolver.mode_for("cowork") is RoutingMode.OFF
|
||||
|
||||
|
||||
def test_service_is_built_once_and_cached_on_the_context() -> None:
|
||||
"""Every surface must share one instance, so future per-surface state (a
|
||||
cool-down, a switch history) is shared rather than duplicated per widget."""
|
||||
class StubCtx:
|
||||
def __init__(self):
|
||||
self.routing_calls = 0
|
||||
self.config = type("Cfg", (), {"routing": {"confirm_timeout_sec": 45}})()
|
||||
|
||||
def routing(self):
|
||||
self.routing_calls += 1
|
||||
return FakeRoutingService(FakeRouteResult(make_decision(should_switch=False)))
|
||||
|
||||
def project_routing_mode(self, surface):
|
||||
return "off"
|
||||
|
||||
ctx = StubCtx()
|
||||
first = build_routing_application_service(ctx)
|
||||
second = build_routing_application_service(ctx)
|
||||
|
||||
assert first is second
|
||||
assert ctx.routing_calls == 1
|
||||
assert isinstance(first, RoutingApplicationService)
|
||||
# The confirm timeout is read from config at call time, not frozen at build.
|
||||
assert first.confirm_timeout() == pytest.approx(45.0)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""R04-T04 — unit tests for the UI-state -> request mapping.
|
||||
|
||||
Three small rules used to sit inline in ``ui/cowork_tab.py::build_job``, where no
|
||||
test could reach them: the turn's prompt is the last message in the working list,
|
||||
the history is everything before it, and the confirm-commands flag becomes a gate
|
||||
mode. Getting any of them wrong is silent (a duplicated user message, a command
|
||||
that stops asking for approval), so they are pinned here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from cowork_local.application.conversations.cowork_turn_request import (
|
||||
build_cowork_turn_request,
|
||||
)
|
||||
|
||||
|
||||
def _build(**overrides):
|
||||
base = {
|
||||
"turn_id": "t3",
|
||||
"session_id": "s1",
|
||||
"messages": [{"role": "user", "content": "make me a report"}],
|
||||
}
|
||||
base.update(overrides)
|
||||
return build_cowork_turn_request(**base)
|
||||
|
||||
|
||||
def test_the_last_message_becomes_the_prompt_and_the_rest_the_history() -> None:
|
||||
request = _build(messages=[
|
||||
{"role": "user", "content": "earlier"},
|
||||
{"role": "assistant", "content": "sure"},
|
||||
{"role": "user", "content": "now this"},
|
||||
])
|
||||
|
||||
assert request.prompt == "now this"
|
||||
assert request.messages == ({"role": "user", "content": "earlier"},
|
||||
{"role": "assistant", "content": "sure"})
|
||||
|
||||
|
||||
def test_an_empty_working_list_yields_an_empty_prompt() -> None:
|
||||
# Defensive: a turn with no message at all must not raise on messages[-1].
|
||||
request = _build(messages=[])
|
||||
|
||||
assert request.prompt == ""
|
||||
assert request.messages == ()
|
||||
|
||||
|
||||
def test_confirming_commands_puts_the_turn_in_confirm_gate_mode() -> None:
|
||||
assert _build(confirm_commands=True).gate_mode == "confirm"
|
||||
assert _build(confirm_commands=False).gate_mode == "auto"
|
||||
assert _build().gate_mode == "auto" # auto-run is the default
|
||||
|
||||
|
||||
def test_the_captured_widget_state_is_carried_into_the_request() -> None:
|
||||
request = _build(
|
||||
surface="cowork", project_id="p7", title="Weekly report",
|
||||
provider_id="anthropic", model="claude-sonnet-4-6",
|
||||
instructions="PROJECT RULES", output_dir="out/.turns/t3",
|
||||
home_output_root="out", agent_role="cowork",
|
||||
)
|
||||
|
||||
assert (request.turn_id, request.session_id) == ("t3", "s1")
|
||||
assert (request.surface, request.project_id, request.title) == \
|
||||
("cowork", "p7", "Weekly report")
|
||||
assert (request.provider_id, request.model) == ("anthropic", "claude-sonnet-4-6")
|
||||
assert request.project_context == "PROJECT RULES"
|
||||
assert request.output_dir == Path("out/.turns/t3")
|
||||
assert request.home_output_root == Path("out")
|
||||
assert request.agent_role == "cowork"
|
||||
|
||||
|
||||
def test_the_prompt_survives_a_message_whose_content_is_missing() -> None:
|
||||
request = _build(messages=[{"role": "user"}])
|
||||
|
||||
assert request.prompt == ""
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Unit tests for FakeProvider and FakeToolExecutor test doubles."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from providers.base import ProviderError
|
||||
from tests.fakes.fake_provider import FakeProvider
|
||||
from tests.fakes.fake_tool_executor import FakeToolExecutor
|
||||
|
||||
|
||||
def test_fake_provider_text_streaming() -> None:
|
||||
"""Verify that FakeProvider streams text chunks to on_text callback."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Hello world", chunks=["Hello ", "world"])
|
||||
|
||||
streamed: list[str] = []
|
||||
response = provider.chat(
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
on_text=lambda piece: streamed.append(piece),
|
||||
)
|
||||
|
||||
assert response["role"] == "assistant"
|
||||
assert response["content"] == "Hello world"
|
||||
assert "".join(streamed) == "Hello world"
|
||||
assert provider.call_count == 1
|
||||
|
||||
|
||||
def test_fake_provider_tool_calls_and_reasoning() -> None:
|
||||
"""Verify reasoning streaming and tool_calls payload emission."""
|
||||
provider = FakeProvider()
|
||||
tool_call = {
|
||||
"id": "call_123",
|
||||
"name": "save_file",
|
||||
"arguments": {"filename": "out.txt", "content": "data"},
|
||||
}
|
||||
provider.queue_response(
|
||||
content="Creating file",
|
||||
tool_calls=[tool_call],
|
||||
reasoning="User wants output in a file",
|
||||
)
|
||||
|
||||
reasoning_chunks: list[str] = []
|
||||
response = provider.chat(
|
||||
messages=[{"role": "user", "content": "Save to out.txt"}],
|
||||
on_reasoning=lambda piece: reasoning_chunks.append(piece),
|
||||
)
|
||||
|
||||
assert response["content"] == "Creating file"
|
||||
assert response["tool_calls"] == [tool_call]
|
||||
assert reasoning_chunks == ["User wants output in a file"]
|
||||
|
||||
|
||||
def test_fake_provider_error_injection() -> None:
|
||||
"""Verify that queued exceptions are raised on demand."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_error(ProviderError("Rate limit exceeded (429)"))
|
||||
|
||||
with pytest.raises(ProviderError, match="Rate limit exceeded"):
|
||||
provider.chat(messages=[{"role": "user", "content": "Hi"}])
|
||||
|
||||
|
||||
def test_fake_provider_cancellation() -> None:
|
||||
"""Verify that cancellation stops execution immediately."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Long reply", chunks=["Part 1", "Part 2"])
|
||||
|
||||
is_cancelled = False
|
||||
|
||||
def cancel_fn() -> bool:
|
||||
return is_cancelled
|
||||
|
||||
is_cancelled = True
|
||||
with pytest.raises(ProviderError, match="aborted by user cancel"):
|
||||
provider.chat(
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
cancel=cancel_fn,
|
||||
)
|
||||
|
||||
|
||||
def test_fake_tool_executor() -> None:
|
||||
"""Verify that FakeToolExecutor records calls and returns expected mock outputs."""
|
||||
executor = FakeToolExecutor()
|
||||
executor.set_mock_response("read_file", {"ok": True, "content": "file contents"})
|
||||
executor.register_handler("calc", lambda args: {"ok": True, "result": args.get("a", 0) + args.get("b", 0)})
|
||||
|
||||
res1 = executor.execute("read_file", {"path": "test.txt"})
|
||||
assert res1["ok"] is True
|
||||
assert res1["content"] == "file contents"
|
||||
|
||||
res2 = executor.execute("calc", {"a": 5, "b": 10})
|
||||
assert res2["result"] == 15
|
||||
|
||||
assert len(executor.call_log) == 2
|
||||
assert executor.get_calls_for("calc")[0]["args"] == {"a": 5, "b": 10}
|
||||
@@ -0,0 +1,204 @@
|
||||
"""R03-T02 — unit tests for ProviderDescriptor and the central ProviderRegistry.
|
||||
|
||||
Covers what the rest of the app now relies on the catalogue for: resolving ids
|
||||
and aliases, resolving a bare model id back to its provider, filling in default
|
||||
models, and refusing to let a duplicate registration silently hijack a built-in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from cowork_local.domain.models.provider_descriptor import (
|
||||
AuthKind,
|
||||
ProviderDescriptor,
|
||||
WireProtocol,
|
||||
)
|
||||
from cowork_local.infrastructure.providers.provider_registry import (
|
||||
BUILTIN_DESCRIPTORS,
|
||||
ProviderNotFoundError,
|
||||
ProviderRegistry,
|
||||
)
|
||||
|
||||
|
||||
def make_descriptor(**overrides) -> ProviderDescriptor:
|
||||
"""A minimal valid descriptor; tests override just the field under test."""
|
||||
fields = dict(
|
||||
provider_id="demo",
|
||||
display_name="Demo provider",
|
||||
wire_protocol=WireProtocol.OPENAI_COMPAT,
|
||||
default_model="demo-small",
|
||||
models=("demo-small", "demo-large"),
|
||||
)
|
||||
fields.update(overrides)
|
||||
return ProviderDescriptor(**fields)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ProviderDescriptor
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_descriptor_rejects_an_empty_id() -> None:
|
||||
"""An id-less descriptor could never be looked up, so it must not exist."""
|
||||
with pytest.raises(ValueError):
|
||||
make_descriptor(provider_id="")
|
||||
|
||||
|
||||
def test_descriptor_rejects_a_non_enum_protocol() -> None:
|
||||
"""The protocol drives adapter selection; a stray string would silently
|
||||
fall through to "no adapter" at build time instead of failing here."""
|
||||
with pytest.raises(TypeError):
|
||||
make_descriptor(wire_protocol="openai_compat")
|
||||
|
||||
|
||||
def test_descriptor_is_immutable() -> None:
|
||||
"""Descriptors are shared process-wide; a mutation would be visible to every
|
||||
other reader mid-iteration."""
|
||||
descriptor = make_descriptor()
|
||||
|
||||
with pytest.raises(Exception):
|
||||
descriptor.default_model = "hacked" # type: ignore[misc]
|
||||
|
||||
|
||||
def test_id_matching_ignores_case_and_honours_aliases() -> None:
|
||||
"""Provider ids come from hand-edited config files and old app versions."""
|
||||
descriptor = make_descriptor(aliases=("legacy-demo",))
|
||||
|
||||
assert descriptor.matches("DEMO")
|
||||
assert descriptor.matches(" legacy-demo ")
|
||||
assert not descriptor.matches("other")
|
||||
|
||||
|
||||
def test_capabilities_use_the_routing_vocabulary() -> None:
|
||||
"""The set must be feedable straight into the routing selector's filter."""
|
||||
descriptor = make_descriptor(supports_vision=True, supports_tools=True,
|
||||
supports_streaming=False)
|
||||
|
||||
assert descriptor.capabilities == frozenset({"vision", "tools"})
|
||||
assert descriptor.has_capability("vision")
|
||||
assert not descriptor.has_capability("streaming")
|
||||
|
||||
|
||||
def test_average_cost_is_none_when_a_price_is_unknown() -> None:
|
||||
"""Unknown prices stay unknown — a guessed number would silently skew the
|
||||
routing scorer's cost term."""
|
||||
assert make_descriptor(cost_per_1k_input=0.5).avg_cost_per_1k is None
|
||||
priced = make_descriptor(cost_per_1k_input=1.0, cost_per_1k_output=3.0)
|
||||
# Same 1:3 input:output weighting as ModelMetadata.avg_cost_per_1k.
|
||||
assert priced.avg_cost_per_1k == pytest.approx((1.0 + 9.0) / 4.0)
|
||||
|
||||
|
||||
def test_resolve_model_prefers_the_caller_then_the_default() -> None:
|
||||
"""One place implements the "picked model or provider default" fallback that
|
||||
every chat surface used to re-implement inline."""
|
||||
descriptor = make_descriptor()
|
||||
|
||||
assert descriptor.resolve_model("demo-large") == "demo-large"
|
||||
assert descriptor.resolve_model("") == "demo-small"
|
||||
assert descriptor.resolve_model(" ") == "demo-small"
|
||||
|
||||
|
||||
def test_with_models_repoints_a_default_that_vanished() -> None:
|
||||
"""After discovery, the default must still name a model that exists."""
|
||||
descriptor = make_descriptor()
|
||||
|
||||
updated = descriptor.with_models(["demo-v2", "demo-v2", "demo-v3"])
|
||||
|
||||
assert updated.models == ("demo-v2", "demo-v3") # de-duplicated, order kept
|
||||
assert updated.default_model == "demo-v2"
|
||||
assert descriptor.models == ("demo-small", "demo-large"), "original was mutated"
|
||||
|
||||
|
||||
def test_with_models_keeps_a_default_that_survived() -> None:
|
||||
"""Discovery must not reshuffle a user's working selection."""
|
||||
updated = make_descriptor().with_models(["demo-large", "demo-small"])
|
||||
|
||||
assert updated.default_model == "demo-small"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ProviderRegistry
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_registry_resolves_ids_aliases_and_reports_unknowns() -> None:
|
||||
"""Lookup must be forgiving about form, but loud about genuinely unknown
|
||||
providers — a typo should fail at the call site, not as a None later."""
|
||||
registry = ProviderRegistry([make_descriptor(aliases=("legacy-demo",))])
|
||||
|
||||
assert registry.get("demo").provider_id == "demo"
|
||||
assert registry.get("legacy-demo").provider_id == "demo"
|
||||
assert registry.find("missing") is None
|
||||
assert "demo" in registry
|
||||
with pytest.raises(ProviderNotFoundError):
|
||||
registry.get("missing")
|
||||
|
||||
|
||||
def test_registry_refuses_to_overwrite_silently_but_replace_works() -> None:
|
||||
"""A second registration of the same id is almost always a bug; updating a
|
||||
descriptor is a deliberate act with its own method."""
|
||||
registry = ProviderRegistry([make_descriptor()])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
registry.register(make_descriptor(display_name="Impostor"))
|
||||
|
||||
registry.replace(make_descriptor(display_name="Renamed"))
|
||||
assert registry.get("demo").display_name == "Renamed"
|
||||
assert len(registry) == 1
|
||||
|
||||
|
||||
def test_registry_re_registering_an_identical_descriptor_is_a_no_op() -> None:
|
||||
"""Idempotent registration keeps repeated bootstrap calls harmless."""
|
||||
registry = ProviderRegistry([make_descriptor()])
|
||||
|
||||
registry.register(make_descriptor())
|
||||
|
||||
assert len(registry) == 1
|
||||
|
||||
|
||||
def test_find_by_model_resolves_a_bare_model_id() -> None:
|
||||
"""Routing decisions and saved conversations sometimes carry only a model
|
||||
name; the registry is what turns that back into a provider."""
|
||||
registry = ProviderRegistry([make_descriptor()])
|
||||
|
||||
assert registry.find_by_model("demo-large").provider_id == "demo"
|
||||
# A gateway model we cannot enumerate offline is a miss, not an error — the
|
||||
# caller falls back to the configured active provider.
|
||||
assert registry.find_by_model("unknown-model") is None
|
||||
assert registry.find_by_model("") is None
|
||||
|
||||
|
||||
def test_builtin_catalogue_covers_every_configured_provider() -> None:
|
||||
"""The catalogue and DEFAULT_CONFIG must not drift: a provider users can
|
||||
configure but the registry cannot build is a dead Settings entry."""
|
||||
from cowork_local.config import DEFAULT_CONFIG
|
||||
|
||||
registry = ProviderRegistry(BUILTIN_DESCRIPTORS)
|
||||
|
||||
for provider_id in DEFAULT_CONFIG["providers"]:
|
||||
assert registry.find(provider_id) is not None, f"{provider_id} missing from registry"
|
||||
|
||||
|
||||
def test_build_fills_in_the_default_model() -> None:
|
||||
"""A half-written config must still produce a usable provider rather than an
|
||||
empty model id that only fails once the request reaches the gateway."""
|
||||
registry = ProviderRegistry(BUILTIN_DESCRIPTORS)
|
||||
|
||||
provider = registry.build("anthropic", {"api_key": "k"})
|
||||
|
||||
assert provider.model == registry.get("anthropic").default_model
|
||||
|
||||
|
||||
def test_build_respects_an_explicit_model() -> None:
|
||||
"""Per-tab model selection must win over the catalogue default."""
|
||||
registry = ProviderRegistry(BUILTIN_DESCRIPTORS)
|
||||
|
||||
provider = registry.build("anthropic", {"api_key": "k", "model": "claude-opus-4-8"})
|
||||
|
||||
assert provider.model == "claude-opus-4-8"
|
||||
|
||||
|
||||
def test_factory_still_raises_provider_error_for_unknown_ids() -> None:
|
||||
"""Existing call sites catch ProviderError; routing lookups through the
|
||||
registry must not change the exception type they see."""
|
||||
from cowork_local.providers import build_provider
|
||||
from cowork_local.providers.base import ProviderError
|
||||
|
||||
with pytest.raises(ProviderError):
|
||||
build_provider("definitely-not-a-provider", {})
|
||||
@@ -0,0 +1,384 @@
|
||||
"""R03-T03 — unit tests for the unified routing decision rules.
|
||||
|
||||
The point of moving these rules out of the three chat widgets is that they can
|
||||
now be exercised without Qt, without the assessment store and without a network:
|
||||
the service talks to two narrow ports, so every mode is driven here by ~10-line
|
||||
fakes. Each test names the behaviour a chat surface depends on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from cowork_local.application.model_routing import (
|
||||
RouteEvaluation,
|
||||
RoutingApplicationService,
|
||||
RoutingMode,
|
||||
RoutingOutcome,
|
||||
RoutingRequest,
|
||||
)
|
||||
|
||||
|
||||
class FakeDecisionPort:
|
||||
"""A routing engine that returns a canned verdict and records its input."""
|
||||
|
||||
def __init__(self, evaluation: RouteEvaluation) -> None:
|
||||
self.evaluation = evaluation
|
||||
self.calls: list = []
|
||||
|
||||
def evaluate(self, request: RoutingRequest, mode: RoutingMode) -> RouteEvaluation:
|
||||
self.calls.append((request, mode))
|
||||
return self.evaluation
|
||||
|
||||
|
||||
class ExplodingDecisionPort:
|
||||
"""An engine that fails — proves routing degrades instead of breaking a turn."""
|
||||
|
||||
def evaluate(self, request: RoutingRequest, mode: RoutingMode) -> RouteEvaluation:
|
||||
raise RuntimeError("assessment store is corrupt")
|
||||
|
||||
|
||||
class FakeModeResolver:
|
||||
"""Per-surface mode lookup, standing in for the workspace settings."""
|
||||
|
||||
def __init__(self, mode) -> None:
|
||||
self.mode = mode
|
||||
self.surfaces: list = []
|
||||
|
||||
def mode_for(self, surface: str):
|
||||
self.surfaces.append(surface)
|
||||
return self.mode
|
||||
|
||||
|
||||
def make_request(**overrides) -> RoutingRequest:
|
||||
"""A representative turn: Cowork chat, currently on a cheap OpenAI model."""
|
||||
fields = dict(
|
||||
surface="cowork",
|
||||
prompt="Refactor this function",
|
||||
current_provider="codex",
|
||||
current_model="gpt-4o-mini",
|
||||
)
|
||||
fields.update(overrides)
|
||||
return RoutingRequest(**fields)
|
||||
|
||||
|
||||
def switch_evaluation(**overrides) -> RouteEvaluation:
|
||||
"""An engine verdict that proposes a switch to a better coding model."""
|
||||
fields = dict(
|
||||
task_type="coding",
|
||||
should_switch=True,
|
||||
target_provider="anthropic",
|
||||
target_model="claude-sonnet-4-6",
|
||||
score_gain=0.21,
|
||||
reason="coding fit 0.88 > current 0.67",
|
||||
decision=object(),
|
||||
)
|
||||
fields.update(overrides)
|
||||
return RouteEvaluation(**fields)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Off
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_off_mode_never_consults_the_engine() -> None:
|
||||
"""Off must be free: no ranking, no store read, no decision at all."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.OFF))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is False
|
||||
assert outcome.provider is None and outcome.model is None
|
||||
assert port.calls == [], "Off mode must not call the routing engine"
|
||||
|
||||
|
||||
def test_missing_mode_resolver_defaults_to_off() -> None:
|
||||
"""Routing stays opt-in: with no way to read the mode, never switch."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port)
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.mode is RoutingMode.OFF
|
||||
assert outcome.switched is False
|
||||
|
||||
|
||||
def test_empty_prompt_is_not_routed() -> None:
|
||||
"""An empty message carries no signal to classify, so the engine is skipped."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.AUTO))
|
||||
|
||||
outcome = service.resolve(make_request(prompt=" "))
|
||||
|
||||
assert outcome.switched is False
|
||||
assert port.calls == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Auto
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_auto_mode_switches_silently() -> None:
|
||||
"""Auto applies the engine's verdict without asking the user."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.AUTO))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is True
|
||||
assert outcome.provider == "anthropic"
|
||||
assert outcome.model == "claude-sonnet-4-6"
|
||||
assert outcome.task_type == "coding"
|
||||
assert outcome.score_gain == pytest.approx(0.21)
|
||||
assert outcome.should_notify is True
|
||||
|
||||
|
||||
def test_auto_mode_keeps_current_when_nothing_is_better() -> None:
|
||||
"""No proposed switch means the surface's own selection is untouched."""
|
||||
port = FakeDecisionPort(switch_evaluation(
|
||||
should_switch=False, reason="current model is already best-fit"))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.AUTO))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is False
|
||||
assert outcome.provider is None
|
||||
assert "already best-fit" in outcome.reason
|
||||
|
||||
|
||||
def test_switch_without_a_target_is_ignored() -> None:
|
||||
"""A verdict that says "switch" but names nothing is not actionable — a
|
||||
surface must never be handed an empty model id."""
|
||||
port = FakeDecisionPort(switch_evaluation(target_provider=None, target_model=None))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.AUTO))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is False
|
||||
|
||||
|
||||
def test_same_provider_switch_keeps_the_current_provider() -> None:
|
||||
"""A model-only switch must not blank out the provider the surface uses."""
|
||||
port = FakeDecisionPort(switch_evaluation(target_provider=None, target_model="o3"))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.AUTO))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is True
|
||||
assert outcome.provider == "codex" # unchanged, from the request
|
||||
assert outcome.model == "o3"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Manual
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_manual_mode_switches_only_after_approval() -> None:
|
||||
"""Manual's contract: ask first, then apply exactly what was approved."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(
|
||||
port, FakeModeResolver(RoutingMode.MANUAL),
|
||||
confirm_timeout_sec=lambda: 30.0,
|
||||
)
|
||||
asked: list = []
|
||||
|
||||
def confirm(decision, timeout):
|
||||
asked.append((decision, timeout))
|
||||
return True
|
||||
|
||||
outcome = service.resolve(make_request(), confirm=confirm)
|
||||
|
||||
assert outcome.switched is True
|
||||
assert len(asked) == 1
|
||||
# The configured timeout must reach the dialog, not a hard-coded default.
|
||||
assert asked[0][1] == pytest.approx(30.0)
|
||||
|
||||
|
||||
def test_manual_mode_decline_is_reported_distinctly() -> None:
|
||||
""""The user said no" must be distinguishable from "nothing better found",
|
||||
so a surface can stay quiet in one case and explain itself in the other."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.MANUAL))
|
||||
|
||||
outcome = service.resolve(make_request(), confirm=lambda decision, timeout: False)
|
||||
|
||||
assert outcome.switched is False
|
||||
assert outcome.declined is True
|
||||
|
||||
|
||||
def test_manual_mode_without_a_callback_never_switches() -> None:
|
||||
"""Silently switching in Manual mode would violate the mode's promise."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.MANUAL))
|
||||
|
||||
outcome = service.resolve(make_request(), confirm=None)
|
||||
|
||||
assert outcome.switched is False
|
||||
|
||||
|
||||
def test_manual_mode_treats_a_broken_dialog_as_a_decline() -> None:
|
||||
"""A crashing confirm dialog must not auto-approve a model change."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.MANUAL))
|
||||
|
||||
def confirm(decision, timeout):
|
||||
raise RuntimeError("dialog blew up")
|
||||
|
||||
outcome = service.resolve(make_request(), confirm=confirm)
|
||||
|
||||
assert outcome.switched is False
|
||||
assert outcome.declined is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fallback
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_fallback_keeps_a_healthy_model_even_when_a_better_one_exists() -> None:
|
||||
"""Fallback is a resilience mode, not an optimiser: a usable pinned model
|
||||
wins over a higher-scoring candidate."""
|
||||
port = FakeDecisionPort(switch_evaluation(current_is_usable=True))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.FALLBACK))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is False
|
||||
assert "healthy" in outcome.reason
|
||||
|
||||
|
||||
def test_fallback_switches_when_the_current_model_cannot_serve_the_turn() -> None:
|
||||
"""The one case Fallback exists for: rescue an unusable selection."""
|
||||
port = FakeDecisionPort(switch_evaluation(current_is_usable=False))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.FALLBACK))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is True
|
||||
assert outcome.model == "claude-sonnet-4-6"
|
||||
|
||||
|
||||
def test_fallback_asks_the_engine_with_auto_semantics() -> None:
|
||||
"""The engine only understands off/auto/manual, so Fallback must reach it as
|
||||
Auto — otherwise the engine would reject the unknown mode and rank nothing."""
|
||||
port = FakeDecisionPort(switch_evaluation(current_is_usable=False))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.FALLBACK))
|
||||
|
||||
service.resolve(make_request())
|
||||
|
||||
assert port.calls[0][1] is RoutingMode.AUTO
|
||||
|
||||
|
||||
def test_fallback_never_confirms_with_the_user() -> None:
|
||||
"""Rescuing an unusable model is not a proposal — it happens silently."""
|
||||
port = FakeDecisionPort(switch_evaluation(current_is_usable=False))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.FALLBACK))
|
||||
asked: list = []
|
||||
|
||||
outcome = service.resolve(
|
||||
make_request(), confirm=lambda decision, timeout: asked.append(1) or True)
|
||||
|
||||
assert outcome.switched is True
|
||||
assert asked == []
|
||||
|
||||
|
||||
def test_fallback_with_no_replacement_keeps_current() -> None:
|
||||
"""Nothing to fall back to means keep going with what we have and let the
|
||||
provider surface the real error, rather than blanking the model."""
|
||||
port = FakeDecisionPort(switch_evaluation(
|
||||
current_is_usable=False, target_provider=None, target_model=None))
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.FALLBACK))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.switched is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Robustness & plumbing
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_engine_failure_degrades_to_keep_current() -> None:
|
||||
"""A broken assessment store must never stop a user sending a message."""
|
||||
service = RoutingApplicationService(
|
||||
ExplodingDecisionPort(), FakeModeResolver(RoutingMode.AUTO))
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert isinstance(outcome, RoutingOutcome)
|
||||
assert outcome.switched is False
|
||||
assert "error" in outcome.reason
|
||||
|
||||
|
||||
def test_mode_resolver_failure_degrades_to_off() -> None:
|
||||
"""An unreadable workspace config must not enable routing by accident."""
|
||||
class BrokenResolver:
|
||||
def mode_for(self, surface):
|
||||
raise OSError("workspace file unreadable")
|
||||
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, BrokenResolver())
|
||||
|
||||
outcome = service.resolve(make_request())
|
||||
|
||||
assert outcome.mode is RoutingMode.OFF
|
||||
assert port.calls == []
|
||||
|
||||
|
||||
def test_explicit_request_mode_overrides_the_resolver() -> None:
|
||||
"""A surface may pin the mode for one turn (tests, replay, admin actions)."""
|
||||
resolver = FakeModeResolver(RoutingMode.OFF)
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, resolver)
|
||||
|
||||
outcome = service.resolve(make_request(mode=RoutingMode.AUTO))
|
||||
|
||||
assert outcome.switched is True
|
||||
assert resolver.surfaces == [], "an explicit mode must skip the resolver"
|
||||
|
||||
|
||||
def test_request_is_forwarded_to_the_engine_unchanged() -> None:
|
||||
"""Surface, prompt and pinned task type must survive the hand-off — AI-Edit
|
||||
relies on its "coding" pin reaching the engine."""
|
||||
port = FakeDecisionPort(switch_evaluation())
|
||||
service = RoutingApplicationService(port, FakeModeResolver(RoutingMode.AUTO))
|
||||
request = make_request(surface="ai_edit", task_type="coding",
|
||||
required_capabilities=("vision",))
|
||||
|
||||
service.resolve(request)
|
||||
|
||||
forwarded = port.calls[0][0]
|
||||
assert forwarded is request
|
||||
assert forwarded.surface == "ai_edit"
|
||||
assert forwarded.task_type == "coding"
|
||||
assert forwarded.required_capabilities == ("vision",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw, expected",
|
||||
[
|
||||
("auto", RoutingMode.AUTO),
|
||||
("MANUAL", RoutingMode.MANUAL),
|
||||
(" fallback ", RoutingMode.FALLBACK),
|
||||
("nonsense", RoutingMode.OFF),
|
||||
("", RoutingMode.OFF),
|
||||
(None, RoutingMode.OFF),
|
||||
],
|
||||
)
|
||||
def test_mode_parsing_is_forgiving(raw, expected) -> None:
|
||||
"""Config values are hand-edited; an unknown one must degrade, not raise."""
|
||||
assert RoutingMode.parse(raw) is expected
|
||||
|
||||
|
||||
def test_confirm_timeout_falls_back_to_the_default_when_unusable() -> None:
|
||||
"""A corrupted timeout must not produce a zero-second dialog that declines
|
||||
every switch before the user can read it."""
|
||||
service = RoutingApplicationService(
|
||||
FakeDecisionPort(switch_evaluation()),
|
||||
FakeModeResolver(RoutingMode.MANUAL),
|
||||
confirm_timeout_sec=lambda: 0.0,
|
||||
)
|
||||
|
||||
assert service.confirm_timeout() == RoutingApplicationService.DEFAULT_CONFIRM_TIMEOUT_SEC
|
||||
|
||||
|
||||
def test_routing_request_is_immutable() -> None:
|
||||
"""The snapshot must not change under a turn that is already in flight."""
|
||||
request = make_request()
|
||||
|
||||
with pytest.raises(Exception):
|
||||
request.prompt = "something else" # type: ignore[misc]
|
||||
@@ -0,0 +1,34 @@
|
||||
"""R04-T05 — unit tests for the unattended-run prompt assembly.
|
||||
|
||||
``_run_agent`` used to build this by rebinding ``prompt`` three times, each with
|
||||
its own ``f"{block}\n\n{prompt}"``. The ORDER that produced is load-bearing (the
|
||||
plan reminder has to lead, the task's own words have to trail) and it was
|
||||
readable only by replaying the rebindings in your head.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cowork_local.core.task_executors import _unattended_prompt
|
||||
|
||||
|
||||
def test_the_plan_reminder_leads_and_the_task_prompt_trails() -> None:
|
||||
built = _unattended_prompt("write the report")
|
||||
|
||||
assert built.startswith("This runs unattended (Schedule Task)")
|
||||
assert built.endswith("write the report")
|
||||
|
||||
|
||||
def test_a_skill_block_sits_between_the_reminder_and_the_agent_persona() -> None:
|
||||
built = _unattended_prompt("write the report", skill_text="SKILL",
|
||||
agent_instructions="PERSONA")
|
||||
|
||||
assert built.index("This runs unattended") < built.index("SKILL")
|
||||
assert built.index("SKILL") < built.index("PERSONA")
|
||||
assert built.index("PERSONA") < built.index("write the report")
|
||||
|
||||
|
||||
def test_absent_blocks_leave_no_extra_blank_lines() -> None:
|
||||
built = _unattended_prompt("do it", skill_text="", agent_instructions=None)
|
||||
|
||||
assert "\n\n\n" not in built
|
||||
assert built.count("do it") == 1
|
||||
@@ -0,0 +1,36 @@
|
||||
"""R04-T04 — unit tests for the shared turn-runtime helpers.
|
||||
|
||||
``combine_instructions`` is the small rule the UI applied inline: a turn's
|
||||
standing instructions are several independent blocks (project context, an Admin
|
||||
agent's persona, a skill's rules, an unattended-run reminder) that must be joined
|
||||
with one blank line, skipping whatever is absent. Two call sites need it (T04's
|
||||
widget and T05's task runner), which is exactly when a rule stops being an inline
|
||||
expression.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cowork_local.application.conversations.turn_runtime import combine_instructions
|
||||
|
||||
|
||||
def test_two_blocks_are_joined_by_a_blank_line() -> None:
|
||||
assert combine_instructions("PROJECT", "AGENT") == "PROJECT\n\nAGENT"
|
||||
|
||||
|
||||
def test_an_absent_block_leaves_no_blank_line_behind() -> None:
|
||||
assert combine_instructions("", "AGENT") == "AGENT"
|
||||
assert combine_instructions("PROJECT", "") == "PROJECT"
|
||||
assert combine_instructions("PROJECT", None) == "PROJECT"
|
||||
|
||||
|
||||
def test_whitespace_only_blocks_do_not_count_as_instructions() -> None:
|
||||
assert combine_instructions(" \n ", "AGENT") == "AGENT"
|
||||
|
||||
|
||||
def test_nothing_to_say_produces_an_empty_string() -> None:
|
||||
assert combine_instructions() == ""
|
||||
assert combine_instructions("", None, " ") == ""
|
||||
|
||||
|
||||
def test_more_than_two_blocks_keep_their_order() -> None:
|
||||
assert combine_instructions("A", "B", "C") == "A\n\nB\n\nC"
|
||||
@@ -0,0 +1,184 @@
|
||||
"""R03-T06 — unit tests for the token-usage telemetry seam.
|
||||
|
||||
The seam exists so provider adapters stop owning telemetry policy. These tests
|
||||
pin the two properties that makes that safe: events reach every subscriber, and
|
||||
no telemetry failure can ever propagate back into the turn that produced it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from cowork_local.infrastructure.telemetry import usage_sink
|
||||
from cowork_local.infrastructure.telemetry.usage_sink import (
|
||||
CompositeUsageSink,
|
||||
InMemoryUsageSink,
|
||||
UsageEvent,
|
||||
UsageTrackerSink,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_sink(monkeypatch):
|
||||
"""Give every test its own process-wide sink.
|
||||
|
||||
Autouse because a leaked sink would let one test's subscriber observe the
|
||||
next test's events — and, worse, let a test write to the developer's real
|
||||
usage files through the default tracker sink.
|
||||
"""
|
||||
monkeypatch.setattr(usage_sink, "_sink", None)
|
||||
yield
|
||||
monkeypatch.setattr(usage_sink, "_sink", None)
|
||||
|
||||
|
||||
def make_event(**overrides) -> UsageEvent:
|
||||
fields = dict(provider="anthropic", model="claude-sonnet-4-6",
|
||||
input_tokens=100, output_tokens=40, cached_tokens=10)
|
||||
fields.update(overrides)
|
||||
return UsageEvent(**fields)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# UsageEvent
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_event_is_immutable() -> None:
|
||||
"""A subscriber must not be able to edit the event the next one receives."""
|
||||
event = make_event()
|
||||
|
||||
with pytest.raises(Exception):
|
||||
event.input_tokens = 0 # type: ignore[misc]
|
||||
|
||||
|
||||
def test_total_tokens_does_not_double_count_cache_reads() -> None:
|
||||
"""Every gateway we support already reports cached tokens inside the input
|
||||
count, so adding them again would inflate the dashboard."""
|
||||
assert make_event().total_tokens == 140
|
||||
|
||||
|
||||
def test_to_dict_uses_the_stored_row_keys() -> None:
|
||||
"""Matching the tracker's short keys lets a caller diff an event against a
|
||||
persisted row without a translation table."""
|
||||
row = make_event(source="cowork", label="Refactor chat").to_dict()
|
||||
|
||||
assert row["in"] == 100 and row["out"] == 40 and row["cache"] == 10
|
||||
assert row["source"] == "cowork" and row["label"] == "Refactor chat"
|
||||
assert row["estimated"] is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fan-out
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_publish_reaches_every_subscriber() -> None:
|
||||
"""The whole point of the seam: extra consumers attach without patching
|
||||
provider code."""
|
||||
first, second = InMemoryUsageSink(), InMemoryUsageSink()
|
||||
usage_sink.set_usage_sink(CompositeUsageSink([first, second]))
|
||||
|
||||
usage_sink.publish(make_event())
|
||||
|
||||
assert len(first.snapshot()) == 1
|
||||
assert len(second.snapshot()) == 1
|
||||
|
||||
|
||||
def test_one_failing_subscriber_does_not_starve_the_others() -> None:
|
||||
"""A buggy consumer must not silently disable the Dashboard."""
|
||||
class Exploding:
|
||||
def emit(self, event):
|
||||
raise RuntimeError("subscriber is broken")
|
||||
|
||||
healthy = InMemoryUsageSink()
|
||||
usage_sink.set_usage_sink(CompositeUsageSink([Exploding(), healthy]))
|
||||
|
||||
usage_sink.publish(make_event())
|
||||
|
||||
assert len(healthy.snapshot()) == 1
|
||||
|
||||
|
||||
def test_subscribe_and_unsubscribe_round_trip() -> None:
|
||||
"""Teardown code calls unsubscribe unconditionally, so removing a sink that
|
||||
was never added must be harmless."""
|
||||
extra = InMemoryUsageSink()
|
||||
|
||||
usage_sink.subscribe(extra)
|
||||
usage_sink.publish(make_event())
|
||||
usage_sink.unsubscribe(extra)
|
||||
usage_sink.unsubscribe(extra) # second removal is a no-op
|
||||
usage_sink.publish(make_event(model="claude-opus-4-8"))
|
||||
|
||||
assert [e.model for e in extra.snapshot()] == ["claude-sonnet-4-6"]
|
||||
|
||||
|
||||
def test_default_sink_is_the_usage_tracker() -> None:
|
||||
"""Out of the box the seam must preserve the existing Dashboard pipeline."""
|
||||
sinks = usage_sink.get_usage_sink().sinks()
|
||||
|
||||
assert any(isinstance(s, UsageTrackerSink) for s in sinks)
|
||||
|
||||
|
||||
def test_in_memory_sink_totals_and_clears() -> None:
|
||||
"""Test-double conveniences the contract suite relies on."""
|
||||
sink = InMemoryUsageSink()
|
||||
sink.emit(make_event())
|
||||
sink.emit(make_event(input_tokens=1, output_tokens=1, cached_tokens=0))
|
||||
|
||||
assert sink.total_tokens == 142
|
||||
sink.clear()
|
||||
assert sink.snapshot() == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# UsageTrackerSink forwarding
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_tracker_sink_forwards_the_counts() -> None:
|
||||
"""The adapter must hand the tracker exactly what the provider measured."""
|
||||
recorded: list = []
|
||||
|
||||
def fake_record(provider, model, tokens_in, tokens_out, cached, estimated=False):
|
||||
recorded.append((provider, model, tokens_in, tokens_out, cached, estimated))
|
||||
|
||||
UsageTrackerSink(recorder=fake_record).emit(make_event(estimated=True))
|
||||
|
||||
assert recorded == [("anthropic", "claude-sonnet-4-6", 100, 40, 10, True)]
|
||||
|
||||
|
||||
def test_tracker_sink_restores_the_thread_context_it_borrowed() -> None:
|
||||
"""An event carrying its own attribution must relabel ONE row, not every
|
||||
later turn that happens to run on the same worker thread."""
|
||||
from cowork_local.core import usage_tracker as tracker
|
||||
|
||||
tracker.set_context("cowork", "original chat")
|
||||
seen: list = []
|
||||
UsageTrackerSink(recorder=lambda *a, **k: seen.append(tracker.current_context())).emit(
|
||||
make_event(source="co4e", label="flow run"))
|
||||
|
||||
assert seen == [("co4e", "flow run")], "event attribution was not applied"
|
||||
assert tracker.current_context() == ("cowork", "original chat")
|
||||
|
||||
|
||||
def test_tracker_sink_swallows_recorder_failures() -> None:
|
||||
"""Telemetry is never allowed to abort an otherwise successful turn."""
|
||||
def boom(*_args, **_kwargs):
|
||||
raise OSError("usage directory is read-only")
|
||||
|
||||
UsageTrackerSink(recorder=boom).emit(make_event()) # must not raise
|
||||
|
||||
|
||||
def test_publish_never_raises_even_with_a_broken_sink() -> None:
|
||||
"""Last line of defence: providers call publish() inside their stream loop."""
|
||||
class Hostile:
|
||||
def emit(self, event):
|
||||
raise RuntimeError("nope")
|
||||
|
||||
def sinks(self):
|
||||
raise RuntimeError("nope")
|
||||
|
||||
usage_sink.set_usage_sink(Hostile())
|
||||
|
||||
usage_sink.publish(make_event()) # must not raise
|
||||
|
||||
|
||||
def test_estimate_tokens_matches_the_tracker_heuristic() -> None:
|
||||
"""Re-exported so adapters need one telemetry import; it must not drift."""
|
||||
from cowork_local.core import usage_tracker as tracker
|
||||
|
||||
for text in ("", "a", "hello world", "x" * 4001):
|
||||
assert usage_sink.estimate_tokens(text) == tracker.estimate_tokens(text)
|
||||
Reference in New Issue
Block a user