"""Provider contract suite (R03-T01). Every provider - the two real adapters and the test double - must honour the same promises declared in ``providers/base.py``: 1. ``chat()`` returns the canonical assistant message ``{"role": "assistant", "content": str, "tool_calls": [...]}``. 2. Answer text is streamed through ``on_text`` and equals the returned content. 3. Private reasoning goes to ``on_reasoning`` ONLY - it must never leak into the answer, or a reasoning model's chain of thought ends up persisted in history. 4. Tool calls come back as ``{"id", "name", "arguments": dict}`` with arguments already parsed - callers must never have to json.loads() them. 5. A failure raises ``ProviderError`` and nothing else, so one except clause in the agent loop covers every provider. The real adapters are exercised WITHOUT network access by replacing ``Provider._request`` with a canned SSE response - which is exactly the seam ``providers/base.py`` documents for its TLS retry, so no production code needed changing to make this testable. """ from __future__ import annotations import json from typing import Any, Dict, List, Optional import pytest from cowork_local.domain.models.provider_descriptor import ProviderCapability from cowork_local.infrastructure.providers.provider_registry import ( BUILT_IN_PROVIDERS, ProviderRegistry, ) from cowork_local.providers.anthropic import AnthropicProvider from cowork_local.providers.base import Provider, ProviderError, ToolSpec from cowork_local.providers.openai_compat import OpenAICompatProvider from tests.fakes import FakeProvider, ScriptedTurn class _StubResponse: """Minimal stand-in for a streamed ``requests.Response``. Only the members the provider code actually touches are implemented; adding more would invite tests that pass against the stub but not against requests. """ def __init__(self, lines: List[str], status_code: int = 200, text: str = "") -> None: self._lines = lines self.status_code = status_code self.text = text self.headers: Dict[str, str] = {} self.encoding = "utf-8" self.closed = False def iter_lines(self, decode_unicode: bool = False): yield from self._lines def close(self) -> None: self.closed = True def json(self) -> Any: return json.loads(self.text or "{}") def _sse(*payloads: Dict[str, Any]) -> List[str]: """Render payloads as SSE ``data:`` lines, the wire shape both adapters parse.""" return [f"data: {json.dumps(p)}" for p in payloads] @pytest.fixture def canned(monkeypatch): """Return a helper that makes every provider request answer with ``lines``.""" def _install(lines: List[str], status_code: int = 200, text: str = "") -> Dict[str, Any]: seen: Dict[str, Any] = {} def fake_request(self, method, url, **kwargs): # Capture the outgoing payload so tests can assert on how the # canonical message list was translated to the provider's wire format. seen["method"] = method seen["url"] = url seen["json"] = kwargs.get("json") return _StubResponse(lines, status_code=status_code, text=text) monkeypatch.setattr(Provider, "_request", fake_request, raising=True) return seen return _install # --------------------------------------------------------------------------- # # Shared base-class behaviour every provider inherits # --------------------------------------------------------------------------- # def _providers_under_test() -> List[Provider]: """One instance of each implementation, configured but never called.""" conf = {"base_url": "https://example.invalid/v1", "api_key": "k", "model": "m"} return [ OpenAICompatProvider(dict(conf)), AnthropicProvider(dict(conf)), FakeProvider(), ] @pytest.mark.parametrize("provider", _providers_under_test(), ids=lambda p: type(p).__name__) def test_every_provider_exposes_the_base_contract(provider): assert isinstance(provider, Provider) assert callable(provider.chat) assert callable(provider.list_models) assert callable(provider.test_connection) # `name` identifies the provider in usage records and audit entries; an # implementation that forgot to set it would silently report as "base". assert provider.name and provider.name != "base" assert isinstance(provider.supports_vision, bool) assert provider.describe() == f"{provider.name}:{provider.model}" @pytest.mark.parametrize("provider", _providers_under_test(), ids=lambda p: type(p).__name__) def test_strip_think_removes_inline_reasoning_from_a_final_answer(provider): """Safety net for gateways that fold reasoning into the content stream: the answer stored in history must never contain a block.""" assert provider.strip_think("secretAnswer") == "Answer" assert provider.strip_think("Plain answer") == "Plain answer" def test_tool_spec_translates_to_both_wire_formats(): """One ToolSpec must render for both protocols - this is what lets the agent loop build its tool catalogue once and reuse it across providers.""" spec = ToolSpec(name="save_file", description="Write a file", parameters={"type": "object", "properties": {}}) openai_shape = spec.to_openai() anthropic_shape = spec.to_anthropic() assert openai_shape["type"] == "function" assert openai_shape["function"]["name"] == "save_file" assert openai_shape["function"]["parameters"] == spec.parameters # Anthropic names the same field `input_schema`; the values must stay equal, # otherwise the same tool would validate differently per provider. assert anthropic_shape["name"] == "save_file" assert anthropic_shape["input_schema"] == spec.parameters # --------------------------------------------------------------------------- # # Streaming contract - real adapters, canned transport # --------------------------------------------------------------------------- # def test_openai_compat_streams_text_and_returns_canonical_message(canned): canned(_sse( {"choices": [{"delta": {"content": "Hel"}}]}, {"choices": [{"delta": {"content": "lo"}}]}, ) + ["data: [DONE]"]) provider = OpenAICompatProvider({"base_url": "https://x.invalid/v1", "api_key": "k", "model": "m"}) chunks: List[str] = [] result = provider.chat([{"role": "user", "content": "hi"}], on_text=chunks.append) assert "".join(chunks) == "Hello" assert result["role"] == "assistant" assert result["content"] == "Hello" assert result["tool_calls"] == [] def test_openai_compat_keeps_reasoning_out_of_the_answer(canned): canned(_sse( {"choices": [{"delta": {"reasoning_content": "hmm..."}}]}, {"choices": [{"delta": {"content": "42"}}]}, ) + ["data: [DONE]"]) provider = OpenAICompatProvider({"base_url": "https://x.invalid/v1", "api_key": "k", "model": "m"}) text: List[str] = [] reasoning: List[str] = [] result = provider.chat([{"role": "user", "content": "q"}], on_text=text.append, on_reasoning=reasoning.append) assert reasoning == ["hmm..."] assert result["content"] == "42" assert "hmm" not in result["content"] def test_openai_compat_returns_tool_calls_with_parsed_arguments(canned): """Arguments arrive as a JSON string split across chunks; the contract says the caller receives a ready-to-use dict.""" canned(_sse( {"choices": [{"delta": {"tool_calls": [ {"index": 0, "id": "call_a", "function": {"name": "save_file", "arguments": '{"filename":'}}]}}]}, {"choices": [{"delta": {"tool_calls": [ {"index": 0, "function": {"arguments": '"a.md"}'}}]}}]}, ) + ["data: [DONE]"]) provider = OpenAICompatProvider({"base_url": "https://x.invalid/v1", "api_key": "k", "model": "m"}) result = provider.chat([{"role": "user", "content": "save it"}]) assert len(result["tool_calls"]) == 1 call = result["tool_calls"][0] assert call["id"] == "call_a" assert call["name"] == "save_file" assert call["arguments"] == {"filename": "a.md"} def test_anthropic_streams_text_and_returns_canonical_message(canned): canned(_sse( {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hel"}}, {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "lo"}}, {"type": "message_stop"}, )) provider = AnthropicProvider({"base_url": "https://x.invalid", "api_key": "k", "model": "m"}) chunks: List[str] = [] result = provider.chat([{"role": "user", "content": "hi"}], on_text=chunks.append) assert "".join(chunks) == "Hello" assert result["content"] == "Hello" assert result["role"] == "assistant" def test_anthropic_keeps_extended_thinking_out_of_the_answer(canned): canned(_sse( {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "reasoning..."}}, {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "42"}}, {"type": "message_stop"}, )) provider = AnthropicProvider({"base_url": "https://x.invalid", "api_key": "k", "model": "m"}) reasoning: List[str] = [] result = provider.chat([{"role": "user", "content": "q"}], on_reasoning=reasoning.append) assert reasoning == ["reasoning..."] assert result["content"] == "42" def test_anthropic_returns_tool_calls_with_parsed_arguments(canned): canned(_sse( {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "save_file"}}, {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"filename":"a.md"}'}}, {"type": "message_stop"}, )) provider = AnthropicProvider({"base_url": "https://x.invalid", "api_key": "k", "model": "m"}) result = provider.chat([{"role": "user", "content": "save"}]) assert result["tool_calls"] == [ {"id": "toolu_1", "name": "save_file", "arguments": {"filename": "a.md"}} ] @pytest.mark.parametrize("factory", [ lambda: OpenAICompatProvider({"base_url": "https://x.invalid/v1", "api_key": "k", "model": "m"}), lambda: AnthropicProvider({"base_url": "https://x.invalid", "api_key": "k", "model": "m"}), ], ids=["openai_compat", "anthropic"]) def test_transport_failure_surfaces_as_provider_error(canned, factory): """Every failure mode must arrive as ProviderError so the agent loop needs exactly one except clause, whichever provider is active.""" canned([], status_code=500, text="boom") with pytest.raises(ProviderError): factory().chat([{"role": "user", "content": "hi"}]) def test_fake_provider_satisfies_the_same_streaming_contract(): """The double is only useful as a stand-in if it keeps the same promises the real adapters are held to above.""" provider = FakeProvider([ScriptedTurn(text="Hello", reasoning="hmm")]) text: List[str] = [] reasoning: List[str] = [] result = provider.chat([{"role": "user", "content": "hi"}], on_text=text.append, on_reasoning=reasoning.append) assert "".join(text) == result["content"] == "Hello" assert reasoning == ["hmm"] assert result["role"] == "assistant" assert result["tool_calls"] == [] def test_fake_provider_raises_provider_error_like_the_real_ones(): provider = FakeProvider([ScriptedTurn(error="gateway exploded")]) with pytest.raises(ProviderError): provider.chat([{"role": "user", "content": "hi"}]) # --------------------------------------------------------------------------- # # Registry <-> implementation agreement # --------------------------------------------------------------------------- # @pytest.mark.parametrize("descriptor", BUILT_IN_PROVIDERS, ids=lambda d: d.id) def test_every_descriptor_builds_a_working_provider(descriptor): """A descriptor that cannot be built is a catalogue lying to the UI: Settings would list the provider and selecting it would fail at the first message.""" registry = ProviderRegistry() conf = {"base_url": "https://x.invalid/v1", "api_key": "k"} provider = registry.build(descriptor.id, conf) assert isinstance(provider, Provider) # The id, not the shared adapter class name: three descriptors map onto # OpenAICompatProvider, and usage/audit records must still tell them apart. assert provider.name == descriptor.id assert provider.model == descriptor.default_model @pytest.mark.parametrize("descriptor", BUILT_IN_PROVIDERS, ids=lambda d: d.id) def test_declared_vision_capability_matches_the_implementation(descriptor): """``supports_vision`` decides whether an image block may be sent. A descriptor claiming vision for an adapter that cannot translate the block would route image turns into a guaranteed failure.""" provider = ProviderRegistry().build(descriptor.id, {"base_url": "u", "api_key": "k"}) if descriptor.supports(ProviderCapability.VISION): assert provider.supports_vision is True def test_registry_build_never_mutates_the_caller_config(): """The routing layer runs one turn on a different model; if build() wrote that model back into the config dict it was handed, the override would silently become the user's saved default.""" registry = ProviderRegistry() conf = {"base_url": "u", "api_key": "k", "model": "configured-model"} provider = registry.build("openai_compat", conf, model="routed-model") assert provider.model == "routed-model" assert conf["model"] == "configured-model" def test_registry_rejects_an_unknown_provider_with_provider_error(): with pytest.raises(ProviderError) as excinfo: ProviderRegistry().build("does_not_exist", {}) # The message lists what IS known, so a typo in config is fixable from the # error alone without opening the source. assert "openai_compat" in str(excinfo.value)